Update build script
- Make script build ICU - ...
This commit is contained in:
parent
2a892b3d48
commit
3170103e4a
SSH key fingerprint:
SHA256:dIg0ywU1OP+ZYifrYxy8c5esO72cIKB+4/9wkZj1VaY
1 changed files with 137 additions and 68 deletions
205
x.py
205
x.py
|
|
@ -2,16 +2,16 @@
|
|||
|
||||
import argparse
|
||||
from enum import Enum
|
||||
from os import walk
|
||||
import os
|
||||
from re import A
|
||||
import subprocess
|
||||
import shutil
|
||||
import shlex
|
||||
import math
|
||||
from pathlib import Path
|
||||
from sys import builtin_module_names
|
||||
|
||||
LLVM_VERSION = '18.1.0'
|
||||
LLVM_VERSION = '19.1.7'
|
||||
ICU_VERSION = '76.1'
|
||||
|
||||
here = Path(__file__).parent.resolve()
|
||||
|
||||
|
|
@ -19,19 +19,25 @@ parser = argparse.ArgumentParser()
|
|||
|
||||
parser.add_argument('--no-ninja', action='store_true', help='Do not use Ninja if present')
|
||||
parser.add_argument('--clang', action='store_true', help='Make sure the code is compiled using Clang ')
|
||||
parser.add_argument('--gcc', action='store_true', help='Make sure the code is compiled using Clang ')
|
||||
parser.add_argument('--gcc', action='store_true', help='Make sure the code is compiled using GCC ')
|
||||
parser.add_argument('--msvc', action='store_true', help='Make sure the code is compiled using the Microsoft Visual C++ compiler')
|
||||
parser.add_argument('--target', action='append', help='CPU target to support. Can be specified multiple times.')
|
||||
parser.add_argument('-j', '--jobs', help='The maximum amount of jobs that build in parallel')
|
||||
parser.add_argument('--no-system-llvm', action='store_true', help='Use a local version of the LLVM compiler framework')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
cache_dir = here / '.cache' / 'bolt-build'
|
||||
download_dir = cache_dir / 'downloads'
|
||||
source_dir = cache_dir / 'source'
|
||||
binary_dir = cache_dir / 'opt'
|
||||
build_dir = cache_dir / 'build'
|
||||
llvm_source_dir = source_dir / 'llvm'
|
||||
llvm_install_dir = binary_dir / 'llvm'
|
||||
llvm_build_dir = build_dir / 'llvm'
|
||||
icu_source_dir = source_dir / 'icu'
|
||||
icu_install_dir = binary_dir / 'icu'
|
||||
icu_build_dir = build_dir / 'icu'
|
||||
bolt_source_dir = here
|
||||
bolt_build_dir = build_dir / 'bolt'
|
||||
|
||||
|
|
@ -85,6 +91,9 @@ def shell(cmd: str, *args, **kwargs):
|
|||
print(cmd)
|
||||
subprocess.run(cmd, shell=True, *args, check=True, **kwargs)
|
||||
|
||||
def stdout(argv: list[str], *args, **kwargs) -> str:
|
||||
return subprocess.run(argv, check=True, stdout=subprocess.PIPE, *args, **kwargs).stdout.decode('utf-8').strip()
|
||||
|
||||
def cmake(
|
||||
src_dir: Path,
|
||||
build_dir: Path,
|
||||
|
|
@ -98,13 +107,16 @@ def cmake(
|
|||
defines = dict()
|
||||
argv = [
|
||||
'cmake',
|
||||
'-S',
|
||||
src_dir,
|
||||
'-B', build_dir,
|
||||
]
|
||||
if generator is not None:
|
||||
argv.extend(['-G', generator.value])
|
||||
if clang_cxx_path is not None:
|
||||
argv.append(f'-DCMAKE_CXX_COMPILER={cmake_encode(clang_cxx_path)}')
|
||||
if cxx_path is not None:
|
||||
argv.append(f'-DCMAKE_CXX_COMPILER={cmake_encode(str(cxx_path))}')
|
||||
if c_path is not None:
|
||||
argv.append(f'-DCMAKE_C_COMPILER={cmake_encode(str(c_path))}')
|
||||
if compile_commands:
|
||||
argv.append('-DCMAKE_EXPORT_COMPILE_COMMANDS=ON')
|
||||
for k, v in defines.items():
|
||||
|
|
@ -122,34 +134,62 @@ def build(*targets: str, build_dir: Path, jobs: int | None = None) -> None:
|
|||
args.extend([ '-t', target ])
|
||||
spawn(args)
|
||||
|
||||
def mkdirp(path: Path) -> None:
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def touch(path: Path) -> None:
|
||||
with open(path, 'w'):
|
||||
pass
|
||||
|
||||
def download_llvm(version: str):
|
||||
download_dir.mkdir(parents=True, exist_ok=True)
|
||||
shell(f'wget https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-{version}.tar.gz', cwd=download_dir)
|
||||
shell(f'tar -xf llvmorg-{version}.tar.gz --directory {llvm_source_dir}', cwd=download_dir)
|
||||
fname = f'llvmorg-{version}.tar.gz'
|
||||
downloaded_path = download_dir / (fname + '.downloaded')
|
||||
extracted_path = download_dir / (fname + '.extracted')
|
||||
if not downloaded_path.exists():
|
||||
mkdirp(download_dir)
|
||||
shell(f'wget --continue https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-{version}.tar.gz', cwd=download_dir)
|
||||
touch(downloaded_path)
|
||||
if not extracted_path.exists():
|
||||
mkdirp(llvm_source_dir)
|
||||
shell(f'tar -xf {fname} --directory {llvm_source_dir} --strip-components=1', cwd=download_dir)
|
||||
touch(extracted_path)
|
||||
|
||||
def build_llvm(target_archs: list[str], jobs: int | None = None):
|
||||
|
||||
return # FIXME
|
||||
|
||||
download_llvm(LLVM_VERSION)
|
||||
|
||||
cmake(
|
||||
llvm_source_dir,
|
||||
llvm_build_dir,
|
||||
defines={
|
||||
'CMAKE_BUILD_TYPE': 'Release',
|
||||
'LLVM_ENABLE_ASSERTIONS': True,
|
||||
'LLVM_TARGETS_TO_BUILD': ';'.join(target_archs),
|
||||
'LLVM_OPTIMIZED_TABLEGEN': True
|
||||
}
|
||||
)
|
||||
cmake_generated_path = llvm_source_dir / '.cmake-generated'
|
||||
cmake_built_path = llvm_source_dir / '.cmake-built'
|
||||
cmake_installed_path = llvm_source_dir / '.cmake-installed'
|
||||
|
||||
build_cmd = 'make' if ninja_path is None else 'ninja'
|
||||
build_argv = [ build_cmd ]
|
||||
if jobs is not None:
|
||||
build_argv.extend([ '-j', str(jobs) ])
|
||||
if not cmake_generated_path.exists():
|
||||
cmake(
|
||||
llvm_source_dir / 'llvm',
|
||||
llvm_build_dir,
|
||||
defines={
|
||||
'CMAKE_INSTALL_PREFIX': str(llvm_install_dir),
|
||||
'CMAKE_BUILD_TYPE': 'Release',
|
||||
'LLVM_ENABLE_ASSERTIONS': True,
|
||||
'LLVM_ENABLE_PROJECTS': '',
|
||||
'LLVM_TARGETS_TO_BUILD': ';'.join(target_archs),
|
||||
'LLVM_OPTIMIZED_TABLEGEN': True,
|
||||
'CMAKE_C_COMPILER': str(c_path),
|
||||
'CMAKE_CXX_COMPILER': str(cxx_path),
|
||||
}
|
||||
)
|
||||
touch(cmake_generated_path)
|
||||
|
||||
spawn(build_argv, cwd=llvm_build_dir)
|
||||
if not cmake_built_path.exists():
|
||||
build_cmd = 'make' if ninja_path is None else 'ninja'
|
||||
build_argv = [ build_cmd ]
|
||||
if jobs is not None:
|
||||
build_argv.extend([ '-j', str(jobs) ])
|
||||
spawn(build_argv, cwd=llvm_build_dir)
|
||||
touch(cmake_built_path)
|
||||
|
||||
if not cmake_installed_path.exists():
|
||||
spawn([ 'cmake', '--install', str(llvm_build_dir) ])
|
||||
touch(cmake_installed_path)
|
||||
|
||||
def ninja(targets: list[str], cwd: Path | None = None) -> None:
|
||||
argv = [ str(ninja_path) ]
|
||||
|
|
@ -158,22 +198,22 @@ def ninja(targets: list[str], cwd: Path | None = None) -> None:
|
|||
argv.extend([ '-C', str(cwd) ])
|
||||
spawn(argv)
|
||||
|
||||
def build_bolt(c_path: str | None = None, cxx_path: str | None = None) -> None:
|
||||
def build_bolt(llvm_root: Path) -> None:
|
||||
|
||||
if newer(bolt_source_dir / 'CMakeLists.txt', bolt_build_dir):
|
||||
|
||||
defines = {
|
||||
defines: dict[str, CMakeValue] = {
|
||||
'CMAKE_EXPORT_COMPILE_COMMANDS': True,
|
||||
'CMAKE_BUILD_TYPE': 'Debug',
|
||||
'BOLT_ENABLE_TESTS': True,
|
||||
'ZEN_ENABLE_TESTS': False,
|
||||
'LLVM_ROOT': str(llvm_install_dir),
|
||||
'ICU_ROOt': str(icu_install_dir),
|
||||
#'LLVM_CONFIG': str(llvm_config_path),
|
||||
'LLVM_TARGETS_TO_BUILD': 'X86',
|
||||
#'LLVM_TARGETS_TO_BUILD': 'X86',
|
||||
}
|
||||
if c_path is not None:
|
||||
defines['CMAKE_C_COMPILER'] = c_path
|
||||
if cxx_path is not None:
|
||||
defines['CMAKE_CXX_COMPILER'] = cxx_path
|
||||
defines['CMAKE_C_COMPILER'] = str(c_path)
|
||||
defines['CMAKE_CXX_COMPILER'] = str(cxx_path)
|
||||
cmake(
|
||||
bolt_source_dir,
|
||||
bolt_build_dir,
|
||||
|
|
@ -182,6 +222,35 @@ def build_bolt(c_path: str | None = None, cxx_path: str | None = None) -> None:
|
|||
|
||||
build('bolt', build_dir=bolt_build_dir)
|
||||
|
||||
def download_icu(version: str) -> None:
|
||||
fname = f'icu4c-{version.replace('.', '_')}-src.tgz'
|
||||
downloaded_path = download_dir / (fname + '.downloaded')
|
||||
extracted_path = download_dir / (fname + '.extracted')
|
||||
if not downloaded_path.exists():
|
||||
mkdirp(download_dir)
|
||||
shell(f'wget --continue https://github.com/unicode-org/icu/releases/download/release-{version.replace('.', '-')}/icu4c-{version.replace('.', '_')}-src.tgz', cwd=download_dir)
|
||||
touch(downloaded_path)
|
||||
if not extracted_path.exists():
|
||||
mkdirp(icu_source_dir)
|
||||
shell(f'tar -xf {fname} --directory {icu_source_dir} --strip-components=1', cwd=download_dir)
|
||||
touch(extracted_path)
|
||||
|
||||
def build_icu(version: str) -> None:
|
||||
download_icu(version)
|
||||
mkdirp(icu_build_dir)
|
||||
env = dict(os.environ)
|
||||
env['CC'] = str(c_path)
|
||||
env['CXX'] = str(cxx_path)
|
||||
env['CPPFLAGS'] = '-DUNISTR_FROM_CHAR_EXPLICIT=explicit -DUNISTR_FROM_STRING_EXPLICIT=explicit -DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1 -DU_HIDE_OBSOLETE_UTF_OLD_H=1'
|
||||
configured_path = icu_source_dir / '.configured'
|
||||
if not configured_path.exists():
|
||||
shell(f'{icu_source_dir}/source/runConfigureICU Linux', cwd=icu_build_dir, env=env)
|
||||
touch(configured_path)
|
||||
built_path = icu_source_dir / '.built'
|
||||
if not built_path.exists():
|
||||
shell(f'make -j{os.cpu_count()}', cwd=icu_build_dir, env=env)
|
||||
touch(built_path)
|
||||
|
||||
enable_ninja = not args.no_ninja
|
||||
|
||||
NONE = 0
|
||||
|
|
@ -193,41 +262,38 @@ force = NONE
|
|||
|
||||
ninja_path = enable_ninja and shutil.which('ninja')
|
||||
|
||||
c_path = None
|
||||
cxx_path = None
|
||||
|
||||
if os.name == 'posix':
|
||||
clang_c_path = shutil.which('clang')
|
||||
clang_cxx_path = shutil.which('clang++')
|
||||
if clang_c_path is not None and clang_cxx_path is not None and (force == NONE or force == CLANG):
|
||||
c_path = clang_c_path
|
||||
cxx_path = clang_cxx_path
|
||||
else:
|
||||
for version in [ '18', '19' ]:
|
||||
clang_c_path = shutil.which(f'clang-{version}')
|
||||
clang_cxx_path = shutil.which(f'clang++-{version}')
|
||||
def detect_compilers() -> tuple[Path, Path] | None:
|
||||
if os.name == 'posix':
|
||||
for suffix in [ '', '-19', '-18' ]:
|
||||
clang_c_path = shutil.which(f'clang{suffix}')
|
||||
clang_cxx_path = shutil.which(f'clang++{suffix}')
|
||||
if clang_c_path is not None and clang_cxx_path is not None and (force == NONE or force == CLANG):
|
||||
c_path = clang_c_path
|
||||
cxx_path = clang_cxx_path
|
||||
break
|
||||
if c_path is None or cxx_path is None:
|
||||
return Path(clang_c_path), Path(clang_cxx_path)
|
||||
gcc_c_path = shutil.which('gcc')
|
||||
gcc_cxx_path = shutil.which('g++')
|
||||
if gcc_c_path is not None and gcc_cxx_path is not None and (force == NONE or force == GCC):
|
||||
c_path = gcc_c_path
|
||||
cxx_path = gcc_cxx_path
|
||||
return Path(gcc_c_path), Path(gcc_cxx_path)
|
||||
c_path = shutil.which('cc')
|
||||
cxx_path = shutil.which('c++')
|
||||
if c_path is not None and cxx_path is not None:
|
||||
print("Warning: falling back to default system compiler. This may not be what you asked.")
|
||||
# TODO determine the compiler type and match with force
|
||||
return Path(c_path), Path(cxx_path)
|
||||
elif os.name == 'nt':
|
||||
msvc_path = shutil.which('cl.exe')
|
||||
if msvc_path is not None and (force == NONE or force == MSVC):
|
||||
return Path(msvc_path), Path(msvc_path)
|
||||
else:
|
||||
print('Going to use platform default compiler')
|
||||
elif os.name == 'nt':
|
||||
msvc_path = shutil.which('cl.exe')
|
||||
if msvc_path is not None and (force == NONE or force == MSVC):
|
||||
c_path = msvc_path
|
||||
cxx_path = msvc_path
|
||||
print("Error: could not detect C/C++ compiler")
|
||||
exit(1)
|
||||
else:
|
||||
print('Going to use platform default compiler')
|
||||
else:
|
||||
print('Platform not supported right now')
|
||||
print('Error: platform not supported right now')
|
||||
|
||||
result = detect_compilers()
|
||||
if result is None:
|
||||
print('Error: no suitable compiler could be detected.')
|
||||
exit(1)
|
||||
c_path, cxx_path = result
|
||||
|
||||
num_jobs = args.jobs
|
||||
llvm_targets = []
|
||||
|
|
@ -239,14 +305,17 @@ else:
|
|||
for target in target_spec.split(','):
|
||||
llvm_targets.append(target)
|
||||
|
||||
llvm_config_path = shutil.which('llvm-config-18')
|
||||
llvm_config_path = shutil.which('llvm-config')
|
||||
|
||||
if llvm_config_path is None:
|
||||
if llvm_config_path is None or args.no_system_llvm:
|
||||
build_llvm(llvm_targets, jobs=num_jobs)
|
||||
llvm_config_path = llvm_build_dir / 'bin' / 'llvm-config'
|
||||
llvm_config_path = llvm_install_dir / 'bin' / 'llvm-config'
|
||||
|
||||
build_bolt(
|
||||
c_path=c_path,
|
||||
cxx_path=cxx_path,
|
||||
)
|
||||
llvm_root = Path(stdout([ str(llvm_config_path), '--cmakedir' ]))
|
||||
|
||||
print(llvm_root)
|
||||
|
||||
build_icu(version=ICU_VERSION)
|
||||
|
||||
build_bolt(llvm_root=llvm_root)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue