From 3170103e4ab75c8811c6c565321fbbc847e1c282 Mon Sep 17 00:00:00 2001 From: Sam Vervaeck Date: Thu, 27 Feb 2025 13:32:45 +0100 Subject: [PATCH] Update build script - Make script build ICU - ... --- x.py | 205 +++++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 137 insertions(+), 68 deletions(-) diff --git a/x.py b/x.py index 52e467f86..37eb7331c 100755 --- a/x.py +++ b/x.py @@ -2,16 +2,16 @@ import argparse from enum import Enum -from os import walk import os -from re import A import subprocess import shutil import shlex import math from pathlib import Path +from sys import builtin_module_names -LLVM_VERSION = '18.1.0' +LLVM_VERSION = '19.1.7' +ICU_VERSION = '76.1' here = Path(__file__).parent.resolve() @@ -19,19 +19,25 @@ parser = argparse.ArgumentParser() parser.add_argument('--no-ninja', action='store_true', help='Do not use Ninja if present') parser.add_argument('--clang', action='store_true', help='Make sure the code is compiled using Clang ') -parser.add_argument('--gcc', action='store_true', help='Make sure the code is compiled using Clang ') +parser.add_argument('--gcc', action='store_true', help='Make sure the code is compiled using GCC ') parser.add_argument('--msvc', action='store_true', help='Make sure the code is compiled using the Microsoft Visual C++ compiler') parser.add_argument('--target', action='append', help='CPU target to support. Can be specified multiple times.') parser.add_argument('-j', '--jobs', help='The maximum amount of jobs that build in parallel') +parser.add_argument('--no-system-llvm', action='store_true', help='Use a local version of the LLVM compiler framework') args = parser.parse_args() cache_dir = here / '.cache' / 'bolt-build' download_dir = cache_dir / 'downloads' source_dir = cache_dir / 'source' +binary_dir = cache_dir / 'opt' build_dir = cache_dir / 'build' llvm_source_dir = source_dir / 'llvm' +llvm_install_dir = binary_dir / 'llvm' llvm_build_dir = build_dir / 'llvm' +icu_source_dir = source_dir / 'icu' +icu_install_dir = binary_dir / 'icu' +icu_build_dir = build_dir / 'icu' bolt_source_dir = here bolt_build_dir = build_dir / 'bolt' @@ -85,6 +91,9 @@ def shell(cmd: str, *args, **kwargs): print(cmd) subprocess.run(cmd, shell=True, *args, check=True, **kwargs) +def stdout(argv: list[str], *args, **kwargs) -> str: + return subprocess.run(argv, check=True, stdout=subprocess.PIPE, *args, **kwargs).stdout.decode('utf-8').strip() + def cmake( src_dir: Path, build_dir: Path, @@ -98,13 +107,16 @@ def cmake( defines = dict() argv = [ 'cmake', + '-S', src_dir, '-B', build_dir, ] if generator is not None: argv.extend(['-G', generator.value]) - if clang_cxx_path is not None: - argv.append(f'-DCMAKE_CXX_COMPILER={cmake_encode(clang_cxx_path)}') + if cxx_path is not None: + argv.append(f'-DCMAKE_CXX_COMPILER={cmake_encode(str(cxx_path))}') + if c_path is not None: + argv.append(f'-DCMAKE_C_COMPILER={cmake_encode(str(c_path))}') if compile_commands: argv.append('-DCMAKE_EXPORT_COMPILE_COMMANDS=ON') for k, v in defines.items(): @@ -122,34 +134,62 @@ def build(*targets: str, build_dir: Path, jobs: int | None = None) -> None: args.extend([ '-t', target ]) spawn(args) +def mkdirp(path: Path) -> None: + path.mkdir(parents=True, exist_ok=True) + +def touch(path: Path) -> None: + with open(path, 'w'): + pass + def download_llvm(version: str): - download_dir.mkdir(parents=True, exist_ok=True) - shell(f'wget https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-{version}.tar.gz', cwd=download_dir) - shell(f'tar -xf llvmorg-{version}.tar.gz --directory {llvm_source_dir}', cwd=download_dir) + fname = f'llvmorg-{version}.tar.gz' + downloaded_path = download_dir / (fname + '.downloaded') + extracted_path = download_dir / (fname + '.extracted') + if not downloaded_path.exists(): + mkdirp(download_dir) + shell(f'wget --continue https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-{version}.tar.gz', cwd=download_dir) + touch(downloaded_path) + if not extracted_path.exists(): + mkdirp(llvm_source_dir) + shell(f'tar -xf {fname} --directory {llvm_source_dir} --strip-components=1', cwd=download_dir) + touch(extracted_path) def build_llvm(target_archs: list[str], jobs: int | None = None): - return # FIXME - download_llvm(LLVM_VERSION) - cmake( - llvm_source_dir, - llvm_build_dir, - defines={ - 'CMAKE_BUILD_TYPE': 'Release', - 'LLVM_ENABLE_ASSERTIONS': True, - 'LLVM_TARGETS_TO_BUILD': ';'.join(target_archs), - 'LLVM_OPTIMIZED_TABLEGEN': True - } - ) + cmake_generated_path = llvm_source_dir / '.cmake-generated' + cmake_built_path = llvm_source_dir / '.cmake-built' + cmake_installed_path = llvm_source_dir / '.cmake-installed' - build_cmd = 'make' if ninja_path is None else 'ninja' - build_argv = [ build_cmd ] - if jobs is not None: - build_argv.extend([ '-j', str(jobs) ]) + if not cmake_generated_path.exists(): + cmake( + llvm_source_dir / 'llvm', + llvm_build_dir, + defines={ + 'CMAKE_INSTALL_PREFIX': str(llvm_install_dir), + 'CMAKE_BUILD_TYPE': 'Release', + 'LLVM_ENABLE_ASSERTIONS': True, + 'LLVM_ENABLE_PROJECTS': '', + 'LLVM_TARGETS_TO_BUILD': ';'.join(target_archs), + 'LLVM_OPTIMIZED_TABLEGEN': True, + 'CMAKE_C_COMPILER': str(c_path), + 'CMAKE_CXX_COMPILER': str(cxx_path), + } + ) + touch(cmake_generated_path) - spawn(build_argv, cwd=llvm_build_dir) + if not cmake_built_path.exists(): + build_cmd = 'make' if ninja_path is None else 'ninja' + build_argv = [ build_cmd ] + if jobs is not None: + build_argv.extend([ '-j', str(jobs) ]) + spawn(build_argv, cwd=llvm_build_dir) + touch(cmake_built_path) + + if not cmake_installed_path.exists(): + spawn([ 'cmake', '--install', str(llvm_build_dir) ]) + touch(cmake_installed_path) def ninja(targets: list[str], cwd: Path | None = None) -> None: argv = [ str(ninja_path) ] @@ -158,22 +198,22 @@ def ninja(targets: list[str], cwd: Path | None = None) -> None: argv.extend([ '-C', str(cwd) ]) spawn(argv) -def build_bolt(c_path: str | None = None, cxx_path: str | None = None) -> None: +def build_bolt(llvm_root: Path) -> None: if newer(bolt_source_dir / 'CMakeLists.txt', bolt_build_dir): - defines = { + defines: dict[str, CMakeValue] = { 'CMAKE_EXPORT_COMPILE_COMMANDS': True, 'CMAKE_BUILD_TYPE': 'Debug', 'BOLT_ENABLE_TESTS': True, 'ZEN_ENABLE_TESTS': False, + 'LLVM_ROOT': str(llvm_install_dir), + 'ICU_ROOt': str(icu_install_dir), #'LLVM_CONFIG': str(llvm_config_path), - 'LLVM_TARGETS_TO_BUILD': 'X86', + #'LLVM_TARGETS_TO_BUILD': 'X86', } - if c_path is not None: - defines['CMAKE_C_COMPILER'] = c_path - if cxx_path is not None: - defines['CMAKE_CXX_COMPILER'] = cxx_path + defines['CMAKE_C_COMPILER'] = str(c_path) + defines['CMAKE_CXX_COMPILER'] = str(cxx_path) cmake( bolt_source_dir, bolt_build_dir, @@ -182,6 +222,35 @@ def build_bolt(c_path: str | None = None, cxx_path: str | None = None) -> None: build('bolt', build_dir=bolt_build_dir) +def download_icu(version: str) -> None: + fname = f'icu4c-{version.replace('.', '_')}-src.tgz' + downloaded_path = download_dir / (fname + '.downloaded') + extracted_path = download_dir / (fname + '.extracted') + if not downloaded_path.exists(): + mkdirp(download_dir) + shell(f'wget --continue https://github.com/unicode-org/icu/releases/download/release-{version.replace('.', '-')}/icu4c-{version.replace('.', '_')}-src.tgz', cwd=download_dir) + touch(downloaded_path) + if not extracted_path.exists(): + mkdirp(icu_source_dir) + shell(f'tar -xf {fname} --directory {icu_source_dir} --strip-components=1', cwd=download_dir) + touch(extracted_path) + +def build_icu(version: str) -> None: + download_icu(version) + mkdirp(icu_build_dir) + env = dict(os.environ) + env['CC'] = str(c_path) + env['CXX'] = str(cxx_path) + env['CPPFLAGS'] = '-DUNISTR_FROM_CHAR_EXPLICIT=explicit -DUNISTR_FROM_STRING_EXPLICIT=explicit -DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1 -DU_HIDE_OBSOLETE_UTF_OLD_H=1' + configured_path = icu_source_dir / '.configured' + if not configured_path.exists(): + shell(f'{icu_source_dir}/source/runConfigureICU Linux', cwd=icu_build_dir, env=env) + touch(configured_path) + built_path = icu_source_dir / '.built' + if not built_path.exists(): + shell(f'make -j{os.cpu_count()}', cwd=icu_build_dir, env=env) + touch(built_path) + enable_ninja = not args.no_ninja NONE = 0 @@ -193,41 +262,38 @@ force = NONE ninja_path = enable_ninja and shutil.which('ninja') -c_path = None -cxx_path = None - -if os.name == 'posix': - clang_c_path = shutil.which('clang') - clang_cxx_path = shutil.which('clang++') - if clang_c_path is not None and clang_cxx_path is not None and (force == NONE or force == CLANG): - c_path = clang_c_path - cxx_path = clang_cxx_path - else: - for version in [ '18', '19' ]: - clang_c_path = shutil.which(f'clang-{version}') - clang_cxx_path = shutil.which(f'clang++-{version}') +def detect_compilers() -> tuple[Path, Path] | None: + if os.name == 'posix': + for suffix in [ '', '-19', '-18' ]: + clang_c_path = shutil.which(f'clang{suffix}') + clang_cxx_path = shutil.which(f'clang++{suffix}') if clang_c_path is not None and clang_cxx_path is not None and (force == NONE or force == CLANG): - c_path = clang_c_path - cxx_path = clang_cxx_path - break - if c_path is None or cxx_path is None: + return Path(clang_c_path), Path(clang_cxx_path) gcc_c_path = shutil.which('gcc') gcc_cxx_path = shutil.which('g++') if gcc_c_path is not None and gcc_cxx_path is not None and (force == NONE or force == GCC): - c_path = gcc_c_path - cxx_path = gcc_cxx_path + return Path(gcc_c_path), Path(gcc_cxx_path) + c_path = shutil.which('cc') + cxx_path = shutil.which('c++') + if c_path is not None and cxx_path is not None: + print("Warning: falling back to default system compiler. This may not be what you asked.") + # TODO determine the compiler type and match with force + return Path(c_path), Path(cxx_path) + elif os.name == 'nt': + msvc_path = shutil.which('cl.exe') + if msvc_path is not None and (force == NONE or force == MSVC): + return Path(msvc_path), Path(msvc_path) else: - print('Going to use platform default compiler') -elif os.name == 'nt': - msvc_path = shutil.which('cl.exe') - if msvc_path is not None and (force == NONE or force == MSVC): - c_path = msvc_path - cxx_path = msvc_path + print("Error: could not detect C/C++ compiler") + exit(1) else: - print('Going to use platform default compiler') -else: - print('Platform not supported right now') + print('Error: platform not supported right now') + +result = detect_compilers() +if result is None: + print('Error: no suitable compiler could be detected.') exit(1) +c_path, cxx_path = result num_jobs = args.jobs llvm_targets = [] @@ -239,14 +305,17 @@ else: for target in target_spec.split(','): llvm_targets.append(target) -llvm_config_path = shutil.which('llvm-config-18') +llvm_config_path = shutil.which('llvm-config') -if llvm_config_path is None: +if llvm_config_path is None or args.no_system_llvm: build_llvm(llvm_targets, jobs=num_jobs) - llvm_config_path = llvm_build_dir / 'bin' / 'llvm-config' + llvm_config_path = llvm_install_dir / 'bin' / 'llvm-config' -build_bolt( - c_path=c_path, - cxx_path=cxx_path, -) +llvm_root = Path(stdout([ str(llvm_config_path), '--cmakedir' ])) + +print(llvm_root) + +build_icu(version=ICU_VERSION) + +build_bolt(llvm_root=llvm_root)