bolt/x.py

332 lines
12 KiB
Python
Raw Normal View History

#!/usr/bin/env python3
import argparse
from enum import Enum
import os
import subprocess
import shutil
import shlex
import math
from pathlib import Path
from sys import builtin_module_names
LLVM_VERSION = '19.1.7'
ICU_VERSION = '76.1'
here = Path(__file__).parent.resolve()
2025-02-28 17:25:32 +01:00
NONE = 0
CLANG = 1
GCC = 2
MSVC = 3
force = None
parser = argparse.ArgumentParser()
parser.add_argument('--no-ninja', action='store_true', help='Do not use Ninja if present')
parser.add_argument('--clang', action='store_true', help='Make sure the code is compiled using Clang ')
parser.add_argument('--gcc', action='store_true', help='Make sure the code is compiled using GCC ')
parser.add_argument('--msvc', action='store_true', help='Make sure the code is compiled using the Microsoft Visual C++ compiler')
parser.add_argument('--target', action='append', help='CPU target to support. Can be specified multiple times.')
parser.add_argument('-j', '--jobs', help='The maximum amount of jobs that build in parallel')
parser.add_argument('--no-system-llvm', action='store_true', help='Use a local version of the LLVM compiler framework')
args = parser.parse_args()
2025-02-28 17:25:32 +01:00
if args.clang:
force = CLANG
elif args.gcc:
force = GCC
cache_dir = here / '.cache' / 'bolt-build'
download_dir = cache_dir / 'downloads'
source_dir = cache_dir / 'source'
binary_dir = cache_dir / 'opt'
build_dir = cache_dir / 'build'
llvm_source_dir = source_dir / 'llvm'
llvm_install_dir = binary_dir / 'llvm'
llvm_build_dir = build_dir / 'llvm'
icu_source_dir = source_dir / 'icu'
icu_install_dir = binary_dir / 'icu'
icu_build_dir = build_dir / 'icu'
bolt_source_dir = here
bolt_build_dir = build_dir / 'bolt'
def newer(a: Path, b: Path) -> bool:
def min_time(path: Path) -> float:
if not path.exists():
return math.inf
if path.is_dir():
min = math.inf
for p in path.iterdir():
m = min_time(p)
if m < min:
min = m
return min
return path.stat().st_mtime
def max_time(path: Path) -> float:
if not path.exists():
return 0
if path.is_dir():
max = 0
for p in path.iterdir():
m = min_time(p)
if m > max:
max = m
return max
return path.stat().st_mtime
return min_time(a) > max_time(b)
class CMakeGenerator(Enum):
ninja = 'Ninja'
make = 'Unix Makefiles'
type CMakeValue = None | bool | int | str
def cmake_encode(value: CMakeValue):
if isinstance(value, str):
return shlex.quote(value)
if value == True:
return 'ON'
if value == False:
return 'OFF'
if isinstance(value, int):
return str(value);
raise NotImplementedError()
def spawn(cmd: list[str], *args, **kwargs):
print(' '.join(str(x) for x in cmd))
subprocess.run(cmd, *args, check=True, **kwargs)
def shell(cmd: str, *args, **kwargs):
print(cmd)
subprocess.run(cmd, shell=True, *args, check=True, **kwargs)
def stdout(argv: list[str], *args, **kwargs) -> str:
return subprocess.run(argv, check=True, stdout=subprocess.PIPE, *args, **kwargs).stdout.decode('utf-8').strip()
def cmake(
src_dir: Path,
build_dir: Path,
generator: CMakeGenerator | None = None,
defines: dict[str, CMakeValue] | None = None,
compile_commands = True,
):
if generator is None and ninja_path is not None:
generator = CMakeGenerator.ninja
if defines is None:
defines = dict()
argv = [
'cmake',
'-S',
src_dir,
'-B', build_dir,
]
if generator is not None:
argv.extend(['-G', generator.value])
if cxx_path is not None:
argv.append(f'-DCMAKE_CXX_COMPILER={cmake_encode(str(cxx_path))}')
if c_path is not None:
argv.append(f'-DCMAKE_C_COMPILER={cmake_encode(str(c_path))}')
if compile_commands:
argv.append('-DCMAKE_EXPORT_COMPILE_COMMANDS=ON')
for k, v in defines.items():
argv.append(f'-D{k}={cmake_encode(v)}')
spawn(argv)
compile_commands_json_path = here / 'compile_commands.json';
if compile_commands and not compile_commands_json_path.exists():
compile_commands_json_path.symlink_to(build_dir / 'compile_commands.json')
def build(*targets: str, build_dir: Path, jobs: int | None = None) -> None:
args = [ 'cmake', '--build', build_dir ]
if jobs is not None:
args.extend(['-j', str(jobs) ])
for target in targets:
args.extend([ '-t', target ])
spawn(args)
def mkdirp(path: Path) -> None:
path.mkdir(parents=True, exist_ok=True)
def touch(path: Path) -> None:
with open(path, 'w'):
pass
def download_llvm(version: str):
fname = f'llvmorg-{version}.tar.gz'
downloaded_path = download_dir / (fname + '.downloaded')
extracted_path = download_dir / (fname + '.extracted')
if not downloaded_path.exists():
mkdirp(download_dir)
shell(f'wget --continue https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-{version}.tar.gz', cwd=download_dir)
touch(downloaded_path)
if not extracted_path.exists():
mkdirp(llvm_source_dir)
shell(f'tar -xf {fname} --directory {llvm_source_dir} --strip-components=1', cwd=download_dir)
touch(extracted_path)
def build_llvm(target_archs: list[str], jobs: int | None = None):
download_llvm(LLVM_VERSION)
cmake_generated_path = llvm_source_dir / '.cmake-generated'
cmake_built_path = llvm_source_dir / '.cmake-built'
cmake_installed_path = llvm_source_dir / '.cmake-installed'
if not cmake_generated_path.exists():
cmake(
llvm_source_dir / 'llvm',
llvm_build_dir,
defines={
'CMAKE_INSTALL_PREFIX': str(llvm_install_dir),
'CMAKE_BUILD_TYPE': 'Release',
'LLVM_ENABLE_ASSERTIONS': True,
'LLVM_ENABLE_PROJECTS': '',
'LLVM_TARGETS_TO_BUILD': ';'.join(target_archs),
'LLVM_OPTIMIZED_TABLEGEN': True,
'CMAKE_C_COMPILER': str(c_path),
'CMAKE_CXX_COMPILER': str(cxx_path),
}
)
touch(cmake_generated_path)
if not cmake_built_path.exists():
build_cmd = 'make' if ninja_path is None else 'ninja'
build_argv = [ build_cmd ]
if jobs is not None:
build_argv.extend([ '-j', str(jobs) ])
spawn(build_argv, cwd=llvm_build_dir)
touch(cmake_built_path)
if not cmake_installed_path.exists():
spawn([ 'cmake', '--install', str(llvm_build_dir) ])
touch(cmake_installed_path)
def ninja(targets: list[str], cwd: Path | None = None) -> None:
argv = [ str(ninja_path) ]
argv.extend(targets)
if cwd is not None:
argv.extend([ '-C', str(cwd) ])
spawn(argv)
2025-02-28 17:25:32 +01:00
def build_bolt(llvm_root: Path, icu_root: Path) -> None:
if newer(bolt_source_dir / 'CMakeLists.txt', bolt_build_dir):
2025-02-28 17:25:32 +01:00
print(icu_root)
defines: dict[str, CMakeValue] = {
'CMAKE_EXPORT_COMPILE_COMMANDS': True,
'CMAKE_BUILD_TYPE': 'Debug',
'BOLT_ENABLE_TESTS': True,
'ZEN_ENABLE_TESTS': False,
2025-02-28 17:25:32 +01:00
'LLVM_ROOT': str(llvm_root),
'ICU_ROOT': str(icu_root),
#'LLVM_CONFIG': str(llvm_config_path),
#'LLVM_TARGETS_TO_BUILD': 'X86',
}
defines['CMAKE_C_COMPILER'] = str(c_path)
defines['CMAKE_CXX_COMPILER'] = str(cxx_path)
cmake(
bolt_source_dir,
bolt_build_dir,
defines=defines,
)
build('bolt', build_dir=bolt_build_dir)
def download_icu(version: str) -> None:
fname = f'icu4c-{version.replace('.', '_')}-src.tgz'
downloaded_path = download_dir / (fname + '.downloaded')
extracted_path = download_dir / (fname + '.extracted')
if not downloaded_path.exists():
mkdirp(download_dir)
shell(f'wget --continue https://github.com/unicode-org/icu/releases/download/release-{version.replace('.', '-')}/icu4c-{version.replace('.', '_')}-src.tgz', cwd=download_dir)
touch(downloaded_path)
if not extracted_path.exists():
mkdirp(icu_source_dir)
shell(f'tar -xf {fname} --directory {icu_source_dir} --strip-components=1', cwd=download_dir)
touch(extracted_path)
def build_icu(version: str) -> None:
download_icu(version)
mkdirp(icu_build_dir)
env = dict(os.environ)
env['CC'] = str(c_path)
env['CXX'] = str(cxx_path)
2025-02-28 17:25:32 +01:00
env['CXXFLAGS'] = '-std=c++17'
#env['CPPFLAGS'] = '-DUNISTR_FROM_CHAR_EXPLICIT=explicit -DUNISTR_FROM_STRING_EXPLICIT=explicit -DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1 -DU_HIDE_OBSOLETE_UTF_OLD_H=1'
env['CPPFLAGS'] = '-DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1 -DU_HIDE_OBSOLETE_UTF_OLD_H=1'
configured_path = icu_source_dir / '.configured'
if not configured_path.exists():
2025-02-28 17:25:32 +01:00
spawn([ f'{icu_source_dir}/source/runConfigureICU', 'Linux', '--enable-static', '--prefix', str(icu_install_dir) ], cwd=icu_build_dir, env=env)
touch(configured_path)
2025-02-28 17:25:32 +01:00
installed_path = icu_source_dir / '.installed'
if not installed_path.exists():
shell(f'make install -j{os.cpu_count()}', cwd=icu_build_dir, env=env)
touch(installed_path)
enable_ninja = not args.no_ninja
ninja_path = enable_ninja and shutil.which('ninja')
def detect_compilers() -> tuple[Path, Path] | None:
if os.name == 'posix':
2025-02-28 17:25:32 +01:00
cxx_path = os.environ.get('CXX')
c_path = os.environ.get('CC')
if c_path is not None and cxx_path is not None:
return Path(c_path).absolute(), Path(cxx_path).absolute()
for suffix in [ '', '-19', '-18' ]:
clang_c_path = shutil.which(f'clang{suffix}')
clang_cxx_path = shutil.which(f'clang++{suffix}')
if clang_c_path is not None and clang_cxx_path is not None and (force == NONE or force == CLANG):
return Path(clang_c_path), Path(clang_cxx_path)
gcc_c_path = shutil.which('gcc')
gcc_cxx_path = shutil.which('g++')
if gcc_c_path is not None and gcc_cxx_path is not None and (force == NONE or force == GCC):
return Path(gcc_c_path), Path(gcc_cxx_path)
c_path = shutil.which('cc')
cxx_path = shutil.which('c++')
if c_path is not None and cxx_path is not None:
print("Warning: falling back to default system compiler. This may not be what you asked.")
# TODO determine the compiler type and match with force
return Path(c_path), Path(cxx_path)
elif os.name == 'nt':
msvc_path = shutil.which('cl.exe')
if msvc_path is not None and (force == NONE or force == MSVC):
return Path(msvc_path), Path(msvc_path)
else:
print("Error: could not detect C/C++ compiler")
exit(1)
else:
print('Error: platform not supported right now')
result = detect_compilers()
if result is None:
print('Error: no suitable compiler could be detected.')
exit(1)
c_path, cxx_path = result
num_jobs = args.jobs
llvm_targets = []
if args.target is None:
llvm_targets.append('host')
else:
for target_spec in args.target:
for target in target_spec.split(','):
llvm_targets.append(target)
llvm_config_path = shutil.which('llvm-config')
if llvm_config_path is None or args.no_system_llvm:
build_llvm(llvm_targets, jobs=num_jobs)
llvm_config_path = llvm_install_dir / 'bin' / 'llvm-config'
llvm_root = Path(stdout([ str(llvm_config_path), '--cmakedir' ]))
build_icu(version=ICU_VERSION)
2025-02-28 17:25:32 +01:00
build_bolt(llvm_root=llvm_root, icu_root=icu_install_dir)