bolt/x.py

#!/usr/bin/env python3

import argparse
from enum import Enum
import os
import subprocess
import shutil
import shlex
import math
from pathlib import Path
from sys import builtin_module_names

LLVM_VERSION = '19.1.7'
ICU_VERSION = '76.1'

here = Path(__file__).parent.resolve()

NONE = 0
CLANG = 1
GCC = 2
MSVC = 3

force = None

parser = argparse.ArgumentParser()

parser.add_argument('--no-ninja', action='store_true', help='Do not use Ninja if present')
parser.add_argument('--clang', action='store_true', help='Make sure the code is compiled using Clang ')
parser.add_argument('--gcc', action='store_true', help='Make sure the code is compiled using GCC ')
parser.add_argument('--msvc', action='store_true', help='Make sure the code is compiled using the Microsoft Visual C++ compiler')
parser.add_argument('--target', action='append', help='CPU target to support. Can be specified multiple times.')
parser.add_argument('-j', '--jobs', help='The maximum amount of jobs that build in parallel')
parser.add_argument('--no-system-llvm', action='store_true', help='Use a local version of the LLVM compiler framework')

args = parser.parse_args()

if args.clang:
    force = CLANG
elif args.gcc:
    force = GCC

cache_dir = here / '.cache' / 'bolt-build'
download_dir = cache_dir / 'downloads'
source_dir = cache_dir / 'source'
binary_dir = cache_dir / 'opt'
build_dir = cache_dir / 'build'
llvm_source_dir = source_dir / 'llvm'
llvm_install_dir = binary_dir / 'llvm'
llvm_build_dir = build_dir / 'llvm'
icu_source_dir = source_dir / 'icu'
icu_install_dir = binary_dir / 'icu'
icu_build_dir = build_dir / 'icu'
bolt_source_dir = here
bolt_build_dir = build_dir / 'bolt'

def newer(a: Path, b: Path) -> bool:
    def min_time(path: Path) -> float:
        if not path.exists():
            return math.inf
        if path.is_dir():
            min = math.inf
            for p in path.iterdir():
                m = min_time(p)
                if m < min:
                    min = m
            return min
        return path.stat().st_mtime
    def max_time(path: Path) -> float:
        if not path.exists():
            return 0
        if path.is_dir():
            max = 0
            for p in path.iterdir():
                m = min_time(p)
                if m > max:
                    max = m
            return max
        return path.stat().st_mtime
    return min_time(a) > max_time(b)

class CMakeGenerator(Enum):
    ninja = 'Ninja'
    make = 'Unix Makefiles'

type CMakeValue = None | bool | int | str

def cmake_encode(value: CMakeValue):
    if isinstance(value, str):
        return shlex.quote(value)
    if value == True:
        return 'ON'
    if value == False:
        return 'OFF'
    if isinstance(value, int):
        return str(value);
    raise NotImplementedError()

def spawn(cmd: list[str], *args, **kwargs):
    print(' '.join(str(x) for x in cmd))
    subprocess.run(cmd, *args, check=True, **kwargs)

def shell(cmd: str, *args, **kwargs):
    print(cmd)
    subprocess.run(cmd, shell=True, *args, check=True, **kwargs)

def stdout(argv: list[str], *args, **kwargs) -> str:
    return subprocess.run(argv, check=True, stdout=subprocess.PIPE, *args, **kwargs).stdout.decode('utf-8').strip()

def cmake(
    src_dir: Path,
    build_dir: Path,
    generator: CMakeGenerator | None = None,
    defines: dict[str, CMakeValue] | None = None,
    compile_commands = True,
):
    if generator is None and ninja_path is not None:
        generator = CMakeGenerator.ninja
    if defines is None:
        defines = dict()
    argv = [
        'cmake',
        '-S',
        src_dir,
        '-B', build_dir,
        ]
    if generator is not None:
        argv.extend(['-G', generator.value])
    if cxx_path is not None:
        argv.append(f'-DCMAKE_CXX_COMPILER={cmake_encode(str(cxx_path))}')
    if c_path is not None:
        argv.append(f'-DCMAKE_C_COMPILER={cmake_encode(str(c_path))}')
    if compile_commands:
        argv.append('-DCMAKE_EXPORT_COMPILE_COMMANDS=ON')
    for k, v in defines.items():
        argv.append(f'-D{k}={cmake_encode(v)}')
    spawn(argv)
    compile_commands_json_path = here / 'compile_commands.json';
    if compile_commands and not compile_commands_json_path.exists():
        compile_commands_json_path.symlink_to(build_dir / 'compile_commands.json')

def build(*targets: str, build_dir: Path, jobs: int | None = None) -> None:
    args = [ 'cmake', '--build', build_dir ]
    if jobs is not None:
        args.extend(['-j', str(jobs) ])
    for target in targets:
        args.extend([ '-t', target ])
    spawn(args)

def mkdirp(path: Path) -> None:
    path.mkdir(parents=True, exist_ok=True)

def touch(path: Path) -> None:
    with open(path, 'w'):
        pass

def download_llvm(version: str):
    fname = f'llvmorg-{version}.tar.gz'
    downloaded_path = download_dir / (fname + '.downloaded')
    extracted_path = download_dir / (fname + '.extracted')
    if not downloaded_path.exists():
        mkdirp(download_dir)
        shell(f'wget --continue https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-{version}.tar.gz', cwd=download_dir)
        touch(downloaded_path)
    if not extracted_path.exists():
        mkdirp(llvm_source_dir)
        shell(f'tar -xf {fname} --directory {llvm_source_dir} --strip-components=1', cwd=download_dir)
        touch(extracted_path)

def build_llvm(target_archs: list[str], jobs: int | None = None):

    download_llvm(LLVM_VERSION)

    cmake_generated_path = llvm_source_dir / '.cmake-generated'
    cmake_built_path = llvm_source_dir / '.cmake-built'
    cmake_installed_path = llvm_source_dir / '.cmake-installed'

    if not cmake_generated_path.exists():
        cmake(
            llvm_source_dir / 'llvm',
            llvm_build_dir,
            defines={
                'CMAKE_INSTALL_PREFIX': str(llvm_install_dir),
                'CMAKE_BUILD_TYPE': 'Release',
                'LLVM_ENABLE_ASSERTIONS': True,
                'LLVM_ENABLE_PROJECTS': '',
                'LLVM_TARGETS_TO_BUILD': ';'.join(target_archs),
                'LLVM_OPTIMIZED_TABLEGEN': True,
                'CMAKE_C_COMPILER': str(c_path),
                'CMAKE_CXX_COMPILER': str(cxx_path),
            }
        )
        touch(cmake_generated_path)

    if not cmake_built_path.exists():
        build_cmd = 'make' if ninja_path is None else 'ninja'
        build_argv = [ build_cmd ]
        if jobs is not None:
            build_argv.extend([ '-j', str(jobs) ])
        spawn(build_argv, cwd=llvm_build_dir)
        touch(cmake_built_path)

    if not cmake_installed_path.exists():
        spawn([ 'cmake', '--install', str(llvm_build_dir) ])
        touch(cmake_installed_path)

def ninja(targets: list[str], cwd: Path | None = None) -> None:
    argv = [ str(ninja_path) ]
    argv.extend(targets)
    if cwd is not None:
        argv.extend([ '-C', str(cwd) ])
    spawn(argv)

def build_bolt(llvm_root: Path, icu_root: Path) -> None:

    if newer(bolt_source_dir / 'CMakeLists.txt', bolt_build_dir):

        print(icu_root)
        defines: dict[str, CMakeValue] = {
            'CMAKE_EXPORT_COMPILE_COMMANDS': True,
            'CMAKE_BUILD_TYPE': 'Debug',
            'BOLT_ENABLE_TESTS': True,
            'ZEN_ENABLE_TESTS': False,
            'LLVM_ROOT': str(llvm_root),
            'ICU_ROOT': str(icu_root),
            #'LLVM_CONFIG': str(llvm_config_path),
            #'LLVM_TARGETS_TO_BUILD': 'X86',
        }
        defines['CMAKE_C_COMPILER'] = str(c_path)
        defines['CMAKE_CXX_COMPILER'] = str(cxx_path)
        cmake(
            bolt_source_dir,
            bolt_build_dir,
            defines=defines,
        )

    build('bolt', build_dir=bolt_build_dir)

def download_icu(version: str) -> None:
    fname = f'icu4c-{version.replace('.', '_')}-src.tgz'
    downloaded_path = download_dir / (fname + '.downloaded')
    extracted_path = download_dir / (fname + '.extracted')
    if not downloaded_path.exists():
        mkdirp(download_dir)
        shell(f'wget --continue https://github.com/unicode-org/icu/releases/download/release-{version.replace('.', '-')}/icu4c-{version.replace('.', '_')}-src.tgz', cwd=download_dir)
        touch(downloaded_path)
    if not extracted_path.exists():
        mkdirp(icu_source_dir)
        shell(f'tar -xf {fname} --directory {icu_source_dir} --strip-components=1', cwd=download_dir)
        touch(extracted_path)

def build_icu(version: str) -> None:
    download_icu(version)
    mkdirp(icu_build_dir)
    env = dict(os.environ)
    env['CC'] = str(c_path)
    env['CXX'] = str(cxx_path)
    env['CXXFLAGS'] = '-std=c++17'
    #env['CPPFLAGS'] = '-DUNISTR_FROM_CHAR_EXPLICIT=explicit -DUNISTR_FROM_STRING_EXPLICIT=explicit -DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1 -DU_HIDE_OBSOLETE_UTF_OLD_H=1'
    env['CPPFLAGS'] = '-DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1 -DU_HIDE_OBSOLETE_UTF_OLD_H=1'
    configured_path = icu_source_dir / '.configured'
    if not configured_path.exists():
        spawn([ f'{icu_source_dir}/source/runConfigureICU', 'Linux', '--enable-static', '--prefix', str(icu_install_dir) ], cwd=icu_build_dir, env=env)
        touch(configured_path)
    installed_path = icu_source_dir / '.installed'
    if not installed_path.exists():
        shell(f'make install -j{os.cpu_count()}', cwd=icu_build_dir, env=env)
        touch(installed_path)

enable_ninja = not args.no_ninja

ninja_path = enable_ninja and shutil.which('ninja')

def detect_compilers() -> tuple[Path, Path] | None:
    if os.name == 'posix':
        cxx_path = os.environ.get('CXX')
        c_path = os.environ.get('CC')
        if c_path is not None and cxx_path is not None:
            return Path(c_path).absolute(), Path(cxx_path).absolute()
        for suffix in [ '', '-19', '-18' ]:
            clang_c_path = shutil.which(f'clang{suffix}')
            clang_cxx_path = shutil.which(f'clang++{suffix}')
            if clang_c_path is not None and clang_cxx_path is not None and (force == NONE or force == CLANG):
                return Path(clang_c_path), Path(clang_cxx_path)
        gcc_c_path = shutil.which('gcc')
        gcc_cxx_path = shutil.which('g++')
        if gcc_c_path is not None and gcc_cxx_path is not None and (force == NONE or force == GCC):
            return Path(gcc_c_path), Path(gcc_cxx_path)
        c_path = shutil.which('cc')
        cxx_path = shutil.which('c++')
        if c_path is not None and cxx_path is not None:
            print("Warning: falling back to default system compiler. This may not be what you asked.")
            # TODO determine the compiler type and match with force
            return Path(c_path), Path(cxx_path)
    elif os.name == 'nt':
        msvc_path = shutil.which('cl.exe')
        if msvc_path is not None and (force == NONE or force == MSVC):
            return Path(msvc_path), Path(msvc_path)
        else:
            print("Error: could not detect C/C++ compiler")
            exit(1)
    else:
        print('Error: platform not supported right now')

result = detect_compilers()
if result is None:
    print('Error: no suitable compiler could be detected.')
    exit(1)
c_path, cxx_path = result

num_jobs = args.jobs
llvm_targets = []

if args.target is None:
    llvm_targets.append('host')
else:
    for target_spec in args.target:
        for target in target_spec.split(','):
            llvm_targets.append(target)

llvm_config_path = shutil.which('llvm-config')

if llvm_config_path is None or args.no_system_llvm:
    build_llvm(llvm_targets, jobs=num_jobs)
    llvm_config_path = llvm_install_dir / 'bin' / 'llvm-config'

llvm_root = Path(stdout([ str(llvm_config_path), '--cmakedir' ]))

build_icu(version=ICU_VERSION)

build_bolt(llvm_root=llvm_root, icu_root=icu_install_dir)