Update build script

- Make script build ICU
 - ...

x.py

@@ -2,16 +2,16 @@
 
 import argparse
 from enum import Enum
-from os import walk
 import os
-from re import A
 import subprocess
 import shutil
 import shlex
 import math
 from pathlib import Path
+from sys import builtin_module_names
 
-LLVM_VERSION = '18.1.0'
+LLVM_VERSION = '19.1.7'
+ICU_VERSION = '76.1'
 
 here = Path(__file__).parent.resolve()
 
@@ -19,19 +19,25 @@ parser = argparse.ArgumentParser()
 
 parser.add_argument('--no-ninja', action='store_true', help='Do not use Ninja if present')
 parser.add_argument('--clang', action='store_true', help='Make sure the code is compiled using Clang ')
-parser.add_argument('--gcc', action='store_true', help='Make sure the code is compiled using Clang ')
+parser.add_argument('--gcc', action='store_true', help='Make sure the code is compiled using GCC ')
 parser.add_argument('--msvc', action='store_true', help='Make sure the code is compiled using the Microsoft Visual C++ compiler')
 parser.add_argument('--target', action='append', help='CPU target to support. Can be specified multiple times.')
 parser.add_argument('-j', '--jobs', help='The maximum amount of jobs that build in parallel')
+parser.add_argument('--no-system-llvm', action='store_true', help='Use a local version of the LLVM compiler framework')
 
 args = parser.parse_args()
 
 cache_dir = here / '.cache' / 'bolt-build'
 download_dir = cache_dir / 'downloads'
 source_dir = cache_dir / 'source'
+binary_dir = cache_dir / 'opt'
 build_dir = cache_dir / 'build'
 llvm_source_dir = source_dir / 'llvm'
+llvm_install_dir = binary_dir / 'llvm'
 llvm_build_dir = build_dir / 'llvm'
+icu_source_dir = source_dir / 'icu'
+icu_install_dir = binary_dir / 'icu'
+icu_build_dir = build_dir / 'icu'
 bolt_source_dir = here
 bolt_build_dir = build_dir / 'bolt'
 
@@ -85,6 +91,9 @@ def shell(cmd: str, *args, **kwargs):
     print(cmd)
     subprocess.run(cmd, shell=True, *args, check=True, **kwargs)
 
+def stdout(argv: list[str], *args, **kwargs) -> str:
+    return subprocess.run(argv, check=True, stdout=subprocess.PIPE, *args, **kwargs).stdout.decode('utf-8').strip()
+
 def cmake(
     src_dir: Path,
     build_dir: Path,
@@ -98,13 +107,16 @@ def cmake(
         defines = dict()
     argv = [
         'cmake',
+        '-S',
         src_dir,
         '-B', build_dir,
         ]
     if generator is not None:
         argv.extend(['-G', generator.value])
-    if clang_cxx_path is not None:
+    if cxx_path is not None:
-        argv.append(f'-DCMAKE_CXX_COMPILER={cmake_encode(clang_cxx_path)}')
+        argv.append(f'-DCMAKE_CXX_COMPILER={cmake_encode(str(cxx_path))}')
+    if c_path is not None:
+        argv.append(f'-DCMAKE_C_COMPILER={cmake_encode(str(c_path))}')
     if compile_commands:
         argv.append('-DCMAKE_EXPORT_COMPILE_COMMANDS=ON')
     for k, v in defines.items():
@@ -122,34 +134,62 @@ def build(*targets: str, build_dir: Path, jobs: int | None = None) -> None:
         args.extend([ '-t', target ])
     spawn(args)
 
+def mkdirp(path: Path) -> None:
+    path.mkdir(parents=True, exist_ok=True)
+
+def touch(path: Path) -> None:
+    with open(path, 'w'):
+        pass
+
 def download_llvm(version: str):
-    download_dir.mkdir(parents=True, exist_ok=True)
+    fname = f'llvmorg-{version}.tar.gz'
-    shell(f'wget https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-{version}.tar.gz', cwd=download_dir)
+    downloaded_path = download_dir / (fname + '.downloaded')
-    shell(f'tar -xf llvmorg-{version}.tar.gz --directory {llvm_source_dir}', cwd=download_dir)
+    extracted_path = download_dir / (fname + '.extracted')
+    if not downloaded_path.exists():
+        mkdirp(download_dir)
+        shell(f'wget --continue https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-{version}.tar.gz', cwd=download_dir)
+        touch(downloaded_path)
+    if not extracted_path.exists():
+        mkdirp(llvm_source_dir)
+        shell(f'tar -xf {fname} --directory {llvm_source_dir} --strip-components=1', cwd=download_dir)
+        touch(extracted_path)
 
 def build_llvm(target_archs: list[str], jobs: int | None = None):
 
-    return # FIXME
-
     download_llvm(LLVM_VERSION)
 
-    cmake(
+    cmake_generated_path = llvm_source_dir / '.cmake-generated'
-        llvm_source_dir,
+    cmake_built_path = llvm_source_dir / '.cmake-built'
-        llvm_build_dir,
+    cmake_installed_path = llvm_source_dir / '.cmake-installed'
-        defines={
-            'CMAKE_BUILD_TYPE': 'Release',
-            'LLVM_ENABLE_ASSERTIONS': True,
-            'LLVM_TARGETS_TO_BUILD': ';'.join(target_archs),
-            'LLVM_OPTIMIZED_TABLEGEN': True
-        }
-    )
 
-    build_cmd = 'make' if ninja_path is None else 'ninja'
+    if not cmake_generated_path.exists():
-    build_argv = [ build_cmd ]
+        cmake(
-    if jobs is not None:
+            llvm_source_dir / 'llvm',
-        build_argv.extend([ '-j', str(jobs) ])
+            llvm_build_dir,
+            defines={
+                'CMAKE_INSTALL_PREFIX': str(llvm_install_dir),
+                'CMAKE_BUILD_TYPE': 'Release',
+                'LLVM_ENABLE_ASSERTIONS': True,
+                'LLVM_ENABLE_PROJECTS': '',
+                'LLVM_TARGETS_TO_BUILD': ';'.join(target_archs),
+                'LLVM_OPTIMIZED_TABLEGEN': True,
+                'CMAKE_C_COMPILER': str(c_path),
+                'CMAKE_CXX_COMPILER': str(cxx_path),
+            }
+        )
+        touch(cmake_generated_path)
 
-    spawn(build_argv, cwd=llvm_build_dir)
+    if not cmake_built_path.exists():
+        build_cmd = 'make' if ninja_path is None else 'ninja'
+        build_argv = [ build_cmd ]
+        if jobs is not None:
+            build_argv.extend([ '-j', str(jobs) ])
+        spawn(build_argv, cwd=llvm_build_dir)
+        touch(cmake_built_path)
+
+    if not cmake_installed_path.exists():
+        spawn([ 'cmake', '--install', str(llvm_build_dir) ])
+        touch(cmake_installed_path)
 
 def ninja(targets: list[str], cwd: Path | None = None) -> None:
     argv = [ str(ninja_path) ]
@@ -158,22 +198,22 @@ def ninja(targets: list[str], cwd: Path | None = None) -> None:
         argv.extend([ '-C', str(cwd) ])
     spawn(argv)
 
-def build_bolt(c_path: str | None = None, cxx_path: str | None = None) -> None:
+def build_bolt(llvm_root: Path) -> None:
 
     if newer(bolt_source_dir / 'CMakeLists.txt', bolt_build_dir):
 
-        defines = {
+        defines: dict[str, CMakeValue] = {
             'CMAKE_EXPORT_COMPILE_COMMANDS': True,
             'CMAKE_BUILD_TYPE': 'Debug',
             'BOLT_ENABLE_TESTS': True,
             'ZEN_ENABLE_TESTS': False,
+            'LLVM_ROOT': str(llvm_install_dir),
+            'ICU_ROOt': str(icu_install_dir),
             #'LLVM_CONFIG': str(llvm_config_path),
-            'LLVM_TARGETS_TO_BUILD': 'X86',
+            #'LLVM_TARGETS_TO_BUILD': 'X86',
         }
-        if c_path is not None:
+        defines['CMAKE_C_COMPILER'] = str(c_path)
-            defines['CMAKE_C_COMPILER'] = c_path
+        defines['CMAKE_CXX_COMPILER'] = str(cxx_path)
-        if cxx_path is not None:
-            defines['CMAKE_CXX_COMPILER'] = cxx_path
         cmake(
             bolt_source_dir,
             bolt_build_dir,
@@ -182,6 +222,35 @@ def build_bolt(c_path: str | None = None, cxx_path: str | None = None) -> None:
 
     build('bolt', build_dir=bolt_build_dir)
 
+def download_icu(version: str) -> None:
+    fname = f'icu4c-{version.replace('.', '_')}-src.tgz'
+    downloaded_path = download_dir / (fname + '.downloaded')
+    extracted_path = download_dir / (fname + '.extracted')
+    if not downloaded_path.exists():
+        mkdirp(download_dir)
+        shell(f'wget --continue https://github.com/unicode-org/icu/releases/download/release-{version.replace('.', '-')}/icu4c-{version.replace('.', '_')}-src.tgz', cwd=download_dir)
+        touch(downloaded_path)
+    if not extracted_path.exists():
+        mkdirp(icu_source_dir)
+        shell(f'tar -xf {fname} --directory {icu_source_dir} --strip-components=1', cwd=download_dir)
+        touch(extracted_path)
+
+def build_icu(version: str) -> None:
+    download_icu(version)
+    mkdirp(icu_build_dir)
+    env = dict(os.environ)
+    env['CC'] = str(c_path)
+    env['CXX'] = str(cxx_path)
+    env['CPPFLAGS'] = '-DUNISTR_FROM_CHAR_EXPLICIT=explicit -DUNISTR_FROM_STRING_EXPLICIT=explicit -DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1 -DU_HIDE_OBSOLETE_UTF_OLD_H=1'
+    configured_path = icu_source_dir / '.configured'
+    if not configured_path.exists():
+        shell(f'{icu_source_dir}/source/runConfigureICU Linux', cwd=icu_build_dir, env=env)
+        touch(configured_path)
+    built_path = icu_source_dir / '.built'
+    if not built_path.exists():
+        shell(f'make -j{os.cpu_count()}', cwd=icu_build_dir, env=env)
+        touch(built_path)
+
 enable_ninja = not args.no_ninja
 
 NONE = 0
@@ -193,41 +262,38 @@ force = NONE
 
 ninja_path = enable_ninja and shutil.which('ninja')
 
-c_path = None
+def detect_compilers() -> tuple[Path, Path] | None:
-cxx_path = None
+    if os.name == 'posix':
-
+        for suffix in [ '', '-19', '-18' ]:
-if os.name == 'posix':
+            clang_c_path = shutil.which(f'clang{suffix}')
-    clang_c_path = shutil.which('clang')
+            clang_cxx_path = shutil.which(f'clang++{suffix}')
-    clang_cxx_path = shutil.which('clang++')
-    if clang_c_path is not None and clang_cxx_path is not None and (force == NONE or force == CLANG):
-        c_path = clang_c_path
-        cxx_path = clang_cxx_path
-    else:
-        for version in [ '18', '19' ]:
-            clang_c_path = shutil.which(f'clang-{version}')
-            clang_cxx_path = shutil.which(f'clang++-{version}')
             if clang_c_path is not None and clang_cxx_path is not None and (force == NONE or force == CLANG):
-                c_path = clang_c_path
+                return Path(clang_c_path), Path(clang_cxx_path)
-                cxx_path = clang_cxx_path
-                break
-    if c_path is None or cxx_path is None:
         gcc_c_path = shutil.which('gcc')
         gcc_cxx_path = shutil.which('g++')
         if gcc_c_path is not None and gcc_cxx_path is not None and (force == NONE or force == GCC):
-            c_path = gcc_c_path
+            return Path(gcc_c_path), Path(gcc_cxx_path)
-            cxx_path = gcc_cxx_path
+        c_path = shutil.which('cc')
+        cxx_path = shutil.which('c++')
+        if c_path is not None and cxx_path is not None:
+            print("Warning: falling back to default system compiler. This may not be what you asked.")
+            # TODO determine the compiler type and match with force
+            return Path(c_path), Path(cxx_path)
+    elif os.name == 'nt':
+        msvc_path = shutil.which('cl.exe')
+        if msvc_path is not None and (force == NONE or force == MSVC):
+            return Path(msvc_path), Path(msvc_path)
         else:
-            print('Going to use platform default compiler')
+            print("Error: could not detect C/C++ compiler")
-elif os.name == 'nt':
+            exit(1)
-    msvc_path = shutil.which('cl.exe')
-    if msvc_path is not None and (force == NONE or force == MSVC):
-        c_path = msvc_path
-        cxx_path = msvc_path
     else:
-        print('Going to use platform default compiler')
+        print('Error: platform not supported right now')
-else:
+
-    print('Platform not supported right now')
+result = detect_compilers()
+if result is None:
+    print('Error: no suitable compiler could be detected.')
     exit(1)
+c_path, cxx_path = result
 
 num_jobs = args.jobs
 llvm_targets = []
@@ -239,14 +305,17 @@ else:
         for target in target_spec.split(','):
             llvm_targets.append(target)
 
-llvm_config_path = shutil.which('llvm-config-18')
+llvm_config_path = shutil.which('llvm-config')
 
-if llvm_config_path is None:
+if llvm_config_path is None or args.no_system_llvm:
     build_llvm(llvm_targets, jobs=num_jobs)
-    llvm_config_path = llvm_build_dir / 'bin' / 'llvm-config'
+    llvm_config_path = llvm_install_dir / 'bin' / 'llvm-config'
 
-build_bolt(
+llvm_root = Path(stdout([ str(llvm_config_path), '--cmakedir' ]))
-    c_path=c_path,
+
-    cxx_path=cxx_path,
+print(llvm_root)
-)
+
+build_icu(version=ICU_VERSION)
+
+build_bolt(llvm_root=llvm_root)