diff --git a/.devcontainer/docker_dev_container/devcontainer.json b/.devcontainer/docker_dev_container/devcontainer.json index 6707a25..082f230 100644 --- a/.devcontainer/docker_dev_container/devcontainer.json +++ b/.devcontainer/docker_dev_container/devcontainer.json @@ -8,6 +8,7 @@ "enableNonRootDocker": true, "moby": false }, - "ghcr.io/devcontainers-extra/features/pre-commit:2": {} + "ghcr.io/devcontainers-extra/features/pre-commit:2": {}, + "ghcr.io/anthropics/devcontainer-features/claude-code:1.0": {} } } diff --git a/Dockerfile.clang.ubuntu b/Dockerfile.clang.ubuntu new file mode 100644 index 0000000..bede1d8 --- /dev/null +++ b/Dockerfile.clang.ubuntu @@ -0,0 +1,100 @@ +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# Dockerfile for building LLVM/Clang from source on Ubuntu 24.04 +# +# This Dockerfile builds a custom Clang compiler tagged with a specific release. +# +# The build is setup to minimize build time, thus it disables unnecessary components +# (tests, examples, docs, static analyzer, etc.) and only targets X86 architecture. +# +# Build arguments: +# LLVM_GIT_URL - Git repository URL (default: https://github.com/llvm/llvm-project.git) +# LLVM_GIT_REF - Git branch/tag/commit to build from (default: llvmorg-21.1.2) +# NUM_JOBS - Number of parallel build jobs (default: 4) +# +# Usage: +# Build from default release tag (llvm 21.1.2) +# docker build -f Dockerfile.clang.ubuntu -t clang-ubuntu:21 . +# +# Build from different release tag: +# docker build -f Dockerfile.clang.ubuntu -t clang-ubuntu:19 --build-arg LLVM_GIT_REF=llvmorg-19.1.7 . +# docker build -f Dockerfile.clang.ubuntu -t clang-ubuntu:20 --build-arg LLVM_GIT_REF=llvmorg-20.1.8 . +# +# Build from release branch (tracks latest stable): +# docker build -f Dockerfile.clang.ubuntu -t clang-ubuntu:20 --build-arg LLVM_GIT_REF=release/20.x . +# docker build -f Dockerfile.clang.ubuntu -t clang-ubuntu:21 --build-arg LLVM_GIT_REF=release/21.x . +# +# Build from main branch (nightly): +# docker build -f Dockerfile.clang.ubuntu -t clang-ubuntu:main --build-arg LLVM_GIT_REF=main . +# +# Build from specific commit: +# docker build -f Dockerfile.clang.ubuntu -t clang-ubuntu:custom --build-arg LLVM_GIT_REF=abc123def . +# +# Build from fork or mirror: +# docker build -f Dockerfile.clang.ubuntu -t clang-ubuntu:custom \ +# --build-arg LLVM_GIT_URL=https://github.com/yourfork/llvm-project.git \ +# --build-arg LLVM_GIT_REF=your-branch . +# +# Build with custom parallelism: +# docker build -f Dockerfile.clang.ubuntu -t clang-ubuntu:21 --build-arg NUM_JOBS=8 . +# +# The built image includes: +# - clang and clang++ compilers at /usr/local + +FROM ubuntu:24.04 + +# Install build dependencies +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + build-essential \ + cmake \ + ninja-build \ + git \ + python3 \ + python3-pip \ + wget \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Clone LLVM source from git +ARG LLVM_GIT_URL=https://github.com/llvm/llvm-project.git +ARG LLVM_GIT_REF=llvmorg-21.1.2 + +WORKDIR /tmp +RUN echo "Building from $LLVM_GIT_URL @ $LLVM_GIT_REF" && \ + git clone --depth 1 --branch "$LLVM_GIT_REF" "$LLVM_GIT_URL" llvm-project-src + +# Build LLVM/Clang (optimized for build speed) +ARG NUM_JOBS=4 +WORKDIR /tmp/llvm-project-src +RUN cmake -S llvm -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DLLVM_ENABLE_PROJECTS="clang" \ + -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ + -DLLVM_TARGETS_TO_BUILD="X86;AArch64" \ + -DLLVM_INCLUDE_BENCHMARKS=OFF \ + -DLLVM_INCLUDE_TESTS=OFF \ + -DLLVM_INCLUDE_EXAMPLES=OFF \ + -DLLVM_INCLUDE_DOCS=OFF \ + -DLLVM_ENABLE_ASSERTIONS=OFF \ + -DLLVM_ENABLE_BACKTRACES=OFF \ + -DLLVM_OPTIMIZED_TABLEGEN=ON \ + -DCLANG_ENABLE_ARCMT=OFF \ + -DCLANG_ENABLE_STATIC_ANALYZER=OFF + +RUN ninja -C build -j${NUM_JOBS} + +RUN ninja -C build install \ + && rm -rf /tmp/llvm-project-src + +# Configure runtime linker to find libc++ and libunwind +RUN echo "/usr/local/lib/aarch64-unknown-linux-gnu" > /etc/ld.so.conf.d/libc++.conf && \ + echo "/usr/local/lib/x86_64-unknown-linux-gnu" >> /etc/ld.so.conf.d/libc++.conf && \ + ldconfig + +# Verify installation +RUN clang --version && clang++ --version + +WORKDIR /workspace +CMD ["/bin/bash"] diff --git a/Dockerfile.devcontainer.clang b/Dockerfile.devcontainer.clang new file mode 100644 index 0000000..0abc868 --- /dev/null +++ b/Dockerfile.devcontainer.clang @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# Dockerfile for building a development container with custom-built Clang +# +# This Dockerfile creates a devcontainer environment with a custom-built Clang compiler +# from the clang-ubuntu base image, along with development tools and pre-commit hooks. +# +# Build arguments: +# CLANG_BASE_IMAGE - Base image with custom Clang (default: clang-ubuntu:21) +# +# Usage: +# Build with default Clang 21: +# docker build -f Dockerfile.devcontainer.clang -t devcontainer-clang:21 . +# +# Build with different Clang version: +# docker build -f Dockerfile.devcontainer.clang -t devcontainer-clang:20 --build-arg CLANG_BASE_IMAGE=clang-ubuntu:20 . + +ARG CLANG_BASE_IMAGE=clang-ubuntu:21 + +# Stage 1: Build custom Clang (reference to separate build) +# This stage is just a reference - the actual clang-ubuntu image should be built separately +# using Dockerfile.clang.ubuntu +FROM ${CLANG_BASE_IMAGE} AS clang-builder + +# Stage 2: Create devcontainer with custom Clang +FROM mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04 + +# Create vscode user if it doesn't exist +RUN bash <<"EOF" + if ! id "vscode" &>/dev/null; then + apt-get update && apt-get install -y sudo adduser + useradd -ms /bin/bash -p "" vscode && usermod -aG sudo vscode + fi +EOF + +# Install CMake from Kitware's official repository +RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null +RUN echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ noble main' | tee /etc/apt/sources.list.d/kitware.list >/dev/null +RUN apt-get update && apt-get install -y cmake + +# Copy custom-built Clang from the clang-ubuntu image +COPY --from=clang-builder /usr/local/bin/clang* /usr/local/bin/ +COPY --from=clang-builder /usr/local/lib/clang /usr/local/lib/clang +COPY --from=clang-builder /usr/local/lib/libclang* /usr/local/lib/ +COPY --from=clang-builder /usr/local/include/clang* /usr/local/include/ +COPY --from=clang-builder /usr/local/share/clang /usr/local/share/clang + +# Install standard library headers and development tools +RUN apt-get update && apt-get install -y \ + libstdc++-14-dev \ + build-essential \ + git \ + pipx \ + && rm -rf /var/lib/apt/lists/* + +# Set up alternatives for clang and clang++ +RUN update-alternatives --install /usr/bin/clang clang /usr/local/bin/clang 100 && \ + update-alternatives --install /usr/bin/clang++ clang++ /usr/local/bin/clang++ 100 + +# Switch to vscode user for user-specific setup +USER vscode +WORKDIR /tmp + +# Pre-commit is beman library's standard linting tool +RUN pipx install pre-commit +ENV PATH="/home/vscode/.local/bin:${PATH}" + +# Verify installation +RUN clang --version && clang++ --version + +WORKDIR /workspace +ENTRYPOINT ["/usr/bin/bash"] diff --git a/Dockerfile.gcc.ubuntu b/Dockerfile.gcc.ubuntu new file mode 100644 index 0000000..d110090 --- /dev/null +++ b/Dockerfile.gcc.ubuntu @@ -0,0 +1,105 @@ +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# Dockerfile for building GCC from source on Ubuntu 24.04 +# +# This Dockerfile builds a custom GCC compiler from source. +# +# The build is setup to minimize build time, thus it disables unnecessary components +# (tests, docs, etc.) and only targets essential languages (C, C++). +# +# Build arguments: +# GCC_GIT_URL - Git repository URL (default: https://gcc.gnu.org/git/gcc.git) +# GCC_GIT_REF - Git branch/tag/commit to build from (default: releases/gcc-14.2.0) +# NUM_JOBS - Number of parallel build jobs (default: 4) +# +# Usage: +# Build from default release tag (GCC 14.2.0) +# docker build -f Dockerfile.gcc.ubuntu -t gcc-ubuntu:14 . +# +# Build from different release tag: +# docker build -f Dockerfile.gcc.ubuntu -t gcc-ubuntu:13 --build-arg GCC_GIT_REF=releases/gcc-13.3.0 . +# docker build -f Dockerfile.gcc.ubuntu -t gcc-ubuntu:14 --build-arg GCC_GIT_REF=releases/gcc-14.2.0 . +# +# Build from release branch (tracks latest stable): +# docker build -f Dockerfile.gcc.ubuntu -t gcc-ubuntu:13 --build-arg GCC_GIT_REF=releases/gcc-13 . +# docker build -f Dockerfile.gcc.ubuntu -t gcc-ubuntu:14 --build-arg GCC_GIT_REF=releases/gcc-14 . +# +# Build from master branch (development): +# docker build -f Dockerfile.gcc.ubuntu -t gcc-ubuntu:master --build-arg GCC_GIT_REF=master . +# +# Build from specific commit: +# docker build -f Dockerfile.gcc.ubuntu -t gcc-ubuntu:custom --build-arg GCC_GIT_REF=abc123def . +# +# Build from fork or mirror: +# docker build -f Dockerfile.gcc.ubuntu -t gcc-ubuntu:custom \ +# --build-arg GCC_GIT_URL=https://github.com/yourfork/gcc.git \ +# --build-arg GCC_GIT_REF=your-branch . +# +# Build with custom parallelism: +# docker build -f Dockerfile.gcc.ubuntu -t gcc-ubuntu:14 --build-arg NUM_JOBS=8 . +# +# The built image includes: +# - gcc and g++ compilers at /usr/local +# - libstdc++ standard library + +FROM ubuntu:24.04 + +# Install build dependencies +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + build-essential \ + git \ + wget \ + ca-certificates \ + libgmp-dev \ + libmpfr-dev \ + libmpc-dev \ + zlib1g-dev \ + flex \ + bison \ + texinfo \ + && rm -rf /var/lib/apt/lists/* + +# Clone GCC source from git +ARG GCC_GIT_URL=https://gcc.gnu.org/git/gcc.git +ARG GCC_GIT_REF=releases/gcc-14.2.0 + +WORKDIR /tmp +RUN echo "Building from $GCC_GIT_URL @ $GCC_GIT_REF" && \ + git clone --depth 1 --branch "$GCC_GIT_REF" "$GCC_GIT_URL" gcc-src + +# Build GCC (optimized for build speed) +ARG NUM_JOBS=4 +WORKDIR /tmp/gcc-build + +RUN /tmp/gcc-src/configure \ + --prefix=/usr/local \ + --enable-languages=c,c++ \ + --disable-multilib \ + --disable-bootstrap \ + --enable-checking=release \ + --with-system-zlib \ + --disable-nls \ + --disable-libsanitizer \ + --disable-libvtv \ + --disable-libquadmath \ + --disable-libgomp \ + --disable-libssp \ + --disable-werror + +RUN make -j${NUM_JOBS} + +RUN make install-strip \ + && rm -rf /tmp/gcc-src /tmp/gcc-build + +# Verify installation +RUN gcc --version && g++ --version + +# Test that libstdc++ works +RUN echo '#include \nint main() { std::cout << "GCC libstdc++ works" << std::endl; }' > /tmp/test.cpp && \ + g++ /tmp/test.cpp -o /tmp/test && \ + /tmp/test && \ + rm /tmp/test.cpp /tmp/test + +WORKDIR /workspace +CMD ["/bin/bash"] diff --git a/build_containers.py b/build_containers.py new file mode 100755 index 0000000..121b716 --- /dev/null +++ b/build_containers.py @@ -0,0 +1,392 @@ +#!/usr/bin/env python3 +""" +Container Build Manager + +This script manages the building of Docker containers for the infra-containers project. +It handles building custom Clang compilers and devcontainer images with proper dependency +management and caching. + +Usage: + # Build Clang 21 + ./build_containers.py clang --version 21 + + # Build Clang from specific git ref + ./build_containers.py clang --version 21 --git-ref llvmorg-21.1.2 + + # Build devcontainer with Clang 21 + ./build_containers.py devcontainer --clang-version 21 + + # Build everything + ./build_containers.py all --clang-version 21 + + # Build with custom parallelism + ./build_containers.py clang --version 21 --jobs 8 + + # Dry run (show what would be built) + ./build_containers.py all --clang-version 21 --dry-run + +Requirements: + pip install docker +""" + +import argparse +import json +import sys +from pathlib import Path +from typing import Optional + +try: + import docker + from docker.errors import BuildError, ImageNotFound, APIError +except ImportError: + print("āŒ Error: docker package not found. Install it with:", file=sys.stderr) + print(" pip install docker", file=sys.stderr) + sys.exit(1) + + +class ContainerBuilder: + """Manages building Docker containers with proper dependency handling.""" + + def __init__(self, dry_run: bool = False, verbose: bool = False): + self.dry_run = dry_run + self.verbose = verbose + self.root_dir = Path(__file__).parent + self.client = docker.from_env() if not dry_run else None + + def _stream_build_output(self, stream, description: str, log_file: Optional[Path] = None): + """Stream Docker build output with progress information.""" + print(f"\nšŸ”Ø {description}") + + log_handle = None + if log_file and not self.dry_run: + log_handle = open(log_file, 'w') + + try: + for chunk in stream: + if 'stream' in chunk: + text = chunk['stream'] + if self.verbose: + print(text, end='', flush=True) + if log_handle: + log_handle.write(text) + log_handle.flush() + elif 'status' in chunk: + # Show pull/push status + if self.verbose: + status = chunk['status'] + if 'id' in chunk: + print(f"{chunk['id']}: {status}") + else: + print(status) + elif 'error' in chunk: + error_msg = chunk['error'] + print(f"āŒ Build error: {error_msg}", file=sys.stderr) + if log_handle: + log_handle.write(f"ERROR: {error_msg}\n") + raise BuildError(error_msg, None) + finally: + if log_handle: + log_handle.close() + print(f"šŸ“ Build log saved to: {log_file}") + + def _image_exists(self, image_tag: str) -> bool: + """Check if a Docker image exists locally.""" + if self.dry_run: + return True + + try: + self.client.images.get(image_tag) + return True + except ImageNotFound: + return False + + def build_clang( + self, + version: int, + git_ref: Optional[str] = None, + git_url: str = "https://github.com/llvm/llvm-project.git", + jobs: int = 4, + tag_suffix: str = "", + ) -> int: + """ + Build a custom Clang compiler image. + + Args: + version: Major version number for tagging + git_ref: Git branch/tag/commit to build from + git_url: Git repository URL + jobs: Number of parallel build jobs + tag_suffix: Additional suffix for the image tag + + Returns: + Exit code (0 for success) + """ + if git_ref is None: + git_ref = f"llvmorg-{version}.1.2" + + tag = f"clang-ubuntu:{version}{tag_suffix}" + log_file = self.root_dir / f"build-clang-{version}.log" + + build_args = { + "LLVM_GIT_REF": git_ref, + "LLVM_GIT_URL": git_url, + "NUM_JOBS": str(jobs), + } + + if self.dry_run: + print(f"\n[DRY RUN] Would build Clang {version}") + print(f" Dockerfile: Dockerfile.clang.ubuntu") + print(f" Tag: {tag}") + print(f" Build args: {build_args}") + print(f" Log file: {log_file}") + return 0 + + try: + stream = self.client.api.build( + path=str(self.root_dir), + dockerfile="Dockerfile.clang.ubuntu", + tag=tag, + buildargs=build_args, + decode=True, + rm=True, + ) + + self._stream_build_output( + stream, + f"Building Clang {version} from {git_ref}", + log_file if not self.verbose else None + ) + + print(f"āœ… Successfully built {tag}") + return 0 + + except BuildError as e: + print(f"āŒ Build failed: {e}", file=sys.stderr) + return 1 + except APIError as e: + print(f"āŒ Docker API error: {e}", file=sys.stderr) + return 1 + except Exception as e: + print(f"āŒ Unexpected error: {e}", file=sys.stderr) + return 1 + + def build_devcontainer( + self, + clang_version: int, + tag_suffix: str = "", + ) -> int: + """ + Build a devcontainer image with custom Clang. + + Args: + clang_version: Clang version to use + tag_suffix: Additional suffix for the image tag + + Returns: + Exit code (0 for success) + """ + clang_base_image = f"clang-ubuntu:{clang_version}{tag_suffix}" + tag = f"devcontainer-clang:{clang_version}{tag_suffix}" + + # Check if base Clang image exists + if not self._image_exists(clang_base_image): + print(f"āŒ Error: Base image {clang_base_image} not found. Build it first with:", file=sys.stderr) + print(f" ./build_containers.py clang --version {clang_version}", file=sys.stderr) + return 1 + + build_args = { + "CLANG_BASE_IMAGE": clang_base_image, + } + + if self.dry_run: + print(f"\n[DRY RUN] Would build devcontainer with Clang {clang_version}") + print(f" Dockerfile: Dockerfile.devcontainer.clang") + print(f" Tag: {tag}") + print(f" Build args: {build_args}") + return 0 + + try: + stream = self.client.api.build( + path=str(self.root_dir), + dockerfile="Dockerfile.devcontainer.clang", + tag=tag, + buildargs=build_args, + decode=True, + rm=True, + ) + + self._stream_build_output( + stream, + f"Building devcontainer with Clang {clang_version}", + None # Don't log devcontainer builds + ) + + print(f"āœ… Successfully built {tag}") + return 0 + + except BuildError as e: + print(f"āŒ Build failed: {e}", file=sys.stderr) + return 1 + except APIError as e: + print(f"āŒ Docker API error: {e}", file=sys.stderr) + return 1 + except Exception as e: + print(f"āŒ Unexpected error: {e}", file=sys.stderr) + return 1 + + def build_all( + self, + clang_version: int, + git_ref: Optional[str] = None, + jobs: int = 4, + ) -> int: + """ + Build both Clang and devcontainer images. + + Args: + clang_version: Clang version to build + git_ref: Git branch/tag/commit to build from + jobs: Number of parallel build jobs + + Returns: + Exit code (0 for success) + """ + print(f"šŸ”Ø Building Clang {clang_version} and devcontainer...") + + # Build Clang first + result = self.build_clang(clang_version, git_ref=git_ref, jobs=jobs) + if result != 0: + return result + + # Then build devcontainer + result = self.build_devcontainer(clang_version) + if result != 0: + return result + + print(f"\nāœ… Successfully built all containers for Clang {clang_version}") + return 0 + + +def main(): + parser = argparse.ArgumentParser( + description="Build Docker containers for custom Clang compilers and devcontainers", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + + subparsers = parser.add_subparsers(dest="command", help="Build target") + + # Clang subcommand + clang_parser = subparsers.add_parser("clang", help="Build custom Clang compiler image") + clang_parser.add_argument( + "--version", + type=int, + required=True, + help="Clang major version (e.g., 19, 20, 21)", + ) + clang_parser.add_argument( + "--git-ref", + help="Git branch/tag/commit to build from (default: llvmorg-{version}.1.2)", + ) + clang_parser.add_argument( + "--git-url", + default="https://github.com/llvm/llvm-project.git", + help="Git repository URL for LLVM", + ) + clang_parser.add_argument( + "--jobs", + type=int, + default=4, + help="Number of parallel build jobs (default: 4)", + ) + clang_parser.add_argument( + "--tag-suffix", + default="", + help="Additional suffix for image tag", + ) + + # Devcontainer subcommand + dev_parser = subparsers.add_parser("devcontainer", help="Build devcontainer image with custom Clang") + dev_parser.add_argument( + "--clang-version", + type=int, + required=True, + help="Clang version to use (must be built first)", + ) + dev_parser.add_argument( + "--tag-suffix", + default="", + help="Additional suffix for image tag", + ) + + # All subcommand + all_parser = subparsers.add_parser("all", help="Build both Clang and devcontainer") + all_parser.add_argument( + "--clang-version", + type=int, + required=True, + help="Clang version to build", + ) + all_parser.add_argument( + "--git-ref", + help="Git branch/tag/commit to build from (default: llvmorg-{version}.1.2)", + ) + all_parser.add_argument( + "--jobs", + type=int, + default=4, + help="Number of parallel build jobs (default: 4)", + ) + + # Global options + parser.add_argument( + "--dry-run", + action="store_true", + help="Print commands without executing them", + ) + parser.add_argument( + "--verbose", + "-v", + action="store_true", + help="Enable verbose output", + ) + + args = parser.parse_args() + + if not args.command: + parser.print_help() + return 1 + + builder = ContainerBuilder(dry_run=args.dry_run, verbose=args.verbose) + + try: + if args.command == "clang": + return builder.build_clang( + version=args.version, + git_ref=args.git_ref, + git_url=args.git_url, + jobs=args.jobs, + tag_suffix=args.tag_suffix, + ) + elif args.command == "devcontainer": + return builder.build_devcontainer( + clang_version=args.clang_version, + tag_suffix=args.tag_suffix, + ) + elif args.command == "all": + return builder.build_all( + clang_version=args.clang_version, + git_ref=args.git_ref, + jobs=args.jobs, + ) + else: + parser.print_help() + return 1 + + except KeyboardInterrupt: + print("\nāŒ Build interrupted by user", file=sys.stderr) + return 130 + + +if __name__ == "__main__": + sys.exit(main())