Skip to content

feat: add reproducible central buildspec generation #1115

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 14 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 27 additions & 2 deletions scripts/release_scripts/run_macaron.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/usr/bin/env bash

# Copyright (c) 2023 - 2024, Oracle and/or its affiliates. All rights reserved.
# Copyright (c) 2023 - 2025, Oracle and/or its affiliates. All rights reserved.
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.

# This script runs the Macaron Docker image.
Expand Down Expand Up @@ -279,7 +279,7 @@ while [[ $# -gt 0 ]]; do
entrypoint+=("macaron")
;;
# Parsing commands for macaron entrypoint.
analyze|dump-defaults|verify-policy)
analyze|dump-defaults|verify-policy|gen-build-spec)
command=$1
shift
break
Expand Down Expand Up @@ -355,6 +355,19 @@ elif [[ $command == "verify-policy" ]]; then
esac
shift
done
elif [[ $command == "gen-build-spec" ]]; then
while [[ $# -gt 0 ]]; do
case $1 in
-d|--database)
gen_build_spec_arg_database="$2"
shift
;;
*)
rest_command+=("$1")
;;
esac
shift
done
elif [[ $command == "dump-defaults" ]]; then
while [[ $# -gt 0 ]]; do
case $1 in
Expand Down Expand Up @@ -531,6 +544,18 @@ if [[ -n "${arg_datalog_policy_file:-}" ]]; then
mount_file "-f/--file" "$datalog_policy_file" "$datalog_policy_file_in_container" "ro,Z"
fi

# MACARON entrypoint - gen-build-spec command argvs
# This is for macaron gen-build-spec command.
# Determine the database path to be mounted into ${MACARON_WORKSPACE}/database/<database_file_name>.
if [[ -n "${gen_build_spec_arg_database:-}" ]]; then
gen_build_spec_database_path="${gen_build_spec_arg_database}"
file_name="$(basename "${gen_build_spec_database_path}")"
gen_build_spec_database_path_in_container="${MACARON_WORKSPACE}/database/${file_name}"

argv_command+=("--database" "$gen_build_spec_database_path_in_container")
mount_file "-d/--database" "$gen_build_spec_database_path" "$gen_build_spec_database_path_in_container" "rw,Z"
fi

# Determine that ~/.gradle/gradle.properties exists to be mounted into ${MACARON_WORKSPACE}/gradle.properties
if [[ -f "$HOME/.gradle/gradle.properties" ]]; then
mounts+=("-v" "$HOME/.gradle/gradle.properties":"${MACARON_WORKSPACE}/gradle.properties:ro,Z")
Expand Down
95 changes: 95 additions & 0 deletions src/macaron/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@
from packageurl import PackageURL

import macaron
from macaron.build_spec_generator.build_spec_generator import (
BuildSpecFormat,
gen_build_spec_str,
)
from macaron.config.defaults import create_defaults, load_defaults
from macaron.config.global_config import global_config
from macaron.errors import ConfigurationError
Expand Down Expand Up @@ -236,6 +240,70 @@ def verify_policy(verify_policy_args: argparse.Namespace) -> int:
return os.EX_USAGE


def gen_build_spec(gen_build_spec_args: argparse.Namespace) -> int:
"""Generate a build spec containing the build information discovered by Macaron.

Returns
-------
int
Returns os.EX_OK if successful or the corresponding error code on failure.
"""
if not os.path.isfile(gen_build_spec_args.database):
logger.critical("The database file does not exist.")
return os.EX_OSFILE

output_format = gen_build_spec_args.output_format

try:
build_spec_format = BuildSpecFormat(output_format)
except ValueError:
logger.error("The output format %s is not supported.", output_format)
return os.EX_USAGE

try:
purl = PackageURL.from_string(gen_build_spec_args.package_url)
except ValueError as error:
logger.error("Cannot parse purl %s. Error %s", gen_build_spec_args.package_url, error)
return os.EX_USAGE

logger.info(
"Generating %s buildspec for PURL %s from %s.",
output_format,
purl,
gen_build_spec_args.database,
)

build_spec_content = gen_build_spec_str(
purl=purl,
database_path=gen_build_spec_args.database,
build_spec_format=build_spec_format,
)

if not build_spec_content:
logger.error("Error while generate reproducible central build spec.")
return os.EX_DATAERR

logger.debug("Build spec content: \n%s", build_spec_content)
build_spec_filepath = os.path.join(global_config.output_path, "macaron.buildspec")
try:
with open(build_spec_filepath, mode="w", encoding="utf-8") as file:
logger.info(
"Generating the %s format build spec to %s.",
build_spec_format.value,
os.path.relpath(build_spec_filepath, os.getcwd()),
)
file.write(build_spec_content)
except OSError as error:
logger.error(
"Could not generate the Buildspec to %s. Error: %s",
os.path.relpath(build_spec_filepath, os.getcwd()),
error,
)
return os.EX_DATAERR

return os.EX_OK


def find_source(find_args: argparse.Namespace) -> int:
"""Perform repo and commit finding for a passed PURL, or commit finding for a passed PURL and repo."""
if repo_finder.find_source(find_args.package_url, find_args.repo_path or None):
Expand Down Expand Up @@ -284,6 +352,9 @@ def perform_action(action_args: argparse.Namespace) -> None:

find_source(action_args)

case "gen-build-spec":
sys.exit(gen_build_spec(action_args))

case _:
logger.error("Macaron does not support command option %s.", action_args.action)
sys.exit(os.EX_USAGE)
Expand Down Expand Up @@ -523,6 +594,30 @@ def main(argv: list[str] | None = None) -> None:
),
)

# Generate a build spec containing rebuild information for a software component.
gen_build_spec_parser = sub_parser.add_parser(name="gen-build-spec")

gen_build_spec_parser.add_argument(
"-purl",
"--package-url",
required=True,
type=str,
help=("The PURL string of the software component to generate build spec for."),
)

gen_build_spec_parser.add_argument(
"--database",
help="Path to the database.",
required=True,
)

gen_build_spec_parser.add_argument(
"--output-format",
type=str,
help=('The output format. Can be rc-buildspec (Reproducible-central build spec) (default "rc-buildspec")'),
default="rc-buildspec",
)

args = main_parser.parse_args(argv)

if not args.action:
Expand Down
2 changes: 2 additions & 0 deletions src/macaron/build_spec_generator/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved.
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
137 changes: 137 additions & 0 deletions src/macaron/build_spec_generator/build_command_patcher.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved.
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.

"""This module contains the implementation of the build command patching."""

import logging
from collections.abc import Mapping, Sequence

from macaron.build_spec_generator.cli_command_parser import CLICommand, CLICommandParser, PatchCommandBuildTool
from macaron.build_spec_generator.cli_command_parser.gradle_cli_parser import (
GradleCLICommandParser,
GradleOptionPatchValueType,
)
from macaron.build_spec_generator.cli_command_parser.maven_cli_parser import (
CommandLineParseError,
MavenCLICommandParser,
MavenOptionPatchValueType,
PatchBuildCommandError,
)
from macaron.build_spec_generator.cli_command_parser.unparsed_cli_command import UnparsedCLICommand

logger: logging.Logger = logging.getLogger(__name__)

MVN_CLI_PARSER = MavenCLICommandParser()
GRADLE_CLI_PARSER = GradleCLICommandParser()

PatchValueType = GradleOptionPatchValueType | MavenOptionPatchValueType


def _patch_commands(
cmds_sequence: Sequence[list[str]],
cli_parsers: Sequence[CLICommandParser],
patches: Mapping[
PatchCommandBuildTool,
Mapping[str, PatchValueType | None],
],
) -> list[CLICommand] | None:
"""Patch the sequence of build commands, using the provided CLICommandParser instances.

For each command in `cmds_sequence`, it will be checked against all CLICommandParser instances until there is
one that can parse it, then a patch from ``patches`` is applied for this command if provided.

If a command doesn't have any corresponding ``CLICommandParser`` instance it will be parsed as UnparsedCLICommand,
which just holds the original command as a list of string, without any changes.
"""
result: list[CLICommand] = []
for cmds in cmds_sequence:
effective_cli_parser = None
for cli_parser in cli_parsers:
if cli_parser.is_build_tool(cmds[0]):
effective_cli_parser = cli_parser
break

if not effective_cli_parser:
result.append(UnparsedCLICommand(original_cmds=cmds))
continue

try:
cli_command = effective_cli_parser.parse(cmds)
except CommandLineParseError as error:
logger.error(
"Failed to parse the mvn command %s. Error %s.",
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
"Failed to parse the mvn command %s. Error %s.",
"Failed to parse the cli command %s. Error %s.",

" ".join(cmds),
error,
)
return None

patch = patches.get(effective_cli_parser.build_tool, None)
if not patch:
result.append(cli_command)
continue

try:
new_cli_command = effective_cli_parser.apply_patch(
cli_command=cli_command,
options_patch=patch,
)
except PatchBuildCommandError as error:
logger.error(
"Failed to patch the mvn command %s. Error %s.",
" ".join(cmds),
error,
)
return None

result.append(new_cli_command)

return result


def patch_commands(
cmds_sequence: Sequence[list[str]],
patches: Mapping[
PatchCommandBuildTool,
Mapping[str, PatchValueType | None],
],
) -> list[list[str]] | None:
"""Patch a sequence of CLI commands.

For each command in this command sequence:

- If the command is not a build command or the build tool is not supported by us, it will be leave intact.

- If the command is a build command supported by us, it will be patch if a patch value is provided to ``patches``.
If no patch value is provided for a build command, it will be leave intact.

`patches` is a mapping with:

- **Key**: an instance of the ``BuildTool`` enum

- **Value**: the patch value provided to ``CLICommandParser.apply_patch``. For more information on the patch value
see the concrete implementations of the ``CLICommandParser.apply_patch`` method.
For example: :class:`macaron.cli_command_parser.maven_cli_parser.MavenCLICommandParser.apply_patch`,
:class:`macaron.cli_command_parser.gradle_cli_parser.GradleCLICommandParser.apply_patch`.

This means that all commands that matches a BuildTool will be apply by the same patch value.

Returns
-------
list[list[str]] | None
The patched command sequence or None if there is an error. The errors that can happen if any command
which we support is invalid in ``cmds_sequence``, or the patch value is valid.
"""
result = []
patch_cli_commands = _patch_commands(
cmds_sequence=cmds_sequence,
cli_parsers=[MVN_CLI_PARSER, GRADLE_CLI_PARSER],
patches=patches,
)

if patch_cli_commands is None:
return None

for patch_cmd in patch_cli_commands:
result.append(patch_cmd.to_cmds())

return result
Loading
Loading