Skip to content

Commit 6b3e170

Browse files
committed
[GR-44299] Move global-common CI config from common.json to ci/common.jsonnet
1 parent 9f45be3 commit 6b3e170

File tree

29 files changed

+376
-552
lines changed

29 files changed

+376
-552
lines changed

.github/workflows/main.yml

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -161,17 +161,18 @@ jobs:
161161
if: ${{ env.MX_RUNS_STYLE == 'true' }}
162162
run: |
163163
sudo apt install python3-pip python-setuptools
164-
jq -r '.deps.common.packages | to_entries[] | select(.key | startswith("pip:")) | (.key | split(":")[1]) + .value' common.json |
165-
xargs sudo pip install
164+
sudo pip install ninja_syntax $(jq -r '.pip.ninja_syntax' common.json)
165+
sudo pip install lazy-object-proxy $(jq -r '.pip["lazy-object-proxy"]' common.json)
166+
sudo pip install pylint $(jq -r '.pip.pylint' common.json)
166167
- name: Install additional pip packages
167168
if: ${{ matrix.env.PIP_PACKAGES != '' }}
168169
run: ${MX_PYTHON} -m pip install ${{ matrix.env.PIP_PACKAGES }}
169170
- name: Download Eclipse
170171
if: ${{ env.MX_RUNS_STYLE == 'true' }}
171172
run: |
172173
ECLIPSE_TAR=eclipse.tar.gz
173-
ECLIPSE_ORG_VERSION=$(jq -r '.downloads.eclipse.eclipse_org.version' common.json)
174-
ECLIPSE_ORG_TIMESTAMP=$(jq -r '.downloads.eclipse.eclipse_org.timestamp' common.json)
174+
ECLIPSE_ORG_VERSION=$(jq -r '.eclipse.short_version' common.json)
175+
ECLIPSE_ORG_TIMESTAMP=$(jq -r '.eclipse.timestamp' common.json)
175176
wget --no-verbose https://archive.eclipse.org/eclipse/downloads/drops4/R-${ECLIPSE_ORG_VERSION}-${ECLIPSE_ORG_TIMESTAMP}/eclipse-SDK-${ECLIPSE_ORG_VERSION}-linux-gtk-x86_64.tar.gz -O $ECLIPSE_TAR
176177
tar -xzf ${ECLIPSE_TAR}
177178
echo "ECLIPSE_EXE=${PWD}/eclipse/eclipse" >> $GITHUB_ENV

ci.jsonnet

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,8 @@ local visualizer = import 'visualizer/ci/ci.jsonnet';
4141
local verify_ci = (import 'ci/ci_common/ci-check.libsonnet').verify_ci;
4242

4343
{
44-
# Ensure that entries in common.jsonnet can be resolved.
45-
_checkCommon: (import 'ci/ci_common/common.jsonnet'),
44+
# Ensure that non-hidden entries in ci/common.jsonnet and ci/ci_common/common.jsonnet can be resolved.
45+
assert std.length(std.toString(import 'ci/ci_common/common.jsonnet')) > 0,
4646
ci_resources:: (import 'ci/ci_common/ci-resources.libsonnet'),
4747
overlay: graal_common.ci.overlay,
4848
specVersion: "3",

ci/README.md

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# CI Shared Files
2+
3+
## Files shared between many repositories
4+
5+
* `common.json`
6+
* `ci/common.jsonnet`
7+
8+
## Files shared between graal and graal-enterprise
9+
10+
* `graal-common.json`
11+
* `ci/ci_common`
12+
* `<suite>/ci/ci_common` in various suites

ci/ci_common/bench-common.libsonnet

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
local common = import "ci/ci_common/common.jsonnet",
2+
local common = import "../../ci/ci_common/common.jsonnet",
33
local utils = import "common-utils.libsonnet",
44

55
# benchmark job base with automatically generated name
@@ -43,26 +43,26 @@
4343
threads_per_node:: if self.is_numa then self.num_threads / std.length(self.numa_nodes) else self.num_threads,
4444
},
4545

46-
x52:: common.linux + common.amd64 + self._bench_machine + {
46+
x52:: common.linux_amd64 + self._bench_machine + {
4747
machine_name:: "x52",
4848
capabilities+: ["no_frequency_scaling", "tmpfs25g"],
4949
numa_nodes:: [0, 1],
5050
default_numa_node:: 0,
5151
num_threads:: 72
5252
},
53-
x82:: common.linux + common.amd64 + self._bench_machine + {
53+
x82:: common.linux_amd64 + self._bench_machine + {
5454
machine_name:: "x82",
5555
capabilities+: ["no_frequency_scaling", "tmpfs25g"],
5656
numa_nodes:: [0, 1],
5757
default_numa_node:: 0,
5858
num_threads:: 96
5959
},
60-
xgene3:: common.linux + common.aarch64 + self._bench_machine + {
60+
xgene3:: common.linux_aarch64 + self._bench_machine + {
6161
machine_name:: "xgene3",
6262
capabilities+: [],
6363
num_threads:: 32
6464
},
65-
a12c:: common.linux + common.aarch64 + self._bench_machine + {
65+
a12c:: common.linux_aarch64 + self._bench_machine + {
6666
machine_name:: "a12c",
6767
capabilities+: ["no_frequency_scaling", "tmpfs25g"],
6868
numa_nodes:: [0, 1],

ci/ci_common/common-utils.libsonnet

Lines changed: 0 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -1,69 +1,4 @@
11
{
2-
# composable: make an object composable
3-
#
4-
# When composing objects with `+`, the RHS overrides the LHS when fields collide,
5-
# unless the RHS defines fields with `+:`. This is especially an issue when
6-
# importing nested objects from JSON, where all fields are implicitly defined
7-
# with `:`. To solve this, `composable` essentially turns `:` fields into `+:`.
8-
#
9-
# See the following example:
10-
#
11-
# input.jsonnet
12-
# <code>
13-
# {
14-
# composable:: ...,
15-
# obj1:: {
16-
# key1: {
17-
# foo: "foo",
18-
# },
19-
# key2: "key2",
20-
# },
21-
# obj2:: {
22-
# key1: {
23-
# baz : "baz",
24-
# },
25-
# key3: "key3",
26-
# },
27-
# res1: self.obj1 + self.obj2,
28-
# res2: self.composable(self.obj1) + self.composable(self.obj2),
29-
# }
30-
# </code>
31-
#
32-
# Output of jsonnet input.jsonnet
33-
# <code>
34-
# {
35-
# "res1": {
36-
# "key1": {
37-
# "baz": "baz"
38-
# },
39-
# "key2": "key2",
40-
# "key3": "key3"
41-
# },
42-
# "res2": {
43-
# "key1": {
44-
# "baz": "baz",
45-
# "foo": "foo"
46-
# },
47-
# "key2": "key2",
48-
# "key3": "key3"
49-
# }
50-
# }
51-
# </code>
52-
# Note the missing `res1.key1.foo`!
53-
#
54-
local _composable(o) =
55-
std.foldl(function(obj, key)
56-
obj +
57-
if std.type(o[key]) == "object" then
58-
{ [key] +: _composable(o[key]) }
59-
else
60-
{ [key] : o[key] },
61-
std.objectFields(o),
62-
{}
63-
),
64-
# exported name
65-
composable(o) :: _composable(o),
66-
672
# prefixes the given number with 'jdk'
683
prefixed_jdk(jdk_version)::
694
if jdk_version == null || std.length(std.toString(jdk_version)) == 0 then

ci/ci_common/common.jsonnet

Lines changed: 19 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -1,77 +1,14 @@
1-
local composable = (import "common-utils.libsonnet").composable;
1+
# This file is only shared between the graal and graal-enterprise repositories.
22

3-
local mx_version = (import "../../common.json").mx_version;
4-
local common_json = composable(import "../../common.json");
3+
local common = import "../common.jsonnet";
54
local repo_config = import '../repo-configuration.libsonnet';
6-
local jdks = common_json.jdks;
7-
local deps = common_json.deps;
8-
local downloads = common_json.downloads;
9-
10-
# Finds the first integer in a string and returns it as an integer.
11-
local find_first_integer(versionString) =
12-
local charToInt(c) =
13-
std.codepoint(c) - std.codepoint("0");
14-
local firstNum(s, i) =
15-
assert std.length(s) > i : "No number found in string " + s;
16-
local n = charToInt(s[i]);
17-
if n >=0 && n < 10 then i else firstNum(s, i + 1);
18-
local lastNum(s, i) =
19-
if i >= std.length(s) then
20-
i
21-
else
22-
local n = charToInt(s[i]);
23-
if n < 0 || n > 9 then i else lastNum(s, i + 1);
24-
local versionIndexStart = firstNum(versionString, 0);
25-
local versionIndexEnd = lastNum(versionString, versionIndexStart);
26-
std.parseInt(versionString[versionIndexStart:versionIndexEnd])
27-
;
28-
# jdk_version is an hidden field that can be used to generate job names
29-
local add_jdk_version(name) =
30-
local jdk = jdks[name];
31-
// this assumes that the version is the first number in the jdk.version string
32-
local version = find_first_integer(jdk.version);
33-
// santity check that the parsed version is also included in the name
34-
assert std.length(std.findSubstr(std.toString(version), name)) == 1 : "Cannot find version %d in name %s" % [version, name];
35-
{ jdk_version:: version }
36-
;
37-
38-
{
39-
40-
mx:: {
41-
packages+: {
42-
mx: mx_version
43-
}
44-
},
45-
46-
eclipse:: downloads.eclipse,
47-
jdt:: downloads.jdt,
48-
devkits:: common_json.devkits,
49-
50-
svm_deps:: common_json.svm.deps + repo_config.native_image.extra_deps,
515

6+
common + common.frequencies + {
527
build_base:: {
538
// holds location of CI resources that can easily be overwritten in an overlay
549
ci_resources:: (import "ci/ci_common/ci-resources.libsonnet"),
5510
},
5611

57-
// Job frequencies
58-
// ***************
59-
on_demand:: {
60-
targets+: ["ondemand"],
61-
},
62-
post_merge:: {
63-
targets+: ["post-merge"],
64-
},
65-
daily:: {
66-
targets+: ["daily"],
67-
},
68-
weekly:: {
69-
targets+: ["weekly"],
70-
},
71-
monthly:: {
72-
targets+: ["monthly"],
73-
},
74-
7512
# Add a guard to `build` that prevents it from running in the gate
7613
# for a PR that only touches *.md files, the docs, are config files for GitHub
7714
add_excludes_guard(build):: build + {
@@ -111,13 +48,7 @@ local add_jdk_version(name) =
11148
}
11249
},
11350

114-
} + {
115-
// JDK definitions
116-
// ***************
117-
// this adds all jdks from common.json
118-
[name]: add_jdk_version(name) + { downloads+: { [if std.endsWith(name, "llvm") then "LLVM_JAVA_HOME" else "JAVA_HOME"] : jdks[name] }},
119-
for name in std.objectFieldsAll(jdks)
120-
} + {
51+
} + common.jdks + {
12152
# Aliases to edition specific labsjdks
12253
labsjdk17:: self["labsjdk-" + repo_config.graalvm_edition + "-17"],
12354
labsjdk19:: self["labsjdk-" + repo_config.graalvm_edition + "-19"],
@@ -130,52 +61,28 @@ local add_jdk_version(name) =
13061
labsjdk20Debug:: self["labsjdk-" + repo_config.graalvm_edition + "-20Debug"],
13162
labsjdk20LLVM:: self["labsjdk-" + repo_config.graalvm_edition + "-20-llvm"],
13263

133-
13464
// Hardware definitions
13565
// ********************
136-
common:: deps.common + self.mx + {
137-
local where = if std.objectHas(self, "name") then " in " + self.name else "",
138-
# enforce self.os (useful for generating job names)
139-
os:: error "self.os not set" + where,
140-
# enforce self.arch (useful for generating job names)
141-
arch:: error "self.arch not set" + where,
142-
capabilities +: [],
143-
catch_files +: common_json.catch_files,
144-
logs +: [
66+
local graal_common_extras = common.deps.pylint + {
67+
logs+: [
14568
"*.bgv",
146-
"./" + repo_config.compiler.compiler_suite + "/graal_dumps/*/*"
147-
]
69+
"./" + repo_config.compiler.compiler_suite + "/graal_dumps/*/*",
70+
],
71+
timelimit: "30:00",
14872
},
149-
150-
ol7:: {
151-
docker+: {
152-
image: "buildslave_ol7",
153-
mount_modules: true,
73+
local linux_deps_extras = {
74+
packages+: {
75+
"apache/ant": ">=1.9.4",
15476
},
15577
},
15678

157-
linux:: deps.linux + self.common + {os::"linux", capabilities+: [self.os]},
158-
darwin:: deps.darwin + self.common + {os::"darwin", capabilities+: [self.os]},
159-
windows:: deps.windows + self.common + {os::"windows", capabilities+: [self.os]},
160-
windows_server_2016:: self.windows + {capabilities+: ["windows_server_2016"]},
161-
162-
amd64:: { arch::"amd64", capabilities+: [self.arch]},
163-
aarch64:: { arch::"aarch64", capabilities+: [self.arch]},
164-
165-
linux_amd64:: self.linux + self.amd64 + self.ol7,
166-
darwin_amd64:: self.darwin + self.amd64,
167-
darwin_aarch64:: self.darwin + self.aarch64 + {
168-
# only needed until GR-22580 is resolved?
169-
python_version: 3,
170-
},
171-
windows_amd64:: self.windows + self.amd64,
172-
windows_server_2016_amd64:: self.windows_server_2016 + self.amd64,
173-
linux_aarch64:: self.linux + self.aarch64,
79+
linux_amd64: linux_deps_extras + common.linux_amd64 + graal_common_extras,
80+
linux_aarch64: linux_deps_extras + common.linux_aarch64 + graal_common_extras,
81+
darwin_amd64: common.darwin_amd64 + graal_common_extras,
82+
darwin_aarch64: common.darwin_aarch64 + graal_common_extras,
83+
windows_amd64: common.windows_amd64 + graal_common_extras,
84+
windows_server_2016_amd64: common.windows_server_2016_amd64 + graal_common_extras,
17485

86+
// Other
17587
mach5_target:: {targets+: ["mach5"]},
176-
177-
// Utils
178-
disable_proxies:: {
179-
setup+: [["unset", "HTTP_PROXY", "HTTPS_PROXY", "FTP_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "ftp_proxy", "no_proxy"]],
180-
},
18188
}

ci/ci_common/run-spec-tools.libsonnet

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ local std_get = (import "../../ci/ci_common/common-utils.libsonnet").std_get;
5555
table
5656
,
5757
// Removes the 'timelimit' property from an object.
58-
// Usually, this is used to remove hard-coded (default) timelimits defined in `common.json`.
58+
// Usually, this is used to remove hard-coded (default) timelimits defined in `ci/ci_common/common.jsonnet`.
5959
// These definitions assume that the os/arch definition comes first and will be refined later.
6060
// With run-spec, however, this is not true in general because the os/arch is only fixed later
6161
// in the pipeline. Thus, hard-coded timelimits would override any previous settings. To resolve

0 commit comments

Comments
 (0)