Skip to content

Commit c08972d

Browse files
robozmeyleborchuk
robozmey
and
leborchuk
authored
Add new workflow (open-gpdb#151)
* Add new workflow --------- Co-authored-by: Leonid <63977577+leborchuk@users.noreply.github.com>
1 parent 480ff6f commit c08972d

3 files changed

Lines changed: 329 additions & 784 deletions

File tree

‎.github/workflows/build-gpdb.yml‎

Lines changed: 329 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,329 @@
1+
# --------------------------------------------------------------------
2+
# GitHub Actions Workflow: Greenplum Build Pipeline
3+
# --------------------------------------------------------------------
4+
5+
name: Greenplum Build
6+
7+
on:
8+
push:
9+
branches: [ "OPENGPDB_STABLE", "OPENGPDB_6_27_STABLE", "MDB_6_25_STABLE_YEZZEY" ]
10+
pull_request:
11+
branches: [ "OPENGPDB_STABLE", "OPENGPDB_6_27_STABLE", "MDB_6_25_STABLE_YEZZEY" ]
12+
13+
jobs:
14+
15+
## ======================================================================
16+
## Job: prepare-test-matrix
17+
## ======================================================================
18+
19+
prepare-test-matrix:
20+
21+
runs-on: ubuntu-latest
22+
23+
outputs:
24+
test-matrix: ${{ steps.set-matrix.outputs.matrix }}
25+
26+
steps:
27+
- id: set-matrix
28+
run: |
29+
echo "=== Matrix Preparation Diagnostics ==="
30+
echo "Event type: ${{ github.event_name }}"
31+
echo "Test selection input: '${{ github.event.inputs.test_selection }}'"
32+
33+
stat -fc %T /sys/fs/cgroup/
34+
35+
# Define defaults
36+
DEFAULT_NUM_PRIMARY_MIRROR_PAIRS=3
37+
DEFAULT_ENABLE_CGROUPS=false
38+
DEFAULT_ENABLE_CORE_CHECK=true
39+
DEFAULT_INSTALL_TARGET=false
40+
DEFAULT_PG_SETTINGS_OPTIMIZER=""
41+
DEFAULT_RUNS_ON="ubuntu-latest"
42+
43+
# Define base test configurations
44+
ALL_TESTS='{
45+
"include": [
46+
{"test":"ic-good-opt-off",
47+
"make_configs":["src/test/regress:installcheck-good"],
48+
"pg_settings":{"optimizer":"off"}
49+
},
50+
{"test":"ic-good-opt-on",
51+
"make_configs":["src/test/regress:installcheck-good"],
52+
"pg_settings":{"optimizer":"on"}
53+
},
54+
{"test":"ic-expandshrink",
55+
"make_configs":["src/test/isolation2:installcheck-expandshrink"]
56+
},
57+
{"test":"ic-resgroup",
58+
"make_configs":["src/test/isolation2:installcheck-resgroup"],
59+
"enable_cgroups":true,
60+
"runs_on":"ubuntu-20.04"
61+
},
62+
{"test":"ic-contrib",
63+
"install_target":true,
64+
"make_configs":["contrib/amcheck:installcheck",
65+
"contrib/auto_explain:installcheck",
66+
"contrib/btree_gin:installcheck",
67+
"contrib/btree_gist:installcheck",
68+
"contrib/citext:installcheck",
69+
"contrib/cube:installcheck",
70+
"contrib/dblink:installcheck",
71+
"contrib/dict_int:installcheck",
72+
"contrib/dict_xsyn:installcheck",
73+
"contrib/earthdistance:installcheck",
74+
"contrib/extprotocol:installcheck",
75+
"contrib/file_fdw:installcheck",
76+
"contrib/formatter_fixedwidth:installcheck",
77+
"contrib/fuzzystrmatch:installcheck",
78+
"contrib/hstore:installcheck",
79+
"contrib/indexscan:installcheck",
80+
"contrib/intarray:installcheck",
81+
"contrib/isn:installcheck",
82+
"contrib/ltree:installcheck",
83+
"contrib/passwordcheck:installcheck",
84+
"contrib/pg_trgm:installcheck",
85+
"contrib/pgcrypto:installcheck",
86+
"contrib/pgstattuple:installcheck",
87+
"contrib/postgres_fdw:installcheck",
88+
"contrib/sasdemo:installcheck",
89+
"contrib/sepgsql:installcheck",
90+
"contrib/sslinfo:installcheck",
91+
"contrib/tablefunc:installcheck",
92+
"contrib/test_decoding:installcheck",
93+
"contrib/test_parser:installcheck",
94+
"contrib/test_shm_mq:installcheck",
95+
"contrib/try_convert:installcheck",
96+
"contrib/tsearch2:installcheck",
97+
"contrib/unaccent:installcheck",
98+
"contrib/uuid-ossp:installcheck",
99+
"contrib/xml2:installcheck"]
100+
},
101+
{"test":"ic-gpcontrib",
102+
"make_configs":["gpcontrib/gp_array_agg:installcheck",
103+
"gpcontrib/gp_debug_numsegments:installcheck",
104+
"gpcontrib/gp_distribution_policy:installcheck",
105+
"gpcontrib/gp_error_handling:installcheck",
106+
"gpcontrib/gp_inject_fault:installcheck",
107+
"gpcontrib/gp_internal_tools:installcheck",
108+
"gpcontrib/gp_legacy_string_agg:installcheck",
109+
"gpcontrib/gp_percentile_agg:installcheck",
110+
"gpcontrib/gp_sparse_vector:installcheck",
111+
"gpcontrib/gp_subtransaction_overflow:installcheck",
112+
"gpcontrib/gpcloud:installcheck",
113+
"gpcontrib/gpmapreduce:installcheck",
114+
"gpcontrib/orafce:installcheck",
115+
"gpcontrib/pgaudit:installcheck",
116+
"gpcontrib/quicklz:installcheck",
117+
"gpcontrib/sr_plan",
118+
"gpcontrib/zstd"]
119+
},
120+
{"test":"ic-parallel-retrieve-cursor",
121+
"make_configs":["src/test/isolation2:installcheck-parallel-retrieve-cursor"]
122+
},
123+
{"test":"ic-mirrorless",
124+
"make_configs":["src/test/isolation2:installcheck-mirrorless"]
125+
}
126+
]
127+
}'
128+
129+
# Function to apply defaults
130+
apply_defaults() {
131+
echo "$1" | jq --arg npm "$DEFAULT_NUM_PRIMARY_MIRROR_PAIRS" \
132+
--argjson ec "$DEFAULT_ENABLE_CGROUPS" \
133+
--argjson ecc "$DEFAULT_ENABLE_CORE_CHECK" \
134+
--argjson emk "$DEFAULT_INSTALL_TARGET" \
135+
--arg opt "$DEFAULT_PG_SETTINGS_OPTIMIZER" \
136+
--arg ro "$DEFAULT_RUNS_ON" \
137+
'def get_defaults:
138+
{
139+
num_primary_mirror_pairs: ($npm|tonumber),
140+
enable_cgroups: $ec,
141+
install_target: $emk,
142+
runs_on: $ro,
143+
enable_core_check: $ecc,
144+
pg_settings: {
145+
optimizer: $opt
146+
}
147+
};
148+
get_defaults * .'
149+
}
150+
151+
# Extract all valid test names from ALL_TESTS
152+
VALID_TESTS=$(echo "$ALL_TESTS" | jq -r '.include[].test')
153+
154+
# Parse input test selection
155+
IFS=',' read -ra SELECTED_TESTS <<< "${{ github.event.inputs.test_selection }}"
156+
157+
# Default to all tests if selection is empty or 'all'
158+
if [[ "${SELECTED_TESTS[*]}" == "all" || -z "${SELECTED_TESTS[*]}" ]]; then
159+
mapfile -t SELECTED_TESTS <<< "$VALID_TESTS"
160+
fi
161+
162+
# Validate and filter selected tests
163+
INVALID_TESTS=()
164+
FILTERED_TESTS=()
165+
for TEST in "${SELECTED_TESTS[@]}"; do
166+
TEST=$(echo "$TEST" | tr -d '[:space:]') # Trim whitespace
167+
if echo "$VALID_TESTS" | grep -qw "$TEST"; then
168+
FILTERED_TESTS+=("$TEST")
169+
else
170+
INVALID_TESTS+=("$TEST")
171+
fi
172+
done
173+
174+
# Handle invalid tests
175+
if [[ ${#INVALID_TESTS[@]} -gt 0 ]]; then
176+
echo "::error::Invalid test(s) selected: ${INVALID_TESTS[*]}"
177+
echo "Valid tests are: $(echo "$VALID_TESTS" | tr '\n' ', ')"
178+
exit 1
179+
fi
180+
181+
# Build result JSON with defaults applied
182+
RESULT='{"include":['
183+
FIRST=true
184+
for TEST in "${FILTERED_TESTS[@]}"; do
185+
CONFIG=$(jq -c --arg test "$TEST" '.include[] | select(.test == $test)' <<< "$ALL_TESTS")
186+
FILTERED_WITH_DEFAULTS=$(apply_defaults "$CONFIG")
187+
if [[ "$FIRST" == true ]]; then
188+
FIRST=false
189+
else
190+
RESULT="${RESULT},"
191+
fi
192+
RESULT="${RESULT}${FILTERED_WITH_DEFAULTS}"
193+
done
194+
RESULT="${RESULT}]}"
195+
196+
# Output the matrix for GitHub Actions
197+
echo "Final matrix configuration:"
198+
echo "$RESULT" | jq .
199+
200+
# Fix: Use block redirection
201+
{
202+
echo "matrix<<EOF"
203+
echo "$RESULT"
204+
echo "EOF"
205+
} >> "$GITHUB_OUTPUT"
206+
207+
echo "=== Matrix Preparation Complete ==="
208+
209+
## ======================================================================
210+
## Job: init-docker-env
211+
## ======================================================================
212+
213+
214+
215+
## ======================================================================
216+
## Job: build
217+
## ======================================================================
218+
219+
build:
220+
env:
221+
JOB_TYPE: build
222+
runs-on: ubuntu-latest
223+
timeout-minutes: 120
224+
outputs:
225+
build_timestamp: ${{ steps.set_timestamp.outputs.timestamp }}
226+
227+
container:
228+
image: ghcr.io/open-gpdb/gpdb-env:jammy-latest
229+
230+
steps:
231+
- name: Set build timestamp
232+
id: set_timestamp # Add an ID to reference this step
233+
run: |
234+
timestamp=$(date +'%Y%m%d_%H%M%S')
235+
echo "timestamp=$timestamp" | tee -a "$GITHUB_OUTPUT" # Use GITHUB_OUTPUT for job outputs
236+
echo "BUILD_TIMESTAMP=$timestamp" | tee -a "$GITHUB_ENV" # Also set as environment variable
237+
238+
- name: Checkout Greenplum
239+
uses: actions/checkout@v4
240+
with:
241+
fetch-depth: 1
242+
243+
- name: Checkout CI Build/Test Scripts
244+
uses: actions/checkout@v4
245+
with:
246+
repository: open-gpdb/gpdb-devops
247+
ref: main
248+
path: gpdb-devops
249+
fetch-depth: 1
250+
251+
- name: Move gpdb-devops directory
252+
run: |
253+
set -ex pipefail
254+
if ! mv "${GITHUB_WORKSPACE}"/gpdb-devops "${GITHUB_WORKSPACE}"/..; then
255+
echo "::error::Container initialization failed"
256+
exit 1
257+
fi
258+
259+
- name: Generate Build Job Summary Start
260+
run: |
261+
{
262+
echo "# Build Job Summary"
263+
echo "## Environment"
264+
echo "- Start Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')"
265+
echo "- ENABLE_DEBUG: ${{ env.ENABLE_DEBUG }}"
266+
echo "- OS Version: $(lsb_release -sd)"
267+
echo "- GCC Version: $(gcc --version | head -n1)"
268+
} >> "$GITHUB_STEP_SUMMARY"
269+
270+
- name: Run Greenplum configure script
271+
env:
272+
SRC_DIR: ${{ github.workspace }}
273+
run: |
274+
set -ex pipefail
275+
chmod +x "${SRC_DIR}"/../gpdb-devops/build_automation/gpdb/scripts/configure-gpdb.sh
276+
cd ${SRC_DIR}
277+
if ! SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} ${SRC_DIR}/../gpdb-devops/build_automation/gpdb/scripts/configure-gpdb.sh; then
278+
echo "::error::Configure script failed"
279+
exit 1
280+
fi
281+
282+
- name: Run Greenplum build script
283+
env:
284+
SRC_DIR: ${{ github.workspace }}
285+
run: |
286+
set -ex pipefail
287+
chmod +x "${SRC_DIR}"/../gpdb-devops/build_automation/gpdb/scripts/build-gpdb.sh
288+
cd ${SRC_DIR}
289+
if ! SRC_DIR=${SRC_DIR} ${SRC_DIR}/../gpdb-devops/build_automation/gpdb/scripts/build-gpdb.sh; then
290+
echo "::error::Build script failed"
291+
exit 1
292+
fi
293+
294+
- name: Run Greenplum debian package build script
295+
env:
296+
SRC_DIR: ${{ github.workspace }}
297+
run: |
298+
set -ex pipefail
299+
300+
chmod +x "${SRC_DIR}"/../gpdb-devops/scripts/build-deb.sh
301+
if ! ${SRC_DIR}/../gpdb-devops/scripts/build-deb.sh; then
302+
echo "::error::Build script failed"
303+
exit 1
304+
fi
305+
306+
- name: Generate Build Job Summary End
307+
if: always()
308+
run: |
309+
{
310+
echo "## Build Results"
311+
echo "- End Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')"
312+
} >> "$GITHUB_STEP_SUMMARY"
313+
314+
- name: Upload Debian Packages
315+
uses: actions/upload-artifact@v4
316+
with:
317+
name: debian-packages-${{ env.BUILD_TIMESTAMP }}
318+
path: |
319+
**/*.deb
320+
321+
- name: Upload build logs
322+
uses: actions/upload-artifact@v4
323+
with:
324+
name: build-logs-${{ env.BUILD_TIMESTAMP }}
325+
path: |
326+
build-logs/
327+
retention-days: ${{ env.LOG_RETENTION_DAYS }}
328+
329+

0 commit comments

Comments
 (0)