Skip to content

Dev release through ci #1773

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 11 commits into
base: master
Choose a base branch
from
+209 −188
Draft

Dev release through ci #1773

Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
a53bf75
Testing source build
pranavrth Jul 4, 2024
d9e5f2e
Added current branch in whitelist for running CI
pranavrth Jul 4, 2024
5e8a5a1
Updated script
pranavrth Jul 4, 2024
f1ddb53
Removed destination
pranavrth Jul 4, 2024
ea29902
Removed destination
pranavrth Jul 4, 2024
56427e3
debug
pranavrth Jul 4, 2024
74c4a4e
debug
pranavrth Jul 4, 2024
866fe16
Debug
pranavrth Jul 4, 2024
f9779f4
Not running wheels task
pranavrth Jul 4, 2024
07b991d
Debug
pranavrth Jul 4, 2024
db635cc
Debug
pranavrth Jul 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .semaphore/project.yml
View file
Original file line number Diff line number Diff line change
@@ -28,6 +28,7 @@ spec:
- main
- /^v\d+\.\d+\.x$/
- /^gh-readonly-queue.*/
- dev_release-through-ci
custom_permissions: true
debug_permissions:
- empty
396 changes: 208 additions & 188 deletions .semaphore/semaphore.yml
View file
Original file line number Diff line number Diff line change
@@ -14,203 +14,220 @@ global_job_config:
- checkout
- mkdir artifacts
blocks:
- name: "Wheels: OSX x64"
run:
when: "tag =~ '.*'"
dependencies: []
task:
agent:
machine:
type: s1-prod-macos-13-5-amd64
env_vars:
- name: OS_NAME
value: osx
- name: ARCH
value: x64
jobs:
- name: Build
commands:
- PIP_INSTALL_OPTIONS="--user" tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse
- tar -czf wheelhouse-macOS-${ARCH}.tgz wheelhouse
- artifact push workflow wheelhouse-macOS-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/
- name: "Wheels: OSX arm64"
run:
when: "tag =~ '.*'"
dependencies: []
task:
agent:
machine:
type: s1-prod-macos-13-5-arm64
env_vars:
- name: OS_NAME
value: osx
- name: CIBW_ARCHS
value: arm64
- name: ARCH
value: arm64
jobs:
- name: Build
commands:
- PIP_INSTALL_OPTIONS="--user" tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse
- tar -czf wheelhouse-macOS-${ARCH}.tgz wheelhouse
- artifact push workflow wheelhouse-macOS-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/
- name: "Wheels: Linux arm64"
run:
when: "tag =~ '.*'"
- name: "Source Build"
# run:
# when: "tag =~ '.*'"
dependencies: []
task:
agent:
machine:
type: s1-prod-ubuntu20-04-arm64-1
env_vars:
- name: OS_NAME
value: linux
- name: ARCH
value: arm64
jobs:
- name: Build
commands:
- ./tools/build-manylinux.sh "${LIBRDKAFKA_VERSION#v}"
- tar -czf wheelhouse-linux-${ARCH}.tgz wheelhouse
- artifact push workflow wheelhouse-linux-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/
- name: "Wheels: Linux x64"
run:
when: "tag =~ '.*'"
dependencies: []
task:
agent:
machine:
type: s1-prod-ubuntu20-04-amd64-3
env_vars:
- name: OS_NAME
value: linux
- name: ARCH
value: x64
jobs:
- name: Build
commands:
- ./tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse
- tar -czf wheelhouse-linux-${ARCH}.tgz wheelhouse
- artifact push workflow wheelhouse-linux-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/
- name: "Wheels: Windows"
run:
when: "tag =~ '.*'"
dependencies: []
task:
agent:
machine:
type: s1-prod-windows
env_vars:
- name: OS_NAME
value: windows
- name: ARCH
value: x64
prologue:
commands:
- ".\\tools\\mingw-w64\\setup-msys2.ps1"
- $env:PATH = 'C:\msys64\usr\bin;' + $env:PATH
- bash -lc './tools/mingw-w64/msys2-dependencies.sh'
jobs:
- name: Build
env_vars:
- name: CHERE_INVOKING
value: 'yes'
- name: MSYSTEM
value: UCRT64
commands:
- bash tools/mingw-w64/semaphore_commands.sh
- bash tools/wheels/install-librdkafka.sh $env:LIBRDKAFKA_VERSION.TrimStart("v") dest
- tools/wheels/build-wheels.bat x64 win_amd64 dest wheelhouse
- tar -czf wheelhouse-windows-${Env:ARCH}.tgz wheelhouse
- artifact push workflow wheelhouse-windows-${Env:ARCH}.tgz --destination artifacts/wheels-${Env:OS_NAME}-${Env:ARCH}.tgz/
- name: "Source package verification and Integration tests with Python 3 (Linux x64)"
dependencies: []
task:
agent:
machine:
type: s1-prod-ubuntu20-04-amd64-2
env_vars:
- name: OS_NAME
value: linux
- name: ARCH
value: x64
prologue:
commands:
- '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY'
jobs:
- name: Build
commands:
- sem-version python 3.8
# use a virtualenv
- python3 -m venv _venv && source _venv/bin/activate
- chmod u+r+x tools/source-package-verification.sh
- tools/source-package-verification.sh
- name: "Source package verification with Python 3 (Linux arm64)"
dependencies: []
task:
agent:
machine:
type: s1-prod-ubuntu20-04-arm64-1
env_vars:
- name: OS_NAME
value: linux
- name: ARCH
value: arm64
jobs:
- name: Build
commands:
- sem-version python 3.8
# use a virtualenv
- python3 -m venv _venv && source _venv/bin/activate
- chmod u+r+x tools/source-package-verification.sh
- tools/source-package-verification.sh
- name: "Source package verification with Python 3 (OSX x64) +docs"
dependencies: []
task:
agent:
machine:
type: s1-prod-macos-13-5-amd64
env_vars:
- name: OS_NAME
value: osx
- name: ARCH
value: x64
jobs:
- name: Build
commands:
- sem-version python 3.8
# use a virtualenv
- python3 -m venv _venv && source _venv/bin/activate
- chmod u+r+x tools/source-package-verification.sh
- tools/source-package-verification.sh
- name: "Source package verification with Python 3 (OSX arm64) +docs"
dependencies: []
task:
agent:
machine:
type: s1-prod-macos-13-5-arm64
env_vars:
- name: OS_NAME
value: osx
- name: ARCH
value: arm64
jobs:
- name: Build
commands:
- sem-version python 3.8
# use a virtualenv
- python3 -m venv _venv && source _venv/bin/activate
- chmod u+r+x tools/source-package-verification.sh
- tools/source-package-verification.sh
- python setup.py sdist
- cd dist
- export SOURCE_BUILD_FILENAME=`ls *.tar.gz`
- cd ..
- artifact push workflow dist/${SOURCE_BUILD_FILENAME} --destination artifacts/${SOURCE_BUILD_FILENAME}
# - name: "Wheels: OSX x64"
# run:
# when: "tag =~ '.*'"
# dependencies: []
# task:
# agent:
# machine:
# type: s1-prod-macos-13-5-amd64
# env_vars:
# - name: OS_NAME
# value: osx
# - name: ARCH
# value: x64
# jobs:
# - name: Build
# commands:
# - PIP_INSTALL_OPTIONS="--user" tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse
# - tar -czf wheelhouse-macOS-${ARCH}.tgz wheelhouse
# - artifact push workflow wheelhouse-macOS-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/
# - name: "Wheels: OSX arm64"
# run:
# when: "tag =~ '.*'"
# dependencies: []
# task:
# agent:
# machine:
# type: s1-prod-macos-13-5-arm64
# env_vars:
# - name: OS_NAME
# value: osx
# - name: CIBW_ARCHS
# value: arm64
# - name: ARCH
# value: arm64
# jobs:
# - name: Build
# commands:
# - PIP_INSTALL_OPTIONS="--user" tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse
# - tar -czf wheelhouse-macOS-${ARCH}.tgz wheelhouse
# - artifact push workflow wheelhouse-macOS-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/
# - name: "Wheels: Linux arm64"
# run:
# when: "tag =~ '.*'"
# dependencies: []
# task:
# agent:
# machine:
# type: s1-prod-ubuntu20-04-arm64-1
# env_vars:
# - name: OS_NAME
# value: linux
# - name: ARCH
# value: arm64
# jobs:
# - name: Build
# commands:
# - ./tools/build-manylinux.sh "${LIBRDKAFKA_VERSION#v}"
# - tar -czf wheelhouse-linux-${ARCH}.tgz wheelhouse
# - artifact push workflow wheelhouse-linux-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/
# - name: "Wheels: Linux x64"
# run:
# when: "tag =~ '.*'"
# dependencies: []
# task:
# agent:
# machine:
# type: s1-prod-ubuntu20-04-amd64-3
# env_vars:
# - name: OS_NAME
# value: linux
# - name: ARCH
# value: x64
# jobs:
# - name: Build
# commands:
# - ./tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse
# - tar -czf wheelhouse-linux-${ARCH}.tgz wheelhouse
# - artifact push workflow wheelhouse-linux-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/
# - name: "Wheels: Windows"
# run:
# when: "tag =~ '.*'"
# dependencies: []
# task:
# agent:
# machine:
# type: s1-prod-windows
# env_vars:
# - name: OS_NAME
# value: windows
# - name: ARCH
# value: x64
# prologue:
# commands:
# - ".\\tools\\mingw-w64\\setup-msys2.ps1"
# - $env:PATH = 'C:\msys64\usr\bin;' + $env:PATH
# - bash -lc './tools/mingw-w64/msys2-dependencies.sh'
# jobs:
# - name: Build
# env_vars:
# - name: CHERE_INVOKING
# value: 'yes'
# - name: MSYSTEM
# value: UCRT64
# commands:
# - bash tools/mingw-w64/semaphore_commands.sh
# - bash tools/wheels/install-librdkafka.sh $env:LIBRDKAFKA_VERSION.TrimStart("v") dest
# - tools/wheels/build-wheels.bat x64 win_amd64 dest wheelhouse
# - tar -czf wheelhouse-windows-${Env:ARCH}.tgz wheelhouse
# - artifact push workflow wheelhouse-windows-${Env:ARCH}.tgz --destination artifacts/wheels-${Env:OS_NAME}-${Env:ARCH}.tgz/
# - name: "Source package verification and Integration tests with Python 3 (Linux x64)"
# dependencies: []
# task:
# agent:
# machine:
# type: s1-prod-ubuntu20-04-amd64-2
# env_vars:
# - name: OS_NAME
# value: linux
# - name: ARCH
# value: x64
# prologue:
# commands:
# - '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY'
# jobs:
# - name: Build
# commands:
# - sem-version python 3.8
# # use a virtualenv
# - python3 -m venv _venv && source _venv/bin/activate
# - chmod u+r+x tools/source-package-verification.sh
# - tools/source-package-verification.sh
# - name: "Source package verification with Python 3 (Linux arm64)"
# dependencies: []
# task:
# agent:
# machine:
# type: s1-prod-ubuntu20-04-arm64-1
# env_vars:
# - name: OS_NAME
# value: linux
# - name: ARCH
# value: arm64
# jobs:
# - name: Build
# commands:
# - sem-version python 3.8
# # use a virtualenv
# - python3 -m venv _venv && source _venv/bin/activate
# - chmod u+r+x tools/source-package-verification.sh
# - tools/source-package-verification.sh
# - name: "Source package verification with Python 3 (OSX x64) +docs"
# dependencies: []
# task:
# agent:
# machine:
# type: s1-prod-macos-13-5-amd64
# env_vars:
# - name: OS_NAME
# value: osx
# - name: ARCH
# value: x64
# jobs:
# - name: Build
# commands:
# - sem-version python 3.8
# # use a virtualenv
# - python3 -m venv _venv && source _venv/bin/activate
# - chmod u+r+x tools/source-package-verification.sh
# - tools/source-package-verification.sh
# - name: "Source package verification with Python 3 (OSX arm64) +docs"
# dependencies: []
# task:
# agent:
# machine:
# type: s1-prod-macos-13-5-arm64
# env_vars:
# - name: OS_NAME
# value: osx
# - name: ARCH
# value: arm64
# jobs:
# - name: Build
# commands:
# - sem-version python 3.8
# # use a virtualenv
# - python3 -m venv _venv && source _venv/bin/activate
# - chmod u+r+x tools/source-package-verification.sh
# - tools/source-package-verification.sh
- name: "Packaging"
run:
when: "tag =~ '.*'"
# run:
# when: "tag =~ '.*'"
dependencies:
- "Wheels: OSX x64"
- "Wheels: OSX arm64"
- "Wheels: Linux arm64"
- "Wheels: Linux x64"
- "Wheels: Windows"
# - "Wheels: OSX x64"
# - "Wheels: OSX arm64"
# - "Wheels: Linux arm64"
# - "Wheels: Linux x64"
# - "Wheels: Windows"
- "Source Build"
task:
agent:
machine:
@@ -220,7 +237,10 @@ blocks:
commands:
- artifact pull workflow artifacts
- cd artifacts
- ls *.tgz |xargs -n1 tar -xvf
# - ls *.tgz |xargs -n1 tar -xvf
- mkdir wheelhouse
- ls -la
- mv *.tar.gz wheelhouse/
- tar cvf confluent-kafka-python-wheels-${SEMAPHORE_GIT_TAG_NAME}-${SEMAPHORE_WORKFLOW_ID}.tgz wheelhouse/
- ls -la
- sha256sum confluent-kafka-python-wheels-${SEMAPHORE_GIT_TAG_NAME}-${SEMAPHORE_WORKFLOW_ID}.tgz