Skip to content

Commit 3e5ee37

Browse files
vaindvaind
authored
ci: collect SDK app metrics (#1034)
1 parent 0acb66c commit 3e5ee37

13 files changed

+315
-0
lines changed

‎.github/workflows/metrics.yml

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
name: SDK metrics
2+
3+
on:
4+
push:
5+
branches: [main]
6+
pull_request:
7+
paths:
8+
- .github/workflows/metrics.yml
9+
- dart/**
10+
- flutter/**
11+
- metrics/**
12+
13+
jobs:
14+
metrics:
15+
name: ${{ matrix.name }}
16+
runs-on: ${{ matrix.host }}
17+
strategy:
18+
# We want that the matrix keeps running, default is to cancel all jobs if a single fails.
19+
fail-fast: false
20+
matrix:
21+
include:
22+
- platform: ios
23+
name: iOS
24+
appPlain: test-app-plain.ipa
25+
host: macos-latest
26+
- platform: android
27+
name: Android
28+
appPlain: metrics/perf-test-app-plain/build/app/outputs/apk/release/app-arm64-v8a-release.apk
29+
host: ubuntu-latest
30+
31+
steps:
32+
- uses: actions/checkout@v3
33+
34+
# Let's stick to an explicit version and update manually because a version change may affect results.
35+
# If it would update implicitly it could confuse people to think the change is actually caused by the PR.
36+
# Instead, we use Updater (update-deps.yml) to create PRs for explicit Flutter SDK update.
37+
- name: Read configured Flutter SDK version
38+
id: conf
39+
run: |
40+
version=$(grep "version" metrics/flutter.properties | cut -d'=' -f2 | xargs)
41+
echo "::set-output name=flutter::$version"
42+
43+
- name: Install Flutter v${{ steps.conf.outputs.flutter }}
44+
uses: subosito/flutter-action@1e6ee87cb840500837bcd50a667fb28815d8e310 # pin@v2
45+
with:
46+
flutter-version: ${{ steps.conf.outputs.flutter }}
47+
48+
- uses: actions/setup-java@v3
49+
if: ${{ matrix.platform == 'android' }}
50+
with:
51+
java-version: '11'
52+
distribution: 'adopt'
53+
54+
- run: ./metrics/prepare.sh
55+
56+
- uses: actions/cache@v3
57+
id: app-plain-cache
58+
with:
59+
path: ${{ matrix.appPlain }}
60+
key: ${{ github.workflow }}-${{ github.job }}-appplain-${{ matrix.platform }}-${{ hashFiles('metrics/perf-test-app-plain/pubspec.yaml') }}
61+
62+
- name: Build
63+
run: ./metrics/build.sh ${{ matrix.platform }}
64+
env:
65+
# Necessary to build an iOS .ipa (using fastlane)
66+
APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }}
67+
APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }}
68+
APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }}
69+
FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }}
70+
MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }}
71+
MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
72+
MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }}
73+
APP_PLAIN: ${{ matrix.appPlain }}
74+
75+
- name: Collect apps metrics
76+
uses: getsentry/action-app-sdk-overhead-metrics@v1
77+
with:
78+
name: ${{ matrix.name }}
79+
config: ./metrics/metrics-${{ matrix.platform }}.yml
80+
sauce-user: ${{ secrets.SAUCE_USERNAME }}
81+
sauce-key: ${{ secrets.SAUCE_ACCESS_KEY }}

‎.github/workflows/update-deps.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,11 @@ jobs:
2525
name: Cocoa SDK
2626
secrets:
2727
api-token: ${{ secrets.CI_DEPLOY_KEY }}
28+
29+
metrics-flutter:
30+
uses: getsentry/github-workflows/.github/workflows/updater.yml@v2
31+
with:
32+
path: metrics/flutter.properties
33+
name: Flutter SDK (metrics)
34+
secrets:
35+
api-token: ${{ secrets.CI_DEPLOY_KEY }}

‎metrics/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
perf-test-app*

‎metrics/build.sh

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
targetDir=$(
5+
cd $(dirname $0)
6+
pwd
7+
)
8+
[[ "$targetDir" != "" ]] || exit 1
9+
10+
if [ "$#" -gt 2 ]; then
11+
shift
12+
args="$@"
13+
target=$1
14+
else
15+
# Use default args if no were given
16+
case "$1" in
17+
ios)
18+
target="ipa"
19+
args="--release --no-codesign"
20+
;;
21+
22+
android)
23+
target="apk"
24+
args="--release --target-platform android-arm64 --split-per-abi"
25+
;;
26+
27+
*)
28+
echo "Unknown platform: '$1'"
29+
exit 1
30+
;;
31+
esac
32+
fi
33+
34+
for dir in "$targetDir/perf-test-app-"*; do
35+
app="$(basename "$dir")"
36+
echo "Building $app"
37+
38+
# Support caching if this is running in CI.
39+
if [[ -n ${CI+x} && $app == *-plain && -f $APP_PLAIN ]]; then
40+
echo "Cached app exists: $APP_PLAIN - skipping build"
41+
continue
42+
fi
43+
44+
(
45+
echo "::group::Flutter build $app"
46+
cd $dir
47+
flutter pub get
48+
if [[ "$1" == "ios" && -f "ios/Podfile" ]]; then
49+
cd ios
50+
pod install --repo-update
51+
cd -
52+
fi
53+
flutter build $target $args
54+
echo '::endgroup::'
55+
)
56+
57+
if [[ "$1" == "ios" ]]; then
58+
flJob=${app//-/_}
59+
(
60+
echo "::group::Fastlane build $app"
61+
cd $targetDir
62+
fastlane "build_$flJob" --verbose
63+
echo '::endgroup::'
64+
)
65+
fi
66+
done

‎metrics/fastlane/.gitignore

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
test_output/
2+
README.md
3+
report.xml
4+
.env
5+
cobertura.xml

‎metrics/fastlane/Appfile

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
itc_team_id("96157806") # App Store Connect Team ID
2+
team_id("97JCY7859U") # Developer Portal Team ID
3+
4+
# For more information about the Appfile, see:
5+
# https://docs.fastlane.tools/advanced/#appfile
6+

‎metrics/fastlane/Fastfile

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
default_platform(:ios)
2+
3+
platform :ios do
4+
desc "Build perf-test app without Sentry"
5+
lane :build_perf_test_app_plain do
6+
setup_ci
7+
8+
sync_code_signing(
9+
type: "development",
10+
readonly: true,
11+
app_identifier: ["io.sentry.dart.perfTestAppPlain"]
12+
)
13+
14+
build_app(
15+
workspace: "perf-test-app-plain/ios/Runner.xcworkspace",
16+
scheme: "Runner",
17+
include_bitcode: false,
18+
include_symbols: false,
19+
export_method: "development",
20+
export_team_id: CredentialsManager::AppfileConfig.try_fetch_value(:team_id),
21+
output_name: "test-app-plain.ipa",
22+
skip_build_archive: true,
23+
archive_path: "perf-test-app-plain/build/ios/archive/Runner.xcarchive",
24+
)
25+
26+
delete_keychain(name: "fastlane_tmp_keychain") unless is_ci
27+
end
28+
29+
desc "Build perf-test app with Sentry"
30+
lane :build_perf_test_app_with_sentry do
31+
setup_ci
32+
33+
sync_code_signing(
34+
type: "development",
35+
readonly: true,
36+
app_identifier: ["io.sentry.dart.perfTestAppWithSentry"]
37+
)
38+
39+
build_app(
40+
workspace: "perf-test-app-with-sentry/ios/Runner.xcworkspace",
41+
scheme: "Runner",
42+
include_bitcode: false,
43+
include_symbols: false,
44+
export_method: "development",
45+
export_team_id: CredentialsManager::AppfileConfig.try_fetch_value(:team_id),
46+
output_name: "test-app-sentry.ipa",
47+
skip_build_archive: true,
48+
archive_path: "perf-test-app-with-sentry/build/ios/archive/Runner.xcarchive",
49+
)
50+
51+
delete_keychain(name: "fastlane_tmp_keychain") unless is_ci
52+
end
53+
54+
end

‎metrics/fastlane/Matchfile

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
git_url("[email protected]:getsentry/codesigning.git")
2+
storage_mode("git")
3+
username("[email protected]") # Your Apple Developer Portal username
4+
5+
# The docs are available on https://docs.fastlane.tools/actions/match

‎metrics/fastlane/Pluginfile

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Autogenerated by fastlane
2+
#
3+
# Ensure this file is checked in to source control!
4+
5+
gem 'fastlane-plugin-sentry'

‎metrics/flutter.properties

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
version = 3.3.3
2+
repo = https://github.com/flutter/flutter

‎metrics/metrics-android.yml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
apps:
2+
- name: io.sentry.dart.perf_test_app_plain
3+
activity: MainActivity
4+
path: perf-test-app-plain/build/app/outputs/apk/release/app-arm64-v8a-release.apk
5+
- name: io.sentry.dart.perf_test_app_with_sentry
6+
activity: MainActivity
7+
path: perf-test-app-with-sentry/build/app/outputs/apk/release/app-arm64-v8a-release.apk
8+
9+
startupTimeTest:
10+
runs: 50
11+
diffMin: 0
12+
diffMax: 150
13+
14+
binarySizeTest:
15+
diffMin: 900 KiB
16+
diffMax: 1100 KiB

‎metrics/metrics-ios.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
apps:
2+
- name: io.sentry.dart.perfTestAppPlain
3+
path: test-app-plain.ipa
4+
- name: io.sentry.dart.perfTestAppWithSentry
5+
path: test-app-sentry.ipa
6+
7+
startupTimeTest:
8+
runs: 50
9+
diffMin: 0
10+
diffMax: 150
11+
12+
binarySizeTest:
13+
diffMin: 900 KiB
14+
diffMax: 1100 KiB

‎metrics/prepare.sh

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
targetDir=$(
5+
cd $(dirname $0)
6+
pwd
7+
)
8+
[[ "$targetDir" != "" ]] || exit 1
9+
10+
flutterCreate() {
11+
name=${1//-/_}
12+
dir=$targetDir/$1
13+
rm -rf $dir
14+
echo "::group::Flutter create $1"
15+
flutter create --template=app --no-pub --org 'io.sentry.dart' --project-name $name "$dir"
16+
echo '::endgroup::'
17+
}
18+
19+
flutterCreate 'perf-test-app-plain'
20+
flutterCreate 'perf-test-app-with-sentry'
21+
22+
echo '::group::Patch perf-test-app-with-sentry'
23+
pubspec="$targetDir/perf-test-app-with-sentry/pubspec.yaml"
24+
echo "Adding dependencies to $pubspec"
25+
cat <<EOF >>"$pubspec"
26+
27+
dependency_overrides:
28+
sentry:
29+
path: ../../dart
30+
sentry_flutter:
31+
path: ../../flutter
32+
33+
EOF
34+
35+
patch -p0 "$targetDir/perf-test-app-with-sentry/lib/main.dart" <<'EOF'
36+
@@ -1,7 +1,12 @@
37+
import 'package:flutter/material.dart';
38+
+import 'package:sentry_flutter/sentry_flutter.dart';
39+
40+
-void main() {
41+
- runApp(const MyApp());
42+
+Future<void> main() async {
43+
+ await SentryFlutter.init(
44+
+ (options) => options.dsn =
45+
+ 'https://[email protected]/5428562',
46+
+ appRunner: () => runApp(const MyApp()),
47+
+ );
48+
}
49+
50+
class MyApp extends StatelessWidget {
51+
EOF
52+
echo '::endgroup::'

0 commit comments

Comments
 (0)