Skip to content

Commit 998da01

Browse files
freef4llfreef4ll
committed
Refactor and add docker scripts
1 parent 0a5b476 commit 998da01

19 files changed

+273
-66
lines changed

‎Benchmarks/.gitignore

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
.DS_Store
2+
/.build
3+
/Packages
4+
xcuserdata/
5+
DerivedData/
6+
.swiftpm/configuration/registries.json
7+
.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata
8+
.netrc
9+
.benchmarkBaselines/

‎Benchmarks/Benchmarks/PrometheusBenchmarks/Benchmarks.swift

Lines changed: 28 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -16,82 +16,49 @@
1616
import Benchmark
1717
import Prometheus
1818

19+
let registry = PrometheusCollectorRegistry()
20+
21+
public func makeLabels(_ idx: Int) -> [(String, String)] {
22+
[
23+
("job", "api_server_\(idx)"),
24+
("handler", "/api/handler_\(idx)"),
25+
("status_code", "200"),
26+
("version", "\(idx).0.0"),
27+
]
28+
}
29+
1930
let benchmarks = {
2031
Benchmark.defaultConfiguration.maxDuration = .seconds(5)
2132
Benchmark.defaultConfiguration.scalingFactor = .kilo
33+
// Benchmark.defaultConfiguration.metrics = [.wallClock, .throughput, .mallocCountTotal]
34+
Benchmark.defaultConfiguration.metrics = [.mallocCountTotal]
2235

23-
let registry = PrometheusCollectorRegistry()
24-
25-
func metricsDimensions(_ idx: Int) -> [(String, String)] {
26-
[
27-
("job", "api_server_\(idx)"),
28-
("handler", "/api/handler_\(idx)"),
29-
("status_code", "200"),
30-
("version", "\(idx).0.0"),
31-
]
36+
Benchmark("Counter #1") { benchmark in
37+
runCounterBench(benchmark.scaledIterations)
3238
}
3339

34-
Benchmark("1 - Metrics: Counter benchmark") { benchmark in
35-
let ctr = registry.makeCounter(name: "counter_1", labels: metricsDimensions(1))
36-
benchmark.startMeasurement()
40+
Benchmark("Counter #2") { benchmark, run in
3741
for _ in benchmark.scaledIterations {
38-
blackHole(ctr.increment())
42+
run()
3943
}
44+
} setup: {
45+
setupCounterBench()
4046
}
4147

42-
Benchmark("2 - Metrics: Gauge benchmark") { benchmark in
43-
let gauge = registry.makeGauge(name: "gauge_1", labels: metricsDimensions(2))
44-
benchmark.startMeasurement()
45-
for _ in benchmark.scaledIterations {
46-
blackHole(gauge.increment())
47-
}
48+
Benchmark("Gauge") { benchmark in
49+
runGaugeBench(benchmark.scaledIterations)
4850
}
4951

50-
Benchmark("3 - Metrics: Histogram benchmark") { benchmark in
51-
let histogram = registry.makeDurationHistogram(name: "histogram_1", labels: metricsDimensions(3),
52-
buckets: [
53-
.milliseconds(100),
54-
.milliseconds(250),
55-
.milliseconds(500),
56-
.seconds(1),
57-
])
58-
benchmark.startMeasurement()
59-
for _ in benchmark.scaledIterations {
60-
histogram.record(Duration.milliseconds(400))
61-
}
52+
Benchmark("DurationHistogram") { benchmark in
53+
runDurationHistogramBench(benchmark.scaledIterations)
6254
}
6355

64-
Benchmark("4 - Metrics: export 5000 metrics",
65-
configuration: .init(scalingFactor: .one)) { benchmark in
66-
let metricsCount = 5000
67-
68-
let registryExport = PrometheusCollectorRegistry()
69-
70-
var counterArray = [Counter]()
71-
var gaugeArray = [Gauge]()
72-
var buffer = [UInt8]()
73-
74-
let counterExportSize = 620_000
75-
counterArray.reserveCapacity(metricsCount)
76-
gaugeArray.reserveCapacity(metricsCount)
77-
buffer.reserveCapacity(counterExportSize)
78-
79-
for i in 0..<(metricsCount / 2) {
80-
let counter = registryExport.makeCounter(name: "http_requests_total", labels: metricsDimensions(i))
81-
counter.increment()
82-
counterArray.append(counter)
83-
84-
let gauge = registryExport.makeGauge(name: "export_gauge_\(i)", labels: metricsDimensions(i))
85-
gauge.increment()
86-
gaugeArray.append(gauge)
87-
}
88-
89-
benchmark.startMeasurement()
56+
Benchmark("RegistryEmit - 5000 metrics",
57+
configuration: .init(scalingFactor: .one)) { benchmark, run in
9058
for _ in benchmark.scaledIterations {
91-
blackHole(registryExport.emit(into: &buffer))
92-
59+
run()
9360
}
94-
benchmark.stopMeasurement()
95-
buffer.removeAll(keepingCapacity: true)
61+
} setup: {
62+
setupRegistryExport(numberOfMetrics: 5000)
9663
}
9764
}

‎Benchmarks/Benchmarks/PrometheusBenchmarks/Counter.swift

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
// swift-tools-version:5.7
2+
//===----------------------------------------------------------------------===//
3+
//
4+
// This source file is part of the SwiftPrometheus open source project
5+
//
6+
// Copyright (c) 2018-2023 SwiftPrometheus project authors
7+
// Licensed under Apache License v2.0
8+
//
9+
// See LICENSE.txt for license information
10+
// See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors
11+
//
12+
// SPDX-License-Identifier: Apache-2.0
13+
//
14+
//===----------------------------------------------------------------------===//
15+
16+
import Benchmark
17+
import Prometheus
18+
19+
public func runCounterBench(_ iterations: Range<Int>) {
20+
let ctr = registry.makeCounter(name: "counter_1", labels: makeLabels(1))
21+
for _ in iterations {
22+
blackHole(ctr.increment())
23+
}
24+
}
25+
26+
public func setupCounterBench() -> () -> Void {
27+
let ctr = registry.makeCounter(name: "counter_2", labels: makeLabels(2))
28+
return {
29+
blackHole(ctr.increment())
30+
}
31+
}

‎Benchmarks/Benchmarks/PrometheusBenchmarks/DurationHistogram.swift

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
// swift-tools-version:5.7
2+
//===----------------------------------------------------------------------===//
3+
//
4+
// This source file is part of the SwiftPrometheus open source project
5+
//
6+
// Copyright (c) 2018-2023 SwiftPrometheus project authors
7+
// Licensed under Apache License v2.0
8+
//
9+
// See LICENSE.txt for license information
10+
// See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors
11+
//
12+
// SPDX-License-Identifier: Apache-2.0
13+
//
14+
//===----------------------------------------------------------------------===//
15+
16+
import Benchmark
17+
import Prometheus
18+
19+
public func runDurationHistogramBench(_ iterations: Range<Int>) {
20+
let histogram = registry.makeDurationHistogram(name: "histogram_1", labels: makeLabels(3),
21+
buckets: [
22+
.milliseconds(100),
23+
.milliseconds(250),
24+
.milliseconds(500),
25+
.seconds(1),
26+
])
27+
for _ in iterations {
28+
blackHole(histogram.record(Duration.milliseconds(400)))
29+
}
30+
}

‎Benchmarks/Benchmarks/PrometheusBenchmarks/Gauge.swift

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
// swift-tools-version:5.7
2+
//===----------------------------------------------------------------------===//
3+
//
4+
// This source file is part of the SwiftPrometheus open source project
5+
//
6+
// Copyright (c) 2018-2023 SwiftPrometheus project authors
7+
// Licensed under Apache License v2.0
8+
//
9+
// See LICENSE.txt for license information
10+
// See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors
11+
//
12+
// SPDX-License-Identifier: Apache-2.0
13+
//
14+
//===----------------------------------------------------------------------===//
15+
16+
import Benchmark
17+
import Prometheus
18+
19+
public func runGaugeBench(_ iterations: Range<Int>) {
20+
let gauge = registry.makeGauge(name: "gauge_1", labels: makeLabels(2))
21+
for _ in iterations {
22+
blackHole(gauge.increment())
23+
}
24+
}
25+
26+
public func setupGaugeBench() -> () -> Void {
27+
let gauge = registry.makeGauge(name: "gauge_1", labels: makeLabels(2))
28+
return {
29+
blackHole(gauge.increment())
30+
}
31+
}

‎Benchmarks/Benchmarks/PrometheusBenchmarks/RegistryEmit.swift

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
// swift-tools-version:5.7
2+
//===----------------------------------------------------------------------===//
3+
//
4+
// This source file is part of the SwiftPrometheus open source project
5+
//
6+
// Copyright (c) 2018-2023 SwiftPrometheus project authors
7+
// Licensed under Apache License v2.0
8+
//
9+
// See LICENSE.txt for license information
10+
// See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors
11+
//
12+
// SPDX-License-Identifier: Apache-2.0
13+
//
14+
//===----------------------------------------------------------------------===//
15+
16+
import Benchmark
17+
import Prometheus
18+
19+
public func setupRegistryExport(numberOfMetrics: Int) -> () -> Void {
20+
let registryExport = PrometheusCollectorRegistry()
21+
22+
var counterArray = [Counter]()
23+
var gaugeArray = [Gauge]()
24+
var buffer = [UInt8]()
25+
26+
let counterExportSize = 620_000
27+
counterArray.reserveCapacity(numberOfMetrics)
28+
gaugeArray.reserveCapacity(numberOfMetrics)
29+
buffer.reserveCapacity(counterExportSize)
30+
31+
for i in 0..<(numberOfMetrics / 2) {
32+
let counter = registryExport.makeCounter(name: "http_requests_total", labels: makeLabels(i))
33+
counter.increment()
34+
counterArray.append(counter)
35+
36+
let gauge = registryExport.makeGauge(name: "export_gauge_\(i)", labels: makeLabels(i))
37+
gauge.increment()
38+
gaugeArray.append(gauge)
39+
}
40+
return {
41+
blackHole(registryExport.emit(into: &buffer))
42+
}
43+
}

‎Benchmarks/Package.swift

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import PackageDescription
1818
let package = Package(
1919
name: "benchmarks",
2020
platforms: [
21-
.macOS("13"),
21+
.macOS("14"),
2222
],
2323
dependencies: [
2424
.package(path: "../"),
@@ -29,10 +29,12 @@ let package = Package(
2929
name: "PrometheusBenchmarks",
3030
dependencies: [
3131
.product(name: "Benchmark", package: "package-benchmark"),
32-
.product(name: "BenchmarkPlugin", package: "package-benchmark"),
3332
.product(name: "Prometheus", package: "swift-prometheus"),
3433
],
35-
path: "Benchmarks/PrometheusBenchmarks"
34+
path: "Benchmarks/PrometheusBenchmarks",
35+
plugins: [
36+
.plugin(name: "BenchmarkPlugin", package: "package-benchmark"),
37+
]
3638
),
3739
]
3840
)

‎Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Counter_#1.p90.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"mallocCountTotal" : 1
3+
}

‎Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Counter_#2.p90.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"mallocCountTotal" : 0
3+
}

‎Benchmarks/Thresholds/5.9/PrometheusBenchmarks.DurationHistogram.p90.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"mallocCountTotal" : 2
3+
}

‎Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Gauge.p90.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"mallocCountTotal" : 1
3+
}

‎Benchmarks/Thresholds/5.9/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"mallocCountTotal" : 0
3+
}

‎README.md

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,17 @@ If you find a bug or have issues, please [create an issue](https://github.com/sw
2222

2323
[Documentation]: https://swiftpackageindex.com/swift-server/swift-prometheus/documentation/prometheus
2424
[SSWG-Incubation]: https://www.swift.org/sswg/incubation-process.html
25+
26+
27+
## Benchmarks
28+
29+
Benchmarks for `swift-prometheus` are in a separate Swift Package in the `Benchmarks` subfolder of this repository.
30+
They use the [`package-benchmark`](https://github.com/ordo-one/package-benchmark) plugin.
31+
Benchmarks depends on the [`jemalloc`](https://jemalloc.net) memory allocation library, which is used by `package-benchmark` to capture memory allocation statistics.
32+
An installation guide can be found in the [Getting Started article](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark/gettingstarted#Installing-Prerequisites-and-Platform-Support) of `package-benchmark`.
33+
Afterwards you can run the benchmarks from CLI by going to the `Benchmarks` subfolder (e.g. `cd Benchmarks`) and invoking:
34+
```
35+
swift package benchmark
36+
```
37+
38+
For more information please refer to `swift package benchmark --help` or the [documentation of `package-benchmark`](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark).

‎dev/update-benchmark-tresholds.sh

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
#!/bin/bash
2+
##===----------------------------------------------------------------------===##
3+
##
4+
## This source file is part of the SwiftNIO open source project
5+
##
6+
## Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors
7+
## Licensed under Apache License v2.0
8+
##
9+
## See LICENSE.txt for license information
10+
## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
11+
##
12+
## SPDX-License-Identifier: Apache-2.0
13+
##
14+
##===----------------------------------------------------------------------===##
15+
##===----------------------------------------------------------------------===##
16+
##
17+
## This source file is part of the SwiftCertificates open source project
18+
##
19+
## Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors
20+
## Licensed under Apache License v2.0
21+
##
22+
## See LICENSE.txt for license information
23+
## See CONTRIBUTORS.txt for the list of SwiftCertificates project authors
24+
##
25+
## SPDX-License-Identifier: Apache-2.0
26+
##
27+
##===----------------------------------------------------------------------===##
28+
29+
set -eu
30+
set -o pipefail
31+
32+
here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
33+
target_repo=${2-"$here/.."}
34+
35+
for f in 57 58 59 510 -nightly; do
36+
echo "swift$f"
37+
38+
docker_file=$(if [[ "$f" == "-nightly" ]]; then f=main; fi && ls "$target_repo/docker/docker-compose."*"$f"*".yaml")
39+
40+
docker-compose -f docker/docker-compose.yaml -f $docker_file run update-benchmark-baseline
41+
done

‎docker/docker-compose.2204.57.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,10 @@ services:
1717
environment: []
1818
#- SANITIZER_ARG=--sanitize=thread
1919

20+
update-benchmark-baseline:
21+
image: prometheus:22.04-5.7
22+
environment:
23+
- SWIFT_VERSION=5.7
24+
2025
shell:
2126
image: prometheus:22.04-5.7

‎docker/docker-compose.2204.58.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,5 +18,10 @@ services:
1818
- IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error
1919
#- SANITIZER_ARG=--sanitize=thread
2020

21+
update-benchmark-baseline:
22+
image: prometheus:22.04-5.8
23+
environment:
24+
- SWIFT_VERSION=5.8
25+
2126
shell:
2227
image: prometheus:22.04-5.8

0 commit comments

Comments
 (0)