diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..021da66 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,20 @@ +stages: + - build + +variables: + DOCKER_DRIVER: overlay + EXPORT_DOCKER_ON: "domain.com" + BUILD_PACKAGES_ON: "domain.com" + +benchmark: + stage: build + tags: [docker] + image: registry.gitlab.com/software-energy-lab/nea:latest + script: + - runner go csv-svg-path + - runner rust-nea csv-svg-path + - runner roc-nea csv-svg-path + artifacts: + paths: + - "*.json" + diff --git a/Cargo.lock b/Cargo.lock index 4f1297b..5350b6f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "anyhow" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" + [[package]] name = "atomic-wait" version = "1.1.0" @@ -48,6 +54,18 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "buildinfy" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "430a88bf7f097544c3f9afb547b3ad3a4ab6ab5e9aa2b349fdceb0a76a6adb16" + [[package]] name = "bytes" version = "1.5.0" @@ -78,18 +96,57 @@ dependencies = [ "num-traits", ] +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "energy-bench" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e37e5e315f82ab81850930eddf28531417d816e60c7c98fe58ac24e3569149f" +dependencies = [ + "anyhow", + "buildinfy", + "ureq", +] + [[package]] name = "equivalent" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "flate2" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + [[package]] name = "futures-channel" version = "0.3.29" @@ -123,6 +180,17 @@ dependencies = [ "pin-utils", ] +[[package]] +name = "getrandom" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "gimli" version = "0.28.0" @@ -193,6 +261,16 @@ dependencies = [ "want", ] +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "indexmap" version = "2.1.0" @@ -286,6 +364,18 @@ dependencies = [ "memchr", ] +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + [[package]] name = "pin-project-lite" version = "0.2.13" @@ -316,12 +406,65 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "runner" +version = "0.1.0" +dependencies = [ + "energy-bench", +] + [[package]] name = "rustc-demangle" version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustls" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" +dependencies = [ + "log 0.4.20", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" + +[[package]] +name = "rustls-webpki" +version = "0.102.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "serde" version = "1.0.193" @@ -360,6 +503,18 @@ dependencies = [ "log 0.1.0", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + [[package]] name = "syn" version = "2.0.39" @@ -371,6 +526,21 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.34.0" @@ -421,12 +591,61 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + [[package]] name = "unicode-ident" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-normalization" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "2.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d11a831e3c0b56e438a28308e7c810799e3c118417f342d30ecec080105395cd" +dependencies = [ + "base64", + "flate2", + "log 0.4.20", + "once_cell", + "rustls", + "rustls-pki-types", + "rustls-webpki", + "url", + "webpki-roots", +] + +[[package]] +name = "url" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + [[package]] name = "want" version = "0.3.1" @@ -442,6 +661,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "webpki-roots" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "windows-sys" version = "0.42.0" @@ -463,7 +691,16 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.5", ] [[package]] @@ -481,6 +718,22 @@ dependencies = [ "windows_x86_64_msvc 0.48.5", ] +[[package]] +name = "windows-targets" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +dependencies = [ + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -493,6 +746,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -505,6 +764,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -517,6 +782,18 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -529,6 +806,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -541,6 +824,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -553,6 +842,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -565,6 +860,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" + [[package]] name = "winnow" version = "0.5.19" @@ -573,3 +874,9 @@ checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b" dependencies = [ "memchr", ] + +[[package]] +name = "zeroize" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" diff --git a/Cargo.toml b/Cargo.toml index 9272a93..8a90cf1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "log", "shared", "nea", + "runner", ] exclude = [ "benchmarks", diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d76458e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM ubuntu:latest + +RUN apt-get update && apt-get install -y curl wget libc-dev binutils siege golang-go clang mold + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + +# Install latest roc nightly +RUN wget https://github.com/roc-lang/roc/releases/download/nightly/roc_nightly-linux_x86_64-latest.tar.gz +RUN tar -xf roc_nightly-linux_x86_64-latest.tar.gz +# Please fix this before 2099... +RUN cd roc_nightly-linux_x86_64-20* && cp roc /usr/local/bin + +ENV PATH="/root/.cargo/bin:${PATH}" + +COPY . /nea + +RUN cd /nea/benchmarks/roc-nea && roc build --optimize csv-svg-path.roc +RUN cd /nea/benchmarks/roc-nea && roc build --optimize send-static-file.roc +RUN cd /nea/benchmarks/roc-nea && roc build --optimize varying-allocations.roc +RUN cd /nea/benchmarks/rust-nea && cargo build --release +RUN cd /nea/benchmarks/rust-tokio && cargo build --release +RUN cd /nea/benchmarks/go && go build csv-svg-path.go +RUN cd /nea/benchmarks/go && go build send-static-file.go +RUN cd /nea/benchmarks/go && go build varying-allocations.go +RUN cd /nea/runner && cargo build --release && cp ../target/release/runner /usr/local/bin \ No newline at end of file diff --git a/benchmarks/csv-svg-path.txt b/benchmarks/csv-svg-path.txt new file mode 100644 index 0000000..e674bf3 --- /dev/null +++ b/benchmarks/csv-svg-path.txt @@ -0,0 +1 @@ +http://127.0.0.1:8080/10 POST < benchmarks/paths.csv diff --git a/benchmarks/paths.csv b/benchmarks/paths.csv new file mode 100644 index 0000000..98372bb --- /dev/null +++ b/benchmarks/paths.csv @@ -0,0 +1,52 @@ +370578, 299450 +128350, 287732 +221014, 519470 +2773, 434218 +720908, 132839 +304647, 478393 +281689, 365608 +972811, 804316 +395740, 478228 +6677, 778501 +463739, 837952 +82383, 200396 +8988, 357228 +264210, 389147 +230326, 616897 +547659, 801711 +545518, 361647 +43147, 515588 +795785, 205692 +506933, 375831 +881354, 253008 +9600, 405302 +128792, 583597 +129494, 563675 +645961, 618280 +378975, 592141 +471886, 996297 +387183, 46028 +24748, 796164 +384854, 120970 +178555, 423542 +184338, 36726 +447700, 810950 +447454, 439655 +208208, 835917 +490757, 296193 +177931, 99597 +217242, 211469 +237675, 318281 +643023, 53680 +545710, 727098 +262493, 133403 +435867, 295392 +339124, 275281 +536855, 337921 +769567, 419610 +99056, 232021 +994014, 39479 +886234, 583161 +661668, 699509 +427439, 1786 +347137, 472237 \ No newline at end of file diff --git a/benchmarks/send-static-file.txt b/benchmarks/send-static-file.txt new file mode 100644 index 0000000..768833d --- /dev/null +++ b/benchmarks/send-static-file.txt @@ -0,0 +1 @@ +http://127.0.0.1:8080/ diff --git a/benchmarks/urls.txt b/benchmarks/urls.txt deleted file mode 100644 index 6564213..0000000 --- a/benchmarks/urls.txt +++ /dev/null @@ -1 +0,0 @@ -127.0.0.1:8000/10 POST 1, 1\n2, 1\n2, 2\n1, 1 diff --git a/benchmarks/varying-allocations.txt b/benchmarks/varying-allocations.txt new file mode 100644 index 0000000..a77670e --- /dev/null +++ b/benchmarks/varying-allocations.txt @@ -0,0 +1,3 @@ +http://127.0.0.1:8080/1 +http://127.0.0.1:8080/10 +http://127.0.0.1:8080/50 \ No newline at end of file diff --git a/runner/Cargo.toml b/runner/Cargo.toml new file mode 100644 index 0000000..83c2cf2 --- /dev/null +++ b/runner/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "runner" +version = "0.1.0" +edition = "2021" + +[dependencies] +energy-bench = { version = "0.1.6"} diff --git a/runner/src/main.rs b/runner/src/main.rs new file mode 100644 index 0000000..bb985a4 --- /dev/null +++ b/runner/src/main.rs @@ -0,0 +1,40 @@ +use std::sync::OnceLock; + +static EXAMPLE: OnceLock = OnceLock::new(); + +fn main() { + let mut it = std::env::args(); + let _ = it.next(); + let implementation = it.next().unwrap(); + let example = it.next().unwrap(); + + let path = match implementation.as_str() { + "go" => format!("benchmarks/go/{}", example), + "roc-nea" => format!("benchmarks/roc-nea/{}", example), + "rust-nea" => format!("benchmarks/rust-nea/target/release/{}", example), + "rust-tokio" => format!("benchmarks/rust-tokio/target/release/{}", example), + _ => unreachable!(), + }; + + let mut child = std::process::Command::new(path) + .env("RUST_LOG", "error") + .spawn() + .unwrap(); + + EXAMPLE.set(example.clone()).unwrap(); + + let json = energy_bench::EnergyTool::from_function(|()| { + let siege = std::process::Command::new("siege") + .args(&["-j", "-c", "4", "-r", "10000", "-i", "-f", &format!("benchmarks/{}.txt", EXAMPLE.get().unwrap())]) + .output() + .unwrap(); + + if siege.status.code() != Some(0) { + panic!("siege failed: {:?}", siege); + } + }).benchmark().unwrap(); + + std::fs::write(format!("benchmarks/{implementation}-{example}.json"), json).unwrap(); + + child.kill().unwrap(); +}