Use oxide-tokio-rt rather than #[tokio::main] (#523)

As described in [RFD 579], we are now recommending certain non-default
Tokio runtime configurations in all software we deploy in production.
The [`oxide-tokio-rt`] crate provides a way to easily get these common
recommended configurations by using its functions to initialize the
runtime rather than `#[tokio::main]`

Presently, `oxide-tokio-rt` does the following:

- Enables DTrace probes for Tokio runtime events using [`tokio-dtrace`]
- Disables the [LIFO slot optimization][lifo], which was the root cause
  of omicron#8334

But, we anticipate that if there are additional Tokio configurations
which we want to recommend in the future, we'll add them here as well.

This commit updates `dendrite` to initialize the Tokio runtime using
`oxide-tokio-rt`. I've added a Clippy config for warning on uses of
`#[tokio::main]`, as well, to avoid forgetting to use `oxide-tokio-rt`
if new binaries are added. All the configs we set currently in
`oxide-tokio-rt` requires the `tokio_unstable` RUSTFLAGS cfg, but it
looks like that's already being set, so that shouldn't be a problem.

I've changed every binary to use `oxide-tokio-rt` with the exception of
`mg-package` --- it didn't seem necessary to use the production-like
config there and maybe avoiding an additional dependency to fetch would
make it build slightly faster, although I'm unconvinced. I did change
the lab binaries to use `oxide-tokio-rt`, as I figured that the DTrace
probes could come in handy there.

[RFD 579]: https://rfd.shared.oxide.computer/rfd/0579
[`oxide-tokio-rt`]: https://github.com/oxidecomputer/oxide-tokio-rt
[`tokio-dtrace`]: https://github.com/oxidecomputer/tokio-dtrace
[lifo]:
    https://rfd.shared.oxide.computer/rfd/0579#_disabling_the_lifo_slot_optimization

Author: hawkw

Cargo.lock

@@ -82,6 +82,7 @@ rand = "0.8.5"
 backoff = "0.4"
 mg-common = { path = "mg-common" }
 chrono = { version = "0.4.41", features = ["serde"] }
+oxide-tokio-rt = "0.1.2"
 oximeter = { git = "https://github.com/oxidecomputer/omicron", branch = "main"}
 oximeter-producer = { git = "https://github.com/oxidecomputer/omicron", branch = "main"}
 oxnet = { version = "0.1.2", default-features = false, features = ["schemars", "serde"] }

Cargo.toml

@@ -0,0 +1,4 @@
+[[disallowed-macros]]
+path = "tokio::main"
+reason = "prefer `oxide_tokio_rt` for production software"
+replacement = "oxide_tokio_rt::run"

clippy.toml

@@ -11,6 +11,7 @@ anstyle.workspace = true
 anyhow.workspace = true
 clap.workspace = true
 colored.workspace = true
+oxide-tokio-rt.workspace = true
 oxnet.workspace = true
 slog-async.workspace = true
 slog-envlogger.workspace = true

ddmadm/Cargo.toml

@@ -89,9 +89,8 @@ struct Peer {
     addr: Ipv6Addr,
 }
 
-#[tokio::main]
-async fn main() -> Result<()> {
-    run().await
+fn main() -> Result<()> {
+    oxide_tokio_rt::run(run())
 }
 
 async fn run() -> Result<()> {

ddmadm/src/main.rs

@@ -9,6 +9,7 @@ mg-common = { path = "../mg-common" }
 anyhow.workspace = true
 clap.workspace = true
 libnet.workspace = true
+oxide-tokio-rt.workspace = true
 slog.workspace = true
 slog-bunyan.workspace = true
 slog-async.workspace = true

ddmd/Cargo.toml

@@ -107,8 +107,11 @@ struct Dendrite {
     port: u16,
 }
 
-#[tokio::main]
-async fn main() {
+fn main() {
+    oxide_tokio_rt::run(run())
+}
+
+async fn run() {
     let arg = Arg::parse();
     let log = init_logger();
 

ddmd/src/main.rs

@@ -7,6 +7,7 @@ edition = "2018"
 libfalcon = { git = "https://github.com/oxidecomputer/falcon", branch = "main" }
 anyhow.workspace = true
 tokio.workspace = true
+oxide-tokio-rt.workspace = true
 
 [[bin]]
 name = "solo"

lab/Cargo.toml

@@ -9,8 +9,11 @@ use libfalcon::error::Error;
 use libfalcon::unit::gb;
 use libfalcon::Runner;
 
-#[tokio::main]
-async fn main() -> Result<(), Error> {
+fn main() -> Result<(), Error> {
+    oxide_tokio_rt::run(main_impl())
+}
+
+async fn main_impl() -> Result<(), Error> {
     let mut d = Runner::new("mg2x2");
 
     // routers

lab/src/2x2/main.rs

@@ -10,8 +10,11 @@ use libfalcon::error::Error;
 use libfalcon::unit::gb;
 use libfalcon::Runner;
 
-#[tokio::main]
-async fn main() -> Result<(), Error> {
+fn main() -> Result<(), Error> {
+    oxide_tokio_rt::run(main_impl())
+}
+
+async fn main_impl() -> Result<(), Error> {
     let mut d = Runner::new("duo");
 
     // nodes