feat: node discovery via DNS (#2045)

## Description

This enables global node discovery over DNS, i.e. dialing nodes by just
their node id.

Current setup is as follows:

* When dialing a node only by its NodeId, the new `DnsDiscovery` service
is invoked. It will lookup a TXT record at (by default)
`_iroh_node.b32encodednodeid.testdns.iroh.link` over regular DNS or
DNS-over-http. Right now the Cloudflare DNS servers are configured. At
`testdns.iroh.link` we run a custom [DNS
server](https://github.com/n0-computer/iroh-dns-server/tree/main)
* Nodes publish their Derp address to this DNS server through Pkarr
signed packets. This is an intermediate step, we decided that the
publishing by default should not happen by the nodes directly but
mediated through the Derp servers. Work for the latter happens in #2052

This PR thus allows for the following:
```sh
# terminal/computer 1
$ iroh console --start
Iroh is running
Node ID: qp2znfedwdij4llc5noizwfemfgba7bzxozvr4bp7hfsdmwqbpua
$ blob add ./myfile
...
Blob: o5uanh5s2zwn2sucy47puqidsfx2advxos7kajq3ajwitcwobhba
...

# terminal/computer 2
iroh console --start
blob get o5uanh5s2zwn2sucy47puqidsfx2advxos7kajq3ajwitcwobhba --node qp2znfedwdij4llc5noizwfemfgba7bzxozvr4bp7hfsdmwqbpua
```


<!-- A summary of what this pull request achieves and a rough list of
changes. -->

## Notes & open questions

* Misses node configuration in the CLI for the node origin domain (right
now hardcoded to `testdns.iroh.link`). How do we want to expose this -
CLI flag? Or in the config file? I'd say the latter.

* Offload publishing to the Derpers - see #2052 

* Right now the records published via pkarr have a TTL of 30s - the
iroh-dns-server will use that TTL as-is when serving the records over
DNS. both can/should change?

* We can also *very* easily allow to lookup nodes not only by NodeId,
but by any domain name. In the `iroh-dns` crate I included an example
`resolve` that does just that. By setting a `CNAME` record you can even
use any domain and simply point to the record hosted at the
`testdns.iroh.link` server.
So if, on your custom domain, you added a record like this
```
_iroh_node.frando.n0.computer CNAME _iroh_node.qp2znfedwdij4llc5noizwfemfgba7bzxozvr4bp7hfsdmwqbpua.iroh.link.
```
You can use this with the example to resolve to the node id and derp
addresses:
```
cargo run --example resolve -- domain frando.n0.computer
```

<!-- Any notes, remarks or open questions you have to make about the PR.
-->

## Change checklist

- [x] Self-review.
- [x] Documentation updates if relevant.
- [x] Tests if relevant.

Closes #1248

---------

Co-authored-by: Kasey <[email protected]>
Co-authored-by: Asmir Avdicevic <[email protected]>
Co-authored-by: Ruediger Klaehn <[email protected]>

Author: 4 people

Cargo.lock

@@ -3,6 +3,7 @@ members = [
   "iroh",
   "iroh-bytes",
   "iroh-base",
+  "iroh-dns-server",
   "iroh-gossip",
   "iroh-metrics",
   "iroh-net",

Cargo.toml

@@ -4,7 +4,7 @@ use anyhow::Context;
 use serde::{Deserialize, Serialize};
 use url::Url;
 
-use crate::key::PublicKey;
+use crate::key::{NodeId, PublicKey};
 
 /// A peer and it's addressing information.
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
@@ -63,6 +63,12 @@ impl From<(PublicKey, Option<RelayUrl>, &[SocketAddr])> for NodeAddr {
     }
 }
 
+impl From<NodeId> for NodeAddr {
+    fn from(node_id: NodeId) -> Self {
+        NodeAddr::new(node_id)
+    }
+}
+
 /// Addressing information to connect to a peer.
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
 pub struct AddrInfo {
@@ -142,6 +148,12 @@ impl FromStr for RelayUrl {
     }
 }
 
+impl From<RelayUrl> for Url {
+    fn from(value: RelayUrl) -> Self {
+        value.0
+    }
+}
+
 /// Dereference to the wrapped [`Url`].
 ///
 /// Note that [`DerefMut`] is not implemented on purpose, so this type has more flexibility

iroh-base/src/node_addr.rs

@@ -236,11 +236,6 @@ impl BlobCommands {
                     return Err(anyhow::anyhow!("The input arguments refer to a collection of blobs and output is set to STDOUT. Only single blobs may be passed in this case."));
                 }
 
-                if node_addr.info.is_empty() {
-                    return Err(anyhow::anyhow!(
-                        "no relay url provided and no direct addresses provided"
-                    ));
-                }
                 let tag = match tag {
                     Some(tag) => SetTagOption::Named(Tag::from(tag)),
                     None => SetTagOption::Auto,

iroh-cli/src/commands/blob.rs

@@ -0,0 +1,54 @@
+[package]
+name = "iroh-dns-server"
+version = "0.13.0"
+edition = "2021"
+description = "A pkarr relay and DNS server"
+license = "MIT OR Apache-2.0"
+authors = ["Frando <[email protected]>", "n0 team"]
+repository = "https://github.com/n0-computer/iroh"
+keywords = ["networking", "pkarr", "dns", "dns-server", "iroh"]
+readme = "README.md"
+
+[dependencies]
+anyhow = "1.0.80"
+async-trait = "0.1.77"
+axum = { version = "0.7.4", features = ["macros"] }
+axum-server = { version = "0.6.0", features = ["tls-rustls"] }
+base64-url = "2.0.2"
+bytes = "1.5.0"
+clap = { version = "4.5.1", features = ["derive"] }
+derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "into", "from"] }
+dirs-next = "2.0.0"
+futures = "0.3.30"
+governor = "0.6.3"
+hickory-proto = "0.24.0"
+hickory-server = { version = "0.24.0", features = ["dns-over-rustls"] }
+http = "1.0.0"
+iroh-metrics = { version = "0.13.0", path = "../iroh-metrics" }
+lru = "0.12.3"
+parking_lot = "0.12.1"
+pkarr = { version = "1.1.2", features = [ "async", "relay"], default_features = false }
+rcgen = "0.12.1"
+redb = "2.0.0"
+regex = "1.10.3"
+rustls = "0.21"
+rustls-pemfile = "1"
+serde = { version = "1.0.197", features = ["derive"] }
+struct_iterable = "0.1.1"
+strum = { version = "0.26.1", features = ["derive"] }
+tokio = { version = "1.36.0", features = ["full"] }
+tokio-rustls = "0.24"
+tokio-rustls-acme = { version = "0.3", features = ["axum"] }
+tokio-stream = "0.1.14"
+tokio-util = "0.7.10"
+toml = "0.8.10"
+tower-http = { version = "0.5.2", features = ["cors", "trace"] }
+tower_governor = "0.3.2"
+tracing = "0.1.40"
+tracing-subscriber = "0.3.18"
+url = "2.5.0"
+z32 = "1.1.1"
+
+[dev-dependencies]
+hickory-resolver = "0.24.0"
+iroh-net = { version = "0.13.0", path = "../iroh-net" }

iroh-dns-server/Cargo.toml

@@ -0,0 +1,38 @@
+# iroh-dns-server
+
+A server that functions as a [pkarr](https://github.com/Nuhvi/pkarr/) relay and
+[DNS](https://de.wikipedia.org/wiki/Domain_Name_System) server.
+
+This server compiles to a binary `iroh-dns-server`. It needs a config file, of
+which there are two examples included:
+
+- [`config.dev.toml`](./config.dev.toml) - suitable for local development
+- [`config.prod.toml`](./config.dev.toml) - suitable for production, after
+  adjusting the domain names and IP addresses
+
+The server will expose the following services:
+
+- A DNS server listening on UDP and TCP for DNS queries
+- A HTTP and/or HTTPS server which provides the following routes:
+  - `/pkarr`: `GET` and `PUT` for pkarr signed packets
+  - `/dns-query`: Answer DNS queries over
+    [DNS-over-HTTPS](https://datatracker.ietf.org/doc/html/rfc8484)
+
+All received and valid pkarr signed packets will be served over DNS. The pkarr
+packet origin will be appended with the origin as configured by this server.
+
+# License
+
+This project is licensed under either of
+
+- Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
+  http://www.apache.org/licenses/LICENSE-2.0)
+- MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
+
+at your option.
+
+### Contribution
+
+Unless you explicitly state otherwise, any contribution intentionally submitted
+for inclusion in this project by you, as defined in the Apache-2.0 license,
+shall be dual licensed as above, without any additional terms or conditions.

iroh-dns-server/README.md

@@ -0,0 +1,18 @@
+[http]
+port = 8080
+bind_addr = "127.0.0.1"
+
+[https]
+port = 8443
+bind_addr = "127.0.0.1"
+domains = ["localhost"]
+cert_mode = "self_signed"
+
+[dns]
+port = 5300
+bind_addr = "127.0.0.1"
+default_soa = "dns1.irohdns.example hostmaster.irohdns.example 0 10800 3600 604800 3600"
+default_ttl = 900
+origins = ["irohdns.example.", "."]
+rr_a = "127.0.0.1"
+rr_ns = "ns1.irohdns.example."

iroh-dns-server/config.dev.toml

@@ -0,0 +1,13 @@
+[https]
+port = 443
+domains = ["irohdns.example.org"]
+cert_mode = "lets_encrypt"
+letsencrypt_prod = true
+
+[dns]
+port = 53
+default_soa = "dns1.irohdns.example.org hostmaster.irohdns.example.org 0 10800 3600 604800 3600"
+default_ttl = 30
+origins = ["irohdns.example.org", "."]
+rr_a = "203.0.10.10"
+rr_ns = "ns1.irohdns.example.org."

iroh-dns-server/config.prod.toml

@@ -0,0 +1,33 @@
+use std::str::FromStr;
+
+use clap::Parser;
+use iroh_net::NodeId;
+
+#[derive(Debug, Parser)]
+struct Cli {
+    #[clap(subcommand)]
+    command: Command,
+}
+
+#[derive(Debug, Parser)]
+enum Command {
+    NodeToPkarr { node_id: String },
+    PkarrToNode { z32_pubkey: String },
+}
+
+fn main() -> anyhow::Result<()> {
+    let args = Cli::parse();
+    match args.command {
+        Command::NodeToPkarr { node_id } => {
+            let node_id = NodeId::from_str(&node_id)?;
+            let public_key = pkarr::PublicKey::try_from(*node_id.as_bytes())?;
+            println!("{}", public_key.to_z32())
+        }
+        Command::PkarrToNode { z32_pubkey } => {
+            let public_key = pkarr::PublicKey::try_from(z32_pubkey.as_str())?;
+            let node_id = NodeId::from_bytes(public_key.as_bytes())?;
+            println!("{}", node_id)
+        }
+    }
+    Ok(())
+}

iroh-dns-server/examples/convert.rs

@@ -0,0 +1,106 @@
+use std::str::FromStr;
+
+use anyhow::{bail, Result};
+use clap::{Parser, ValueEnum};
+use iroh_net::{
+    discovery::{
+        dns::N0_DNS_NODE_ORIGIN,
+        pkarr_publish::{PkarrRelayClient, N0_DNS_PKARR_RELAY},
+    },
+    dns::node_info::{to_z32, NodeInfo, IROH_TXT_NAME},
+    key::SecretKey,
+    NodeId,
+};
+use url::Url;
+
+const LOCALHOST_PKARR: &str = "http://localhost:8080/pkarr";
+const EXAMPLE_ORIGIN: &str = "irohdns.example";
+
+#[derive(ValueEnum, Clone, Debug, Default, Copy, strum::Display)]
+#[strum(serialize_all = "kebab-case")]
+pub enum Env {
+    /// Use the pkarr relay run by number0.
+    #[default]
+    Default,
+    /// Use a relay listening at http://localhost:8080
+    Dev,
+}
+
+/// Publish a record to an irohdns server.
+///
+/// You have to set the IROH_SECRET environment variable to the node secret for which to publish.
+#[derive(Parser, Debug)]
+struct Cli {
+    /// Environment to publish to.
+    #[clap(value_enum, short, long, default_value_t = Env::Default)]
+    env: Env,
+    /// Pkarr Relay URL. If set, the --env option will be ignored.
+    #[clap(long, conflicts_with = "env")]
+    pkarr_relay: Option<Url>,
+    /// Home relay server to publish for this node
+    relay_url: Url,
+    /// Create a new node secret if IROH_SECRET is unset. Only for development / debugging.
+    #[clap(short, long)]
+    create: bool,
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    tracing_subscriber::fmt::init();
+    let args = Cli::parse();
+
+    let secret_key = match std::env::var("IROH_SECRET") {
+        Ok(s) => SecretKey::from_str(&s)?,
+        Err(_) if args.create => {
+            let s = SecretKey::generate();
+            println!("Generated a new node secret. To reuse, set");
+            println!("IROH_SECRET={s}");
+            s
+        }
+        Err(_) => {
+            bail!("Environtment variable IROH_SECRET is not set. To create a new secret, use the --create option.")
+        }
+    };
+
+    let node_id = secret_key.public();
+    let pkarr_relay = match (args.pkarr_relay, args.env) {
+        (Some(pkarr_relay), _) => pkarr_relay,
+        (None, Env::Default) => N0_DNS_PKARR_RELAY.parse().expect("valid url"),
+        (None, Env::Dev) => LOCALHOST_PKARR.parse().expect("valid url"),
+    };
+
+    println!("announce {node_id}:");
+    println!("    relay={}", args.relay_url);
+    println!();
+    println!("publish to {pkarr_relay} ...");
+
+    let pkarr = PkarrRelayClient::new(pkarr_relay);
+    let node_info = NodeInfo::new(node_id, Some(args.relay_url));
+    let signed_packet = node_info.to_pkarr_signed_packet(&secret_key, 30)?;
+    pkarr.publish(&signed_packet).await?;
+
+    println!("signed packet published.");
+    println!("resolve with:");
+
+    match args.env {
+        Env::Default => {
+            println!("   cargo run --example resolve -- node {}", node_id);
+            println!("   dig {} TXT", fmt_domain(&node_id, N0_DNS_NODE_ORIGIN))
+        }
+        Env::Dev => {
+            println!(
+                "    cargo run --example resolve -- --env dev node {}",
+                node_id
+            );
+            println!(
+                "    dig @localhost -p 5300 {} TXT",
+                fmt_domain(&node_id, EXAMPLE_ORIGIN)
+            )
+        }
+    }
+    Ok(())
+}
+
+fn fmt_domain(node_id: &NodeId, origin: &str) -> String {
+    format!("{}.{}.{}", IROH_TXT_NAME, to_z32(node_id), origin)
+}