[DRAFT] mmds: Smoother migration from v1 to v2

CHANGELOG.md

@@ -16,6 +16,16 @@ and this project adheres to
 - [#5175](https://github.com/firecracker-microvm/firecracker/pull/5175): Allow
   including a custom cpu template directly in the json configuration file passed
   to `--config-file` under the `cpu_config` key.
+- [#5290](https://github.com/firecracker-microvm/firecracker/pull/5290):
+  Extended MMDS to support the EC2 IMDS-compatible session token headers (i.e.
+  "X-aws-ec2-metadata-token" and "X-aws-ec2-metadata-token-ttl-seconds")
+  alongside the MMDS-specific ones.
+- [#5290](https://github.com/firecracker-microvm/firecracker/pull/5290): Added
+  `mmds.rx_invalid_token` and `mmds.rx_no_token` metrics to track the number of
+  GET requests that were rejected due to token validation failures in MMDS
+  version 2. These metrics also count requests that would be rejected in MMDS
+  version 2 when MMDS version 1 is configured. They helps users assess readiness
+  for migrating to MMDS version 2.
 
 ### Changed
 
@@ -24,6 +34,14 @@ and this project adheres to
   Incremental snapshots remain in developer preview.
 - [#5282](https://github.com/firecracker-microvm/firecracker/pull/5282): Updated
   jailer to no longer require the executable file name to contain `firecracker`.
+- [#5290](https://github.com/firecracker-microvm/firecracker/pull/5290): Changed
+  MMDS to validate the value of "X-metadata-token-ttl-seconds" header only if it
+  is a PUT request to /latest/api/token, as in EC2 IMDS.
+- [#5290](https://github.com/firecracker-microvm/firecracker/pull/5290): Changed
+  MMDS version 1 to support the session oriented method as in version 2,
+  allowing easier migration to version 2. Note that MMDS version 1 accepts a GET
+  request even with no token or an invalid token so that existing workloads
+  continue to work.
 
 ### Deprecated
 
@@ -40,6 +58,9 @@ and this project adheres to
 - [#5260](https://github.com/firecracker-microvm/firecracker/pull/5260): Fixed a
   bug allowing the block device to starve all other devices when backed by a
   sufficiently slow drive.
+- [#XXXX](https://github.com/firecracker-microvm/firecracker/pull/XXXX): Fixed
+  MMDS to reject PUT requests containing `X-Forwarded-For` header regardless of
+  its casing (e.g. `x-forwarded-for`).
 
 ## [1.12.0]
 

docs/mmds/mmds-user-guide.md

@@ -231,8 +231,16 @@ must be issued. The requested resource can be referenced by its corresponding
 the MMDS request. The HTTP response content will contain the referenced metadata
 resource.
 
-The only HTTP method supported by MMDS version 1 is `GET`. Requests containing
-any other HTTP method will receive **405 Method Not Allowed** error.
+As in version 2, version 1 also supports a session oriented method in order to
+make the migration easier. See [the next section](#version-2) for the session
+oriented method. Note that version 1 returns a successful response to a `GET`
+request even with an invalid token or no token not to break existing workloads.
+`mmds.rx_invalid_token` and `mmds.rx_no_token` metrics track the number of `GET`
+requests with invalid tokens and missing tokens respectively, helping users
+evaluate their readiness for migrating to MMDS version 2.
+
+Requests containing any other HTTP methods than `GET` and `PUT` will receive
+**405 Method Not Allowed** error.
 
 ```bash
 MMDS_IPV4_ADDR=169.254.170.2
@@ -252,9 +260,9 @@ token. In order to be successful, the request must respect the following
 constraints:
 
 - must be directed towards `/latest/api/token` path
-- must contain a `X-metadata-token-ttl-seconds` header specifying the token
-  lifetime in seconds. The value cannot be lower than 1 or greater than 21600 (6
-  hours).
+- must contain a `X-metadata-token-ttl-seconds` or
+  `X-aws-ec2-metadata-token-ttl-seconds` header specifying the token lifetime in
+  seconds. The value cannot be lower than 1 or greater than 21600 (6 hours).
 - must not contain a `X-Forwarded-For` header.
 
 ```bash
@@ -266,8 +274,8 @@ TOKEN=`curl -X PUT "http://${MMDS_IPV4_ADDR}/latest/api/token" \
 The HTTP response from MMDS is a plaintext containing the session token.
 
 During the duration specified by the token's time to live value, all subsequent
-`GET` requests must specify the session token through the `X-metadata-token`
-header in order to fetch data from MMDS.
+`GET` requests must specify the session token through the `X-metadata-token` or
+`X-aws-ec2-metadata-token` header in order to fetch data from MMDS.
 
 ```bash
 MMDS_IPV4_ADDR=169.254.170.2

resources/chroot.sh

@@ -11,7 +11,7 @@ PS4='+\t '
 
 cp -ruv $rootfs/* /
 
-packages="udev systemd-sysv openssh-server iproute2 curl socat python3-minimal iperf3 iputils-ping fio kmod tmux hwloc-nox vim-tiny trace-cmd linuxptp strace"
+packages="udev systemd-sysv openssh-server iproute2 curl socat python3-minimal iperf3 iputils-ping fio kmod tmux hwloc-nox vim-tiny trace-cmd linuxptp strace python3-boto3"
 
 # msr-tools is only supported on x86-64.
 arch=$(uname -m)

resources/rebuild.sh

@@ -65,6 +65,12 @@ for d in $dirs; do tar c "/$d" | tar x -C $rootfs; done
 mkdir -pv $rootfs/{dev,proc,sys,run,tmp,var/lib/systemd}
 # So apt works
 mkdir -pv $rootfs/var/lib/dpkg/
+
+# Install AWS CLI v2
+curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+unzip awscliv2.zip
+./aws/install --install-dir $rootfs/usr/local/aws-cli --bin-dir $rootfs/usr/local/bin
+rm -rf awscliv2.zip aws
 EOF
 
     # TBD what abt /etc/hosts?

src/vmm/src/builder.rs

@@ -987,7 +987,7 @@ pub(crate) mod tests {
         net_builder.build(net_config).unwrap();
         let net = net_builder.iter().next().unwrap();
         let mut mmds = Mmds::default();
-        mmds.set_version(mmds_version).unwrap();
+        mmds.set_version(mmds_version);
         net.lock().unwrap().configure_mmds_network_stack(
             MmdsNetworkStack::default_ipv4_addr(),
             Arc::new(Mutex::new(mmds)),

src/vmm/src/device_manager/persist.rs

@@ -569,7 +569,7 @@
             // If there's at least one network device having an mmds_ns, it means
             // that we are restoring from a version that did not persist the `MmdsVersionState`.
             // Init with the default.
-            constructor_args.vm_resources.mmds_or_default();
+            constructor_args.vm_resources.mmds_or_default()?;
         }
 
         for net_state in &state.net_devices {

src/vmm/src/logger/metrics.rs

@@ -556,6 +556,10 @@ pub struct MmdsMetrics {
     pub rx_accepted_unusual: SharedIncMetric,
     /// The number of buffers which couldn't be parsed as valid Ethernet frames by the MMDS.
     pub rx_bad_eth: SharedIncMetric,
+    /// The number of GET requests with invalid tokens.
+    pub rx_invalid_token: SharedIncMetric,
+    /// The number of GET requests with no tokens.
+    pub rx_no_token: SharedIncMetric,
     /// The total number of successful receive operations by the MMDS.
     pub rx_count: SharedIncMetric,
     /// The total number of bytes sent by the MMDS.
@@ -579,6 +583,8 @@ impl MmdsMetrics {
             rx_accepted_err: SharedIncMetric::new(),
             rx_accepted_unusual: SharedIncMetric::new(),
             rx_bad_eth: SharedIncMetric::new(),
+            rx_invalid_token: SharedIncMetric::new(),
+            rx_no_token: SharedIncMetric::new(),
             rx_count: SharedIncMetric::new(),
             tx_bytes: SharedIncMetric::new(),
             tx_count: SharedIncMetric::new(),

src/vmm/src/mmds/data_store.rs

@@ -12,9 +12,9 @@ use crate::mmds::token::{MmdsTokenError as TokenError, TokenAuthority};
 /// The Mmds is the Microvm Metadata Service represented as an untyped json.
 #[derive(Debug)]
 pub struct Mmds {
+    version: MmdsVersion,
     data_store: Value,
-    // None when MMDS V1 is configured, Some for MMDS V2.
-    token_authority: Option<TokenAuthority>,
+    token_authority: TokenAuthority,
     is_initialized: bool,
     data_store_limit: usize,
 }
@@ -65,19 +65,20 @@ pub enum MmdsDatastoreError {
 // Used for ease of use in tests.
 impl Default for Mmds {
     fn default() -> Self {
-        Self::default_with_limit(51200)
+        Self::try_new(51200).unwrap()
     }
 }
 
 impl Mmds {
     /// MMDS default instance with limit `data_store_limit`
-    pub fn default_with_limit(data_store_limit: usize) -> Self {
-        Mmds {
+    pub fn try_new(data_store_limit: usize) -> Result<Self, MmdsDatastoreError> {
+        Ok(Mmds {
+            version: MmdsVersion::V1,
             data_store: Value::default(),
-            token_authority: None,
+            token_authority: TokenAuthority::try_new()?,
             is_initialized: false,
             data_store_limit,
-        }
+        })
     }
 
     /// This method is needed to check if data store is initialized.
@@ -92,52 +93,29 @@ impl Mmds {
     }
 
     /// Set the MMDS version.
-    pub fn set_version(&mut self, version: MmdsVersion) -> Result<(), MmdsDatastoreError> {
-        match version {
-            MmdsVersion::V1 => {
-                self.token_authority = None;
-                Ok(())
-            }
-            MmdsVersion::V2 => {
-                if self.token_authority.is_none() {
-                    self.token_authority = Some(TokenAuthority::new()?);
-                }
-                Ok(())
-            }
-        }
+    pub fn set_version(&mut self, version: MmdsVersion) {
+        self.version = version;
     }
 
-    /// Return the MMDS version by checking the token authority field.
+    /// Get the MMDS version.
     pub fn version(&self) -> MmdsVersion {
-        if self.token_authority.is_none() {
-            MmdsVersion::V1
-        } else {
-            MmdsVersion::V2
-        }
+        self.version
     }
 
     /// Sets the Additional Authenticated Data to be used for encryption and
-    /// decryption of the session token when MMDS version 2 is enabled.
+    /// decryption of the session token.
     pub fn set_aad(&mut self, instance_id: &str) {
-        if let Some(ta) = self.token_authority.as_mut() {
-            ta.set_aad(instance_id);
-        }
+        self.token_authority.set_aad(instance_id);
     }
 
     /// Checks if the provided token has not expired.
-    pub fn is_valid_token(&self, token: &str) -> Result<bool, TokenError> {
-        self.token_authority
-            .as_ref()
-            .ok_or(TokenError::InvalidState)
-            .map(|ta| ta.is_valid(token))
+    pub fn is_valid_token(&self, token: &str) -> bool {
+        self.token_authority.is_valid(token)
     }
 
     /// Generate a new Mmds token using the token authority.
     pub fn generate_token(&mut self, ttl_seconds: u32) -> Result<String, TokenError> {
-        self.token_authority
-            .as_mut()
-            .ok_or(TokenError::InvalidState)
-            .and_then(|ta| ta.generate_token_secret(ttl_seconds))
+        self.token_authority.generate_token_secret(ttl_seconds)
     }
 
     /// set MMDS data store limit to `data_store_limit`
@@ -304,11 +282,11 @@ mod tests {
         assert_eq!(mmds.version(), MmdsVersion::V1);
 
         // Test setting MMDS version to v2.
-        mmds.set_version(MmdsVersion::V2).unwrap();
+        mmds.set_version(MmdsVersion::V2);
         assert_eq!(mmds.version(), MmdsVersion::V2);
 
-        // Test setting MMDS version back to default.
-        mmds.set_version(MmdsVersion::V1).unwrap();
+        // Test setting MMDS version back to v1.
+        mmds.set_version(MmdsVersion::V1);
         assert_eq!(mmds.version(), MmdsVersion::V1);
     }
 
@@ -593,37 +571,4 @@ mod tests {
 
         assert_eq!(mmds.get_data_str().len(), 2);
     }
-
-    #[test]
-    fn test_is_valid() {
-        let mut mmds = Mmds::default();
-        // Set MMDS version to V2.
-        mmds.set_version(MmdsVersion::V2).unwrap();
-        assert_eq!(mmds.version(), MmdsVersion::V2);
-
-        assert!(!mmds.is_valid_token("aaa").unwrap());
-
-        mmds.token_authority = None;
-        assert_eq!(
-            mmds.is_valid_token("aaa").unwrap_err().to_string(),
-            TokenError::InvalidState.to_string()
-        )
-    }
-
-    #[test]
-    fn test_generate_token() {
-        let mut mmds = Mmds::default();
-        // Set MMDS version to V2.
-        mmds.set_version(MmdsVersion::V2).unwrap();
-        assert_eq!(mmds.version(), MmdsVersion::V2);
-
-        let token = mmds.generate_token(1).unwrap();
-        assert!(mmds.is_valid_token(&token).unwrap());
-
-        mmds.token_authority = None;
-        assert_eq!(
-            mmds.generate_token(1).err().unwrap().to_string(),
-            TokenError::InvalidState.to_string()
-        );
-    }
 }