contest: remove: add support for env checks

Driver tests seem to fail quite a lot due to other tests not cleaning
up after themselves. Add a hook point and a script for checking that
the env is sane. The expectation is that we'll run
contest/scripts/env_check.py in each netns before the test (as part
of setup), and then after each test. If post-test check fails VM
is destroyed and new one gets started.

Signed-off-by: Jakub Kicinski <[email protected]>

Author: kuba-moo

contest/remote/lib/vm.py

@@ -149,8 +149,11 @@ def _set_env(self):
             self.cmd("export LD_LIBRARY_PATH=" + self.config.get('vm', 'ld_paths') + ':$LD_LIBRARY_PATH')
             self.drain_to_prompt()
 
-        if self.config.get('vm', 'setup', fallback=None):
-            self.cmd(self.config.get('vm', 'setup'))
+        setup_scripts = self.config.get('vm', 'setup', fallback='').split(',')
+        for setup in setup_scripts:
+            if not setup:
+                continue
+            self.cmd(setup)
             self.drain_to_prompt()
 
         exports = self.config.get('vm', 'exports', fallback=None)

contest/remote/vmksft-p.py

@@ -222,7 +222,20 @@ def _vm_thread(config, results_path, thr_id, hard_stop, in_queue, out_queue):
 
             print(f"INFO: thr-{thr_id} {prog} >> nested tests: {len(nested_tests)}")
 
-        if not is_retry and result == 'fail':
+        can_retry = not is_retry
+
+        post_check = config.get('ksft', 'post_check', fallback=None)
+        if post_check and not vm.fail_state:
+            vm.cmd(post_check)
+            vm.drain_to_prompt()
+            pc = vm.bash_prev_retcode()
+            if pc != 0:
+                vm.fail_state = "env-check-fail"
+                if result == 'pass':
+                    result = 'fail'
+                    can_retry = False  # Don't waste time, the test is buggy
+
+        if can_retry and result == 'fail':
             in_queue.put(outcome)
         else:
             out_queue.put(outcome)
@@ -232,7 +245,7 @@ def _vm_thread(config, results_path, thr_id, hard_stop, in_queue, out_queue):
                           "found": indicators, "vm_state": vm.fail_state})
 
         if vm.fail_state:
-            print(f"INFO: thr-{thr_id} VM kernel crashed, destroying it")
+            print(f"INFO: thr-{thr_id} VM {vm.fail_state}, destroying it")
             vm.stop()
             vm.dump_log(results_path + f'/vm-stop-thr{thr_id}-{vm_id}')
             vm = None

contest/scripts/env_check.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+"""
+Collect system state info. Save it to a JSON file,
+if file already exists, compare it first and report deltas.
+"""
+
+import json
+import os
+import subprocess
+import sys
+
+
+def run_cmd_text(cmd):
+    """Execute a shell command and return its output as text."""
+    result = subprocess.run(cmd, shell=True, check=False,
+                            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                            universal_newlines=True)
+    return result.stdout
+
+
+def run_cmd_json(cmd):
+    """Execute a shell command and return its output parsed as JSON."""
+    result = subprocess.run(cmd, shell=True, check=False,
+                            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                            universal_newlines=True)
+    if result.returncode != 0:
+        return {"error": result.stderr.strip()}
+
+    ret = json.loads(result.stdout)
+    # "decapsulate" the one element arrays that ip and ethtool like return
+    if isinstance(ret, list) and len(ret) == 1:
+        ret = ret[0]
+    return ret
+
+
+def collect_system_state():
+    """Collect network interface information."""
+    state = {
+        "links": {},
+        "chans": {},
+        "feat": {},
+        "rings": {},
+        "rss": {},
+        "ntuple": {},
+    }
+
+    interfaces = run_cmd_json("ip -j -d link show")
+
+    for iface in interfaces:
+        ifname = iface['ifname']
+
+        state["links"][ifname] = iface
+
+        state["chans"][ifname] = run_cmd_json(f"ethtool -j -l {ifname}")
+        state["feat" ][ifname] = run_cmd_json(f"ethtool -j -k {ifname}")
+        state["rings"][ifname] = run_cmd_json(f"ethtool -j -g {ifname}")
+        state["rss"  ][ifname] = run_cmd_json(f"ethtool -j -x {ifname}")
+        if "rss-hash-key" in state["rss"][ifname]:
+            del state["rss"][ifname]["rss-hash-key"]
+        state["ntuple"][ifname] = run_cmd_text(f"ethtool -n {ifname}")
+
+    return state
+
+
+def compare_states(current, saved, path=""):
+    """Compare current system state with saved state."""
+
+    ret = 0
+
+    if isinstance(current, dict) and isinstance(saved, dict):
+        for k in current.keys() | saved.keys():
+            if k in current and k in saved:
+                ret |= compare_states(current[k], saved[k], path=f"{path}.{k}")
+            else:
+                print(f"Saved {path}.{k}:", saved.get(k))
+                print(f"Current {path}.{k}:", current.get(k))
+                ret = 1
+    else:
+        if current != saved:
+            print(f"Saved {path}:", saved)
+            print(f"Current {path}:", current)
+            ret = 1
+
+    return ret
+
+
+def main():
+    """Main function to collect and compare network interface states."""
+    output_file = "/tmp/nipa-env-state.json"
+    if len(sys.argv) > 1:
+        output_file = sys.argv[1]
+
+    # Collect current system state
+    current_state = collect_system_state()
+    exit_code = 0
+
+    # Check if the file already exists
+    if os.path.exists(output_file):
+        print("Comparing to existing state file: ", end="")
+        try:
+            with open(output_file, 'r', encoding='utf-8') as f:
+                saved_state = json.load(f)
+
+            # Compare states
+            exit_code = compare_states(current_state, saved_state)
+            if exit_code == 0:
+                print("no differences detected.")
+        except (json.JSONDecodeError, IOError, OSError) as e:
+            print("Error loading or comparing:")
+            print(e)
+    # Save current state to file
+    with open(output_file, 'w', encoding='utf-8') as f:
+        json.dump(current_state, f, indent=2)
+    print(f"Current system state saved to {output_file}")
+
+    sys.exit(exit_code)
+
+
+if __name__ == "__main__":
+    main()