Add support for bug hashes reported by RESTler (#57)

- If Fuzz or Test modality of RESTler, then do not post bugs if they are in the ignoreBugHashes list
- If Replay modality of RESTler then do not replay the bugs that are in the ignoreBugHashes list
- Add Replay all fuzz bugs after running petstore sample
- Use bug hash as the replay summary key

Co-authored-by: stas <[email protected]>

Author: stishkin

cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json

@@ -35,7 +35,7 @@
   "tasks": [
     {
       "toolName": "RESTler",
-      "outputFolder": "fuzz",
+      "outputFolder": "{outputFolder}",
       "duration": "00:10:00",
       "toolConfiguration": {
         "task": "Fuzz",
@@ -44,7 +44,11 @@
             "Port" : 8080 
           },
           "useSsl" : false,
-          "inputFolderPath": "/job-compile/compile"
+          "inputFolderPath": "/job-compile/compile",
+          "IgnoreBugHashes" : [
+            "PayloadBodyChecker_500_3d6e2d1e897967f3bc14cd8f08252a98b25c4d48",
+            "PayloadBodyChecker_500_6a4f4fb484d57121a271a4c65948ba8149e92dee"
+          ]
         }
       }
     }

cli/samples/restler/self-contained/swagger-petstore/restler.replay-all-fuzz-bugs.json

@@ -0,0 +1,44 @@
+{
+  "readOnlyFileShareMounts": [
+    {
+      "FileShareName": "{jobId}",
+      "MountPath": "/job-run"
+    }
+  ],
+ "testTargets" : { 
+    "targets" : [
+      {
+        "Container" : "swaggerapi/petstore",
+        "Port" : 8080,
+        "ExpectedDurationUntilReady" : "00:02:00",
+        "Shell" : "/bin/sh",
+        "OutputFolder" : "petstore",
+        "PostRun" : {
+          "Command" : "/bin/sh", 
+          "Arguments" : ["-c", "cp /var/log/*-requests.log $RAFT_WORK_DIRECTORY"],
+          "ExpectedRunDuration" : "00:00:10"
+        }
+      }
+    ]
+  },
+  "tasks": [
+    {
+      "toolName": "RESTler",
+      "outputFolder": "RESTler-replay-{experiment}-all-bug-buckets",
+      "toolConfiguration": {
+        "task": "Replay",
+        "runConfiguration": {
+          "targetEndpointConfiguration" : {
+            "Port" : 8080 
+          },
+          "useSsl" : false,
+          "inputFolderPath": "/job-run/{outputFolder}/RestlerResults/{experiment}/bug_buckets",
+          "ignoreBugHashes" : [
+            "PayloadBodyChecker_500_acb1c0dcc79e949c24bec51f0ea3f841c61c38a3",
+            "PayloadBodyChecker_500_b8ebb5eaf46cdad18ac1d8e9ed67e2cdc729db3d"
+          ]
+        }
+      }
+    }
+  ]
+}

cli/samples/restler/self-contained/swagger-petstore/restler.test.json

@@ -36,15 +36,19 @@
   "tasks": [
     {
       "toolName": "RESTler",
-      "outputFolder": "test-fuzz-lean",
+      "outputFolder": "test",
       "toolConfiguration": {
         "task": "Test",
         "runConfiguration": {
           "useSsl" : false,
           "targetEndpointConfiguration" : {
             "Port" : 8080
           },
-          "inputFolderPath": "/job-compile/compile"
+          "inputFolderPath": "/job-compile/compile",
+          "ignoreBugHashes" : [
+            "PayloadBodyChecker_500_7c2ae3b8c78eabad55a8075b487322dbbe146e46",
+            "PayloadBodyChecker_500_6349a559db300ca67b3caa316142c02a6bea0fbe" 
+          ]
         }
       }
     }

cli/samples/restler/self-contained/swagger-petstore/run.py

@@ -9,10 +9,9 @@
 sys.path.append(os.path.join(cur_dir, '..', '..', '..', '..'))
 from raft_sdk.raft_service import RaftCLI, RaftJobConfig, RaftJobError
 
-def run(compile, test, fuzz):
+def run(compile, test, fuzz, replay):
     # instantiate RAFT CLI
     cli = RaftCLI()
-
     # Create compilation job configuration
     compile_job_config = RaftJobConfig(file_path=compile)
     # add webhook metadata that will be included in every triggered webhook by Compile job
@@ -23,8 +22,8 @@ def run(compile, test, fuzz):
     # wait for a job with ID from compile_job to finish the run
     cli.poll(compile_job['jobId'])
 
-    # use compile job as input for fuzz job
     subs = {}
+    # use compile job as input for fuzz job
     subs['{compile.jobId}'] = compile_job['jobId']
 
     test_job_config = RaftJobConfig(file_path=test, substitutions=subs)
@@ -34,6 +33,7 @@ def run(compile, test, fuzz):
     # wait for job ID from fuzz_job to finish the run
     cli.poll(test_job['jobId'])
 
+    subs['{outputFolder}'] = 'fuzz'
     # create a new job config with Fuzz configuration JSON
     fuzz_job_config = RaftJobConfig(file_path=fuzz, substitutions=subs)
     print('Fuzz')
@@ -45,10 +45,28 @@ def run(compile, test, fuzz):
     # wait for job ID from fuzz_job to finish the run
     cli.poll(fuzz_job['jobId'])
 
+    status = cli.job_status(fuzz_job['jobId'])
+    experiment = None
+    for s in status:
+        if (s['agentName'] != s['jobId']):
+            if (int(s['details']['numberOfBugsFound']) > 0):
+                experiment = s['details']['experiment']
+            else:
+                print("Did not find any bugs")
+
+    subs['{experiment}'] = experiment
+    subs['{jobId}'] = fuzz_job['jobId']
+
+    replay_job_config = RaftJobConfig(file_path=replay, substitutions=subs)
+    replay_job = cli.new_job(replay_job_config)
+    cli.poll(replay_job['jobId'])
+
+
 if __name__ == "__main__":
     try:
         run(os.path.join(cur_dir, "restler.compile.json"),
             os.path.join(cur_dir, "restler.test.json"), 
-            os.path.join(cur_dir, "restler.fuzz.json"))
+            os.path.join(cur_dir, "restler.fuzz.json"),
+            os.path.join(cur_dir, "restler.replay-all-fuzz-bugs.json"))
     except RaftJobError as ex:
         print(f'ERROR: {ex.message}')

src/Agent/RESTlerAgent/AgentMain.fs

@@ -547,10 +547,17 @@ let main argv =
                 do! Raft.RESTlerDriver.compile restlerPath workDirectory compilerConfig
             }
 
-        let report state (summary: Raft.JobEvents.RunSummary option) =
+        let report state (experiment : string option, summary:Raft.JobEvents.RunSummary option) =
             async {
                 printfn "Reporting summary [%A]: %A" state summary
                 let! bugsList = Raft.RESTlerDriver.getListOfBugs workDirectory globalRunStartTime
+                let bugsListLen = match bugsList with None -> 0 | Some xs -> Seq.length xs
+
+                let details =
+                    match experiment with
+                    | Some e -> Map.empty.Add("Experiment", e)
+                    | None -> Map.empty
+
                 do! jobEventSender.SendRaftJobEvent jobId
                                                 ({
                                                     AgentName = agentName
@@ -561,7 +568,7 @@ let main argv =
 
                                                     Metrics = summary
                                                     UtcEventTime = System.DateTime.UtcNow
-                                                    Details = Some <| Map.empty.Add("numberOfBugsFound", sprintf "%d" (Seq.length bugsList))
+                                                    Details = Some( details.Add("numberOfBugsFound", sprintf "%d" bugsListLen))
                                                 } : Raft.JobEvents.JobStatus)
             }
 
@@ -617,7 +624,13 @@ let main argv =
                 let targetIp, targetPort = getIpAndPort jobConfiguration.TargetEndpointConfiguration
                 let engineParameters = createRESTlerEngineParameters (targetIp, targetPort) grammarPy dictJson task checkerOptions jobConfiguration
                 printfn "Starting RESTler test task"
-                do! Raft.RESTlerDriver.test testType restlerPath workDirectory engineParameters onBugFound (report Raft.JobEvents.JobState.Running) (globalRunStartTime, resultAnalyzerReportInterval)
+
+                let ignoreBugHashes =
+                    match jobConfiguration.IgnoreBugHashes with
+                    | None -> Set.empty
+                    | Some hs -> Set.ofArray hs
+
+                do! Raft.RESTlerDriver.test testType restlerPath workDirectory engineParameters ignoreBugHashes onBugFound (report Raft.JobEvents.JobState.Running) (globalRunStartTime, resultAnalyzerReportInterval)
             }
 
         let fuzz (fuzzType:string) checkerOptions (jobConfiguration: RunConfiguration) =
@@ -626,7 +639,13 @@ let main argv =
                 let targetIp, targetPort = getIpAndPort jobConfiguration.TargetEndpointConfiguration
                 let engineParameters = createRESTlerEngineParameters (targetIp, targetPort) grammarPy dictJson task checkerOptions jobConfiguration
                 printfn "Starting RESTler fuzz task"
-                do! Raft.RESTlerDriver.fuzz fuzzType restlerPath workDirectory engineParameters onBugFound (report Raft.JobEvents.JobState.Running) (globalRunStartTime, resultAnalyzerReportInterval)
+
+                let ignoreBugHashes =
+                    match jobConfiguration.IgnoreBugHashes with
+                    | None -> Set.empty
+                    | Some hs -> Set.ofArray hs
+
+                do! Raft.RESTlerDriver.fuzz fuzzType restlerPath workDirectory engineParameters ignoreBugHashes onBugFound (report Raft.JobEvents.JobState.Running) (globalRunStartTime, resultAnalyzerReportInterval)
             }
 
         let replay replayLogFile (jobConfiguration: RunConfiguration) =
@@ -658,7 +677,7 @@ let main argv =
         // Start RESTler process with following flags
         // compile, test, fuzz 
         // --telemetryRootDirPath
-        let! state, details, exitCode, summary =
+        let! state, details, exitCode, (experiment, summary) =
             async {
                 try
                     match restlerPayload.Task with
@@ -697,7 +716,7 @@ let main argv =
                         | None -> return failwithf "Job-run configuration is not set for Test task for job payload: %A" restlerPayload
                         | Some jobConfiguration ->
                             copyDir jobConfiguration.InputFolderPath workDirectory (set [taskConfigurationPath])
-                            do! report Raft.JobEvents.Running None
+                            do! report Raft.JobEvents.Running (None, None)
                             do! test "directed-smoke-test" [] jobConfiguration
                             let! summary = Raft.RESTlerDriver.processRunSummary workDirectory globalRunStartTime
                             return Raft.JobEvents.Completed, None, 0, summary
@@ -708,7 +727,7 @@ let main argv =
                         | None -> return failwithf "Job-run configuration is not set for Test task for job payload: %A" restlerPayload
                         | Some jobConfiguration ->
                             copyDir jobConfiguration.InputFolderPath workDirectory (set [taskConfigurationPath])
-                            do! report Raft.JobEvents.Running None
+                            do! report Raft.JobEvents.Running (None, None)
                             let fuzzLeanCheckers = 
                                 [
                                     ("--enable_checkers", "*")
@@ -724,7 +743,7 @@ let main argv =
                         | None -> return failwithf "Job-run configuration is not set for Compile task for job payload: %A" restlerPayload
                         | Some jobConfiguration ->
                             copyDir jobConfiguration.InputFolderPath workDirectory (set [taskConfigurationPath])
-                            do! report Raft.JobEvents.Running None
+                            do! report Raft.JobEvents.Running (None, None)
                             let allCheckers = ["--enable_checkers", "*"]
                             do! fuzz "bfs" allCheckers jobConfiguration
                             let! summary = Raft.RESTlerDriver.processRunSummary workDirectory globalRunStartTime
@@ -736,7 +755,7 @@ let main argv =
                         | None -> return failwithf "Job-run configuration is not set for Test task for job payload: %A" restlerPayload
                         | Some jobConfiguration ->
                             copyDir jobConfiguration.InputFolderPath workDirectory (set [taskConfigurationPath])
-                            do! report Raft.JobEvents.Running None
+                            do! report Raft.JobEvents.Running (None, None)
                             let allCheckers = ["--enable_checkers", "*"]
                             do! fuzz "bfs-cheap" allCheckers jobConfiguration
                             let! summary = Raft.RESTlerDriver.processRunSummary workDirectory globalRunStartTime
@@ -748,7 +767,7 @@ let main argv =
                         | None -> return failwithf "Job-run configuration is not set for Fuzz task for job payload: %A" restlerPayload
                         | Some jobConfiguration ->
                             copyDir jobConfiguration.InputFolderPath workDirectory (set [taskConfigurationPath])
-                            do! report Raft.JobEvents.Running None
+                            do! report Raft.JobEvents.Running (None, None)
                             let allCheckers = ["--enable_checkers", "*"]
                             do! fuzz "random-walk" allCheckers jobConfiguration
                             let! summary = Raft.RESTlerDriver.processRunSummary workDirectory globalRunStartTime
@@ -759,23 +778,59 @@ let main argv =
                         | None -> return failwithf "Job-run configuration is not set Replay task for job payload: %A" restlerPayload
                         | Some replayRunConfiguration ->
                             let replaySourcePath = replayRunConfiguration.InputFolderPath
-                            do! report Raft.JobEvents.Running None
+                            do! report Raft.JobEvents.Running (None, None)
 
                             let replayAndReport (bugs: IO.FileInfo seq) =
                                 async {
+                                    let! existingBugBuckets = Raft.RESTlerDriver.getListOfBugsFromBugBuckets replayRunConfiguration.InputFolderPath
+
+                                    let remapBugHashes (bugHashes : Raft.RESTlerTypes.Logs.BugHashes) =
+                                        bugHashes
+                                        |> Seq.map(fun (KeyValue(k, v)) -> v.file_path, k)
+                                        |> Map.ofSeq
+
+                                    let bugs, fileNameToBugHashMap =
+                                        match replayRunConfiguration.IgnoreBugHashes, existingBugBuckets with
+                                        | None, None | Some _, None -> bugs, Map.empty
+                                         | None, Some existingBugs ->
+                                            bugs, (remapBugHashes existingBugs)
+
+                                        | Some ignoreHashes, Some existingBugs ->
+                                            let filesToIgnore = 
+                                                ignoreHashes 
+                                                |> Array.map(fun h ->
+                                                    match Map.tryFind h existingBugs with
+                                                    | None -> None
+                                                    | Some b -> Some b.file_path
+                                                )
+                                                |> Array.filter Option.isSome
+                                                |> Array.map Option.get
+                                                |> Set.ofArray
+
+                                            let filteredBugs =
+                                                bugs
+                                                |> Seq.filter (fun b ->
+                                                    not (filesToIgnore.Contains b.Name)
+                                                )
+                                            filteredBugs, (remapBugHashes existingBugs)
+
                                     let replaySummaryDetails = ResizeArray<string * string>()
 
                                     for bug in bugs do
                                         printfn "Running replay on %s" bug.FullName
-                                        let! replaySummary = replay bug.FullName replayRunConfiguration
+                                        let! (experiment, replaySummary) = replay bug.FullName replayRunConfiguration
                                         let summary =
                                             match replaySummary with
                                             | None ->
                                                 sprintf "%s : No results" bug.Name
                                             | Some s ->
-                                                sprintf "%s: %A" bug.Name (s.ResponseCodeCounts |> Map.toList)
+                                                sprintf "%s/%s: %A" (Option.defaultValue "ExperimentNotSet" experiment) bug.Name (s.ResponseCodeCounts |> Map.toList)
 
-                                        replaySummaryDetails.Add(bug.Name, summary)
+                                        let bugKey =
+                                            match Map.tryFind bug.Name fileNameToBugHashMap with
+                                            | None -> bug.Name
+                                            | Some h -> h
+                                        replaySummaryDetails.Add(bugKey, summary)
                                         do! jobEventSender.SendRaftJobEvent jobId
                                                     ({
                                                         AgentName = agentName
@@ -823,7 +878,7 @@ let main argv =
                                             return Some details
                                         | None -> return! replayAll()
                                     }
-                            return Raft.JobEvents.Completed, (details |> Option.map Map.ofSeq), 0, None
+                            return Raft.JobEvents.Completed, (details |> Option.map Map.ofSeq), 0, (None, None)
                 with
                 | ex ->
                     printfn "%A" ex
@@ -832,17 +887,24 @@ let main argv =
             }
 
         let! bugsList = Raft.RESTlerDriver.getListOfBugs workDirectory globalRunStartTime
+        let bugsListLen = match bugsList with None -> 0 | Some xs -> Seq.length xs
 
         let testingSummary = Raft.RESTlerDriver.loadTestRunSummary workDirectory globalRunStartTime
 
+        let details =
+            let d = Option.defaultValue Map.empty details
+            match experiment with
+            | Some e -> Some (d.Add("Experiment", e))
+            | None -> Some d
+
         let details =
             match testingSummary with
             | None -> details
             | Some status ->
                 let d = Option.defaultValue Map.empty details
                 Some( d
                         .Add("finalSpecCoverage", status.final_spec_coverage)
-                        .Add("numberOfBugsFound", sprintf "%d" (Seq.length bugsList))
+                        .Add("numberOfBugsFound", sprintf "%d" bugsListLen)
                         //.Add("renderedRequests", status.rendered_requests)
                         //.Add("renderedRequestsValidStatus", status.rendered_requests_valid_status)
                         //.Add("numFullyValid", sprintf "%d" status.num_fully_valid)

src/Agent/RESTlerAgent/RESTlerAgentConfigTypes.fs

@@ -102,8 +102,11 @@ type RunConfiguration =
 
         /// Path regex for filtering tested endpoints
         PathRegex : string option
-    }
 
+        // In context of Replay - do not replay bugs specified in the list
+        // In context of Test or Fuzz - do not post onBugFound events if they are in the list
+        IgnoreBugHashes : string array option
+    }
 
 
 type AgentConfiguration =