Dont fail error suites (#540)

* Dont't fail error suites

Co-authored-by: Malofeev Mikhail <[email protected]>

Author: ladisgin

server/proto/testgen.proto

@@ -90,13 +90,20 @@ message ProjectContext {
     string buildDirRelativePath = 4;
 }
 
+enum ErrorMode {
+    FAILING = 0;
+    PASSING_IN_TARGET_ONLY = 1;
+    PASSING = 2;
+}
+
 message SettingsContext {
     bool generateForStaticFunctions = 1;
     bool verbose = 2;
     int32 timeoutPerFunction = 3;
     int32 timeoutPerTest = 4;
     bool useDeterministicSearcher = 5;
     bool useStubs = 6;
+    ErrorMode errorMode = 7;
 }
 
 message SnippetRequest {

server/src/KleeGenerator.cpp

@@ -323,7 +323,8 @@ void KleeGenerator::parseKTestsToFinalCode(
     const std::unordered_map<std::string, types::Type> &methodNameToReturnTypeMap,
     const std::vector<MethodKtests> &kleeOutput,
     const std::shared_ptr<LineInfo> &lineInfo,
-    bool verbose) {
+    bool verbose,
+    ErrorMode errorMode) {
     for (const auto &batch : kleeOutput) {
         bool filterByFlag = (lineInfo != nullptr && !lineInfo->forMethod && !lineInfo->forClass &&
                              !lineInfo->predicateInfo.has_value());
@@ -349,8 +350,7 @@ void KleeGenerator::parseKTestsToFinalCode(
         }
         auto predicate =
             lineInfo ? lineInfo->predicateInfo : std::optional<LineInfo::PredicateInfo>{};
-
-        testsPrinter.genCode(methodDescription, predicate, verbose);
+        testsPrinter.genCode(methodDescription, predicate, verbose, errorMode);
     }
 
     printer::HeaderPrinter(Paths::getSourceLanguage(tests.sourceFilePath))

server/src/KleeGenerator.h

@@ -112,12 +112,13 @@ class KleeGenerator {
      * @throws ExecutionProcessException if a Clang call returns non-zero code.
      */
     void parseKTestsToFinalCode(
-        const utbot::ProjectContext &projectContext,
-        tests::Tests &tests,
-        const std::unordered_map<std::string, types::Type> &methodNameToReturnTypeMap,
-        const std::vector<MethodKtests> &kleeOutput,
-        const std::shared_ptr<LineInfo> &lineInfo = nullptr,
-        bool verbose = false);
+            const utbot::ProjectContext &projectContext,
+            tests::Tests &tests,
+            const std::unordered_map<std::string, types::Type> &methodNameToReturnTypeMap,
+            const std::vector<MethodKtests> &kleeOutput,
+            const std::shared_ptr<LineInfo> &lineInfo = nullptr,
+            bool verbose = false,
+            ErrorMode errorMode = ErrorMode::FAILING);
 
     [[nodiscard]] fs::path getBitcodeFile(const fs::path &sourcePath) const;
 

server/src/KleeRunner.cpp

@@ -105,7 +105,7 @@ void KleeRunner::runKlee(const std::vector<tests::TestMethod> &testMethods,
         }
         auto kleeStats = writeKleeStats(Paths::kleeOutDirForFilePath(projectContext, filePath));
         generator->parseKTestsToFinalCode(projectContext, tests, methodNameToReturnTypeMap, ktests,
-                                          lineInfo, settingsContext.verbose);
+                                          lineInfo, settingsContext.verbose, settingsContext.errorMode);
         generationStats.addFileStats(kleeStats, tests);
 
         sarif::sarifAddTestsToResults(projectContext, tests, sarifResults);

server/src/Paths.cpp

@@ -125,7 +125,7 @@ namespace Paths {
         return replaceExtension(path, StringUtils::stringFormat(".%s.err", suffix));
     }
 
-    static bool errorFileExists(const fs::path &path, std::string const& suffix) {
+    bool errorFileExists(const fs::path &path, std::string const& suffix) {
         return fs::exists(errorFile(path, suffix));
     }
 

server/src/Paths.h

@@ -42,6 +42,8 @@ namespace Paths {
                           const std::vector<fs::path> &dirNames,
                           const std::function<bool(const fs::path &path)> &filter);
 
+    bool errorFileExists(const fs::path &path, std::string const& suffix);
+
     static inline void setOptPath(fs::path &path, const std::string &value) {
         path = fs::path(value);
     }
@@ -427,6 +429,7 @@ namespace Paths {
         return "stubs" / relativeTestDirPath;
     }
 
+    bool hasUncaughtException(const fs::path &path);
     //endregion
 
     //region utbot_report
@@ -447,6 +450,6 @@ namespace Paths {
     //endregion
 
     bool isHeadersEqual(const fs::path &srcPath, const fs::path &headerPath);
-}
+} // Paths
 
 #endif //UNITTESTBOT_PATHS_H

server/src/SettingsContext.cpp

@@ -8,7 +8,8 @@ namespace utbot {
                                      int32_t timeoutPerFunction,
                                      int32_t timeoutPerTest,
                                      bool useDeterministicSearcher,
-                                     bool useStubs)
+                                     bool useStubs,
+                                     testsgen::ErrorMode errorMode)
         : generateForStaticFunctions(generateForStaticFunctions),
           verbose(verbose),
           timeoutPerFunction(timeoutPerFunction > 0
@@ -17,14 +18,16 @@ namespace utbot {
          timeoutPerTest(timeoutPerTest > 0
          ? std::make_optional(std::chrono::seconds{ timeoutPerTest })
          : std::nullopt),
-          useDeterministicSearcher(useDeterministicSearcher), useStubs(useStubs) {
+          useDeterministicSearcher(useDeterministicSearcher), useStubs(useStubs),
+          errorMode(errorMode) {
     }
     SettingsContext::SettingsContext(const testsgen::SettingsContext &settingsContext)
         : SettingsContext(settingsContext.generateforstaticfunctions(),
                           settingsContext.verbose(),
                           settingsContext.timeoutperfunction(),
                           settingsContext.timeoutpertest(),
                           settingsContext.usedeterministicsearcher(),
-                          settingsContext.usestubs()) {
+                          settingsContext.usestubs(),
+                          settingsContext.errormode()) {
     }
 }

server/src/SettingsContext.h

@@ -3,6 +3,7 @@
 
 #include <chrono>
 #include <optional>
+#include <protobuf/testgen.grpc.pb.h>
 
 namespace testsgen {
     class SettingsContext;
@@ -18,13 +19,15 @@ namespace utbot {
                         int32_t timeoutPerFunction,
                         int32_t timeoutPerTest,
                         bool useDeterministicSearcher,
-                        bool useStubs);
+                        bool useStubs,
+                        testsgen::ErrorMode errorMode);
 
         const bool generateForStaticFunctions;
         const bool verbose;
         const std::optional<std::chrono::seconds> timeoutPerFunction, timeoutPerTest;
         const bool useDeterministicSearcher;
         const bool useStubs;
+        testsgen::ErrorMode errorMode;
     };
 }
 

server/src/Tests.cpp

@@ -744,6 +744,7 @@ void KTestObjectParser::parseTestCases(const UTBotKTestList &cases,
 
             testCase.errorDescriptors = case_.errorDescriptors;
 
+            testCase.errorInfo = testCaseDescription.errorInfo;
             if (filterByLineFlag) {
                 auto view = testCaseDescription.kleePathFlagSymbolicValue.view;
                 if (!view || view->getEntryValue(nullptr) != "1") {
@@ -824,6 +825,7 @@ Tests::TestCaseDescription KTestObjectParser::parseTestCaseParams(
 
     Tests::TestCaseDescription testCaseDescription;
     testCaseDescription.objects = ktest.objects;
+    testCaseDescription.errorInfo = ktest.errorInfo;
 
     for (const auto &obj : testCaseDescription.objects) {
         if (obj.name != LAZYNAME) {

server/src/Tests.h

@@ -13,6 +13,7 @@
 #include <klee/TestCase.h>
 #include <tsl/ordered_map.h>
 #include <tsl/ordered_set.h>
+#include "Paths.h"
 
 #include <cassert>
 #include <climits>
@@ -25,6 +26,7 @@
 #include <type_traits>
 #include <utility>
 #include <vector>
+#include <utils/ErrorInfo.h>
 
 using json = nlohmann::json;
 
@@ -78,6 +80,7 @@ namespace tests {
         std::vector<UTBotKTestObject> objects;
         Status status;
         std::vector<std::string> errorDescriptors;
+        ErrorInfo errorInfo;
 
         UTBotKTest(std::vector<UTBotKTestObject> objects,
                    const Status &status,
@@ -420,6 +423,7 @@ namespace tests {
             std::optional<std::vector<FileInfo>> filesValues;
             std::optional<TestCaseParamValue> classPreValues;
             std::optional<TestCaseParamValue> classPostValues;
+            ErrorInfo errorInfo;
         };
 
         struct MethodTestCase {
@@ -447,6 +451,7 @@ namespace tests {
             std::optional<TestCaseParamValue> classPreValues;
             std::optional<TestCaseParamValue> classPostValues;
             std::vector<std::string> errorDescriptors;
+            ErrorInfo errorInfo;
 
             [[nodiscard]] bool isError() const;
 
@@ -911,5 +916,5 @@ namespace tests {
                                         const std::string &typeName,
                                         size_t offsetInBits,
                                         size_t lenInBits);
-}
+} // tests
 #endif // UNITTESTBOT_TESTS_H