Skip to content

Commit b17516c

Browse files
ladisginMalofeev Mikhail
ladisgin
and
Malofeev Mikhail
authored
Dont fail error suites (#540)
* Dont't fail error suites Co-authored-by: Malofeev Mikhail <[email protected]>
1 parent 24e480c commit b17516c

39 files changed

+691
-115
lines changed

‎server/proto/testgen.proto

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,13 +90,20 @@ message ProjectContext {
9090
string buildDirRelativePath = 4;
9191
}
9292

93+
enum ErrorMode {
94+
FAILING = 0;
95+
PASSING_IN_TARGET_ONLY = 1;
96+
PASSING = 2;
97+
}
98+
9399
message SettingsContext {
94100
bool generateForStaticFunctions = 1;
95101
bool verbose = 2;
96102
int32 timeoutPerFunction = 3;
97103
int32 timeoutPerTest = 4;
98104
bool useDeterministicSearcher = 5;
99105
bool useStubs = 6;
106+
ErrorMode errorMode = 7;
100107
}
101108

102109
message SnippetRequest {

‎server/src/KleeGenerator.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -323,7 +323,8 @@ void KleeGenerator::parseKTestsToFinalCode(
323323
const std::unordered_map<std::string, types::Type> &methodNameToReturnTypeMap,
324324
const std::vector<MethodKtests> &kleeOutput,
325325
const std::shared_ptr<LineInfo> &lineInfo,
326-
bool verbose) {
326+
bool verbose,
327+
ErrorMode errorMode) {
327328
for (const auto &batch : kleeOutput) {
328329
bool filterByFlag = (lineInfo != nullptr && !lineInfo->forMethod && !lineInfo->forClass &&
329330
!lineInfo->predicateInfo.has_value());
@@ -349,8 +350,7 @@ void KleeGenerator::parseKTestsToFinalCode(
349350
}
350351
auto predicate =
351352
lineInfo ? lineInfo->predicateInfo : std::optional<LineInfo::PredicateInfo>{};
352-
353-
testsPrinter.genCode(methodDescription, predicate, verbose);
353+
testsPrinter.genCode(methodDescription, predicate, verbose, errorMode);
354354
}
355355

356356
printer::HeaderPrinter(Paths::getSourceLanguage(tests.sourceFilePath))

‎server/src/KleeGenerator.h

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -112,12 +112,13 @@ class KleeGenerator {
112112
* @throws ExecutionProcessException if a Clang call returns non-zero code.
113113
*/
114114
void parseKTestsToFinalCode(
115-
const utbot::ProjectContext &projectContext,
116-
tests::Tests &tests,
117-
const std::unordered_map<std::string, types::Type> &methodNameToReturnTypeMap,
118-
const std::vector<MethodKtests> &kleeOutput,
119-
const std::shared_ptr<LineInfo> &lineInfo = nullptr,
120-
bool verbose = false);
115+
const utbot::ProjectContext &projectContext,
116+
tests::Tests &tests,
117+
const std::unordered_map<std::string, types::Type> &methodNameToReturnTypeMap,
118+
const std::vector<MethodKtests> &kleeOutput,
119+
const std::shared_ptr<LineInfo> &lineInfo = nullptr,
120+
bool verbose = false,
121+
ErrorMode errorMode = ErrorMode::FAILING);
121122

122123
[[nodiscard]] fs::path getBitcodeFile(const fs::path &sourcePath) const;
123124

‎server/src/KleeRunner.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ void KleeRunner::runKlee(const std::vector<tests::TestMethod> &testMethods,
105105
}
106106
auto kleeStats = writeKleeStats(Paths::kleeOutDirForFilePath(projectContext, filePath));
107107
generator->parseKTestsToFinalCode(projectContext, tests, methodNameToReturnTypeMap, ktests,
108-
lineInfo, settingsContext.verbose);
108+
lineInfo, settingsContext.verbose, settingsContext.errorMode);
109109
generationStats.addFileStats(kleeStats, tests);
110110

111111
sarif::sarifAddTestsToResults(projectContext, tests, sarifResults);

‎server/src/Paths.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ namespace Paths {
125125
return replaceExtension(path, StringUtils::stringFormat(".%s.err", suffix));
126126
}
127127

128-
static bool errorFileExists(const fs::path &path, std::string const& suffix) {
128+
bool errorFileExists(const fs::path &path, std::string const& suffix) {
129129
return fs::exists(errorFile(path, suffix));
130130
}
131131

‎server/src/Paths.h

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ namespace Paths {
4242
const std::vector<fs::path> &dirNames,
4343
const std::function<bool(const fs::path &path)> &filter);
4444

45+
bool errorFileExists(const fs::path &path, std::string const& suffix);
46+
4547
static inline void setOptPath(fs::path &path, const std::string &value) {
4648
path = fs::path(value);
4749
}
@@ -427,6 +429,7 @@ namespace Paths {
427429
return "stubs" / relativeTestDirPath;
428430
}
429431

432+
bool hasUncaughtException(const fs::path &path);
430433
//endregion
431434

432435
//region utbot_report
@@ -447,6 +450,6 @@ namespace Paths {
447450
//endregion
448451

449452
bool isHeadersEqual(const fs::path &srcPath, const fs::path &headerPath);
450-
}
453+
} // Paths
451454

452455
#endif //UNITTESTBOT_PATHS_H

‎server/src/SettingsContext.cpp

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ namespace utbot {
88
int32_t timeoutPerFunction,
99
int32_t timeoutPerTest,
1010
bool useDeterministicSearcher,
11-
bool useStubs)
11+
bool useStubs,
12+
testsgen::ErrorMode errorMode)
1213
: generateForStaticFunctions(generateForStaticFunctions),
1314
verbose(verbose),
1415
timeoutPerFunction(timeoutPerFunction > 0
@@ -17,14 +18,16 @@ namespace utbot {
1718
timeoutPerTest(timeoutPerTest > 0
1819
? std::make_optional(std::chrono::seconds{ timeoutPerTest })
1920
: std::nullopt),
20-
useDeterministicSearcher(useDeterministicSearcher), useStubs(useStubs) {
21+
useDeterministicSearcher(useDeterministicSearcher), useStubs(useStubs),
22+
errorMode(errorMode) {
2123
}
2224
SettingsContext::SettingsContext(const testsgen::SettingsContext &settingsContext)
2325
: SettingsContext(settingsContext.generateforstaticfunctions(),
2426
settingsContext.verbose(),
2527
settingsContext.timeoutperfunction(),
2628
settingsContext.timeoutpertest(),
2729
settingsContext.usedeterministicsearcher(),
28-
settingsContext.usestubs()) {
30+
settingsContext.usestubs(),
31+
settingsContext.errormode()) {
2932
}
3033
}

‎server/src/SettingsContext.h

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
#include <chrono>
55
#include <optional>
6+
#include <protobuf/testgen.grpc.pb.h>
67

78
namespace testsgen {
89
class SettingsContext;
@@ -18,13 +19,15 @@ namespace utbot {
1819
int32_t timeoutPerFunction,
1920
int32_t timeoutPerTest,
2021
bool useDeterministicSearcher,
21-
bool useStubs);
22+
bool useStubs,
23+
testsgen::ErrorMode errorMode);
2224

2325
const bool generateForStaticFunctions;
2426
const bool verbose;
2527
const std::optional<std::chrono::seconds> timeoutPerFunction, timeoutPerTest;
2628
const bool useDeterministicSearcher;
2729
const bool useStubs;
30+
testsgen::ErrorMode errorMode;
2831
};
2932
}
3033

‎server/src/Tests.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -744,6 +744,7 @@ void KTestObjectParser::parseTestCases(const UTBotKTestList &cases,
744744

745745
testCase.errorDescriptors = case_.errorDescriptors;
746746

747+
testCase.errorInfo = testCaseDescription.errorInfo;
747748
if (filterByLineFlag) {
748749
auto view = testCaseDescription.kleePathFlagSymbolicValue.view;
749750
if (!view || view->getEntryValue(nullptr) != "1") {
@@ -824,6 +825,7 @@ Tests::TestCaseDescription KTestObjectParser::parseTestCaseParams(
824825

825826
Tests::TestCaseDescription testCaseDescription;
826827
testCaseDescription.objects = ktest.objects;
828+
testCaseDescription.errorInfo = ktest.errorInfo;
827829

828830
for (const auto &obj : testCaseDescription.objects) {
829831
if (obj.name != LAZYNAME) {

‎server/src/Tests.h

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#include <klee/TestCase.h>
1414
#include <tsl/ordered_map.h>
1515
#include <tsl/ordered_set.h>
16+
#include "Paths.h"
1617

1718
#include <cassert>
1819
#include <climits>
@@ -25,6 +26,7 @@
2526
#include <type_traits>
2627
#include <utility>
2728
#include <vector>
29+
#include <utils/ErrorInfo.h>
2830

2931
using json = nlohmann::json;
3032

@@ -78,6 +80,7 @@ namespace tests {
7880
std::vector<UTBotKTestObject> objects;
7981
Status status;
8082
std::vector<std::string> errorDescriptors;
83+
ErrorInfo errorInfo;
8184

8285
UTBotKTest(std::vector<UTBotKTestObject> objects,
8386
const Status &status,
@@ -420,6 +423,7 @@ namespace tests {
420423
std::optional<std::vector<FileInfo>> filesValues;
421424
std::optional<TestCaseParamValue> classPreValues;
422425
std::optional<TestCaseParamValue> classPostValues;
426+
ErrorInfo errorInfo;
423427
};
424428

425429
struct MethodTestCase {
@@ -447,6 +451,7 @@ namespace tests {
447451
std::optional<TestCaseParamValue> classPreValues;
448452
std::optional<TestCaseParamValue> classPostValues;
449453
std::vector<std::string> errorDescriptors;
454+
ErrorInfo errorInfo;
450455

451456
[[nodiscard]] bool isError() const;
452457

@@ -911,5 +916,5 @@ namespace tests {
911916
const std::string &typeName,
912917
size_t offsetInBits,
913918
size_t lenInBits);
914-
}
919+
} // tests
915920
#endif // UNITTESTBOT_TESTS_H

‎server/src/commands/Commands.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -408,6 +408,10 @@ bool Commands::SettingsContextOptionGroup::withStubs() const {
408408
return !noStubs;
409409
}
410410

411+
ErrorMode Commands::SettingsContextOptionGroup::getErrorMode() const {
412+
return errorMode;
413+
}
414+
411415
Commands::RunTestsCommands::RunTestsCommands(Commands::MainCommands &commands) {
412416
runCommand = commands.getRunTestsCommand();
413417

‎server/src/commands/Commands.h

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
#include <CLI11.hpp>
1111
#include <string>
1212

13-
13+
using namespace ::testsgen;
1414

1515
namespace Commands {
1616
extern uint32_t threadsPerUser;
@@ -243,6 +243,8 @@ namespace Commands {
243243

244244
[[nodiscard]] bool withStubs() const;
245245

246+
[[nodiscard]] ErrorMode getErrorMode() const;
247+
246248
private:
247249
CLI::Option_group *settingsContextOptions;
248250
bool generateForStaticFunctions = true;
@@ -251,6 +253,7 @@ namespace Commands {
251253
int32_t timeoutPerTest = 30;
252254
bool noDeterministicSearcher = false;
253255
bool noStubs = false;
256+
ErrorMode errorMode = ErrorMode::FAILING;
254257
};
255258
};
256259

‎server/src/coverage/TestRunner.cpp

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -65,18 +65,16 @@ std::vector<UnitTest> TestRunner::getTestsFromMakefile(const fs::path &makefile,
6565
StringUtils::trim(s);
6666
}
6767
std::string testSuite;
68-
std::vector<std::string> testsList;
69-
for (const std::string &s : gtestListTestsOutput) {
68+
std::vector<UnitTest> testList;
69+
for (const std::string &s: gtestListTestsOutput) {
7070
if (s.back() == '.') {
7171
testSuite = s;
7272
testSuite.pop_back();
7373
} else {
74-
testsList.push_back(s);
74+
testList.push_back({testFilePath, testSuite, s});
7575
}
7676
}
77-
return CollectionUtils::transform(testsList, [&testFilePath, &testSuite](std::string const &name) {
78-
return UnitTest{ testFilePath, testSuite, name };
79-
});
77+
return testList;
8078
}
8179

8280
std::vector<UnitTest> TestRunner::getTestsToLaunch() {

0 commit comments

Comments
 (0)