diff --git a/.github/workflows/matrix.json b/.github/workflows/matrix.json index 0b10822ff..64f7d23d6 100644 --- a/.github/workflows/matrix.json +++ b/.github/workflows/matrix.json @@ -2,7 +2,7 @@ "include": [ { "DOCKER_TAG": "24-03-2022", - "OPERATING_SYSTEM_TAG": "18.04", + "OPERATING_SYSTEM_TAG": "20.04", "LLVM_VERSION_MAJOR": "10" } ] diff --git a/.gitmodules b/.gitmodules index ec3f420a3..5a5c95ed9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,6 @@ [submodule "parallel-hashmap"] path = submodules/parallel-hashmap url = https://github.com/greg7mdp/parallel-hashmap +[submodule "submodules/kcov"] + path = submodules/kcov + url = https://github.com/SimonKagstrom/kcov diff --git a/docker/Dockerfile_base b/docker/Dockerfile_base index 26d33f96d..3a3e078b2 100644 --- a/docker/Dockerfile_base +++ b/docker/Dockerfile_base @@ -200,6 +200,11 @@ RUN chmod +x /usr/local/bin/lit RUN sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 100 +#Install kcov coverage tool and xml parser +RUN apt update && apt install -y kcov +RUN apt install -y libpugixml-dev +RUN apt install -y libpugixml1v5 + # Download CLI11 and Rang libs for nice CLI RUN wget https://github.com/CLIUtils/CLI11/releases/download/v1.9.1/CLI11.hpp -P $UTBOT_ALL/cli RUN wget https://github.com/agauniyal/rang/releases/download/v3.1.0/rang.hpp -P $UTBOT_ALL/cli diff --git a/docker/utbot_docker_dev.sh b/docker/utbot_docker_dev.sh index 45921bf3c..27751a9ee 100755 --- a/docker/utbot_docker_dev.sh +++ b/docker/utbot_docker_dev.sh @@ -10,14 +10,8 @@ CONTAINER_NAME=$USER-utbot-dev MOUNT_NAME=$USER-utbot MOUNT_LOCAL_NAME=$MOUNT_NAME-local-mnt -read -e -p "Enter base image tag: " IMAGE_TAG -IMAGE="ghcr.io/unittestbot/utbotcpp/base_env:$IMAGE_TAG" +IMAGE="kcov_image:latest" -echo "Pulling docker image '$IMAGE'" -if ! docker pull $IMAGE ; then - echo "Failed to fetch the image. Aborting.." - exit 1 -fi set +e docker exec "$CONTAINER_NAME" ls > /dev/null 2>&1 if [ $? -eq 0 ] diff --git a/firstmap b/firstmap new file mode 160000 index 000000000..703bd9caa --- /dev/null +++ b/firstmap @@ -0,0 +1 @@ +Subproject commit 703bd9caab50b139428cea1aaff9974ebee5742e diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 93ff5d4ba..f8de88cf5 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -120,6 +120,9 @@ include_directories("${CMAKE_CURRENT_BINARY_DIR}") find_package(run_klee REQUIRED) +# +find_package(pugixml REQUIRED) + option(ENABLE_PRECOMPILED_HEADERS "Enable precompiled headers" ON) if (ENABLE_PRECOMPILED_HEADERS) @@ -141,6 +144,7 @@ target_link_libraries(UnitTestBotLib PUBLIC clangTooling clangBasic clangASTMatc protobuf::libprotobuf loguru kleeRunner + pugixml ) if (ENABLE_PRECOMPILED_HEADERS) target_precompile_headers(UnitTestBotLib PUBLIC pch.h) @@ -153,6 +157,7 @@ add_llvm_executable(utbot main.cpp) target_link_libraries(utbot PUBLIC UnitTestBotLib loguru + pugixml ######### ) ################################################################################ @@ -180,6 +185,7 @@ if (ENABLE_UNIT_TESTS) PUBLIC gtest UnitTestBotLib + pugixml ) if (ENABLE_PRECOMPILED_HEADERS) diff --git a/server/src/Paths.cpp b/server/src/Paths.cpp index f8a1d20e5..202ccedc6 100644 --- a/server/src/Paths.cpp +++ b/server/src/Paths.cpp @@ -214,6 +214,10 @@ namespace Paths { fs::path getGccCoverageDir(const utbot::ProjectContext &projectContext) { return getCoverageDir(projectContext) / "gcov"; } + /////////////////////////////////////////////////////// + fs::path getKcovReportDir(const utbot::ProjectContext &projectContext) { + return getBuildDir(projectContext) / "report"; + } fs::path getTestExecutable(const utbot::ProjectContext &projectContext, const fs::path &filePath) { return getTestExecDir(projectContext) / filePath.stem(); diff --git a/server/src/Paths.h b/server/src/Paths.h index bfa14fa39..cc8874715 100644 --- a/server/src/Paths.h +++ b/server/src/Paths.h @@ -295,6 +295,8 @@ namespace Paths { fs::path getGccCoverageDir(const utbot::ProjectContext &projectContext); + fs::path getKcovReportDir(const utbot::ProjectContext &projectContext); + fs::path getTestExecutable(const utbot::ProjectContext &projectContext, const fs::path &filePath); fs::path getGeneratedHeaderPath(const utbot::ProjectContext &projectContext, const fs::path &sourceFilePath); diff --git a/server/src/coverage/CoverageAndResultsGenerator.cpp b/server/src/coverage/CoverageAndResultsGenerator.cpp index b58c72fd0..3e17fc447 100644 --- a/server/src/coverage/CoverageAndResultsGenerator.cpp +++ b/server/src/coverage/CoverageAndResultsGenerator.cpp @@ -34,7 +34,7 @@ grpc::Status CoverageAndResultsGenerator::generate(bool withCoverage, utbot::SettingsContext &settingsContext) { MEASURE_FUNCTION_EXECUTION_TIME try { - init(withCoverage); + init(withCoverage, true); runTests(withCoverage, settingsContext.timeoutPerTest); if (withCoverage) { collectCoverage(); @@ -82,30 +82,5 @@ nlohmann::json const &CoverageAndResultsGenerator::getTotals() { void CoverageAndResultsGenerator::collectCoverage() { MEASURE_FUNCTION_EXECUTION_TIME - if (testsToLaunch.empty()) { - return; - } - std::vector coverageCommands = coverageTool->getCoverageCommands( - CollectionUtils::filterToVector(testsToLaunch, [this](const UnitTest &testToLaunch) { - return testStatusMap[testToLaunch.testFilePath][testToLaunch.testname] != - testsgen::TEST_INTERRUPTED; - })); - if (coverageCommands.empty()) { - return; - } - ExecUtils::doWorkWithProgress( - coverageCommands, coverageAndResultsWriter, "Collecting coverage", - [this](ShellExecTask &task) { - auto [out, status, path] = task.run(); - if (status != 0) { - exceptions.emplace_back( - StringUtils::stringFormat("Command: %s\nOutput: %s", task.toString(), out), - path.value()); - } - }); - - LOG_S(DEBUG) << "All coverage commands were executed"; - coverageMap = coverageTool->getCoverageInfo(); - totals = coverageTool->getTotals(); } diff --git a/server/src/coverage/CoverageTool.cpp b/server/src/coverage/CoverageTool.cpp index 1d0088572..57900b56b 100644 --- a/server/src/coverage/CoverageTool.cpp +++ b/server/src/coverage/CoverageTool.cpp @@ -9,6 +9,7 @@ #include "exceptions/CoverageGenerationException.h" #include "utils/CompilationUtils.h" #include "utils/StringUtils.h" +#include "KcovCoverageTool.h" using namespace CompilationUtils; @@ -17,7 +18,12 @@ CoverageTool::CoverageTool(ProgressWriter const *progressWriter) : progressWrite std::unique_ptr getCoverageTool(const std::string &compileCommandsJsonPath, utbot::ProjectContext projectContext, - ProgressWriter const *progressWriter) { + ProgressWriter const *progressWriter, + bool withKcov) { + if (withKcov) { + return std::make_unique(projectContext, progressWriter); + } + auto compilationDatabase = CompilationUtils::getCompilationDatabase(compileCommandsJsonPath); fs::path compilerPath = compilationDatabase->getBuildCompilerPath(); CompilerName compilerName = CompilationUtils::getCompilerName(compilerPath); diff --git a/server/src/coverage/CoverageTool.h b/server/src/coverage/CoverageTool.h index 2176a2f12..a93b70c77 100644 --- a/server/src/coverage/CoverageTool.h +++ b/server/src/coverage/CoverageTool.h @@ -54,6 +54,7 @@ class CoverageTool { std::unique_ptr getCoverageTool(const std::string &compileCommandsJsonPath, utbot::ProjectContext projectContext, - const ProgressWriter *progressWriter); + const ProgressWriter *progressWriter, + bool withKcov); #endif // UNITTESTBOT_COVERAGETOOL_H diff --git a/server/src/coverage/KcovCoverageTool.cpp b/server/src/coverage/KcovCoverageTool.cpp new file mode 100644 index 000000000..449f861ea --- /dev/null +++ b/server/src/coverage/KcovCoverageTool.cpp @@ -0,0 +1,131 @@ +// +// Created by andrey on 05.06.2022. +// + +#include "KcovCoverageTool.h" + +#include "Coverage.h" +#include "Paths.h" +#include "TimeExecStatistics.h" +#include "environment/EnvironmentPaths.h" +#include "exceptions/CoverageGenerationException.h" +#include "utils/ArgumentsUtils.h" +#include "utils/CollectionUtils.h" +#include "utils/FileSystemUtils.h" +#include "utils/JsonUtils.h" +#include "utils/MakefileUtils.h" +#include "utils/StringUtils.h" +#include "utils/path/FileSystemPath.h" +#include "pugixml.hpp" +#include + +#include "loguru.h" + +using Coverage::CoverageMap; +using Coverage::FileCoverage; + + +KcovCoverageTool::KcovCoverageTool(utbot::ProjectContext projectContext, + ProgressWriter const *progressWriter) + : CoverageTool(progressWriter), projectContext(projectContext) { +} + +std::vector +KcovCoverageTool::getBuildRunCommands(const std::vector &testsToLaunch, bool withCoverage) { + ExecUtils::throwIfCancelled(); + + std::vector result; + ExecUtils::doWorkWithProgress( + testsToLaunch, progressWriter, "Collecting build and run commands", + [&](UnitTest const &testToLaunch) { + auto makefile = Paths::getMakefilePathFromSourceFilePath( + projectContext, + Paths::testPathToSourcePath(projectContext, testToLaunch.testFilePath)); + auto gtestFlags = getTestFilter(testToLaunch); + auto buildCommand = + MakefileUtils::makefileCommand(projectContext, makefile, "build", gtestFlags); + auto runCommand = + MakefileUtils::makefileCommand(projectContext, makefile, "run", gtestFlags); + result.push_back({testToLaunch, buildCommand, runCommand}); + }); + return result; +} + +std::vector +KcovCoverageTool::getCoverageCommands(const std::vector &testsToLaunch) { + return {}; +} + + +static void addLine(uint32_t lineNumber, bool covered, FileCoverage &fileCoverage) { + assert(lineNumber > 0); + if (covered) { + fileCoverage.fullCoverageLines.insert({lineNumber}); + } else { + fileCoverage.noCoverageLines.insert({lineNumber}); + } +} + +static void setLineNumbers(pugi::xml_node lines, FileCoverage &fileCoverage) { + for (pugi::xml_node line: lines.children("line")) { + uint32_t lineNumber = line.attribute("number").as_int(); + bool covered = line.attribute("hits").as_int() > 0; + addLine(lineNumber, covered, fileCoverage); + } +} + +Coverage::CoverageMap KcovCoverageTool::getCoverageInfo() const { + ExecUtils::throwIfCancelled(); + + CoverageMap coverageMap; + + auto coverageReportDirPath = Paths::getKcovReportDir(projectContext); + if (!fs::exists(coverageReportDirPath)) { + std::string message = "Couldn't find coverage directory at " + coverageReportDirPath.string(); + LOG_S(ERROR) << message; + throw CoverageGenerationException(message); + } + LOG_S(INFO) << "Reading coverage file"; + + + std::string pathToReport = ""; + for (const auto &entry: fs::directory_iterator(coverageReportDirPath.string())) { + std::regex reg("(calc.).*$"); + if (std::regex_search(entry.path().string(), reg)) { + pathToReport = entry.path().string() + "/cov.xml"; + break; + } + } + + auto path = pathToReport.c_str(); + + pugi::xml_document doc; + pugi::xml_parse_result result = doc.load_file(path); + auto tools = doc.child("coverage").child("packages").child("package").child("classes"); + + + for (pugi::xml_node fileIter = tools.child("class"); fileIter; fileIter = fileIter.next_sibling("class")) { + std::string filePathString = fileIter.attribute("filename").as_string(); + fs::path filePath(filePathString); + + pugi::xml_node lines = fileIter.child("lines"); + + setLineNumbers(lines, coverageMap[filePath]); + //setFunctionBorders(lines, coverageMap[filePath]); + } + + return coverageMap; +} + + +nlohmann::json KcovCoverageTool::getTotals() const { + return nlohmann::json(); +} + +void KcovCoverageTool::cleanCoverage() const { + +} + +fs::path KcovCoverageTool::getKcovReportFile() const { + return nullptr; +} \ No newline at end of file diff --git a/server/src/coverage/KcovCoverageTool.h b/server/src/coverage/KcovCoverageTool.h new file mode 100644 index 000000000..d9ead1ba5 --- /dev/null +++ b/server/src/coverage/KcovCoverageTool.h @@ -0,0 +1,30 @@ +// +// Created by andrey on 05.06.2022. +// + +#ifndef UNITTESTBOT_KCOVCOVERAGETOOL_H +#define UNITTESTBOT_KCOVCOVERAGETOOL_H + +#include "CoverageAndResultsGenerator.h" +#include "CoverageTool.h" + +class KcovCoverageTool : public CoverageTool { +public: + KcovCoverageTool(utbot::ProjectContext projectContext, const ProgressWriter *progressWriter); + + std::vector getBuildRunCommands(const std::vector &testsToLaunch, + bool withCoverage) override; + + std::vector getCoverageCommands(const std::vector &testFilePath) override; + + [[nodiscard]] Coverage::CoverageMap getCoverageInfo() const override; + [[nodiscard]] nlohmann::json getTotals() const override; + void cleanCoverage() const override; +private: + const utbot::ProjectContext projectContext; + + fs::path getKcovReportFile() const; +}; + + +#endif //UNITTESTBOT_KCOVCOVERAGETOOL_H diff --git a/server/src/coverage/TestRunner.cpp b/server/src/coverage/TestRunner.cpp index 0692e6472..44e00eb20 100644 --- a/server/src/coverage/TestRunner.cpp +++ b/server/src/coverage/TestRunner.cpp @@ -152,11 +152,11 @@ grpc::Status TestRunner::runTests(bool withCoverage, const std::optional getTestsToLaunch(); diff --git a/server/src/printers/NativeMakefilePrinter.cpp b/server/src/printers/NativeMakefilePrinter.cpp index cbce3762e..787386949 100644 --- a/server/src/printers/NativeMakefilePrinter.cpp +++ b/server/src/printers/NativeMakefilePrinter.cpp @@ -377,7 +377,9 @@ namespace printer { declareTarget("bin", { FORCE }, { stringFormat("echo %s", coverageInfoBinary) }); - utbot::RunCommand testRunCommand{ { testExecutablePath.string(), "$(GTEST_FLAGS)" }, + std::string kcovRunCommand = "kcov " + (projectContext.buildDir.string() + "/report ") + testExecutablePath.string(); + + utbot::RunCommand testRunCommand{ { kcovRunCommand, "$(GTEST_FLAGS)" }, buildDirectory }; testRunCommand.addEnvironmentVariable("PATH", "$$PATH:$(pwd)"); if (primaryCompilerName == CompilationUtils::CompilerName::GCC) { diff --git a/server/src/utils/ArgumentsUtils.cpp b/server/src/utils/ArgumentsUtils.cpp index 32a7e9e51..da3506a6d 100644 --- a/server/src/utils/ArgumentsUtils.cpp +++ b/server/src/utils/ArgumentsUtils.cpp @@ -82,10 +82,10 @@ namespace CompilationUtils { switch (compilerName) { case CompilerName::GCC: case CompilerName::GXX: - return { "--coverage" }; + return { "-g", "--coverage" }; case CompilerName::CLANG: case CompilerName::CLANGXX: - return { "-fprofile-instr-generate", "-fcoverage-mapping" }; + return { "-g", "-fprofile-instr-generate", "-fcoverage-mapping" }; default: break; } diff --git a/submodules/kcov b/submodules/kcov new file mode 160000 index 000000000..e615f997c --- /dev/null +++ b/submodules/kcov @@ -0,0 +1 @@ +Subproject commit e615f997c0b299e1dcab6ee5d90032a6419b2cdb