testutils.cpp

Go to the documentation of this file.

#include "./testutils.h"
 
#include "../conversion/stringbuilder.h"
#include "../conversion/stringconversion.h"
#include "../io/ansiescapecodes.h"
#include "../io/misc.h"
#include "../io/nativefilestream.h"
#include "../io/path.h"
#include "../misc/parseerror.h"
 
#include <cerrno>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <initializer_list>
#include <iostream>
#include <limits>
 
#ifdef PLATFORM_UNIX
#ifdef CPP_UTILITIES_USE_STANDARD_FILESYSTEM
#include <filesystem>
#endif
#include <poll.h>
#include <sys/stat.h>
#include <sys/wait.h>
#include <unistd.h>
#endif
 
#ifdef CPP_UTILITIES_BOOST_PROCESS
#include <boost/asio/buffers_iterator.hpp>
#include <boost/asio/io_context.hpp>
#include <boost/asio/streambuf.hpp>
#include <boost/process/async.hpp>
#include <boost/process/child.hpp>
#include <boost/process/env.hpp>
#include <boost/process/environment.hpp>
#include <boost/process/group.hpp>
#include <boost/process/io.hpp>
#include <boost/process/search_path.hpp>
#endif
 
#ifdef PLATFORM_WINDOWS
#include <windows.h>
#endif
 
using namespace std;
using namespace CppUtilities::EscapeCodes;
 
namespace CppUtilities {
 
static bool fileSystemItemExists(const string &path)
{
#ifdef PLATFORM_UNIX
    struct stat res;
    return stat(path.data(), &res) == 0;
#else
    const auto widePath(convertMultiByteToWide(path));
    if (!widePath.first) {
        return false;
    }
    const auto fileType(GetFileAttributesW(widePath.first.get()));
    return fileType != INVALID_FILE_ATTRIBUTES;
#endif
}
 
static bool fileExists(const string &path)
{
#ifdef PLATFORM_UNIX
    struct stat res;
    return stat(path.data(), &res) == 0 && !S_ISDIR(res.st_mode);
#else
    const auto widePath(convertMultiByteToWide(path));
    if (!widePath.first) {
        return false;
    }
    const auto fileType(GetFileAttributesW(widePath.first.get()));
    return (fileType != INVALID_FILE_ATTRIBUTES) && !(fileType & FILE_ATTRIBUTE_DIRECTORY) && !(fileType & FILE_ATTRIBUTE_DEVICE);
#endif
}
 
static bool dirExists(const string &path)
{
#ifdef PLATFORM_UNIX
    struct stat res;
    return stat(path.data(), &res) == 0 && S_ISDIR(res.st_mode);
#else
    const auto widePath(convertMultiByteToWide(path));
    if (!widePath.first) {
        return false;
    }
    const auto fileType(GetFileAttributesW(widePath.first.get()));
    return (fileType != INVALID_FILE_ATTRIBUTES) && (fileType & FILE_ATTRIBUTE_DIRECTORY);
#endif
}
 
static bool makeDir(const string &path)
{
#ifdef PLATFORM_UNIX
    return mkdir(path.data(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) == 0;
#else
    const auto widePath(convertMultiByteToWide(path));
    if (!widePath.first) {
        return false;
    }
    return CreateDirectoryW(widePath.first.get(), nullptr) || GetLastError() == ERROR_ALREADY_EXISTS;
#endif
}
 
TestApplication *TestApplication::s_instance = nullptr;
 
TestApplication::TestApplication()
    : TestApplication(0, nullptr)
{
}
 
TestApplication::TestApplication(int argc, const char *const *argv)
    : m_listArg("list", 'l', "lists available test units")
    , m_runArg("run", 'r', "runs the tests")
    , m_testFilesPathArg("test-files-path", 'p', "specifies the path of the directory with test files", { "path" })
    , m_applicationPathArg("app-path", 'a', "specifies the path of the application to be tested", { "path" })
    , m_workingDirArg("working-dir", 'w', "specifies the directory to store working copies of test files", { "path" })
    , m_unitsArg("units", 'u', "specifies the units to test; omit to test all units", { "unit1", "unit2", "unit3" })
{
    // check whether there is already an instance
    if (s_instance) {
        throw runtime_error("only one TestApplication instance allowed at a time");
    }
    s_instance = this;
 
    // handle specified arguments (if present)
    if (argc && argv) {
        // setup argument parser
        m_testFilesPathArg.setRequiredValueCount(Argument::varValueCount);
        m_unitsArg.setRequiredValueCount(Argument::varValueCount);
        m_runArg.setImplicit(true);
        m_runArg.setSubArguments({ &m_testFilesPathArg, &m_applicationPathArg, &m_workingDirArg, &m_unitsArg });
        m_parser.setMainArguments({ &m_runArg, &m_listArg, &m_parser.noColorArg(), &m_parser.helpArg() });
 
        // parse arguments
        try {
            m_parser.parseArgs(argc, argv, ParseArgumentBehavior::CheckConstraints | ParseArgumentBehavior::InvokeCallbacks);
        } catch (const ParseError &failure) {
            cerr << failure;
            m_valid = false;
            return;
        }
 
        // print help
        if (m_parser.helpArg().isPresent()) {
            exit(0);
        }
    }
 
    // set paths for testfiles
    // -> set paths set via CLI argument
    if (m_testFilesPathArg.isPresent()) {
        for (const char *const testFilesPath : m_testFilesPathArg.values()) {
            if (*testFilesPath) {
                m_testFilesPaths.emplace_back(argsToString(testFilesPath, '/'));
            } else {
                m_testFilesPaths.emplace_back("./");
            }
        }
    }
    // -> read TEST_FILE_PATH environment variable
    bool hasTestFilePathFromEnv;
    if (auto testFilePathFromEnv = readTestfilePathFromEnv(); (hasTestFilePathFromEnv = !testFilePathFromEnv.empty())) {
        m_testFilesPaths.emplace_back(std::move(testFilePathFromEnv));
    }
    // -> find source directory
    if (auto testFilePathFromSrcDirRef = readTestfilePathFromSrcRef(); !testFilePathFromSrcDirRef.empty()) {
        m_testFilesPaths.insert(m_testFilesPaths.end(), std::make_move_iterator(testFilePathFromSrcDirRef.begin()),
            std::make_move_iterator(testFilePathFromSrcDirRef.end()));
    }
    // -> try testfiles directory in working directory
    m_testFilesPaths.emplace_back("./testfiles/");
    for (const auto &testFilesPath : m_testFilesPaths) {
        cerr << testFilesPath << '\n';
    }
 
    // set path for working-copy
    if (m_workingDirArg.isPresent()) {
        if (*m_workingDirArg.values().front()) {
            (m_workingDir = m_workingDirArg.values().front()) += '/';
        } else {
            m_workingDir = "./";
        }
    } else if (const char *const workingDirEnv = getenv("WORKING_DIR")) {
        if (*workingDirEnv) {
            m_workingDir = argsToString(workingDirEnv, '/');
        }
    } else {
        if ((m_testFilesPathArg.isPresent() && !m_testFilesPathArg.values().empty()) || hasTestFilePathFromEnv) {
            m_workingDir = m_testFilesPaths.front() + "workingdir/";
        } else {
            m_workingDir = "./testfiles/workingdir/";
        }
    }
    cerr << "Directory used to store working copies:\n" << m_workingDir << '\n';
 
    // clear list of all additional profiling files created when forking the test application
    if (const char *const profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) {
        ofstream(profrawListFile, ios_base::trunc);
    }
 
    m_valid = true;
}
 
TestApplication::~TestApplication()
{
    s_instance = nullptr;
}
 
std::string TestApplication::testFilePath(const std::string &relativeTestFilePath) const
{
    std::string path;
    for (const auto &testFilesPath : m_testFilesPaths) {
        if (fileExists(path = testFilesPath + relativeTestFilePath)) {
            return path;
        }
    }
    throw std::runtime_error("The test file \"" % relativeTestFilePath % "\" can not be located. Was looking under:\n"
        + joinStrings(m_testFilesPaths, "\n", false, " - ", relativeTestFilePath));
}
 
std::string TestApplication::testDirPath(const std::string &relativeTestDirPath) const
{
    std::string path;
    for (const auto &testFilesPath : m_testFilesPaths) {
        if (dirExists(path = testFilesPath + relativeTestDirPath)) {
            return path;
        }
    }
    throw std::runtime_error("The test directory \"" % relativeTestDirPath % "\" can not be located. Was looking under:\n"
        + joinStrings(m_testFilesPaths, "\n", false, " - ", relativeTestDirPath));
}
 
string TestApplication::workingCopyPath(const string &relativeTestFilePath, WorkingCopyMode mode) const
{
    return workingCopyPathAs(relativeTestFilePath, relativeTestFilePath, mode);
}
 
string TestApplication::workingCopyPathAs(
    const std::string &relativeTestFilePath, const std::string &relativeWorkingCopyPath, WorkingCopyMode mode) const
{
    // ensure working directory is present
    auto workingCopyPath = std::string();
    if (!dirExists(m_workingDir) && !makeDir(m_workingDir)) {
        cerr << Phrases::Error << "Unable to create working copy for \"" << relativeTestFilePath << "\": can't create working directory \""
             << m_workingDir << "\"." << Phrases::EndFlush;
        return workingCopyPath;
    }
 
    // ensure subdirectory exists
    const auto parts = splitString<vector<string>>(relativeWorkingCopyPath, "/", EmptyPartsTreat::Omit);
    if (!parts.empty()) {
        // create subdirectory level by level
        string currentLevel;
        currentLevel.reserve(m_workingDir.size() + relativeWorkingCopyPath.size() + 1);
        currentLevel.assign(m_workingDir);
        for (auto i = parts.cbegin(), end = parts.end() - 1; i != end; ++i) {
            if (currentLevel.back() != '/') {
                currentLevel += '/';
            }
            currentLevel += *i;
 
            // continue if subdirectory level already exists or we can successfully create the directory
            if (dirExists(currentLevel) || makeDir(currentLevel)) {
                continue;
            }
            // fail otherwise
            cerr << Phrases::Error << "Unable to create working copy for \"" << relativeWorkingCopyPath << "\": can't create directory \""
                 << currentLevel << "\" (inside working directory)." << Phrases::EndFlush;
            return workingCopyPath;
        }
    }
 
    workingCopyPath = m_workingDir + relativeWorkingCopyPath;
    switch (mode) {
    case WorkingCopyMode::NoCopy:
        // just return the path if we don't want to actually create a copy
        return workingCopyPath;
    case WorkingCopyMode::Cleanup:
        // ensure the file does not exist in cleanup mode
        if (std::remove(workingCopyPath.data()) != 0 && errno != ENOENT) {
            const auto error = std::strerror(errno);
            cerr << Phrases::Error << "Unable to delete \"" << workingCopyPath << "\": " << error << Phrases::EndFlush;
            workingCopyPath.clear();
        }
        return workingCopyPath;
    default:;
    }
 
    // copy the file
    const auto origFilePath = testFilePath(relativeTestFilePath);
    size_t workingCopyPathAttempt = 0;
    NativeFileStream origFile, workingCopy;
    origFile.open(origFilePath, ios_base::in | ios_base::binary);
    if (origFile.fail()) {
        cerr << Phrases::Error << "Unable to create working copy for \"" << relativeTestFilePath
             << "\": an IO error occurred when opening original file \"" << origFilePath << "\"." << Phrases::EndFlush;
        cerr << "error: " << std::strerror(errno) << endl;
        workingCopyPath.clear();
        return workingCopyPath;
    }
    workingCopy.open(workingCopyPath, ios_base::out | ios_base::binary | ios_base::trunc);
    while (workingCopy.fail() && fileSystemItemExists(workingCopyPath)) {
        // adjust the working copy path if the target file already exists and can not be truncated
        workingCopyPath = argsToString(m_workingDir, relativeWorkingCopyPath, '.', ++workingCopyPathAttempt);
        workingCopy.clear();
        workingCopy.open(workingCopyPath, ios_base::out | ios_base::binary | ios_base::trunc);
    }
    if (workingCopy.fail()) {
        cerr << Phrases::Error << "Unable to create working copy for \"" << relativeTestFilePath
             << "\": an IO error occurred when opening target file \"" << workingCopyPath << "\"." << Phrases::EndFlush;
        cerr << "error: " << strerror(errno) << endl;
        workingCopyPath.clear();
        return workingCopyPath;
    }
    workingCopy << origFile.rdbuf();
    workingCopy.close();
    if (!origFile.fail() && !workingCopy.fail()) {
        return workingCopyPath;
    }
 
    cerr << Phrases::Error << "Unable to create working copy for \"" << relativeTestFilePath << "\": ";
    if (origFile.fail()) {
        cerr << "an IO error occurred when reading original file \"" << origFilePath << "\"";
        workingCopyPath.clear();
        return workingCopyPath;
    }
    if (workingCopy.fail()) {
        if (origFile.fail()) {
            cerr << " and ";
        }
        cerr << " an IO error occurred when writing to target file \"" << workingCopyPath << "\".";
    }
    cerr << "error: " << strerror(errno) << endl;
    workingCopyPath.clear();
    return workingCopyPath;
}
 
#ifdef CPP_UTILITIES_HAS_EXEC_APP
 
#if defined(CPP_UTILITIES_BOOST_PROCESS)
inline static std::string streambufToString(boost::asio::streambuf &buf)
{
    const auto begin = boost::asio::buffers_begin(buf.data());
    return std::string(begin, begin + static_cast<std::ptrdiff_t>(buf.size()));
}
#endif
 
static int execAppInternal(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout,
    const std::string &newProfilingPath, bool enableSearchPath = false)
{
    // print log message
    if (!suppressLogging) {
        // print actual appPath and skip first argument instead
        cout << '-' << ' ' << appPath;
        if (*args) {
            for (const char *const *i = args + 1; *i; ++i) {
                cout << ' ' << *i;
            }
        }
        cout << endl;
    }
 
#if defined(CPP_UTILITIES_BOOST_PROCESS)
    auto path = enableSearchPath ? boost::process::search_path(appPath) : boost::process::filesystem::path(appPath);
    auto ctx = boost::asio::io_context();
    auto group = boost::process::group();
    auto argsAsVector =
#if defined(PLATFORM_WINDOWS)
        std::vector<std::wstring>();
#else
        std::vector<std::string>();
#endif
    if (*args) {
        for (const char *const *arg = args + 1; *arg; ++arg) {
#if defined(PLATFORM_WINDOWS)
            auto ec = std::error_code();
            argsAsVector.emplace_back(convertMultiByteToWide(ec, std::string_view(*arg)));
            if (ec) {
                throw std::runtime_error(argsToString("unable to convert arg \"", *arg, "\" to wide string"));
            }
#else
            argsAsVector.emplace_back(*arg);
#endif
        }
    }
    auto outputBuffer = boost::asio::streambuf(), errorBuffer = boost::asio::streambuf();
    auto env = boost::process::environment(boost::this_process::environment());
    if (!newProfilingPath.empty()) {
        env["LLVM_PROFILE_FILE"] = newProfilingPath;
    }
    auto child
        = boost::process::child(ctx, group, path, argsAsVector, env, boost::process::std_out > outputBuffer, boost::process::std_err > errorBuffer);
    if (timeout > 0) {
        ctx.run_for(std::chrono::milliseconds(timeout));
    } else {
        ctx.run();
    }
    output = streambufToString(outputBuffer);
    errors = streambufToString(errorBuffer);
    child.wait();
    group.wait();
    return child.exit_code();
 
#elif defined(PLATFORM_UNIX)
    // create pipes
    int coutPipes[2], cerrPipes[2];
    pipe(coutPipes);
    pipe(cerrPipes);
    const auto readCoutPipe = coutPipes[0], writeCoutPipe = coutPipes[1];
    const auto readCerrPipe = cerrPipes[0], writeCerrPipe = cerrPipes[1];
 
    // create child process
    if (const auto child = fork()) {
        // parent process: read stdout and stderr from child
        close(writeCoutPipe);
        close(writeCerrPipe);
 
        try {
            if (child == -1) {
                throw runtime_error("Unable to create fork");
            }
 
            // init file descriptor set for poll
            struct pollfd fileDescriptorSet[2];
            fileDescriptorSet[0].fd = readCoutPipe;
            fileDescriptorSet[1].fd = readCerrPipe;
            fileDescriptorSet[0].events = fileDescriptorSet[1].events = POLLIN;
 
            // init variables for reading
            char buffer[512];
            output.clear();
            errors.clear();
 
            // poll as long as at least one pipe is open
            do {
                const auto retpoll = poll(fileDescriptorSet, 2, timeout);
                if (retpoll == 0) {
                    throw runtime_error("Poll time-out");
                }
                if (retpoll < 0) {
                    throw runtime_error("Poll failed");
                }
                if (fileDescriptorSet[0].revents & POLLIN) {
                    const auto count = read(readCoutPipe, buffer, sizeof(buffer));
                    if (count > 0) {
                        output.append(buffer, static_cast<size_t>(count));
                    }
                } else if (fileDescriptorSet[0].revents & POLLHUP) {
                    close(readCoutPipe);
                    fileDescriptorSet[0].fd = -1;
                }
                if (fileDescriptorSet[1].revents & POLLIN) {
                    const auto count = read(readCerrPipe, buffer, sizeof(buffer));
                    if (count > 0) {
                        errors.append(buffer, static_cast<size_t>(count));
                    }
                } else if (fileDescriptorSet[1].revents & POLLHUP) {
                    close(readCerrPipe);
                    fileDescriptorSet[1].fd = -1;
                }
            } while (fileDescriptorSet[0].fd >= 0 || fileDescriptorSet[1].fd >= 0);
        } catch (...) {
            // ensure all pipes are closed in the error case
            close(readCoutPipe);
            close(readCerrPipe);
            throw;
        }
 
        // get return code
        int childReturnCode;
        waitpid(child, &childReturnCode, 0);
        waitpid(-child, nullptr, 0);
        return childReturnCode;
    } else {
        // child process
        // -> set pipes to be used for stdout/stderr
        dup2(writeCoutPipe, STDOUT_FILENO);
        dup2(writeCerrPipe, STDERR_FILENO);
        close(readCoutPipe);
        close(writeCoutPipe);
        close(readCerrPipe);
        close(writeCerrPipe);
 
        // -> create process group
        if (setpgid(0, 0)) {
            cerr << Phrases::Error << "Unable create process group: " << std::strerror(errno) << Phrases::EndFlush;
            exit(EXIT_FAILURE);
        }
 
        // -> modify environment variable LLVM_PROFILE_FILE to apply new path for profiling output
        if (!newProfilingPath.empty()) {
            setenv("LLVM_PROFILE_FILE", newProfilingPath.data(), true);
        }
 
        // -> execute application
        if (enableSearchPath) {
            execvp(appPath, const_cast<char *const *>(args));
        } else {
            execv(appPath, const_cast<char *const *>(args));
        }
        cerr << Phrases::Error << "Unable to execute \"" << appPath << "\": " << std::strerror(errno) << Phrases::EndFlush;
        exit(EXIT_FAILURE);
    }
 
#else
    throw std::runtime_error("lauching test applications is not supported on this platform");
#endif
}
 
int TestApplication::execApp(const char *const *args, string &output, string &errors, bool suppressLogging, int timeout) const
{
    // increase counter used for giving profiling files unique names
    static unsigned int invocationCount = 0;
    ++invocationCount;
 
    // determine the path of the application to be tested
    const char *appPath = m_applicationPathArg.firstValue();
    auto fallbackAppPath = string();
    if (!appPath || !*appPath) {
        // try to find the path by removing "_tests"-suffix from own executable path
        // (the own executable path is the path of the test application and its name is usually the name of the application
        //  to be tested with "_tests"-suffix)
        const char *const testAppPath = m_parser.executable();
        const auto testAppPathLength = strlen(testAppPath);
        if (testAppPathLength > 6 && !strcmp(testAppPath + testAppPathLength - 6, "_tests")) {
            fallbackAppPath.assign(testAppPath, testAppPathLength - 6);
            appPath = fallbackAppPath.data();
            // TODO: it would not hurt to verify whether "fallbackAppPath" actually exists and is executable
        } else {
            throw runtime_error("Unable to execute application to be tested: no application path specified");
        }
    }
 
    // determine new path for profiling output (to not override profiling output of parent and previous invocations)
    const auto newProfilingPath = [appPath] {
        auto path = string();
        const char *const llvmProfileFile = getenv("LLVM_PROFILE_FILE");
        if (!llvmProfileFile) {
            return path;
        }
        // replace eg. "/some/path/tageditor_tests.profraw" with "/some/path/tageditor0.profraw"
        const char *const llvmProfileFileEnd = strstr(llvmProfileFile, ".profraw");
        if (!llvmProfileFileEnd) {
            return path;
        }
        const auto llvmProfileFileWithoutExtension = string(llvmProfileFile, llvmProfileFileEnd);
        // extract application name from path
        const char *appName = strrchr(appPath, '/');
        appName = appName ? appName + 1 : appPath;
        // concat new path
        path = argsToString(llvmProfileFileWithoutExtension, '_', appName, invocationCount, ".profraw");
        // append path to profiling list file
        if (const char *const profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) {
            ofstream(profrawListFile, ios_base::app) << path << endl;
        }
        return path;
    }();
 
    return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, newProfilingPath);
}
 
int execHelperApp(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout)
{
    return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, string());
}
 
int execHelperAppInSearchPath(
    const char *appName, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout)
{
    return execAppInternal(appName, args, output, errors, suppressLogging, timeout, string(), true);
}
#endif
 
string TestApplication::readTestfilePathFromEnv()
{
    const char *const testFilesPathEnv = getenv("TEST_FILE_PATH");
    if (!testFilesPathEnv || !*testFilesPathEnv) {
        return string();
    }
    return argsToString(testFilesPathEnv, '/');
}
 
std::vector<std::string> TestApplication::readTestfilePathFromSrcRef()
{
    // find the path of the current executable on platforms supporting "/proc/self/exe"; otherwise assume the current working directory
    // is the executable path
    auto res = std::vector<std::string>();
    auto binaryPath = std::string();
#if defined(CPP_UTILITIES_USE_STANDARD_FILESYSTEM) && defined(PLATFORM_UNIX)
    try {
        binaryPath = std::filesystem::read_symlink("/proc/self/exe").parent_path();
        binaryPath += '/';
    } catch (const std::filesystem::filesystem_error &e) {
        cerr << Phrases::Warning << "Unable to detect binary path for finding \"srcdirref\": " << e.what() << Phrases::EndFlush;
    }
#endif
    const auto srcdirrefPath = binaryPath + "srcdirref";
    try {
        // read "srcdirref" file which should contain the path of the source directory
        const auto srcDirContent = readFile(srcdirrefPath, 2 * 1024);
        if (srcDirContent.empty()) {
            cerr << Phrases::Warning << "The file \"srcdirref\" is empty." << Phrases::EndFlush;
            return res;
        }
 
        // check whether the referenced source directories contain a "testfiles" directory
        const auto srcPaths = splitStringSimple<std::vector<std::string_view>>(srcDirContent, "\n");
        for (const auto &srcPath : srcPaths) {
            auto testfilesPath = argsToString(srcPath, "/testfiles/");
            if (dirExists(testfilesPath)) {
                res.emplace_back(std::move(testfilesPath));
            } else {
                cerr << Phrases::Warning
                     << "The source directory referenced by the file \"srcdirref\" does not contain a \"testfiles\" directory or does not exist."
                     << Phrases::End << "Referenced source directory: " << testfilesPath << endl;
            }
        }
        return res;
 
    } catch (const std::ios_base::failure &e) {
        cerr << Phrases::Warning << "The file \"" << srcdirrefPath << "\" can not be opened: " << e.what() << Phrases::EndFlush;
    }
    return res;
}
} // namespace CppUtilities