From 1f3f0b0df1884b5d5f6582ae42c54b5f034fe2bc Mon Sep 17 00:00:00 2001 From: Martchus Date: Mon, 18 Apr 2022 20:39:02 +0200 Subject: [PATCH] Avoid high memory usage when reloading database Especially when enabling files DBs it is quite problematic to store all package objects of a database within one big array. This change avoids the array and instead adds the packages directly. The disadvantage is that clearing the database isn't as simple anymore. --- libpkg/data/database.cpp | 17 ++++++++++++----- libpkg/data/package.h | 1 + libpkg/parser/package.cpp | 22 ++++++++++++++++++++-- librepomgr/buildactions/reloaddatabase.cpp | 18 +++++++++++++++--- 4 files changed, 48 insertions(+), 10 deletions(-) diff --git a/libpkg/data/database.cpp b/libpkg/data/database.cpp index 6db090f..78b34ce 100644 --- a/libpkg/data/database.cpp +++ b/libpkg/data/database.cpp @@ -34,6 +34,7 @@ struct PackageUpdaterPrivate { bool clear = false; std::unique_lock lock; PackageStorage::RWTransaction packagesTxn; + std::unordered_set handledIds; AffectedDeps affectedProvidedDeps; AffectedDeps affectedRequiredDeps; AffectedLibs affectedProvidedLibs; @@ -660,10 +661,6 @@ PackageUpdaterPrivate::PackageUpdaterPrivate(DatabaseStorage &storage, bool clea , lock(storage.updateMutex) , packagesTxn(storage.packages.getRWTransaction()) { - if (clear) { - storage.packageCache.clearCacheOnly(storage); - packagesTxn.clear(); - } } void PackageUpdaterPrivate::update(const PackageCache::StoreResult &res, const std::shared_ptr &package) @@ -671,6 +668,7 @@ void PackageUpdaterPrivate::update(const PackageCache::StoreResult &res, const s if (!res.id) { return; } + handledIds.emplace(res.id); update(res.id, false, package); if (!clear && res.oldEntry) { update(res.id, true, res.oldEntry); @@ -804,7 +802,16 @@ StorageID PackageUpdater::update(const std::shared_ptr &package) void PackageUpdater::commit() { const auto &storage = m_database.m_storage; - m_d->packagesTxn.commit(); + auto &pkgTxn = m_d->packagesTxn; + if (m_d->clear) { + const auto &toPreserve = m_d->handledIds; + for (auto i = pkgTxn.begin(); i != pkgTxn.end(); ++i) { + if (!toPreserve.contains(i.getID())) { + i.del(); + } + } + } + pkgTxn.commit(); { auto txn = storage->providedDeps.getRWTransaction(); if (m_d->clear) { diff --git a/libpkg/data/package.h b/libpkg/data/package.h index 74c20be..031f22a 100644 --- a/libpkg/data/package.h +++ b/libpkg/data/package.h @@ -398,6 +398,7 @@ struct LIBPKG_EXPORT Package : public ReflectiveRapidJSON::JsonSerializable fromInfo(const std::string &info, bool isPackageInfo = false); static std::shared_ptr fromDescription(const std::vector &descriptionParts); static std::vector> fromDatabaseFile(FileMap &&databaseFile); + static void fromDatabaseFile(FileMap &&databaseFile, const std::function)> &visitor); static std::shared_ptr fromPkgFile(const std::string &path); static std::tuple fileNameComponents(std::string_view fileName); static std::shared_ptr fromPkgFileName(std::string_view fileName); diff --git a/libpkg/parser/package.cpp b/libpkg/parser/package.cpp index 7730ecf..e1a7680 100644 --- a/libpkg/parser/package.cpp +++ b/libpkg/parser/package.cpp @@ -3,6 +3,8 @@ #include "./binary.h" #include "./utils.h" +#include "../data/database.h" + #include #include #include @@ -718,17 +720,33 @@ std::vector> Package::fromDatabaseFile(FileMap &&databa { std::vector> packages; packages.reserve(databaseFile.size()); + std::vector descriptionParts; for (auto &dir : databaseFile) { - vector descriptionParts; + descriptionParts.clear(); descriptionParts.reserve(dir.second.size()); for (auto &file : dir.second) { - descriptionParts.emplace_back(move(file.content)); + descriptionParts.emplace_back(std::move(file.content)); } packages.emplace_back(Package::fromDescription(descriptionParts)); } return packages; } +void Package::fromDatabaseFile(FileMap &&databaseFile, const std::function)> &visitor) +{ + std::vector descriptionParts; + for (auto &dir : databaseFile) { + descriptionParts.clear(); + descriptionParts.reserve(dir.second.size()); + for (auto &file : dir.second) { + descriptionParts.emplace_back(std::move(file.content)); + } + if (visitor(Package::fromDescription(descriptionParts))) { + return; + } + } +} + bool Package::isPkgInfoFileOrBinary(const char *filePath, const char *fileName, mode_t mode) { return !strcmp(fileName, ".PKGINFO") || mode == S_IXUSR || strstr(filePath, "usr/bin") == filePath || strstr(filePath, "usr/lib") == filePath diff --git a/librepomgr/buildactions/reloaddatabase.cpp b/librepomgr/buildactions/reloaddatabase.cpp index a203cdb..b153032 100644 --- a/librepomgr/buildactions/reloaddatabase.cpp +++ b/librepomgr/buildactions/reloaddatabase.cpp @@ -94,19 +94,31 @@ void ReloadDatabase::run() } } auto dbFile = LibPkg::extractFiles(dbPath, &LibPkg::Database::isFileRelevant); - auto packages = LibPkg::Package::fromDatabaseFile(move(dbFile)); dbFileLock.lock().unlock(); + m_buildAction->appendOutput( Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from local file \"", dbPath, "\"\n"); - const auto configLock = m_setup.config.lockToRead(); + + auto configLock = m_setup.config.lockToRead(); auto *const destinationDb = m_setup.config.findDatabase(dbName, dbArch); if (!destinationDb) { + configLock.unlock(); m_buildAction->appendOutput( Phrases::ErrorMessage, "Loaded database file for \"", dbName, '@', dbArch, "\" but it no longer exists; discarding\n"); session->addResponse(std::move(dbName)); return; } - destinationDb->replacePackages(packages, lastModified); + + auto updater = LibPkg::PackageUpdater(*destinationDb, true); + LibPkg::Package::fromDatabaseFile(std::move(dbFile), [&updater](std::shared_ptr package) { + if (const auto [id, existingPackage] = updater.findPackageWithID(package->name); existingPackage) { + package->addDepsAndProvidesFromOtherPackage(*existingPackage); + } + updater.update(package); + return false; + }); + updater.commit(); + destinationDb->lastUpdate = lastModified; } catch (const std::runtime_error &e) { m_buildAction->appendOutput(Phrases::ErrorMessage, "An error occurred when reloading database \"", dbName, '@', dbArch, "\" from local file \"", dbPath, "\": ", e.what(), '\n');