Avoid high memory usage when reloading database

Especially when enabling files DBs it is quite problematic to store all
package objects of a database within one big array. This change avoids the
array and instead adds the packages directly. The disadvantage is that
clearing the database isn't as simple anymore.
This commit is contained in:
Martchus 2022-04-18 20:39:02 +02:00
parent f07ef9f147
commit 1f3f0b0df1
4 changed files with 48 additions and 10 deletions

View File

@ -34,6 +34,7 @@ struct PackageUpdaterPrivate {
bool clear = false; bool clear = false;
std::unique_lock<std::mutex> lock; std::unique_lock<std::mutex> lock;
PackageStorage::RWTransaction packagesTxn; PackageStorage::RWTransaction packagesTxn;
std::unordered_set<StorageID> handledIds;
AffectedDeps affectedProvidedDeps; AffectedDeps affectedProvidedDeps;
AffectedDeps affectedRequiredDeps; AffectedDeps affectedRequiredDeps;
AffectedLibs affectedProvidedLibs; AffectedLibs affectedProvidedLibs;
@ -660,10 +661,6 @@ PackageUpdaterPrivate::PackageUpdaterPrivate(DatabaseStorage &storage, bool clea
, lock(storage.updateMutex) , lock(storage.updateMutex)
, packagesTxn(storage.packages.getRWTransaction()) , packagesTxn(storage.packages.getRWTransaction())
{ {
if (clear) {
storage.packageCache.clearCacheOnly(storage);
packagesTxn.clear();
}
} }
void PackageUpdaterPrivate::update(const PackageCache::StoreResult &res, const std::shared_ptr<Package> &package) void PackageUpdaterPrivate::update(const PackageCache::StoreResult &res, const std::shared_ptr<Package> &package)
@ -671,6 +668,7 @@ void PackageUpdaterPrivate::update(const PackageCache::StoreResult &res, const s
if (!res.id) { if (!res.id) {
return; return;
} }
handledIds.emplace(res.id);
update(res.id, false, package); update(res.id, false, package);
if (!clear && res.oldEntry) { if (!clear && res.oldEntry) {
update(res.id, true, res.oldEntry); update(res.id, true, res.oldEntry);
@ -804,7 +802,16 @@ StorageID PackageUpdater::update(const std::shared_ptr<Package> &package)
void PackageUpdater::commit() void PackageUpdater::commit()
{ {
const auto &storage = m_database.m_storage; const auto &storage = m_database.m_storage;
m_d->packagesTxn.commit(); auto &pkgTxn = m_d->packagesTxn;
if (m_d->clear) {
const auto &toPreserve = m_d->handledIds;
for (auto i = pkgTxn.begin<std::unique_ptr>(); i != pkgTxn.end(); ++i) {
if (!toPreserve.contains(i.getID())) {
i.del();
}
}
}
pkgTxn.commit();
{ {
auto txn = storage->providedDeps.getRWTransaction(); auto txn = storage->providedDeps.getRWTransaction();
if (m_d->clear) { if (m_d->clear) {

View File

@ -398,6 +398,7 @@ struct LIBPKG_EXPORT Package : public ReflectiveRapidJSON::JsonSerializable<Pack
static std::vector<PackageSpec> fromInfo(const std::string &info, bool isPackageInfo = false); static std::vector<PackageSpec> fromInfo(const std::string &info, bool isPackageInfo = false);
static std::shared_ptr<Package> fromDescription(const std::vector<std::string> &descriptionParts); static std::shared_ptr<Package> fromDescription(const std::vector<std::string> &descriptionParts);
static std::vector<std::shared_ptr<Package>> fromDatabaseFile(FileMap &&databaseFile); static std::vector<std::shared_ptr<Package>> fromDatabaseFile(FileMap &&databaseFile);
static void fromDatabaseFile(FileMap &&databaseFile, const std::function<bool(std::shared_ptr<Package>)> &visitor);
static std::shared_ptr<Package> fromPkgFile(const std::string &path); static std::shared_ptr<Package> fromPkgFile(const std::string &path);
static std::tuple<std::string_view, std::string_view, std::string_view> fileNameComponents(std::string_view fileName); static std::tuple<std::string_view, std::string_view, std::string_view> fileNameComponents(std::string_view fileName);
static std::shared_ptr<Package> fromPkgFileName(std::string_view fileName); static std::shared_ptr<Package> fromPkgFileName(std::string_view fileName);

View File

@ -3,6 +3,8 @@
#include "./binary.h" #include "./binary.h"
#include "./utils.h" #include "./utils.h"
#include "../data/database.h"
#include <c++utilities/conversion/stringbuilder.h> #include <c++utilities/conversion/stringbuilder.h>
#include <c++utilities/conversion/stringconversion.h> #include <c++utilities/conversion/stringconversion.h>
#include <c++utilities/io/ansiescapecodes.h> #include <c++utilities/io/ansiescapecodes.h>
@ -718,17 +720,33 @@ std::vector<std::shared_ptr<Package>> Package::fromDatabaseFile(FileMap &&databa
{ {
std::vector<std::shared_ptr<Package>> packages; std::vector<std::shared_ptr<Package>> packages;
packages.reserve(databaseFile.size()); packages.reserve(databaseFile.size());
std::vector<std::string> descriptionParts;
for (auto &dir : databaseFile) { for (auto &dir : databaseFile) {
vector<string> descriptionParts; descriptionParts.clear();
descriptionParts.reserve(dir.second.size()); descriptionParts.reserve(dir.second.size());
for (auto &file : dir.second) { for (auto &file : dir.second) {
descriptionParts.emplace_back(move(file.content)); descriptionParts.emplace_back(std::move(file.content));
} }
packages.emplace_back(Package::fromDescription(descriptionParts)); packages.emplace_back(Package::fromDescription(descriptionParts));
} }
return packages; return packages;
} }
void Package::fromDatabaseFile(FileMap &&databaseFile, const std::function<bool(std::shared_ptr<Package>)> &visitor)
{
std::vector<std::string> descriptionParts;
for (auto &dir : databaseFile) {
descriptionParts.clear();
descriptionParts.reserve(dir.second.size());
for (auto &file : dir.second) {
descriptionParts.emplace_back(std::move(file.content));
}
if (visitor(Package::fromDescription(descriptionParts))) {
return;
}
}
}
bool Package::isPkgInfoFileOrBinary(const char *filePath, const char *fileName, mode_t mode) bool Package::isPkgInfoFileOrBinary(const char *filePath, const char *fileName, mode_t mode)
{ {
return !strcmp(fileName, ".PKGINFO") || mode == S_IXUSR || strstr(filePath, "usr/bin") == filePath || strstr(filePath, "usr/lib") == filePath return !strcmp(fileName, ".PKGINFO") || mode == S_IXUSR || strstr(filePath, "usr/bin") == filePath || strstr(filePath, "usr/lib") == filePath

View File

@ -94,19 +94,31 @@ void ReloadDatabase::run()
} }
} }
auto dbFile = LibPkg::extractFiles(dbPath, &LibPkg::Database::isFileRelevant); auto dbFile = LibPkg::extractFiles(dbPath, &LibPkg::Database::isFileRelevant);
auto packages = LibPkg::Package::fromDatabaseFile(move(dbFile));
dbFileLock.lock().unlock(); dbFileLock.lock().unlock();
m_buildAction->appendOutput( m_buildAction->appendOutput(
Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from local file \"", dbPath, "\"\n"); Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from local file \"", dbPath, "\"\n");
const auto configLock = m_setup.config.lockToRead();
auto configLock = m_setup.config.lockToRead();
auto *const destinationDb = m_setup.config.findDatabase(dbName, dbArch); auto *const destinationDb = m_setup.config.findDatabase(dbName, dbArch);
if (!destinationDb) { if (!destinationDb) {
configLock.unlock();
m_buildAction->appendOutput( m_buildAction->appendOutput(
Phrases::ErrorMessage, "Loaded database file for \"", dbName, '@', dbArch, "\" but it no longer exists; discarding\n"); Phrases::ErrorMessage, "Loaded database file for \"", dbName, '@', dbArch, "\" but it no longer exists; discarding\n");
session->addResponse(std::move(dbName)); session->addResponse(std::move(dbName));
return; return;
} }
destinationDb->replacePackages(packages, lastModified);
auto updater = LibPkg::PackageUpdater(*destinationDb, true);
LibPkg::Package::fromDatabaseFile(std::move(dbFile), [&updater](std::shared_ptr<LibPkg::Package> package) {
if (const auto [id, existingPackage] = updater.findPackageWithID(package->name); existingPackage) {
package->addDepsAndProvidesFromOtherPackage(*existingPackage);
}
updater.update(package);
return false;
});
updater.commit();
destinationDb->lastUpdate = lastModified;
} catch (const std::runtime_error &e) { } catch (const std::runtime_error &e) {
m_buildAction->appendOutput(Phrases::ErrorMessage, "An error occurred when reloading database \"", dbName, '@', dbArch, m_buildAction->appendOutput(Phrases::ErrorMessage, "An error occurred when reloading database \"", dbName, '@', dbArch,
"\" from local file \"", dbPath, "\": ", e.what(), '\n'); "\" from local file \"", dbPath, "\": ", e.what(), '\n');