Martchus
/
arch-repo-manager
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Avoid high memory usage when reloading database 

Especially when enabling files DBs it is quite problematic to store all
package objects of a database within one big array. This change avoids the
array and instead adds the packages directly. The disadvantage is that
clearing the database isn't as simple anymore.
This commit is contained in:
Martchus 2022-04-18 20:39:02 +02:00
parent f07ef9f147
commit 1f3f0b0df1
4 changed files with 48 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
libpkg/data/database.cpp
View File

@ -34,6 +34,7 @@ struct PackageUpdaterPrivate {
bool clear = false;
std::unique_lock<std::mutex> lock;
PackageStorage::RWTransaction packagesTxn;
std::unordered_set<StorageID> handledIds;
AffectedDeps affectedProvidedDeps;
AffectedDeps affectedRequiredDeps;
AffectedLibs affectedProvidedLibs;
@ -660,10 +661,6 @@ PackageUpdaterPrivate::PackageUpdaterPrivate(DatabaseStorage &storage, bool clea
, lock(storage.updateMutex)
, packagesTxn(storage.packages.getRWTransaction())
{
if (clear) {
storage.packageCache.clearCacheOnly(storage);
packagesTxn.clear();
}
}
void PackageUpdaterPrivate::update(const PackageCache::StoreResult &res, const std::shared_ptr<Package> &package)
@ -671,6 +668,7 @@ void PackageUpdaterPrivate::update(const PackageCache::StoreResult &res, const s
if (!res.id) {
return;
}
handledIds.emplace(res.id);
update(res.id, false, package);
if (!clear && res.oldEntry) {
update(res.id, true, res.oldEntry);
@ -804,7 +802,16 @@ StorageID PackageUpdater::update(const std::shared_ptr<Package> &package)
void PackageUpdater::commit()
{
const auto &storage = m_database.m_storage;
m_d->packagesTxn.commit();
auto &pkgTxn = m_d->packagesTxn;
if (m_d->clear) {
const auto &toPreserve = m_d->handledIds;
for (auto i = pkgTxn.begin<std::unique_ptr>(); i != pkgTxn.end(); ++i) {
if (!toPreserve.contains(i.getID())) {
i.del();
}
}
}
pkgTxn.commit();
{
auto txn = storage->providedDeps.getRWTransaction();
if (m_d->clear) {

1
libpkg/data/package.h
View File

@ -398,6 +398,7 @@ struct LIBPKG_EXPORT Package : public ReflectiveRapidJSON::JsonSerializable<Pack
static std::vector<PackageSpec> fromInfo(const std::string &info, bool isPackageInfo = false);
static std::shared_ptr<Package> fromDescription(const std::vector<std::string> &descriptionParts);
static std::vector<std::shared_ptr<Package>> fromDatabaseFile(FileMap &&databaseFile);
static void fromDatabaseFile(FileMap &&databaseFile, const std::function<bool(std::shared_ptr<Package>)> &visitor);
static std::shared_ptr<Package> fromPkgFile(const std::string &path);
static std::tuple<std::string_view, std::string_view, std::string_view> fileNameComponents(std::string_view fileName);
static std::shared_ptr<Package> fromPkgFileName(std::string_view fileName);

22
libpkg/parser/package.cpp
View File

@ -3,6 +3,8 @@
#include "./binary.h"
#include "./utils.h"
#include "../data/database.h"
#include <c++utilities/conversion/stringbuilder.h>
#include <c++utilities/conversion/stringconversion.h>
#include <c++utilities/io/ansiescapecodes.h>
@ -718,17 +720,33 @@ std::vector<std::shared_ptr<Package>> Package::fromDatabaseFile(FileMap &&databa
{
std::vector<std::shared_ptr<Package>> packages;
packages.reserve(databaseFile.size());
std::vector<std::string> descriptionParts;
for (auto &dir : databaseFile) {
vector<string> descriptionParts;
descriptionParts.clear();
descriptionParts.reserve(dir.second.size());
for (auto &file : dir.second) {
descriptionParts.emplace_back(move(file.content));
descriptionParts.emplace_back(std::move(file.content));
}
packages.emplace_back(Package::fromDescription(descriptionParts));
}
return packages;
}
void Package::fromDatabaseFile(FileMap &&databaseFile, const std::function<bool(std::shared_ptr<Package>)> &visitor)
{
std::vector<std::string> descriptionParts;
for (auto &dir : databaseFile) {
descriptionParts.clear();
descriptionParts.reserve(dir.second.size());
for (auto &file : dir.second) {
descriptionParts.emplace_back(std::move(file.content));
}
if (visitor(Package::fromDescription(descriptionParts))) {
return;
}
}
}
bool Package::isPkgInfoFileOrBinary(const char *filePath, const char *fileName, mode_t mode)
{
return !strcmp(fileName, ".PKGINFO") || mode == S_IXUSR || strstr(filePath, "usr/bin") == filePath || strstr(filePath, "usr/lib") == filePath

18
librepomgr/buildactions/reloaddatabase.cpp
View File

@ -94,19 +94,31 @@ void ReloadDatabase::run()
}
}
auto dbFile = LibPkg::extractFiles(dbPath, &LibPkg::Database::isFileRelevant);
auto packages = LibPkg::Package::fromDatabaseFile(move(dbFile));
dbFileLock.lock().unlock();
m_buildAction->appendOutput(
Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from local file \"", dbPath, "\"\n");
const auto configLock = m_setup.config.lockToRead();
auto configLock = m_setup.config.lockToRead();
auto *const destinationDb = m_setup.config.findDatabase(dbName, dbArch);
if (!destinationDb) {
configLock.unlock();
m_buildAction->appendOutput(
Phrases::ErrorMessage, "Loaded database file for \"", dbName, '@', dbArch, "\" but it no longer exists; discarding\n");
session->addResponse(std::move(dbName));
return;
}
destinationDb->replacePackages(packages, lastModified);
auto updater = LibPkg::PackageUpdater(*destinationDb, true);
LibPkg::Package::fromDatabaseFile(std::move(dbFile), [&updater](std::shared_ptr<LibPkg::Package> package) {
if (const auto [id, existingPackage] = updater.findPackageWithID(package->name); existingPackage) {
package->addDepsAndProvidesFromOtherPackage(*existingPackage);
}
updater.update(package);
return false;
});
updater.commit();
destinationDb->lastUpdate = lastModified;
} catch (const std::runtime_error &e) {
m_buildAction->appendOutput(Phrases::ErrorMessage, "An error occurred when reloading database \"", dbName, '@', dbArch,
"\" from local file \"", dbPath, "\": ", e.what(), '\n');