Avoid high memory usage when reloading database
Especially when enabling files DBs it is quite problematic to store all package objects of a database within one big array. This change avoids the array and instead adds the packages directly. The disadvantage is that clearing the database isn't as simple anymore.
This commit is contained in:
parent
f07ef9f147
commit
1f3f0b0df1
|
@ -34,6 +34,7 @@ struct PackageUpdaterPrivate {
|
||||||
bool clear = false;
|
bool clear = false;
|
||||||
std::unique_lock<std::mutex> lock;
|
std::unique_lock<std::mutex> lock;
|
||||||
PackageStorage::RWTransaction packagesTxn;
|
PackageStorage::RWTransaction packagesTxn;
|
||||||
|
std::unordered_set<StorageID> handledIds;
|
||||||
AffectedDeps affectedProvidedDeps;
|
AffectedDeps affectedProvidedDeps;
|
||||||
AffectedDeps affectedRequiredDeps;
|
AffectedDeps affectedRequiredDeps;
|
||||||
AffectedLibs affectedProvidedLibs;
|
AffectedLibs affectedProvidedLibs;
|
||||||
|
@ -660,10 +661,6 @@ PackageUpdaterPrivate::PackageUpdaterPrivate(DatabaseStorage &storage, bool clea
|
||||||
, lock(storage.updateMutex)
|
, lock(storage.updateMutex)
|
||||||
, packagesTxn(storage.packages.getRWTransaction())
|
, packagesTxn(storage.packages.getRWTransaction())
|
||||||
{
|
{
|
||||||
if (clear) {
|
|
||||||
storage.packageCache.clearCacheOnly(storage);
|
|
||||||
packagesTxn.clear();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void PackageUpdaterPrivate::update(const PackageCache::StoreResult &res, const std::shared_ptr<Package> &package)
|
void PackageUpdaterPrivate::update(const PackageCache::StoreResult &res, const std::shared_ptr<Package> &package)
|
||||||
|
@ -671,6 +668,7 @@ void PackageUpdaterPrivate::update(const PackageCache::StoreResult &res, const s
|
||||||
if (!res.id) {
|
if (!res.id) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
handledIds.emplace(res.id);
|
||||||
update(res.id, false, package);
|
update(res.id, false, package);
|
||||||
if (!clear && res.oldEntry) {
|
if (!clear && res.oldEntry) {
|
||||||
update(res.id, true, res.oldEntry);
|
update(res.id, true, res.oldEntry);
|
||||||
|
@ -804,7 +802,16 @@ StorageID PackageUpdater::update(const std::shared_ptr<Package> &package)
|
||||||
void PackageUpdater::commit()
|
void PackageUpdater::commit()
|
||||||
{
|
{
|
||||||
const auto &storage = m_database.m_storage;
|
const auto &storage = m_database.m_storage;
|
||||||
m_d->packagesTxn.commit();
|
auto &pkgTxn = m_d->packagesTxn;
|
||||||
|
if (m_d->clear) {
|
||||||
|
const auto &toPreserve = m_d->handledIds;
|
||||||
|
for (auto i = pkgTxn.begin<std::unique_ptr>(); i != pkgTxn.end(); ++i) {
|
||||||
|
if (!toPreserve.contains(i.getID())) {
|
||||||
|
i.del();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pkgTxn.commit();
|
||||||
{
|
{
|
||||||
auto txn = storage->providedDeps.getRWTransaction();
|
auto txn = storage->providedDeps.getRWTransaction();
|
||||||
if (m_d->clear) {
|
if (m_d->clear) {
|
||||||
|
|
|
@ -398,6 +398,7 @@ struct LIBPKG_EXPORT Package : public ReflectiveRapidJSON::JsonSerializable<Pack
|
||||||
static std::vector<PackageSpec> fromInfo(const std::string &info, bool isPackageInfo = false);
|
static std::vector<PackageSpec> fromInfo(const std::string &info, bool isPackageInfo = false);
|
||||||
static std::shared_ptr<Package> fromDescription(const std::vector<std::string> &descriptionParts);
|
static std::shared_ptr<Package> fromDescription(const std::vector<std::string> &descriptionParts);
|
||||||
static std::vector<std::shared_ptr<Package>> fromDatabaseFile(FileMap &&databaseFile);
|
static std::vector<std::shared_ptr<Package>> fromDatabaseFile(FileMap &&databaseFile);
|
||||||
|
static void fromDatabaseFile(FileMap &&databaseFile, const std::function<bool(std::shared_ptr<Package>)> &visitor);
|
||||||
static std::shared_ptr<Package> fromPkgFile(const std::string &path);
|
static std::shared_ptr<Package> fromPkgFile(const std::string &path);
|
||||||
static std::tuple<std::string_view, std::string_view, std::string_view> fileNameComponents(std::string_view fileName);
|
static std::tuple<std::string_view, std::string_view, std::string_view> fileNameComponents(std::string_view fileName);
|
||||||
static std::shared_ptr<Package> fromPkgFileName(std::string_view fileName);
|
static std::shared_ptr<Package> fromPkgFileName(std::string_view fileName);
|
||||||
|
|
|
@ -3,6 +3,8 @@
|
||||||
#include "./binary.h"
|
#include "./binary.h"
|
||||||
#include "./utils.h"
|
#include "./utils.h"
|
||||||
|
|
||||||
|
#include "../data/database.h"
|
||||||
|
|
||||||
#include <c++utilities/conversion/stringbuilder.h>
|
#include <c++utilities/conversion/stringbuilder.h>
|
||||||
#include <c++utilities/conversion/stringconversion.h>
|
#include <c++utilities/conversion/stringconversion.h>
|
||||||
#include <c++utilities/io/ansiescapecodes.h>
|
#include <c++utilities/io/ansiescapecodes.h>
|
||||||
|
@ -718,17 +720,33 @@ std::vector<std::shared_ptr<Package>> Package::fromDatabaseFile(FileMap &&databa
|
||||||
{
|
{
|
||||||
std::vector<std::shared_ptr<Package>> packages;
|
std::vector<std::shared_ptr<Package>> packages;
|
||||||
packages.reserve(databaseFile.size());
|
packages.reserve(databaseFile.size());
|
||||||
|
std::vector<std::string> descriptionParts;
|
||||||
for (auto &dir : databaseFile) {
|
for (auto &dir : databaseFile) {
|
||||||
vector<string> descriptionParts;
|
descriptionParts.clear();
|
||||||
descriptionParts.reserve(dir.second.size());
|
descriptionParts.reserve(dir.second.size());
|
||||||
for (auto &file : dir.second) {
|
for (auto &file : dir.second) {
|
||||||
descriptionParts.emplace_back(move(file.content));
|
descriptionParts.emplace_back(std::move(file.content));
|
||||||
}
|
}
|
||||||
packages.emplace_back(Package::fromDescription(descriptionParts));
|
packages.emplace_back(Package::fromDescription(descriptionParts));
|
||||||
}
|
}
|
||||||
return packages;
|
return packages;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void Package::fromDatabaseFile(FileMap &&databaseFile, const std::function<bool(std::shared_ptr<Package>)> &visitor)
|
||||||
|
{
|
||||||
|
std::vector<std::string> descriptionParts;
|
||||||
|
for (auto &dir : databaseFile) {
|
||||||
|
descriptionParts.clear();
|
||||||
|
descriptionParts.reserve(dir.second.size());
|
||||||
|
for (auto &file : dir.second) {
|
||||||
|
descriptionParts.emplace_back(std::move(file.content));
|
||||||
|
}
|
||||||
|
if (visitor(Package::fromDescription(descriptionParts))) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
bool Package::isPkgInfoFileOrBinary(const char *filePath, const char *fileName, mode_t mode)
|
bool Package::isPkgInfoFileOrBinary(const char *filePath, const char *fileName, mode_t mode)
|
||||||
{
|
{
|
||||||
return !strcmp(fileName, ".PKGINFO") || mode == S_IXUSR || strstr(filePath, "usr/bin") == filePath || strstr(filePath, "usr/lib") == filePath
|
return !strcmp(fileName, ".PKGINFO") || mode == S_IXUSR || strstr(filePath, "usr/bin") == filePath || strstr(filePath, "usr/lib") == filePath
|
||||||
|
|
|
@ -94,19 +94,31 @@ void ReloadDatabase::run()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
auto dbFile = LibPkg::extractFiles(dbPath, &LibPkg::Database::isFileRelevant);
|
auto dbFile = LibPkg::extractFiles(dbPath, &LibPkg::Database::isFileRelevant);
|
||||||
auto packages = LibPkg::Package::fromDatabaseFile(move(dbFile));
|
|
||||||
dbFileLock.lock().unlock();
|
dbFileLock.lock().unlock();
|
||||||
|
|
||||||
m_buildAction->appendOutput(
|
m_buildAction->appendOutput(
|
||||||
Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from local file \"", dbPath, "\"\n");
|
Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from local file \"", dbPath, "\"\n");
|
||||||
const auto configLock = m_setup.config.lockToRead();
|
|
||||||
|
auto configLock = m_setup.config.lockToRead();
|
||||||
auto *const destinationDb = m_setup.config.findDatabase(dbName, dbArch);
|
auto *const destinationDb = m_setup.config.findDatabase(dbName, dbArch);
|
||||||
if (!destinationDb) {
|
if (!destinationDb) {
|
||||||
|
configLock.unlock();
|
||||||
m_buildAction->appendOutput(
|
m_buildAction->appendOutput(
|
||||||
Phrases::ErrorMessage, "Loaded database file for \"", dbName, '@', dbArch, "\" but it no longer exists; discarding\n");
|
Phrases::ErrorMessage, "Loaded database file for \"", dbName, '@', dbArch, "\" but it no longer exists; discarding\n");
|
||||||
session->addResponse(std::move(dbName));
|
session->addResponse(std::move(dbName));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
destinationDb->replacePackages(packages, lastModified);
|
|
||||||
|
auto updater = LibPkg::PackageUpdater(*destinationDb, true);
|
||||||
|
LibPkg::Package::fromDatabaseFile(std::move(dbFile), [&updater](std::shared_ptr<LibPkg::Package> package) {
|
||||||
|
if (const auto [id, existingPackage] = updater.findPackageWithID(package->name); existingPackage) {
|
||||||
|
package->addDepsAndProvidesFromOtherPackage(*existingPackage);
|
||||||
|
}
|
||||||
|
updater.update(package);
|
||||||
|
return false;
|
||||||
|
});
|
||||||
|
updater.commit();
|
||||||
|
destinationDb->lastUpdate = lastModified;
|
||||||
} catch (const std::runtime_error &e) {
|
} catch (const std::runtime_error &e) {
|
||||||
m_buildAction->appendOutput(Phrases::ErrorMessage, "An error occurred when reloading database \"", dbName, '@', dbArch,
|
m_buildAction->appendOutput(Phrases::ErrorMessage, "An error occurred when reloading database \"", dbName, '@', dbArch,
|
||||||
"\" from local file \"", dbPath, "\": ", e.what(), '\n');
|
"\" from local file \"", dbPath, "\": ", e.what(), '\n');
|
||||||
|
|
Loading…
Reference in New Issue