Make storing lib deps more efficient and allow for intermediate reads
This commit is contained in:
parent
4b70682ab5
commit
66f59fecb9
|
@ -718,6 +718,11 @@ PackageUpdater::~PackageUpdater()
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
|
LibPkg::PackageSpec LibPkg::PackageUpdater::findPackageWithID(const std::string &packageName)
|
||||||
|
{
|
||||||
|
return m_database.m_storage->packageCache.retrieve(*m_database.m_storage, &m_d->packagesTxn, packageName);
|
||||||
|
}
|
||||||
|
|
||||||
StorageID PackageUpdater::update(const std::shared_ptr<Package> &package)
|
StorageID PackageUpdater::update(const std::shared_ptr<Package> &package)
|
||||||
{
|
{
|
||||||
const auto res = m_database.m_storage->packageCache.store(*m_database.m_storage, m_d->packagesTxn, package);
|
const auto res = m_database.m_storage->packageCache.store(*m_database.m_storage, m_d->packagesTxn, package);
|
||||||
|
|
|
@ -102,6 +102,7 @@ struct LIBPKG_EXPORT PackageUpdater {
|
||||||
explicit PackageUpdater(Database &database);
|
explicit PackageUpdater(Database &database);
|
||||||
~PackageUpdater();
|
~PackageUpdater();
|
||||||
|
|
||||||
|
PackageSpec findPackageWithID(const std::string &packageName);
|
||||||
StorageID update(const std::shared_ptr<Package> &package);
|
StorageID update(const std::shared_ptr<Package> &package);
|
||||||
void commit();
|
void commit();
|
||||||
|
|
||||||
|
|
|
@ -83,7 +83,7 @@ auto StorageCache<StorageEntriesType, StorageType, SpecType>::retrieve(Storage &
|
||||||
}
|
}
|
||||||
|
|
||||||
template <typename StorageEntriesType, typename StorageType, typename SpecType>
|
template <typename StorageEntriesType, typename StorageType, typename SpecType>
|
||||||
auto StorageCache<StorageEntriesType, StorageType, SpecType>::retrieve(Storage &storage, const std::string &entryName) -> SpecType
|
auto StorageCache<StorageEntriesType, StorageType, SpecType>::retrieve(Storage &storage, RWTxn *txn, const std::string &entryName) -> SpecType
|
||||||
{
|
{
|
||||||
// check for package in cache
|
// check for package in cache
|
||||||
using CacheRef = typename Entries::Ref;
|
using CacheRef = typename Entries::Ref;
|
||||||
|
@ -95,8 +95,7 @@ auto StorageCache<StorageEntriesType, StorageType, SpecType>::retrieve(Storage &
|
||||||
lock.unlock();
|
lock.unlock();
|
||||||
// check for package in storage, populate cache entry
|
// check for package in storage, populate cache entry
|
||||||
auto entry = std::make_shared<Entry>();
|
auto entry = std::make_shared<Entry>();
|
||||||
auto txn = storage.packages.getROTransaction();
|
if (auto id = txn ? txn->template get<0>(entryName, *entry) : storage.packages.getROTransaction().template get<0>(entryName, *entry)) {
|
||||||
if (auto id = txn.template get<0>(entryName, *entry)) {
|
|
||||||
using CacheEntry = typename Entries::StorageEntry;
|
using CacheEntry = typename Entries::StorageEntry;
|
||||||
auto newCacheEntry = CacheEntry(ref, id);
|
auto newCacheEntry = CacheEntry(ref, id);
|
||||||
newCacheEntry.entry = entry;
|
newCacheEntry.entry = entry;
|
||||||
|
@ -108,6 +107,12 @@ auto StorageCache<StorageEntriesType, StorageType, SpecType>::retrieve(Storage &
|
||||||
return SpecType(0, std::shared_ptr<Entry>());
|
return SpecType(0, std::shared_ptr<Entry>());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
template <typename StorageEntriesType, typename StorageType, typename SpecType>
|
||||||
|
auto StorageCache<StorageEntriesType, StorageType, SpecType>::retrieve(Storage &storage, const std::string &entryName) -> SpecType
|
||||||
|
{
|
||||||
|
return retrieve(storage, nullptr, entryName);
|
||||||
|
}
|
||||||
|
|
||||||
template <typename StorageEntriesType, typename StorageType, typename SpecType>
|
template <typename StorageEntriesType, typename StorageType, typename SpecType>
|
||||||
auto StorageCache<StorageEntriesType, StorageType, SpecType>::store(Storage &storage, const std::shared_ptr<Entry> &entry, bool force) -> StoreResult
|
auto StorageCache<StorageEntriesType, StorageType, SpecType>::store(Storage &storage, const std::shared_ptr<Entry> &entry, bool force) -> StoreResult
|
||||||
{
|
{
|
||||||
|
|
|
@ -145,6 +145,7 @@ template <typename StorageEntriesType, typename StorageType, typename SpecType>
|
||||||
|
|
||||||
SpecType retrieve(Storage &storage, ROTxn *, StorageID storageID);
|
SpecType retrieve(Storage &storage, ROTxn *, StorageID storageID);
|
||||||
SpecType retrieve(Storage &storage, StorageID storageID);
|
SpecType retrieve(Storage &storage, StorageID storageID);
|
||||||
|
SpecType retrieve(Storage &storage, RWTxn *, const std::string &entryName);
|
||||||
SpecType retrieve(Storage &storage, const std::string &entryName);
|
SpecType retrieve(Storage &storage, const std::string &entryName);
|
||||||
StoreResult store(Storage &storage, const std::shared_ptr<Entry> &entry, bool force);
|
StoreResult store(Storage &storage, const std::shared_ptr<Entry> &entry, bool force);
|
||||||
StoreResult store(Storage &storage, RWTxn &txn, const std::shared_ptr<Entry> &entry);
|
StoreResult store(Storage &storage, RWTxn &txn, const std::shared_ptr<Entry> &entry);
|
||||||
|
|
|
@ -337,19 +337,20 @@ void ReloadLibraryDependencies::loadPackageInfoFromContents()
|
||||||
// store the information in the database
|
// store the information in the database
|
||||||
m_buildAction->appendOutput(Phrases::SuccessMessage, "Adding parsed information to databases ...\n");
|
m_buildAction->appendOutput(Phrases::SuccessMessage, "Adding parsed information to databases ...\n");
|
||||||
std::size_t counter = 0;
|
std::size_t counter = 0;
|
||||||
auto configWritelock = m_setup.config.lockToWrite();
|
|
||||||
for (DatabaseToConsider &relevantDb : m_relevantPackagesByDatabase) {
|
for (DatabaseToConsider &relevantDb : m_relevantPackagesByDatabase) {
|
||||||
|
auto configWritelock = m_setup.config.lockToWrite(); // acquire lock within loop to allow intermediate reads
|
||||||
auto *const db = m_setup.config.findDatabase(relevantDb.name, relevantDb.arch);
|
auto *const db = m_setup.config.findDatabase(relevantDb.name, relevantDb.arch);
|
||||||
if (!db) {
|
if (!db) {
|
||||||
continue; // the whole database has been removed while we were loading package contents
|
continue; // the whole database has been removed while we were loading package contents
|
||||||
}
|
}
|
||||||
|
auto updater = LibPkg::PackageUpdater(*db);
|
||||||
for (PackageToConsider &package : relevantDb.packages) {
|
for (PackageToConsider &package : relevantDb.packages) {
|
||||||
// skip if package info could not be parsed from package contents
|
// skip if package info could not be parsed from package contents
|
||||||
if (package.info.origin != LibPkg::PackageOrigin::PackageContents) {
|
if (package.info.origin != LibPkg::PackageOrigin::PackageContents) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
// find the package in the database again
|
// find the package in the database again
|
||||||
const auto [packageID, existingPackage] = db->findPackageWithID(package.info.name);
|
const auto [packageID, existingPackage] = updater.findPackageWithID(package.info.name);
|
||||||
if (!existingPackage) {
|
if (!existingPackage) {
|
||||||
continue; // the package has been removed while we were loading package contents
|
continue; // the package has been removed while we were loading package contents
|
||||||
}
|
}
|
||||||
|
@ -362,11 +363,11 @@ void ReloadLibraryDependencies::loadPackageInfoFromContents()
|
||||||
existingPackage->timestamp = package.lastModified;
|
existingPackage->timestamp = package.lastModified;
|
||||||
}
|
}
|
||||||
// add the new dependencies on database-level
|
// add the new dependencies on database-level
|
||||||
db->forceUpdatePackage(existingPackage);
|
updater.update(existingPackage);
|
||||||
++counter;
|
++counter;
|
||||||
}
|
}
|
||||||
|
updater.commit();
|
||||||
}
|
}
|
||||||
configWritelock.unlock();
|
|
||||||
|
|
||||||
m_buildAction->appendOutput(Phrases::SuccessMessage, "Added dependency information for ", counter, " packages\n");
|
m_buildAction->appendOutput(Phrases::SuccessMessage, "Added dependency information for ", counter, " packages\n");
|
||||||
conclude();
|
conclude();
|
||||||
|
|
Loading…
Reference in New Issue