Martchus
/
arch-repo-manager
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Speed up deserialization when serving providing packages as well

This commit is contained in:
Martchus 2022-05-31 19:40:12 +02:00
parent 68d67f543f
commit b74392c34d
5 changed files with 80 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
libpkg/algo/search.cpp
View File

@ -258,6 +258,19 @@ void Config::providingPackages(const Dependency &dependency, bool reverse, const
}
}
void Config::providingPackagesBase(const Dependency &dependency, bool reverse, const DatabaseVisitor &databaseVisitor, const PackageVisitorBase &visitor)
{
for (auto &db : databases) {
if (databaseVisitor && databaseVisitor(db)) {
continue;
}
auto visited = std::unordered_set<LibPkg::StorageID>();
db.providingPackagesBase(dependency, reverse, [&](StorageID packageID, std::shared_ptr<PackageBase> &&package) {
return visited.emplace(packageID).second ? visitor(db, packageID, std::move(package)) : false;
});
}
}
void Config::providingPackages(
const std::string &libraryName, bool reverse, const DatabaseVisitor &databaseVisitor, const PackageVisitorConst &visitor)
{
@ -272,4 +285,18 @@ void Config::providingPackages(
}
}
void Config::providingPackagesBase(
const std::string &libraryName, bool reverse, const DatabaseVisitor &databaseVisitor, const PackageVisitorBase &visitor)
{
for (auto &db : databases) {
if (databaseVisitor && databaseVisitor(db)) {
continue;
}
auto visited = std::unordered_set<LibPkg::StorageID>();
db.providingPackagesBase(libraryName, reverse, [&](StorageID packageID, std::shared_ptr<PackageBase> &&package) {
return visited.emplace(packageID).second ? visitor(db, packageID, std::move(package)) : false;
});
}
}
} // namespace LibPkg

2
libpkg/data/config.h
View File

@ -171,7 +171,9 @@ struct LIBPKG_EXPORT Config : public Lockable, public ReflectiveRapidJSON::Binar
void packagesByName(const DatabaseVisitor &databaseVisitor, const PackageVisitorByName &visitor);
void packagesByName(const DatabaseVisitor &databaseVisitor, const PackageVisitorByNameBase &visitor);
void providingPackages(const Dependency &dependency, bool reverse, const DatabaseVisitor &databaseVisitor, const PackageVisitorConst &visitor);
void providingPackagesBase(const Dependency &dependency, bool reverse, const DatabaseVisitor &databaseVisitor, const PackageVisitorBase &visitor);
void providingPackages(const std::string &libraryName, bool reverse, const DatabaseVisitor &databaseVisitor, const PackageVisitorConst &visitor);
void providingPackagesBase(const std::string &libraryName, bool reverse, const DatabaseVisitor &databaseVisitor, const PackageVisitorBase &visitor);
std::vector<Database> databases;
Database aur = Database("aur");

45
libpkg/data/database.cpp
View File

@ -358,6 +358,34 @@ void Database::providingPackages(const Dependency &dependency, bool reverse, con
}
}
void Database::providingPackagesBase(const Dependency &dependency, bool reverse, const PackageVisitorBase &visitor)
{
if (dependency.name.empty()) {
return;
}
auto providesTxn = (reverse ? m_storage->requiredDeps : m_storage->providedDeps).getROTransaction();
auto packagesTxn = m_storage->packages.getROTransaction();
auto package = std::shared_ptr<PackageBase>();
for (auto [i, end] = providesTxn.equal_range<0>(dependency.name); i != end; ++i) {
const auto &providedDependency = i.value();
const auto &asDependency = static_cast<const Dependency &>(providedDependency);
if (!Dependency::matches(dependency.mode, dependency.version, asDependency.version)) {
continue;
}
for (const auto packageID : providedDependency.relevantPackages) {
if (!package) {
package = std::make_shared<PackageBase>();
} else {
package->clear();
}
;
if (packagesTxn.get<PackageBase>(packageID, *package) && visitor(packageID, std::move(package))) {
return;
}
}
}
}
void Database::providingPackages(const std::string &libraryName, bool reverse, const PackageVisitorConst &visitor)
{
if (libraryName.empty()) {
@ -375,6 +403,23 @@ void Database::providingPackages(const std::string &libraryName, bool reverse, c
}
}
void Database::providingPackagesBase(const std::string &libraryName, bool reverse, const PackageVisitorBase &visitor)
{
if (libraryName.empty()) {
return;
}
auto providesTxn = (reverse ? m_storage->requiredLibs : m_storage->providedLibs).getROTransaction();
auto packagesTxn = m_storage->packages.getROTransaction();
auto package = std::shared_ptr<PackageBase>();
for (auto [i, end] = providesTxn.equal_range<0>(libraryName); i != end; ++i) {
for (const auto packageID : i->relevantPackages) {
if (packagesTxn.get<PackageBase>(packageID, *package) && visitor(packageID, std::move(package))) {
return;
}
}
}
}
bool Database::provides(const Dependency &dependency, bool reverse) const
{
if (dependency.name.empty()) {

2
libpkg/data/database.h
View File

@ -174,7 +174,9 @@ struct LIBPKG_EXPORT Database : public ReflectiveRapidJSON::JsonSerializable<Dat
void allPackagesByName(const PackageVisitorByNameBase &visitor);
std::size_t packageCount() const;
void providingPackages(const Dependency &dependency, bool reverse, const PackageVisitorConst &visitor);
void providingPackagesBase(const Dependency &dependency, bool reverse, const PackageVisitorBase &visitor);
void providingPackages(const std::string &libraryName, bool reverse, const PackageVisitorConst &visitor);
void providingPackagesBase(const std::string &libraryName, bool reverse, const PackageVisitorBase &visitor);
bool provides(const Dependency &dependency, bool reverse = false) const;
bool provides(const std::string &libraryName, bool reverse = false) const;
std::shared_ptr<Package> findPackage(StorageID packageID);

8
librepomgr/webapi/routes.cpp
View File

@ -227,7 +227,7 @@ void getPackages(const Params &params, ResponseHandler &&handler)
const auto dbIterator = dbs.find(db.name);
return dbIterator == dbs.end() || dbIterator->second.find(db.arch) == dbIterator->second.end();
});
const auto pushPackage = LibPkg::Config::PackageVisitorBase([&array, &document, &limit](Database &db, LibPkg::StorageID id, const std::shared_ptr<PackageBase> &pkg) {
const auto pushSharedBasePackage = LibPkg::Config::PackageVisitorBase([&array, &document, &limit](Database &db, LibPkg::StorageID id, const std::shared_ptr<PackageBase> &pkg) {
ReflectiveRapidJSON::JsonReflector::push(LibPkg::PackageBaseSearchResult(db, *pkg, id), array, document.GetAllocator());
return array.Size() >= limit;
});
@ -267,7 +267,7 @@ void getPackages(const Params &params, ResponseHandler &&handler)
if (details) {
params.setup.config.packages(dbName, dbArch, packageNameStr, visitDb, pushPackageDetails);
} else {
params.setup.config.packages(dbName, dbArch, packageNameStr, visitDb, pushPackage);
params.setup.config.packages(dbName, dbArch, packageNameStr, visitDb, pushSharedBasePackage);
}
}
break;
@ -320,11 +320,11 @@ void getPackages(const Params &params, ResponseHandler &&handler)
}
case Mode::Provides:
case Mode::Depends:
params.setup.config.providingPackages(Dependency::fromString(name), mode == Mode::Depends, visitDb, pushPackage);
params.setup.config.providingPackagesBase(Dependency::fromString(name), mode == Mode::Depends, visitDb, pushSharedBasePackage);
break;
case Mode::LibProvides:
case Mode::LibDepends:
params.setup.config.providingPackages(name, mode == Mode::LibDepends, visitDb, pushPackage);
params.setup.config.providingPackagesBase(name, mode == Mode::LibDepends, visitDb, pushSharedBasePackage);
break;
default:;
}