Skip to content

WIP: Sqlite quick search #146

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
julian-klode wants to merge 4 commits into
base: master
Choose a base branch
from
Open

WIP: Sqlite quick search #146

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
dae22bf
build: Replace libept with sqlite
julian-klode Apr 3, 2025
0a71eac
WIP: Implement on-demand quick search with sqlite
julian-klode Apr 3, 2025
61510d0
sqlite search: run main event loop between items
julian-klode Apr 3, 2025
48cfdbb
Implement caching
julian-klode Apr 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion common/Makefile.am
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
noinst_LIBRARIES = libsynaptic.a

AM_CPPFLAGS = -I/usr/include/apt-pkg @RPM_HDRS@ @DEB_HDRS@ \
$(LIBEPT_CFLAGS) \
$(SQLITE_CFLAGS) \
-DSYNAPTICLOCALEDIR=\""$(synapticlocaledir)"\" \
-DSYNAPTICSTATEDIR=\""$(localstatedir)"\"

Expand Down
261 changes: 102 additions & 159 deletions common/rpackagelister.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,9 +81,6 @@ using namespace std;

RPackageLister::RPackageLister()
: _records(0), _progMeter(new OpProgress)
#ifdef WITH_EPT
, _xapianDatabase(0)
#endif
{
_cache = new RPackageCache();

Expand All @@ -103,8 +100,8 @@ RPackageLister::RPackageLister()
_views.push_back(_searchView);
// its import that we use "_packages" here instead of _nativeArchPackages
_views.push_back(new RPackageViewArchitecture(_packages));
#ifdef WITH_EPT
openXapianIndex();
#ifdef WITH_SQLITE
//openXapianIndex();
#endif

if (_viewMode >= _views.size())
Expand Down Expand Up @@ -435,48 +432,6 @@ bool RPackageLister::openCache()
return true;
}

#ifdef WITH_EPT
bool RPackageLister::xapianIndexNeedsUpdate()
{
struct stat buf;

if(_config->FindB("Debug::Synaptic::Xapian",false))
std::cerr << "xapainIndexNeedsUpdate()" << std::endl;

// check the xapian index
if(FileExists("/usr/sbin/update-apt-xapian-index") &&
(!_xapianDatabase )) {
if(_config->FindB("Debug::Synaptic::Xapian",false))
std::cerr << "xapain index not build yet" << std::endl;
return true;
}

// compare timestamps, rebuild everytime, its now cheap(er)
// because we use u-a-x-i --update
stat(_config->FindFile("Dir::Cache::pkgcache").c_str(), &buf);
if(ept::axi::timestamp() < buf.st_mtime) {
if(_config->FindB("Debug::Synaptic::Xapian",false))
std::cerr << "xapian outdated "
<< buf.st_mtime - ept::axi::timestamp() << std::endl;
return true;
}

return false;
}

bool RPackageLister::openXapianIndex()
{
if(_xapianDatabase)
delete _xapianDatabase;
try {
_xapianDatabase = new Xapian::Database(ept::axi::path_db());
} catch (Xapian::DatabaseOpeningError) {
return false;
};
return true;
}
#endif

void RPackageLister::applyInitialSelection()
{
_roptions->rereadOrphaned();
Expand Down Expand Up @@ -1974,131 +1929,119 @@ bool RPackageLister::addArchiveToCache(string archive, string &pkgname)
}


#ifdef WITH_EPT
bool RPackageLister::limitBySearch(string searchString)
#ifdef WITH_SQLITE
bool RPackageLister::limitBySearch(string searchString, void (*iter)())
{
//cerr << "limitBySearch(): " << searchString << endl;
if (ept::axi::timestamp() == 0)
return false;
return xapianSearch(searchString);
if (not xapianSearch(searchString, iter))
_error->DumpErrors();
return true;
}

bool RPackageLister::xapianSearch(string unsplitSearchString)
{
//std::cerr << "RPackageLister::xapianSearch()" << std::endl;
static const int defaultQualityCutoff = 15;
int qualityCutoff = _config->FindI("Synaptic::Xapian::qualityCutoff",
defaultQualityCutoff);
if (ept::axi::timestamp() == 0)
return false;

try {
int maxItems = _xapianDatabase->get_doccount();
Xapian::Enquire enquire(*_xapianDatabase);
Xapian::QueryParser parser;
parser.set_database(*_xapianDatabase);
parser.add_prefix("name","XP");
parser.add_prefix("section","XS");
// default op is AND to narrow down the resultset
parser.set_default_op( Xapian::Query::OP_AND );

/* Workaround to allow searching an hyphenated package name using a prefix (name:)
* LP: #282995
* Xapian currently doesn't support wildcard for boolean prefix and
* doesn't handle implicit wildcards at the end of hypenated phrases.
*
* e.g searching for name:ubuntu-res will be equivalent to 'name:ubuntu res*'
* however 'name:(ubuntu* res*) won't return any result because the
* index is built with the full package name
*/
// Always search for the package name
string xpString = "name:";
string::size_type pos = unsplitSearchString.find_first_of(" ,;");
if (pos > 0) {
xpString += unsplitSearchString.substr(0,pos);
} else {
xpString += unsplitSearchString;
}
Xapian::Query xpQuery = parser.parse_query(xpString);

pos = 0;
while ( (pos = unsplitSearchString.find("-", pos)) != string::npos ) {
unsplitSearchString.replace(pos, 1, " ");
pos+=1;
}
bool RPackageLister::xapianSearch(string unsplitSearchString, void (*iter)())
{
static sqlite3 *database = nullptr;
if (not database)
{
char *err;
std::string dir;
if (auto d = getenv("XDG_RUNTIME_DIRECTORY"))
dir = d;
else if (getuid())
dir = flCombine("/run/user/", std::to_string(getuid()));
else
dir = std::string{"/run"};

std::string file = ":memory:";
if (struct stat st; !dir.empty() || access(dir.c_str(), X_OK) != 0)
file = flCombine(dir, "synaptic.db");

struct stat cacheStat{};
struct stat dbStat{};
stat(_config->FindFile("Dir::Cache::pkgcache").c_str(), &cacheStat);
stat(file.c_str(), &dbStat);

if (dbStat.st_mtime && dbStat.st_mtime != cacheStat.st_mtime)
unlink(file.c_str());

if (sqlite3_open(file.c_str(), &database) != SQLITE_OK &&
sqlite3_open(":memory:", &database) != SQLITE_OK)
return _error->Error("Cannot open database");
if (sqlite3_exec(database, "CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(name, description);", nullptr, nullptr, &err) != SQLITE_OK)
return _error->Error("Cannot create table: %s", err);

static sqlite3_stmt *insert_stmt = nullptr;
if (not insert_stmt && sqlite3_prepare_v2(database, "INSERT INTO fts VALUES(?, ?)", -1, &insert_stmt, NULL) != SQLITE_OK)
return _error->Error("Cannot prepare insert statement");

// We abuse the callback here, if we have no rows, the callback returns 0, and then
// sqlite3_exec() returns SQLITE_DONE; whereas if we have a row, it returns 1, and
// the exec returns SQLITE_ABORT.
if (sqlite3_exec(database, "SELECT name FROM fts LIMIT 1", [](void*, int nrows, char**, char**) -> int {
return nrows > 0;
}, nullptr, &err) != SQLITE_ABORT)
{

if (sqlite3_exec(database, "BEGIN", nullptr, nullptr, &err) != SQLITE_OK)
return _error->Error("Cannot begin: %s", err);

for (auto pkg : _packages)
{
if (auto name = pkg->name(); sqlite3_bind_text(insert_stmt, 1, name, strlen(name), nullptr) != SQLITE_OK)
return _error->Error("Cannot bind name '%s': %s", name, sqlite3_errmsg(database));
if (auto desc = pkg->description(); sqlite3_bind_text(insert_stmt, 2, desc, strlen(desc), nullptr) != SQLITE_OK)
return _error->Error("Cannot bind desc '%s': %s", desc, sqlite3_errmsg(database));

if (sqlite3_step(insert_stmt) != SQLITE_DONE)
return _error->Error("Insert failed of '%s': %s", pkg->name(), sqlite3_errmsg(database));

sqlite3_reset(insert_stmt);
iter();
}

if(_config->FindB("Debug::Synaptic::Xapian",false))
std::cerr << "searching for : " << unsplitSearchString << std::endl;

// Build the query
// apply a weight factor to XP term to increase relevancy on package name
Xapian::Query query = parser.parse_query(unsplitSearchString,
Xapian::QueryParser::FLAG_WILDCARD |
Xapian::QueryParser::FLAG_BOOLEAN |
Xapian::QueryParser::FLAG_PARTIAL);
query = Xapian::Query(Xapian::Query::OP_OR, query,
Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT, xpQuery, 3));
enquire.set_query(query);
Xapian::MSet matches = enquire.get_mset(0, maxItems);

if(_config->FindB("Debug::Synaptic::Xapian",false)) {
cerr << "enquire: " << enquire.get_description() << endl;
cerr << "matches estimated: " << matches.get_matches_estimated() << " results found" << endl;
}
if (sqlite3_exec(database, "END", nullptr, nullptr, &err) != SQLITE_OK)
return _error->Error("Cannot begin: %s", err);

// Retrieve the results
int top_percent = 0;
_viewPackages.clear();
for (Xapian::MSetIterator i = matches.begin(); i != matches.end(); ++i)
{
RPackage* pkg = getPackage(i.get_document().get_data());
// Filter out results that apt doesn't know
if (!pkg || !_selectedView->hasPackage(pkg))
if (cacheStat.st_mtime != 0)
{
struct timeval tv[]{timeval{time(NULL), 0}, timeval{cacheStat.st_mtime, 0}};
utimes(file.c_str(), tv);
}
}


}
static sqlite3_stmt *select_stmt = nullptr;
if (select_stmt)
sqlite3_reset(select_stmt);
else if (sqlite3_prepare_v2(database, "SELECT DISTINCT name FROM fts WHERE fts MATCH ?" , -1, &select_stmt, NULL) != SQLITE_OK)
return _error->Error("Cannot prepare statement");

if (sqlite3_bind_text(select_stmt, 1, unsplitSearchString.c_str(), unsplitSearchString.size(), nullptr) != SQLITE_OK)
return _error->Error("Cannot bind statement");

int rc;
_viewPackages.clear();
while(SQLITE_ROW == (rc = sqlite3_step(select_stmt))) {
iter();
const char *name = (const char*)sqlite3_column_text(select_stmt, 0);
auto pkg = getPackage(name);
if (!pkg || !_selectedView->hasPackage(pkg))
continue;

// Save the confidence interval of the top value, to use it as
// a reference to compute an adaptive quality cutoff
if (top_percent == 0)
top_percent = i.get_percent();

// Stop producing if the quality goes below a cutoff point
if (i.get_percent() < qualityCutoff * top_percent / 100)
{
cerr << "Discarding: " << i.get_percent() << " over " << qualityCutoff * top_percent / 100 << endl;
break;
}

if(_config->FindB("Debug::Synaptic::Xapian",false))
cerr << i.get_rank() + 1 << ": " << i.get_percent() << "% docid=" << *i << " [" << i.get_document().get_data() << "]" << endl;
_viewPackages.push_back(pkg);
}
// re-apply sort criteria only if an explicit search is set
if (_sortMode != LIST_SORT_DEFAULT)
sortPackages(_sortMode);
return true;
} catch (const Xapian::Error & error) {
/* We are here if a Xapian call failed. The main cause is a parser exception.
* The error message is always in English currently.
* The possible parser errors are:
* Unknown range operation
* parse error
* Syntax: <expression> AND <expression>
* Syntax: <expression> AND NOT <expression>
* Syntax: <expression> NOT <expression>
* Syntax: <expression> OR <expression>
* Syntax: <expression> XOR <expression>
*/
cerr << "Exception in RPackageLister::xapianSearch():" << error.get_msg() << endl;
return false;
}
_viewPackages.push_back(pkg);
}
if(SQLITE_DONE != rc)
return _error->Error("select statement didn't finish with DONE (%i): %s\n", rc, sqlite3_errmsg(database));
sortPackages(_sortMode);
return true;
}
#else
bool RPackageLister::limitBySearch(string searchString)
bool RPackageLister::limitBySearch(string searchString, void (*iter)())
{
return false;
}

bool RPackageLister::xapianSearch(string searchString)
bool RPackageLister::xapianSearch(string searchString, void (*iter)())
{
return false;
}
Expand Down
18 changes: 4 additions & 14 deletions common/rpackagelister.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@
#include <apt-pkg/acquire.h>
#include <apt-pkg/progress.h>

#ifdef WITH_EPT
#include <ept/axi/axi.h>
#ifdef WITH_SQLITE
#include <sqlite3.h>
#endif

#include "rpackagecache.h"
Expand Down Expand Up @@ -107,11 +107,6 @@ class RPackageLister {
pkgRecords *_records;
OpProgress *_progMeter;

#ifdef WITH_EPT
Xapian::Database *_xapianDatabase;
#endif


// Other members.
vector<RPackage *> _packages;
vector<int> _packagesIndex;
Expand Down Expand Up @@ -140,7 +135,7 @@ class RPackageLister {
RPackageViewSearch *_searchView; // the package view that does the (simple) search

// helper for the limitBySearch() code
bool xapianSearch(string searchString);
bool xapianSearch(string searchString, void (*iter)());

public:

Expand Down Expand Up @@ -210,7 +205,7 @@ class RPackageLister {

public:
// limit what the current view displays
bool limitBySearch(string searchString);
bool limitBySearch(string searchString, void (*iter)());

// clean files older than "Synaptic::delHistory"
void cleanCommitLog();
Expand Down Expand Up @@ -348,11 +343,6 @@ class RPackageLister {
bool writeSelections(ostream &out, bool fullState);

RPackageCache* getCache() { return _cache; }
#ifdef WITH_EPT
Xapian::Database* xapiandatabase() { return _xapianDatabase; }
bool xapianIndexNeedsUpdate();
bool openXapianIndex();
#endif

RPackageLister();
~RPackageLister();
Expand Down
4 changes: 0 additions & 4 deletions common/rpackageview.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,6 @@
#include <string>
#include <map>

#ifdef WITH_EPT
#include <ept/axi/axi.h>
#endif

#include "rpackage.h"
#include "rpackagefilter.h"

Expand Down
Loading