From dae22bff2b365b0428b2d2b92d2d40e966e73fe7 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Thu, 3 Apr 2025 21:22:36 +0200 Subject: [PATCH 1/4] build: Replace libept with sqlite --- common/Makefile.am | 2 +- common/rpackagelister.cc | 8 ++++---- common/rpackagelister.h | 6 +++--- common/rpackageview.h | 2 +- config.h.in | 2 +- configure.ac | 10 +++++----- gtk/Makefile.am | 4 ++-- gtk/rgmainwindow.cc | 6 +++--- tests/Makefile.am | 4 ++-- 9 files changed, 22 insertions(+), 22 deletions(-) diff --git a/common/Makefile.am b/common/Makefile.am index fa4f37035..bbbfa85a7 100644 --- a/common/Makefile.am +++ b/common/Makefile.am @@ -3,7 +3,7 @@ noinst_LIBRARIES = libsynaptic.a AM_CPPFLAGS = -I/usr/include/apt-pkg @RPM_HDRS@ @DEB_HDRS@ \ - $(LIBEPT_CFLAGS) \ + $(SQLITE_CFLAGS) \ -DSYNAPTICLOCALEDIR=\""$(synapticlocaledir)"\" \ -DSYNAPTICSTATEDIR=\""$(localstatedir)"\" diff --git a/common/rpackagelister.cc b/common/rpackagelister.cc index ccaf173c9..e2402aa3c 100644 --- a/common/rpackagelister.cc +++ b/common/rpackagelister.cc @@ -81,7 +81,7 @@ using namespace std; RPackageLister::RPackageLister() : _records(0), _progMeter(new OpProgress) -#ifdef WITH_EPT +#ifdef WITH_SQLITE , _xapianDatabase(0) #endif { @@ -103,7 +103,7 @@ RPackageLister::RPackageLister() _views.push_back(_searchView); // its import that we use "_packages" here instead of _nativeArchPackages _views.push_back(new RPackageViewArchitecture(_packages)); -#ifdef WITH_EPT +#ifdef WITH_SQLITE openXapianIndex(); #endif @@ -435,7 +435,7 @@ bool RPackageLister::openCache() return true; } -#ifdef WITH_EPT +#ifdef WITH_SQLITE bool RPackageLister::xapianIndexNeedsUpdate() { struct stat buf; @@ -1974,7 +1974,7 @@ bool RPackageLister::addArchiveToCache(string archive, string &pkgname) } -#ifdef WITH_EPT +#ifdef WITH_SQLITE bool RPackageLister::limitBySearch(string searchString) { //cerr << "limitBySearch(): " << searchString << endl; diff --git a/common/rpackagelister.h b/common/rpackagelister.h index 1f1dd1d63..17f34d476 100644 --- a/common/rpackagelister.h +++ b/common/rpackagelister.h @@ -37,7 +37,7 @@ #include #include -#ifdef WITH_EPT +#ifdef WITH_SQLITE #include #endif @@ -107,7 +107,7 @@ class RPackageLister { pkgRecords *_records; OpProgress *_progMeter; -#ifdef WITH_EPT +#ifdef WITH_SQLITE Xapian::Database *_xapianDatabase; #endif @@ -348,7 +348,7 @@ class RPackageLister { bool writeSelections(ostream &out, bool fullState); RPackageCache* getCache() { return _cache; } -#ifdef WITH_EPT +#ifdef WITH_SQLITE Xapian::Database* xapiandatabase() { return _xapianDatabase; } bool xapianIndexNeedsUpdate(); bool openXapianIndex(); diff --git a/common/rpackageview.h b/common/rpackageview.h index 343f4ee49..89698d920 100644 --- a/common/rpackageview.h +++ b/common/rpackageview.h @@ -29,7 +29,7 @@ #include #include -#ifdef WITH_EPT +#ifdef WITH_SQLITE #include #endif diff --git a/config.h.in b/config.h.in index a7c05526e..ce7cc03aa 100644 --- a/config.h.in +++ b/config.h.in @@ -130,7 +130,7 @@ #undef WITH_DPKG_STATUSFD /* build with libept */ -#undef WITH_EPT +#undef WITH_SQLITE /* build with launchpad-integration */ #undef WITH_LAUNCHPAD_INTEGRATION diff --git a/configure.ac b/configure.ac index 3ed47505f..5e51e39cf 100644 --- a/configure.ac +++ b/configure.ac @@ -118,12 +118,12 @@ AC_CHECK_HEADER(apt-pkg/cdrom.h, [whether apt-pkg/cdrom.h is present]) ) # check and use libept if available -PKG_CHECK_MODULES([LIBEPT], [libept >= 1.0], - [AC_DEFINE(WITH_EPT, 1, [build with libept]) - AC_SUBST(LIBEPT_CFLAGS) - AC_SUBST(LIBEPT_LIBS) +PKG_CHECK_MODULES([SQLITE], [sqlite3 >= 1.0], + [AC_DEFINE(WITH_SQLITE, 1, [build with sqlite]) + AC_SUBST(SQLITE_CFLAGS) + AC_SUBST(SQLITE_LIBS) ], - [AC_MSG_NOTICE([no libept found, building without]) + [AC_MSG_NOTICE([no sqlite found, building without]) ]) AC_LANG([C]) diff --git a/gtk/Makefile.am b/gtk/Makefile.am index d2b22d591..9f25d1eac 100644 --- a/gtk/Makefile.am +++ b/gtk/Makefile.am @@ -12,7 +12,7 @@ AM_CPPFLAGS = -I${top_srcdir}/common \ @VTE_CFLAGS@ \ @LP_CFLAGS@ \ @GTK_DISABLE_DEPRECATED@ \ - $(LIBEPT_CFLAGS) + $(SQLITE_CFLAGS) sbin_PROGRAMS = synaptic @@ -25,7 +25,7 @@ synaptic_LDADD = \ @VTE_LIBS@ @LP_LIBS@\ -lutil \ -lpthread \ - $(LIBEPT_LIBS) + $(SQLITE_LIBS) synaptic_SOURCES= \ gsynaptic.cc\ diff --git a/gtk/rgmainwindow.cc b/gtk/rgmainwindow.cc index f23db586f..6aff319f1 100644 --- a/gtk/rgmainwindow.cc +++ b/gtk/rgmainwindow.cc @@ -852,7 +852,7 @@ RGMainWindow::RGMainWindow(RPackageLister *packLister, string name) RGPreferencesWindow::applyProxySettings(); } -#ifdef WITH_EPT +#ifdef WITH_SQLITE gboolean RGMainWindow::xapianDoIndexUpdate(void *data) { RGMainWindow *me = (RGMainWindow *) data; @@ -910,7 +910,7 @@ void RGMainWindow::xapianIndexUpdateFinished(GPid pid, gint status, void* data) if(_config->FindB("Debug::Synaptic::Xapian",false)) std::cerr << "xapianIndexUpdateFinished: " << WEXITSTATUS(status) << std::endl; -#ifdef WITH_EPT +#ifdef WITH_SQLITE me->_lister->openXapianIndex(); #endif gtk_label_set_text(GTK_LABEL(gtk_builder_get_object(me->_builder, @@ -1566,7 +1566,7 @@ void RGMainWindow::buildInterface() (_builder, "entry_fast_search")); // only enable fast search if its usable -#ifdef WITH_EPT +#ifdef WITH_SQLITE if(!_lister->xapiandatabase() || !FileExists("/usr/sbin/update-apt-xapian-index")) { gtk_widget_hide(GTK_WIDGET( diff --git a/tests/Makefile.am b/tests/Makefile.am index 4888444f8..b40df421a 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,6 @@ AM_CPPFLAGS = -I${top_srcdir}/common -I${top_srcdir}/gtk \ - @GTK_CFLAGS@ @VTE_CFLAGS@ @LP_CFLAGS@ $(LIBTAGCOLL_CFLAGS) $(LIBEPT_CFLAGS) -O0 -g3 + @GTK_CFLAGS@ @VTE_CFLAGS@ @LP_CFLAGS@ $(LIBTAGCOLL_CFLAGS) $(SQLITE_CFLAGS) -O0 -g3 noinst_PROGRAMS = test_rpackage test_rpackageview test_gtkpkglist test_rpackagefilter @@ -8,7 +8,7 @@ LDADD = \ ${top_builddir}/common/libsynaptic.a\ -lapt-pkg -lX11 @RPM_LIBS@ @DEB_LIBS@ \ @GTK_LIBS@ @VTE_LIBS@ @LP_LIBS@\ - -lpthread $(LIBEPT_LIBS) + -lpthread $(SQLITE_LIBS) test_rpackage_SOURCES= test_rpackage.cc From 0a71eac8cc1468703c2413c2dbd2af9875b0a65d Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Thu, 3 Apr 2025 22:28:33 +0200 Subject: [PATCH 2/4] WIP: Implement on-demand quick search with sqlite This blocks the main thread; we really need to run the main loop both when building and inserting, but otherwise works. --- common/rpackagelister.cc | 207 ++++++++++----------------------------- common/rpackagelister.h | 12 +-- common/rpackageview.h | 4 - config.h.in | 32 +++--- gtk/rgmainwindow.cc | 65 ------------ 5 files changed, 71 insertions(+), 249 deletions(-) diff --git a/common/rpackagelister.cc b/common/rpackagelister.cc index e2402aa3c..a2b678fb0 100644 --- a/common/rpackagelister.cc +++ b/common/rpackagelister.cc @@ -81,9 +81,6 @@ using namespace std; RPackageLister::RPackageLister() : _records(0), _progMeter(new OpProgress) -#ifdef WITH_SQLITE - , _xapianDatabase(0) -#endif { _cache = new RPackageCache(); @@ -104,7 +101,7 @@ RPackageLister::RPackageLister() // its import that we use "_packages" here instead of _nativeArchPackages _views.push_back(new RPackageViewArchitecture(_packages)); #ifdef WITH_SQLITE - openXapianIndex(); + //openXapianIndex(); #endif if (_viewMode >= _views.size()) @@ -435,48 +432,6 @@ bool RPackageLister::openCache() return true; } -#ifdef WITH_SQLITE -bool RPackageLister::xapianIndexNeedsUpdate() -{ - struct stat buf; - - if(_config->FindB("Debug::Synaptic::Xapian",false)) - std::cerr << "xapainIndexNeedsUpdate()" << std::endl; - - // check the xapian index - if(FileExists("/usr/sbin/update-apt-xapian-index") && - (!_xapianDatabase )) { - if(_config->FindB("Debug::Synaptic::Xapian",false)) - std::cerr << "xapain index not build yet" << std::endl; - return true; - } - - // compare timestamps, rebuild everytime, its now cheap(er) - // because we use u-a-x-i --update - stat(_config->FindFile("Dir::Cache::pkgcache").c_str(), &buf); - if(ept::axi::timestamp() < buf.st_mtime) { - if(_config->FindB("Debug::Synaptic::Xapian",false)) - std::cerr << "xapian outdated " - << buf.st_mtime - ept::axi::timestamp() << std::endl; - return true; - } - - return false; -} - -bool RPackageLister::openXapianIndex() -{ - if(_xapianDatabase) - delete _xapianDatabase; - try { - _xapianDatabase = new Xapian::Database(ept::axi::path_db()); - } catch (Xapian::DatabaseOpeningError) { - return false; - }; - return true; -} -#endif - void RPackageLister::applyInitialSelection() { _roptions->rereadOrphaned(); @@ -1978,119 +1933,63 @@ bool RPackageLister::addArchiveToCache(string archive, string &pkgname) bool RPackageLister::limitBySearch(string searchString) { //cerr << "limitBySearch(): " << searchString << endl; - if (ept::axi::timestamp() == 0) - return false; - return xapianSearch(searchString); + if (not xapianSearch(searchString)) + _error->DumpErrors(); + return true; } bool RPackageLister::xapianSearch(string unsplitSearchString) { - //std::cerr << "RPackageLister::xapianSearch()" << std::endl; - static const int defaultQualityCutoff = 15; - int qualityCutoff = _config->FindI("Synaptic::Xapian::qualityCutoff", - defaultQualityCutoff); - if (ept::axi::timestamp() == 0) - return false; - - try { - int maxItems = _xapianDatabase->get_doccount(); - Xapian::Enquire enquire(*_xapianDatabase); - Xapian::QueryParser parser; - parser.set_database(*_xapianDatabase); - parser.add_prefix("name","XP"); - parser.add_prefix("section","XS"); - // default op is AND to narrow down the resultset - parser.set_default_op( Xapian::Query::OP_AND ); - - /* Workaround to allow searching an hyphenated package name using a prefix (name:) - * LP: #282995 - * Xapian currently doesn't support wildcard for boolean prefix and - * doesn't handle implicit wildcards at the end of hypenated phrases. - * - * e.g searching for name:ubuntu-res will be equivalent to 'name:ubuntu res*' - * however 'name:(ubuntu* res*) won't return any result because the - * index is built with the full package name - */ - // Always search for the package name - string xpString = "name:"; - string::size_type pos = unsplitSearchString.find_first_of(" ,;"); - if (pos > 0) { - xpString += unsplitSearchString.substr(0,pos); - } else { - xpString += unsplitSearchString; - } - Xapian::Query xpQuery = parser.parse_query(xpString); - - pos = 0; - while ( (pos = unsplitSearchString.find("-", pos)) != string::npos ) { - unsplitSearchString.replace(pos, 1, " "); - pos+=1; - } - - if(_config->FindB("Debug::Synaptic::Xapian",false)) - std::cerr << "searching for : " << unsplitSearchString << std::endl; - - // Build the query - // apply a weight factor to XP term to increase relevancy on package name - Xapian::Query query = parser.parse_query(unsplitSearchString, - Xapian::QueryParser::FLAG_WILDCARD | - Xapian::QueryParser::FLAG_BOOLEAN | - Xapian::QueryParser::FLAG_PARTIAL); - query = Xapian::Query(Xapian::Query::OP_OR, query, - Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT, xpQuery, 3)); - enquire.set_query(query); - Xapian::MSet matches = enquire.get_mset(0, maxItems); - - if(_config->FindB("Debug::Synaptic::Xapian",false)) { - cerr << "enquire: " << enquire.get_description() << endl; - cerr << "matches estimated: " << matches.get_matches_estimated() << " results found" << endl; - } - - // Retrieve the results - int top_percent = 0; - _viewPackages.clear(); - for (Xapian::MSetIterator i = matches.begin(); i != matches.end(); ++i) - { - RPackage* pkg = getPackage(i.get_document().get_data()); - // Filter out results that apt doesn't know - if (!pkg || !_selectedView->hasPackage(pkg)) + static sqlite3 *database = nullptr; + if (not database) + { + char *err; + printf("Trying to open\n"); + if (sqlite3_open(":memory:", &database) != SQLITE_OK) + return _error->Error("Cannot open database"); + if (sqlite3_exec(database, "CREATE VIRTUAL TABLE fts USING fts5(name, description);", nullptr, nullptr, &err) != SQLITE_OK) + return _error->Error("Cannot create table: %s", err); + + static sqlite3_stmt *insert_stmt = nullptr; + if (not insert_stmt && sqlite3_prepare_v2(database, "INSERT INTO fts VALUES(?, ?)", -1, &insert_stmt, NULL) != SQLITE_OK) + return _error->Error("Cannot prepare insert statement"); + for (auto pkg : _packages) + { + if (auto name = pkg->name(); sqlite3_bind_text(insert_stmt, 1, name, strlen(name), nullptr) != SQLITE_OK) + return _error->Error("Cannot bind name '%s': %s", name, sqlite3_errmsg(database)); + if (auto desc = pkg->description(); sqlite3_bind_text(insert_stmt, 2, desc, strlen(desc), nullptr) != SQLITE_OK) + return _error->Error("Cannot bind desc '%s': %s", desc, sqlite3_errmsg(database)); + + if (sqlite3_step(insert_stmt) != SQLITE_DONE) + return _error->Error("Insert failed of '%s': %s", pkg->name(), sqlite3_errmsg(database)); + + sqlite3_reset(insert_stmt); + } + + } + printf("HELLO\n"); + static sqlite3_stmt *select_stmt = nullptr; + if (select_stmt) + sqlite3_reset(select_stmt); + else if (sqlite3_prepare_v2(database, "SELECT name FROM fts WHERE fts MATCH ?" , -1, &select_stmt, NULL) != SQLITE_OK) + return _error->Error("Cannot prepare statement"); + + if (sqlite3_bind_text(select_stmt, 1, unsplitSearchString.c_str(), unsplitSearchString.size(), nullptr) != SQLITE_OK) + return _error->Error("Cannot bind statement"); + + int rc; + _viewPackages.clear(); + while(SQLITE_ROW == (rc = sqlite3_step(select_stmt))) { + const char *name = (const char*)sqlite3_column_text(select_stmt, 0); + auto pkg = getPackage(name); + if (!pkg || !_selectedView->hasPackage(pkg)) continue; - - // Save the confidence interval of the top value, to use it as - // a reference to compute an adaptive quality cutoff - if (top_percent == 0) - top_percent = i.get_percent(); - - // Stop producing if the quality goes below a cutoff point - if (i.get_percent() < qualityCutoff * top_percent / 100) - { - cerr << "Discarding: " << i.get_percent() << " over " << qualityCutoff * top_percent / 100 << endl; - break; - } - - if(_config->FindB("Debug::Synaptic::Xapian",false)) - cerr << i.get_rank() + 1 << ": " << i.get_percent() << "% docid=" << *i << " [" << i.get_document().get_data() << "]" << endl; - _viewPackages.push_back(pkg); - } - // re-apply sort criteria only if an explicit search is set - if (_sortMode != LIST_SORT_DEFAULT) - sortPackages(_sortMode); - return true; - } catch (const Xapian::Error & error) { - /* We are here if a Xapian call failed. The main cause is a parser exception. - * The error message is always in English currently. - * The possible parser errors are: - * Unknown range operation - * parse error - * Syntax: AND - * Syntax: AND NOT - * Syntax: NOT - * Syntax: OR - * Syntax: XOR - */ - cerr << "Exception in RPackageLister::xapianSearch():" << error.get_msg() << endl; - return false; - } + _viewPackages.push_back(pkg); + } + if(SQLITE_DONE != rc) + return _error->Error("select statement didn't finish with DONE (%i): %s\n", rc, sqlite3_errmsg(database)); + sortPackages(_sortMode); + return true; } #else bool RPackageLister::limitBySearch(string searchString) diff --git a/common/rpackagelister.h b/common/rpackagelister.h index 17f34d476..6636ea459 100644 --- a/common/rpackagelister.h +++ b/common/rpackagelister.h @@ -38,7 +38,7 @@ #include #ifdef WITH_SQLITE -#include +#include #endif #include "rpackagecache.h" @@ -107,11 +107,6 @@ class RPackageLister { pkgRecords *_records; OpProgress *_progMeter; -#ifdef WITH_SQLITE - Xapian::Database *_xapianDatabase; -#endif - - // Other members. vector _packages; vector _packagesIndex; @@ -348,11 +343,6 @@ class RPackageLister { bool writeSelections(ostream &out, bool fullState); RPackageCache* getCache() { return _cache; } -#ifdef WITH_SQLITE - Xapian::Database* xapiandatabase() { return _xapianDatabase; } - bool xapianIndexNeedsUpdate(); - bool openXapianIndex(); -#endif RPackageLister(); ~RPackageLister(); diff --git a/common/rpackageview.h b/common/rpackageview.h index 89698d920..f84617fc6 100644 --- a/common/rpackageview.h +++ b/common/rpackageview.h @@ -29,10 +29,6 @@ #include #include -#ifdef WITH_SQLITE -#include -#endif - #include "rpackage.h" #include "rpackagefilter.h" diff --git a/config.h.in b/config.h.in index ce7cc03aa..a471a3bb0 100644 --- a/config.h.in +++ b/config.h.in @@ -9,7 +9,7 @@ /* whether apt-pkg/cdrom.h is present */ #undef HAVE_APTPKG_CDROM -/* Define to 1 if you have the `bind_textdomain_codeset' function. */ +/* Define to 1 if you have the 'bind_textdomain_codeset' function. */ #undef HAVE_BIND_TEXTDOMAIN_CODESET /* Define to 1 if you have the Mac OS X function CFLocaleCopyCurrent in the @@ -20,13 +20,13 @@ the CoreFoundation framework. */ #undef HAVE_CFPREFERENCESCOPYAPPVALUE -/* Define to 1 if you have the `dcgettext' function. */ +/* Define to 1 if you have the 'dcgettext' function. */ #undef HAVE_DCGETTEXT /* Define if the GNU gettext() function is already present or preinstalled. */ #undef HAVE_GETTEXT -/* Define to 1 if you have the `iconv' function. */ +/* Define to 1 if you have the 'iconv' function. */ #undef HAVE_ICONV /* Define to 1 if you have the header file. */ @@ -44,10 +44,7 @@ /* Define to 1 if you have the header file. */ #undef HAVE_LOCALE_H -/* Define to 1 if you have the header file. */ -#undef HAVE_MEMORY_H - -/* Define to 1 if you have the `regcomp' function. */ +/* Define to 1 if you have the 'regcomp' function. */ #undef HAVE_REGCOMP /* wheter RPM is present */ @@ -56,14 +53,17 @@ /* Define to 1 if you have the header file. */ #undef HAVE_STDINT_H +/* Define to 1 if you have the header file. */ +#undef HAVE_STDIO_H + /* Define to 1 if you have the header file. */ #undef HAVE_STDLIB_H -/* Define to 1 if you have the `strcoll' function and it is properly defined. +/* Define to 1 if you have the 'strcoll' function and it is properly defined. */ #undef HAVE_STRCOLL -/* Define to 1 if you have the `strdup' function. */ +/* Define to 1 if you have the 'strdup' function. */ #undef HAVE_STRDUP /* Define to 1 if you have the header file. */ @@ -108,7 +108,9 @@ /* Define to the version of this package. */ #undef PACKAGE_VERSION -/* Define to 1 if you have the ANSI C header files. */ +/* Define to 1 if all of the C89 standard headers exist (not just the ones + required in a freestanding environment). This macro is provided for + backward compatibility; new code need not use it. */ #undef STDC_HEADERS /* build with package pin feature */ @@ -129,19 +131,19 @@ /* build with dpkg progress bar */ #undef WITH_DPKG_STATUSFD -/* build with libept */ -#undef WITH_SQLITE - /* build with launchpad-integration */ #undef WITH_LAUNCHPAD_INTEGRATION /* Define if you want to enable the extension system. */ #undef WITH_LUA -/* Define to empty if `const' does not conform to ANSI C. */ +/* build with sqlite */ +#undef WITH_SQLITE + +/* Define to empty if 'const' does not conform to ANSI C. */ #undef const -/* Define to `__inline__' or `__inline' if that's what the C compiler +/* Define to '__inline__' or '__inline' if that's what the C compiler calls it, or to nothing if 'inline' is not supported under any name. */ #ifndef __cplusplus #undef inline diff --git a/gtk/rgmainwindow.cc b/gtk/rgmainwindow.cc index 6aff319f1..d25f2c9ea 100644 --- a/gtk/rgmainwindow.cc +++ b/gtk/rgmainwindow.cc @@ -852,57 +852,10 @@ RGMainWindow::RGMainWindow(RPackageLister *packLister, string name) RGPreferencesWindow::applyProxySettings(); } -#ifdef WITH_SQLITE gboolean RGMainWindow::xapianDoIndexUpdate(void *data) { - RGMainWindow *me = (RGMainWindow *) data; - if(_config->FindB("Debug::Synaptic::Xapian",false)) - std::cerr << "xapianDoIndexUpdate()" << std::endl; - - // no need to update if we run non-interactive - if(_config->FindB("Volatile::Non-Interactive", false) == true) - return false; - - // check if we need a update - if(!me->_lister->xapianIndexNeedsUpdate()) { - // if the cache is not open, check back when it is - if (me->_lister->packagesSize() == 0) - g_timeout_add_seconds(30, xapianDoIndexUpdate, me); - return false; - } - - // do not run if we don't have it - if(!FileExists("/usr/sbin/update-apt-xapian-index")) - return false; - // no permission - if (getuid() != 0) - return false; - - // if we make it to this point, we need a xapian update - if(_config->FindB("Debug::Synaptic::Xapian",false)) - std::cerr << "running update-apt-xapian-index" << std::endl; - GPid pid; - const char *argp[] = {"/usr/bin/nice", - "/usr/bin/ionice","-c3", - "/usr/sbin/update-apt-xapian-index", - "--update", "-q", - NULL}; - if(g_spawn_async(NULL, const_cast(argp), NULL, - (GSpawnFlags)(G_SPAWN_DO_NOT_REAP_CHILD), - NULL, NULL, &pid, NULL)) { - g_child_watch_add(pid, (GChildWatchFunc)xapianIndexUpdateFinished, me); - gtk_label_set_text(GTK_LABEL(gtk_builder_get_object(me->_builder, - "label_fast_search")), - _("Rebuilding search index")); - } return false; } -#else -gboolean RGMainWindow::xapianDoIndexUpdate(void *data) -{ - return false; -} -#endif void RGMainWindow::xapianIndexUpdateFinished(GPid pid, gint status, void* data) { @@ -910,9 +863,6 @@ void RGMainWindow::xapianIndexUpdateFinished(GPid pid, gint status, void* data) if(_config->FindB("Debug::Synaptic::Xapian",false)) std::cerr << "xapianIndexUpdateFinished: " << WEXITSTATUS(status) << std::endl; -#ifdef WITH_SQLITE - me->_lister->openXapianIndex(); -#endif gtk_label_set_text(GTK_LABEL(gtk_builder_get_object(me->_builder, "label_fast_search")), _("Quick filter")); @@ -1565,21 +1515,6 @@ void RGMainWindow::buildInterface() _entry_fast_search = GTK_WIDGET(gtk_builder_get_object (_builder, "entry_fast_search")); - // only enable fast search if its usable -#ifdef WITH_SQLITE - if(!_lister->xapiandatabase() || - !FileExists("/usr/sbin/update-apt-xapian-index")) { - gtk_widget_hide(GTK_WIDGET( - gtk_builder_get_object(_builder, "toolitem_fast_search"))); - gtk_box_set_center_widget(GTK_BOX( - gtk_builder_get_object(_builder, "hbox_button_toolbar")), NULL); - } -#else - gtk_widget_hide(GTK_WIDGET( - gtk_builder_get_object(_builder, "toolitem_fast_search"))); - gtk_box_set_center_widget(GTK_BOX( - gtk_builder_get_object(_builder, "hbox_button_toolbar")), NULL); -#endif // stuff for the non-root mode if(getuid() != 0) { GtkWidget *menu; From 61510d08d41b2902aba7426ecf4cb66e692dceb6 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Thu, 3 Apr 2025 22:43:53 +0200 Subject: [PATCH 3/4] sqlite search: run main event loop between items This keeps the UI responsive while we are busy searching. Though perhaps we should show some modal dialog? --- common/rpackagelister.cc | 13 +++++++------ common/rpackagelister.h | 4 ++-- gtk/rgmainwindow.cc | 5 ++++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/common/rpackagelister.cc b/common/rpackagelister.cc index a2b678fb0..a0d1021a1 100644 --- a/common/rpackagelister.cc +++ b/common/rpackagelister.cc @@ -1930,15 +1930,15 @@ bool RPackageLister::addArchiveToCache(string archive, string &pkgname) #ifdef WITH_SQLITE -bool RPackageLister::limitBySearch(string searchString) +bool RPackageLister::limitBySearch(string searchString, void (*iter)()) { //cerr << "limitBySearch(): " << searchString << endl; - if (not xapianSearch(searchString)) + if (not xapianSearch(searchString, iter)) _error->DumpErrors(); return true; } -bool RPackageLister::xapianSearch(string unsplitSearchString) +bool RPackageLister::xapianSearch(string unsplitSearchString, void (*iter)()) { static sqlite3 *database = nullptr; if (not database) @@ -1964,8 +1964,8 @@ bool RPackageLister::xapianSearch(string unsplitSearchString) return _error->Error("Insert failed of '%s': %s", pkg->name(), sqlite3_errmsg(database)); sqlite3_reset(insert_stmt); + iter(); } - } printf("HELLO\n"); static sqlite3_stmt *select_stmt = nullptr; @@ -1980,6 +1980,7 @@ bool RPackageLister::xapianSearch(string unsplitSearchString) int rc; _viewPackages.clear(); while(SQLITE_ROW == (rc = sqlite3_step(select_stmt))) { + iter(); const char *name = (const char*)sqlite3_column_text(select_stmt, 0); auto pkg = getPackage(name); if (!pkg || !_selectedView->hasPackage(pkg)) @@ -1992,12 +1993,12 @@ bool RPackageLister::xapianSearch(string unsplitSearchString) return true; } #else -bool RPackageLister::limitBySearch(string searchString) +bool RPackageLister::limitBySearch(string searchString, void (*iter)()) { return false; } -bool RPackageLister::xapianSearch(string searchString) +bool RPackageLister::xapianSearch(string searchString, void (*iter)()) { return false; } diff --git a/common/rpackagelister.h b/common/rpackagelister.h index 6636ea459..c05ea95f6 100644 --- a/common/rpackagelister.h +++ b/common/rpackagelister.h @@ -135,7 +135,7 @@ class RPackageLister { RPackageViewSearch *_searchView; // the package view that does the (simple) search // helper for the limitBySearch() code - bool xapianSearch(string searchString); + bool xapianSearch(string searchString, void (*iter)()); public: @@ -205,7 +205,7 @@ class RPackageLister { public: // limit what the current view displays - bool limitBySearch(string searchString); + bool limitBySearch(string searchString, void (*iter)()); // clean files older than "Synaptic::delHistory" void cleanCommitLog(); diff --git a/gtk/rgmainwindow.cc b/gtk/rgmainwindow.cc index d25f2c9ea..17630e489 100644 --- a/gtk/rgmainwindow.cc +++ b/gtk/rgmainwindow.cc @@ -311,7 +311,10 @@ void RGMainWindow::refreshTable(RPackage *selectedPkg, bool setAdjustment) if(str != NULL && strlen(str) > 1) { if(_config->FindB("Debug::Synaptic::View",false)) cerr << "RGMainWindow::refreshTable: rerun limitBySearch" << endl; - _lister->limitBySearch(str); + _lister->limitBySearch(str, [](){ + while (gtk_events_pending ()) + gtk_main_iteration (); + }); } if(_pkgList == NULL) From 48cfdbb1ea76f09f1e57399472f62b0684033f21 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Thu, 3 Apr 2025 23:42:30 +0200 Subject: [PATCH 4/4] Implement caching --- common/rpackagelister.cc | 71 ++++++++++++++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 14 deletions(-) diff --git a/common/rpackagelister.cc b/common/rpackagelister.cc index a0d1021a1..c30e56318 100644 --- a/common/rpackagelister.cc +++ b/common/rpackagelister.cc @@ -1944,34 +1944,77 @@ bool RPackageLister::xapianSearch(string unsplitSearchString, void (*iter)()) if (not database) { char *err; - printf("Trying to open\n"); - if (sqlite3_open(":memory:", &database) != SQLITE_OK) + std::string dir; + if (auto d = getenv("XDG_RUNTIME_DIRECTORY")) + dir = d; + else if (getuid()) + dir = flCombine("/run/user/", std::to_string(getuid())); + else + dir = std::string{"/run"}; + + std::string file = ":memory:"; + if (struct stat st; !dir.empty() || access(dir.c_str(), X_OK) != 0) + file = flCombine(dir, "synaptic.db"); + + struct stat cacheStat{}; + struct stat dbStat{}; + stat(_config->FindFile("Dir::Cache::pkgcache").c_str(), &cacheStat); + stat(file.c_str(), &dbStat); + + if (dbStat.st_mtime && dbStat.st_mtime != cacheStat.st_mtime) + unlink(file.c_str()); + + if (sqlite3_open(file.c_str(), &database) != SQLITE_OK && + sqlite3_open(":memory:", &database) != SQLITE_OK) return _error->Error("Cannot open database"); - if (sqlite3_exec(database, "CREATE VIRTUAL TABLE fts USING fts5(name, description);", nullptr, nullptr, &err) != SQLITE_OK) + if (sqlite3_exec(database, "CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(name, description);", nullptr, nullptr, &err) != SQLITE_OK) return _error->Error("Cannot create table: %s", err); static sqlite3_stmt *insert_stmt = nullptr; if (not insert_stmt && sqlite3_prepare_v2(database, "INSERT INTO fts VALUES(?, ?)", -1, &insert_stmt, NULL) != SQLITE_OK) return _error->Error("Cannot prepare insert statement"); - for (auto pkg : _packages) + + // We abuse the callback here, if we have no rows, the callback returns 0, and then + // sqlite3_exec() returns SQLITE_DONE; whereas if we have a row, it returns 1, and + // the exec returns SQLITE_ABORT. + if (sqlite3_exec(database, "SELECT name FROM fts LIMIT 1", [](void*, int nrows, char**, char**) -> int { + return nrows > 0; + }, nullptr, &err) != SQLITE_ABORT) { - if (auto name = pkg->name(); sqlite3_bind_text(insert_stmt, 1, name, strlen(name), nullptr) != SQLITE_OK) - return _error->Error("Cannot bind name '%s': %s", name, sqlite3_errmsg(database)); - if (auto desc = pkg->description(); sqlite3_bind_text(insert_stmt, 2, desc, strlen(desc), nullptr) != SQLITE_OK) - return _error->Error("Cannot bind desc '%s': %s", desc, sqlite3_errmsg(database)); - if (sqlite3_step(insert_stmt) != SQLITE_DONE) - return _error->Error("Insert failed of '%s': %s", pkg->name(), sqlite3_errmsg(database)); + if (sqlite3_exec(database, "BEGIN", nullptr, nullptr, &err) != SQLITE_OK) + return _error->Error("Cannot begin: %s", err); + + for (auto pkg : _packages) + { + if (auto name = pkg->name(); sqlite3_bind_text(insert_stmt, 1, name, strlen(name), nullptr) != SQLITE_OK) + return _error->Error("Cannot bind name '%s': %s", name, sqlite3_errmsg(database)); + if (auto desc = pkg->description(); sqlite3_bind_text(insert_stmt, 2, desc, strlen(desc), nullptr) != SQLITE_OK) + return _error->Error("Cannot bind desc '%s': %s", desc, sqlite3_errmsg(database)); + + if (sqlite3_step(insert_stmt) != SQLITE_DONE) + return _error->Error("Insert failed of '%s': %s", pkg->name(), sqlite3_errmsg(database)); + + sqlite3_reset(insert_stmt); + iter(); + } - sqlite3_reset(insert_stmt); - iter(); + if (sqlite3_exec(database, "END", nullptr, nullptr, &err) != SQLITE_OK) + return _error->Error("Cannot begin: %s", err); + + if (cacheStat.st_mtime != 0) + { + struct timeval tv[]{timeval{time(NULL), 0}, timeval{cacheStat.st_mtime, 0}}; + utimes(file.c_str(), tv); + } } + + } - printf("HELLO\n"); static sqlite3_stmt *select_stmt = nullptr; if (select_stmt) sqlite3_reset(select_stmt); - else if (sqlite3_prepare_v2(database, "SELECT name FROM fts WHERE fts MATCH ?" , -1, &select_stmt, NULL) != SQLITE_OK) + else if (sqlite3_prepare_v2(database, "SELECT DISTINCT name FROM fts WHERE fts MATCH ?" , -1, &select_stmt, NULL) != SQLITE_OK) return _error->Error("Cannot prepare statement"); if (sqlite3_bind_text(select_stmt, 1, unsplitSearchString.c_str(), unsplitSearchString.size(), nullptr) != SQLITE_OK)