From 61f4a15a6978953b41d9df6a40b723a614b90d3d Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 9 Oct 2012 10:26:42 +1000 Subject: [PATCH 01/12] Changed sphinxsearch config to vendor/bundles. --- Resources/config/sphinxsearch.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resources/config/sphinxsearch.xml b/Resources/config/sphinxsearch.xml index 4cb8a8a..98cd059 100644 --- a/Resources/config/sphinxsearch.xml +++ b/Resources/config/sphinxsearch.xml @@ -16,7 +16,7 @@ - %kernel.root_dir%/../src/Search/SphinxsearchBundle/Services/Search/SphinxAPI.php + %kernel.root_dir%/../vendor/bundles/Search/SphinxsearchBundle/Services/Search/SphinxAPI.php %search.sphinxsearch.searchd.host% %search.sphinxsearch.searchd.port% %search.sphinxsearch.searchd.socket% From 69c342ab35d5df33ce63c3db36cd720f255bb361 Mon Sep 17 00:00:00 2001 From: hieuapl Date: Fri, 12 Oct 2012 16:47:37 +1000 Subject: [PATCH 02/12] Changed how search function works to search multiple indexes at one time as per Sphinx API --- Services/Search/Sphinxsearch.php | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/Services/Search/Sphinxsearch.php b/Services/Search/Sphinxsearch.php index 68d4c86..f4a2b34 100644 --- a/Services/Search/Sphinxsearch.php +++ b/Services/Search/Sphinxsearch.php @@ -119,6 +119,7 @@ public function search($query, array $indexes, $escapeQuery = true) if( $escapeQuery ) $query = $this->sphinx->escapeString($query); + $indexnames = ''; $results = array(); foreach( $indexes as $label => $options ) { /** @@ -139,14 +140,20 @@ public function search($query, array $indexes, $escapeQuery = true) if( isset($options['field_weights']) ) $this->sphinx->setFieldWeights($options['field_weights']); - /** - * Perform the query. - */ - $results[$label] = $this->sphinx->query($query, $this->indexes[$label]); - if( $results[$label]['status'] !== SEARCHD_OK ) - throw new \RuntimeException(sprintf('Searching index "%s" for "%s" failed with error "%s".', $label, $query, $this->sphinx->getLastError())); + /* + * Create string of index names for query function. + */ + $indexnames .= $this->indexes[$label].' '; } + /** + * Perform the query. + */ + $results = $this->sphinx->query($query, $indexnames); + + if( $results['status'] !== SEARCHD_OK ) + throw new \RuntimeException(sprintf('Searching index "%s" for "%s" failed with error "%s".', $label, $query, $this->sphinx->getLastError())); + /** * If only one index was searched, return that index's results directly. */ From 7bea3b2e771df0a715838465e18c8565f4783825 Mon Sep 17 00:00:00 2001 From: hieuapl Date: Mon, 15 Oct 2012 09:01:17 +1000 Subject: [PATCH 03/12] Changed search function to match SphinxAPI query intended behaviour --- Services/Search/Sphinxsearch.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Services/Search/Sphinxsearch.php b/Services/Search/Sphinxsearch.php index f4a2b34..7e27774 100644 --- a/Services/Search/Sphinxsearch.php +++ b/Services/Search/Sphinxsearch.php @@ -141,7 +141,7 @@ public function search($query, array $indexes, $escapeQuery = true) $this->sphinx->setFieldWeights($options['field_weights']); /* - * Create string of index names for query function. + * Create string of index names for SphinxAPI query function. */ $indexnames .= $this->indexes[$label].' '; } From b58b5a28914cdbdf976ddf7863c01e44a22b4c87 Mon Sep 17 00:00:00 2001 From: Chris Gordon Date: Wed, 30 Jan 2013 12:55:43 +1000 Subject: [PATCH 04/12] Add batch queries --- Services/Search/Sphinxsearch.php | 49 +++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/Services/Search/Sphinxsearch.php b/Services/Search/Sphinxsearch.php index 7e27774..5cc671a 100644 --- a/Services/Search/Sphinxsearch.php +++ b/Services/Search/Sphinxsearch.php @@ -132,7 +132,7 @@ public function search($query, array $indexes, $escapeQuery = true) * Set the offset and limit for the returned results. */ if( isset($options['result_offset']) && isset($options['result_limit']) ) - $this->sphinx->setLimits($options['result_offset'], $options['result_limit']); + $this->sphinx->setLimits($options['result_offset'], $options['result_limit'],5000); /** * Weight the individual fields. @@ -165,4 +165,51 @@ public function search($query, array $indexes, $escapeQuery = true) */ return $results; } + + /** + * Reset all previously set filters. + * + */ + public function resetFilters() { + $this->sphinx->resetFilters(); + } + + /** + * Adds query with the current settings to multi-query batch. This method doesn't affect current settings (sorting, filtering, grouping etc.) in any way. + * @param string $query The query string that we are searching for. + * @param array $indexes The indexes to perform the search on. + * + * $indexes = array( + * 'IndexLabel', + * 'IndexLabel2' + * ); + */ + public function addQuery($search_str, $indexes) { + $indexnames = ''; + + //Create string on index names + foreach( $indexes as $label) { + + /** + * Ensure that the label corresponds to a defined index. + */ + if(isset($this->indexes[$label])){ + + /* + * Create string of index names for SphinxAPI query function. + */ + $indexnames .= $this->indexes[$label].' '; + } + } + + $this->sphinx->addQuery($search_str, $indexnames); + } + + /** + * Connects to searchd, runs a batch of all queries added using SphinxClient::addQuery, obtains and returns the result sets. + * + */ + public function runQueries() { + return $this->sphinx->runQueries(); + } } From 3a962b81e0770df665e6c7989fa9d76597806319 Mon Sep 17 00:00:00 2001 From: Chris Gordon Date: Thu, 22 Aug 2013 09:05:16 +1000 Subject: [PATCH 05/12] Updated composer json so that I can add this customised bundle to symfony23 --- composer.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/composer.json b/composer.json index 8fa3328..ea143d7 100644 --- a/composer.json +++ b/composer.json @@ -1,9 +1,9 @@ { - "name": "search/sphinxsearch-bundle", + "name": "chrisapl/search-sphinxsearchbundle", "type": "symfony-bundle", "description": "Sphinx search bundle for Symfony 2", "keywords": ["sphinx", "sphinxsearch", "symfony2"], - "homepage": "https://github.com/timewasted/Search-SphinxsearchBundle", + "homepage": "https://github.com/chrisapl/Search-SphinxsearchBundle", "license": "BSD-2-Clause", "authors": [ { From 99b620d2f31132788035d45782a35feff8a85377 Mon Sep 17 00:00:00 2001 From: Chris Gordon Date: Mon, 26 Aug 2013 09:21:16 +1000 Subject: [PATCH 06/12] Fixed searchapi location --- Resources/config/sphinxsearch.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resources/config/sphinxsearch.xml b/Resources/config/sphinxsearch.xml index 98cd059..b71f820 100644 --- a/Resources/config/sphinxsearch.xml +++ b/Resources/config/sphinxsearch.xml @@ -16,7 +16,7 @@ - %kernel.root_dir%/../vendor/bundles/Search/SphinxsearchBundle/Services/Search/SphinxAPI.php + %kernel.root_dir%/../vendor/chrisapl/search-sphinxsearchbundle/Search/SphinxsearchBundle/Services/Search/SphinxAPI.php %search.sphinxsearch.searchd.host% %search.sphinxsearch.searchd.port% %search.sphinxsearch.searchd.socket% From a760fb5d80c0e6256f9fc8aac1fce6036191fd2f Mon Sep 17 00:00:00 2001 From: PRD Server Date: Thu, 12 Sep 2013 14:43:35 +1000 Subject: [PATCH 07/12] Removed extra space at end of index name when searching single index. --- Services/Search/Sphinxsearch.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Services/Search/Sphinxsearch.php b/Services/Search/Sphinxsearch.php index 5cc671a..0047ad6 100644 --- a/Services/Search/Sphinxsearch.php +++ b/Services/Search/Sphinxsearch.php @@ -121,7 +121,9 @@ public function search($query, array $indexes, $escapeQuery = true) $indexnames = ''; $results = array(); + $i = 0; foreach( $indexes as $label => $options ) { + /** * Ensure that the label corresponds to a defined index. */ @@ -143,7 +145,11 @@ public function search($query, array $indexes, $escapeQuery = true) /* * Create string of index names for SphinxAPI query function. */ - $indexnames .= $this->indexes[$label].' '; + if($i == 1){ + $indexnames .= ' '; + } + $indexnames .= $this->indexes[$label]; + $i++; } /** From b077f238fb84a7c7e1d69b78e148c89efe3b6d9d Mon Sep 17 00:00:00 2001 From: hieuapl Date: Mon, 23 Sep 2013 09:19:23 +1000 Subject: [PATCH 08/12] increase the max_matches --- Services/Search/Sphinxsearch.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Services/Search/Sphinxsearch.php b/Services/Search/Sphinxsearch.php index 0047ad6..56a9ec8 100644 --- a/Services/Search/Sphinxsearch.php +++ b/Services/Search/Sphinxsearch.php @@ -134,7 +134,7 @@ public function search($query, array $indexes, $escapeQuery = true) * Set the offset and limit for the returned results. */ if( isset($options['result_offset']) && isset($options['result_limit']) ) - $this->sphinx->setLimits($options['result_offset'], $options['result_limit'],5000); + $this->sphinx->setLimits($options['result_offset'], $options['result_limit'],20000); /** * Weight the individual fields. From f63b4ffe51f5925bed20e22dec0bce6e5c1aab20 Mon Sep 17 00:00:00 2001 From: Chris Gordon Date: Tue, 22 Oct 2013 13:06:08 +1000 Subject: [PATCH 09/12] Mapped bundles search service to sphinx APIs setSortMode function --- Services/Search/Sphinxsearch.php | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/Services/Search/Sphinxsearch.php b/Services/Search/Sphinxsearch.php index 56a9ec8..3ab2fe8 100644 --- a/Services/Search/Sphinxsearch.php +++ b/Services/Search/Sphinxsearch.php @@ -211,11 +211,27 @@ public function addQuery($search_str, $indexes) { $this->sphinx->addQuery($search_str, $indexnames); } - /** + /** * Connects to searchd, runs a batch of all queries added using SphinxClient::addQuery, obtains and returns the result sets. * - */ - public function runQueries() { + */ + public function runQueries() { return $this->sphinx->runQueries(); } + + /** + * Set sort mode for search. + * @param $mode Sortmode as specifed in Sphinx API Doc on of: + * SPH_SORT_RELEVANCE Sort by relevance in descending order (best matches first). + * SPH_SORT_ATTR_DESC Sort by an attribute in descending order (bigger attribute values first). + * SPH_SORT_ATTR_ASC Sort by an attribute in ascending order (smaller attribute values first). + * SPH_SORT_TIME_SEGMENTS Sort by time segments (last hour/day/week/month) in descending order, and then by relevance in descending order. + * SPH_SORT_EXTENDED Sort by SQL-like combination of columns in ASC/DESC order. + * SPH_SORT_EXPR + * @param $sortby String name of field to sort. + */ + public function setSortMode($mode, $sortby){ + $this->sphinx->setSortMode($mode, $sortby); + } } + From d4da72572b27ccdaf1cb09b089a222451b17b9ac Mon Sep 17 00:00:00 2001 From: Chris Gordon Date: Tue, 22 Oct 2013 13:06:45 +1000 Subject: [PATCH 10/12] Updated location of Process class to match Symfony23s update location for this class --- Services/Indexer/Indexer.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Services/Indexer/Indexer.php b/Services/Indexer/Indexer.php index 1775fbe..6aedcc6 100644 --- a/Services/Indexer/Indexer.php +++ b/Services/Indexer/Indexer.php @@ -2,7 +2,7 @@ namespace Search\SphinxsearchBundle\Services\Indexer; -use Assetic\Util\ProcessBuilder; +use Symfony\Component\Process\ProcessBuilder; class Indexer { @@ -65,6 +65,7 @@ public function rotate($indexes) if( is_array($indexes) ) { foreach( $indexes as &$label ) { if( isset($this->indexes[$label]) ) + $pb->add($this->indexes[$label]['index_name']); } } elseif( is_string($indexes) ) { From d127468ef2a757fba6b91ff31b1b70f70aeb7f98 Mon Sep 17 00:00:00 2001 From: Chris Gordon Date: Mon, 28 Oct 2013 11:00:58 +1000 Subject: [PATCH 11/12] Added copy of Sphinx PHP test to aid testing distributed indexes --- Services/Search/SearchTest.php | 168 +++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 Services/Search/SearchTest.php diff --git a/Services/Search/SearchTest.php b/Services/Search/SearchTest.php new file mode 100644 index 0000000..73dced1 --- /dev/null +++ b/Services/Search/SearchTest.php @@ -0,0 +1,168 @@ +\tconnect to searchd at host HOST\n" ); + print ( "-p, --port\t\tconnect to searchd at port PORT\n" ); + print ( "-i, --index \tsearch through index(es) specified by IDX\n" ); + print ( "-s, --sortby \tsort matches by 'CLAUSE' in sort_extended mode\n" ); + print ( "-S, --sortexpr \tsort matches by 'EXPR' DESC in sort_expr mode\n" ); + print ( "-a, --any\t\tuse 'match any word' matching mode\n" ); + print ( "-b, --boolean\t\tuse 'boolean query' matching mode\n" ); + print ( "-e, --extended\t\tuse 'extended query' matching mode\n" ); + print ( "-ph,--phrase\t\tuse 'exact phrase' matching mode\n" ); + print ( "-f, --filter \tfilter by attribute 'ATTR' (default is 'group_id')\n" ); + print ( "-fr,--filterrange \n\t\t\tadd specified range filter\n" ); + print ( "-v, --value \tadd VAL to allowed 'group_id' values list\n" ); + print ( "-g, --groupby \tgroup matches by 'EXPR'\n" ); + print ( "-gs,--groupsort \tsort groups by 'EXPR'\n" ); + print ( "-d, --distinct \tcount distinct values of 'ATTR''\n" ); + print ( "-l, --limit \tretrieve COUNT matches (default: 20)\n" ); + print ( "--select \tuse 'EXPRLIST' as select-list (default: *)\n" ); + exit; +} + +$args = array(); +foreach ( $_SERVER["argv"] as $arg ) + $args[] = $arg; + +$cl = new SphinxClient (); + +$q = ""; +$sql = ""; +$mode = SPH_MATCH_ALL; +$host = "localhost"; +$port = 9312; +$index = "*"; +$groupby = ""; +$groupsort = "@group desc"; +$filter = "group_id"; +$filtervals = array(); +$distinct = ""; +$sortby = ""; +$sortexpr = ""; +$limit = 20; +$ranker = SPH_RANK_PROXIMITY_BM25; +$select = ""; +for ( $i=0; $iSetFilterRange ( $args[++$i], $args[++$i], $args[++$i] ); + else if ( $arg=="-r" ) + { + $arg = strtolower($args[++$i]); + if ( $arg=="bm25" ) $ranker = SPH_RANK_BM25; + if ( $arg=="none" ) $ranker = SPH_RANK_NONE; + if ( $arg=="wordcount" )$ranker = SPH_RANK_WORDCOUNT; + if ( $arg=="fieldmask" )$ranker = SPH_RANK_FIELDMASK; + if ( $arg=="sph04" ) $ranker = SPH_RANK_SPH04; + } + else + $q .= $args[$i] . " "; +} + +//////////// +// do query +//////////// + +$cl->SetServer ( $host, $port ); +$cl->SetConnectTimeout ( 1 ); +$cl->SetArrayResult ( true ); +$cl->SetMatchMode ( $mode ); +if ( count($filtervals) ) $cl->SetFilter ( $filter, $filtervals ); +if ( $groupby ) $cl->SetGroupBy ( $groupby, SPH_GROUPBY_ATTR, $groupsort ); +if ( $sortby ) $cl->SetSortMode ( SPH_SORT_EXTENDED, $sortby ); +if ( $sortexpr ) $cl->SetSortMode ( SPH_SORT_EXPR, $sortexpr ); +if ( $distinct ) $cl->SetGroupDistinct ( $distinct ); +if ( $select ) $cl->SetSelect ( $select ); +if ( $limit ) $cl->SetLimits ( 0, $limit, ( $limit>1000 ) ? $limit : 1000 ); +$cl->SetRankingMode ( $ranker ); +$res = $cl->Query ( $q, $index ); + +//////////////// +// print me out +//////////////// + +if ( $res===false ) +{ + print "Query failed: " . $cl->GetLastError() . ".\n"; + +} else +{ + if ( $cl->GetLastWarning() ) + print "WARNING: " . $cl->GetLastWarning() . "\n\n"; + + print "Query '$q' retrieved $res[total] of $res[total_found] matches in $res[time] sec.\n"; + print "Query stats:\n"; + if ( is_array($res["words"]) ) + foreach ( $res["words"] as $word => $info ) + print " '$word' found $info[hits] times in $info[docs] documents\n"; + print "\n"; + + if ( is_array($res["matches"]) ) + { + $n = 1; + print "Matches:\n"; + foreach ( $res["matches"] as $docinfo ) + { + print "$n. doc_id=$docinfo[id], weight=$docinfo[weight]"; + foreach ( $res["attrs"] as $attrname => $attrtype ) + { + $value = $docinfo["attrs"][$attrname]; + if ( $attrtype==SPH_ATTR_MULTI || $attrtype==SPH_ATTR_MULTI64 ) + { + $value = "(" . join ( ",", $value ) .")"; + } else + { + if ( $attrtype==SPH_ATTR_TIMESTAMP ) + $value = date ( "Y-m-d H:i:s", $value ); + } + print ", $attrname=$value"; + } + print "\n"; + $n++; + } + } +} + +// +// $Id$ +// + +?> From 47a7c2a8eed2109f3d933503d786bafd74c057ee Mon Sep 17 00:00:00 2001 From: PRD Server Date: Tue, 19 Nov 2013 09:59:47 +1000 Subject: [PATCH 12/12] Increase search limits --- Services/Search/Sphinxsearch.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Services/Search/Sphinxsearch.php b/Services/Search/Sphinxsearch.php index 3ab2fe8..fdfa9f7 100644 --- a/Services/Search/Sphinxsearch.php +++ b/Services/Search/Sphinxsearch.php @@ -134,7 +134,7 @@ public function search($query, array $indexes, $escapeQuery = true) * Set the offset and limit for the returned results. */ if( isset($options['result_offset']) && isset($options['result_limit']) ) - $this->sphinx->setLimits($options['result_offset'], $options['result_limit'],20000); + $this->sphinx->setLimits($options['result_offset'], $options['result_limit'], 100000); /** * Weight the individual fields.