Added Fuzzy approx. feature + Stemming support + improved faceting

VincentFoulon80 · VincentFoulon80 · commit c8b68b027600 · 2019-06-07T19:12:24.000+02:00
diff --git a/Engine.php b/Engine.php
@@ -115,7 +115,8 @@ private function getDefaultConfig(){
                 "var_dir" => $_SERVER['DOCUMENT_ROOT'].DIRECTORY_SEPARATOR."var",
                 "index_dir" => DIRECTORY_SEPARATOR."engine".DIRECTORY_SEPARATOR."index",
                 "documents_dir" => DIRECTORY_SEPARATOR."engine".DIRECTORY_SEPARATOR."documents",
-                "cache_dir" => DIRECTORY_SEPARATOR."engine".DIRECTORY_SEPARATOR."cache"
+                "cache_dir" => DIRECTORY_SEPARATOR."engine".DIRECTORY_SEPARATOR."cache",
+                "fuzzy_cost" => 1
             ],
             "schemas" => [
                 "example-post" => [
diff --git a/Services/Index.php b/Services/Index.php
@@ -12,6 +12,11 @@
 
 class Index
 {
+    /**
+     * @var array $config
+     */
+    private $config;
+
     /**
      * @var Directory $index
      */
@@ -52,6 +57,7 @@ class Index
      */
     public function __construct($config, $schemas, $types)
     {
+        $this->config = $config;
         $this->schemas = $schemas;
         $this->types = $types;
         try {
@@ -223,6 +229,18 @@ public function search($query, $filters = [])
             arsort($results);
             $documents = [];
 
+            $facets = [];
+            if(isset($filters['facets']) && !empty($filters['facets'])) {
+                foreach ($filters['facets'] as $facet) {
+                    if ($this->index->open("facet_" . $facet, false) !== null) {
+                        $array = $this->index->open("facet_" . $facet, false)->getContent();
+                        foreach ($array as $token => $ids) {
+                            $facets[$facet][$token] = count(array_intersect_key(array_flip($ids), $results));
+                        }
+                    }
+                }
+            }
+
             $i = 0;
             foreach($results as $doc => $score){
                 if($i < $filters['offset']){
@@ -235,8 +253,7 @@ public function search($query, $filters = [])
                 $i++;
             }
 
-            $facets = [];
-            if(isset($filters['facets'])){
+            if(empty($query) && isset($filters['facets'])){
                 foreach($filters['facets'] as $facet){
                     if($this->index->open("facet_".$facet,false) !== null){
                         $array = $this->index->open("facet_".$facet,false)->getContent();
@@ -323,8 +340,9 @@ public function suggest($token, $providePonderations = false){
         $tokens = array_keys($all->getContent());
         $matching = [];
         foreach($tokens as $indexToken){
-            if(strpos($indexToken, $token) !== false){
-                $matching[$indexToken] = strpos($indexToken, $token);
+            $strPos = strpos($indexToken, $token);
+            if($strPos !== false){
+                $matching[$indexToken] = $strPos;
             }
         }
         asort($matching);
@@ -339,9 +357,11 @@ public function suggest($token, $providePonderations = false){
      * @return array
      * @throws Exception
      */
-    private function fuzzyFind($token){
-        if(empty($token)) return [];
+    private function fuzzyFind($token){if(empty($token)) return [];
         $matching = $this->suggest($token, true);
+        if(empty($matching)){
+            $matching = $this->approximate($token, $this->config['fuzzy_cost']);
+        }
         $found = [];
         if(!empty($matching)){
             reset($matching);
@@ -358,6 +378,51 @@ private function fuzzyFind($token){
         return $found;
     }
 
+    /**
+     *
+     * @param $term
+     * @param $cost
+     * @param array $positions
+     * @return array|mixed
+     * @throws Exception
+     */
+    private function approximate($term, $cost, $positions = []){
+        $cached = $this->getCache("approx_".$term);
+        if(!empty($cached)){
+            return $cached;
+        }
+        $termL = strlen($term);
+        if($termL <= 1) return []; // we shouldn't approximate one character
+        $cost = max($cost, $termL-1); // The cost can't be more than the term's length itself
+        $tokens = array_keys($this->index->open("all")->getContent());
+        $matching = [];
+        for($i=0;$i<$termL;$i++){
+            $termToFind = substr_replace($term, '', $i,1);
+            foreach($tokens as $token){
+                $originalToken = $token;
+                if(!empty($positions)){
+                    foreach($positions as $position){
+                        $token = substr_replace($token, '', $position,1);
+                    }
+                }
+                if(strlen($token) >= $termL){
+                    $tokenToLink = substr_replace($token, '', $i,1);
+                    $strPos = strpos($tokenToLink,$termToFind);
+                    if($strPos !== false){
+                        $matching[$originalToken] = $strPos;
+                    }
+                }
+            }
+            if($cost > 1){
+                $positions[$cost] = $i;
+                $matching = array_replace($matching, $this->approximate($termToFind,$cost-1, $positions));
+            }
+        }
+        asort($matching);
+        $this->setCache("approx_".$term, $matching);
+        return $matching;
+    }
+
     /**
      * @param $data
      * @param $schema
diff --git a/Tokenizers/DanishStemmingTokenizer.php b/Tokenizers/DanishStemmingTokenizer.php
@@ -0,0 +1,17 @@
+<?php
+
+namespace VFou\Search\Tokenizers;
+
+use Wamania\Snowball\Danish;
+
+class DanishStemmingTokenizer implements TokenizerInterface
+{
+
+    public static function tokenize($data)
+    {
+        return array_map(function($value){
+            $stemmer = new Danish();
+            return $stemmer->stem($value);
+        }, $data);
+    }
+}
diff --git a/Tokenizers/DutchStemmingTokenizer.php b/Tokenizers/DutchStemmingTokenizer.php
@@ -0,0 +1,17 @@
+<?php
+
+namespace VFou\Search\Tokenizers;
+
+use Wamania\Snowball\Dutch;
+
+class DutchStemmingTokenizer implements TokenizerInterface
+{
+
+    public static function tokenize($data)
+    {
+        return array_map(function($value){
+            $stemmer = new Dutch();
+            return $stemmer->stem($value);
+        }, $data);
+    }
+}
diff --git a/Tokenizers/EnglishStemmingTokenizer.php b/Tokenizers/EnglishStemmingTokenizer.php
@@ -0,0 +1,17 @@
+<?php
+
+namespace VFou\Search\Tokenizers;
+
+use Wamania\Snowball\English;
+
+class EnglishStemmingTokenizer implements TokenizerInterface
+{
+
+    public static function tokenize($data)
+    {
+        return array_map(function($value){
+            $stemmer = new English();
+            return $stemmer->stem($value);
+        }, $data);
+    }
+}
diff --git a/Tokenizers/FrenchStemmingTokenizer.php b/Tokenizers/FrenchStemmingTokenizer.php
@@ -0,0 +1,17 @@
+<?php
+
+namespace VFou\Search\Tokenizers;
+
+use Wamania\Snowball\French;
+
+class FrenchStemmingTokenizer implements TokenizerInterface
+{
+
+    public static function tokenize($data)
+    {
+        return array_map(function($value){
+            $stemmer = new French();
+            return $stemmer->stem($value);
+        }, $data);
+    }
+}
diff --git a/Tokenizers/GermanStemmingTokenizer.php b/Tokenizers/GermanStemmingTokenizer.php
@@ -0,0 +1,17 @@
+<?php
+
+namespace VFou\Search\Tokenizers;
+
+use Wamania\Snowball\German;
+
+class GermanStemmingTokenizer implements TokenizerInterface
+{
+
+    public static function tokenize($data)
+    {
+        return array_map(function($value){
+            $stemmer = new German();
+            return $stemmer->stem($value);
+        }, $data);
+    }
+}
diff --git a/Tokenizers/ItalianStemmingTokenizer.php b/Tokenizers/ItalianStemmingTokenizer.php
@@ -0,0 +1,17 @@
+<?php
+
+namespace VFou\Search\Tokenizers;
+
+use Wamania\Snowball\Italian;
+
+class ItalianStemmingTokenizer implements TokenizerInterface
+{
+
+    public static function tokenize($data)
+    {
+        return array_map(function($value){
+            $stemmer = new Italian();
+            return $stemmer->stem($value);
+        }, $data);
+    }
+}
diff --git a/Tokenizers/NorwegianStemmingTokenizer.php b/Tokenizers/NorwegianStemmingTokenizer.php
@@ -0,0 +1,17 @@
+<?php
+
+namespace VFou\Search\Tokenizers;
+
+use Wamania\Snowball\Norwegian;
+
+class NorwegianStemmingTokenizer implements TokenizerInterface
+{
+
+    public static function tokenize($data)
+    {
+        return array_map(function($value){
+            $stemmer = new Norwegian();
+            return $stemmer->stem($value);
+        }, $data);
+    }
+}
diff --git a/Tokenizers/PortugueseStemmingTokenizer.php b/Tokenizers/PortugueseStemmingTokenizer.php
@@ -0,0 +1,17 @@
+<?php
+
+namespace VFou\Search\Tokenizers;
+
+use Wamania\Snowball\Portuguese;
+
+class PortugueseStemmingTokenizer implements TokenizerInterface
+{
+
+    public static function tokenize($data)
+    {
+        return array_map(function($value){
+            $stemmer = new Portuguese();
+            return $stemmer->stem($value);
+        }, $data);
+    }
+}
diff --git a/Tokenizers/RomanianStemmingTokenizer.php b/Tokenizers/RomanianStemmingTokenizer.php
@@ -0,0 +1,17 @@
+<?php
+
+namespace VFou\Search\Tokenizers;
+
+use Wamania\Snowball\Romanian;
+
+class RomanianStemmingTokenizer implements TokenizerInterface
+{
+
+    public static function tokenize($data)
+    {
+        return array_map(function($value){
+            $stemmer = new Romanian();
+            return $stemmer->stem($value);
+        }, $data);
+    }
+}
diff --git a/Tokenizers/RussianStemmingTokenizer.php b/Tokenizers/RussianStemmingTokenizer.php
@@ -0,0 +1,17 @@
+<?php
+
+namespace VFou\Search\Tokenizers;
+
+use Wamania\Snowball\Russian;
+
+class RussianStemmingTokenizer implements TokenizerInterface
+{
+
+    public static function tokenize($data)
+    {
+        return array_map(function($value){
+            $stemmer = new Russian();
+            return $stemmer->stem($value);
+        }, $data);
+    }
+}
diff --git a/Tokenizers/SpanishStemmingTokenizer.php b/Tokenizers/SpanishStemmingTokenizer.php
@@ -0,0 +1,17 @@
+<?php
+
+namespace VFou\Search\Tokenizers;
+
+use Wamania\Snowball\Spanish;
+
+class SpanishStemmingTokenizer implements TokenizerInterface
+{
+
+    public static function tokenize($data)
+    {
+        return array_map(function($value){
+            $stemmer = new Spanish();
+            return $stemmer->stem($value);
+        }, $data);
+    }
+}
diff --git a/Tokenizers/SwedishStemmingTokenizer.php b/Tokenizers/SwedishStemmingTokenizer.php
@@ -0,0 +1,17 @@
+<?php
+
+namespace VFou\Search\Tokenizers;
+
+use Wamania\Snowball\Swedish;
+
+class SwedishStemmingTokenizer implements TokenizerInterface
+{
+
+    public static function tokenize($data)
+    {
+        return array_map(function($value){
+            $stemmer = new Swedish();
+            return $stemmer->stem($value);
+        }, $data);
+    }
+}
diff --git a/composer.json b/composer.json
@@ -13,7 +13,8 @@
     }
   ],
   "require": {
-    "php": "^7.0"
+    "php": "^7.0",
+    "wamania/php-stemmer": "^1.2"
   },
   "require-dev": {
   },
