Merge pull request #815 from VesnaT/keywords_pp_fix

[FIX] Keywords: Always invoke preprocessor __call__

Author: PrimozGodec

orangecontrib/text/widgets/owkeywords.py

@@ -94,7 +94,9 @@ def callback(i: float, status=""):
         # Normalize words
         for preprocessor in corpus.used_preprocessor.preprocessors:
             if isinstance(preprocessor, BaseNormalizer):
-                words = [preprocessor.normalizer(w) for w in words]
+                dummy = Corpus(Domain((), metas=[StringVariable("Words")]),
+                               metas=np.array(words)[:, None])
+                words = list(preprocessor(dummy).tokens.flatten())
 
         # Filter scores using words
         existing_words = [w for w in set(words) if w in scores.index]

orangecontrib/text/widgets/tests/test_owkeywords.py

@@ -85,7 +85,7 @@ def test_run_with_words(self):
         self.assertEqual(len(results.scores), 42)
 
     def test_run_normalize_words(self):
-        normalizer = WordNetLemmatizer()
+        normalizer = LemmagenLemmatizer()
         corpus = normalizer(self.corpus)
 
         words = ["minor", "tree"]