remove extra-phrase marker from index_tokenizer_with_stopwords

Author: alok1304

src/licensedcode/data/rules/bsd-new_578.RULE

@@ -16,7 +16,7 @@ are met:
    copyright notice, this list of conditions and the following
    disclaimer in the documentation and/or other materials provided
    with the distribution.
- * Neither the name of nor the names of its
+ * Neither the name of  nor the names of its
    contributors may be used to endorse or promote products derived
    from this software without specific prior written permission.
 

src/licensedcode/match.py

@@ -597,7 +597,7 @@ def score(self):
         computed from the number of matched tokens, the number of query tokens
         in the matched range (including unknowns and unmatched) and the matched
         rule relevance.
-        """       
+        """
         # relevance is a number between 0 and 100. Divide by 100
         relevance = self.rule.relevance / 100
         if not relevance:

src/licensedcode/tokenize.py

@@ -85,6 +85,9 @@ def query_lines(
 extra_phrase_pattern = r'(?:' + query_pattern + r'|\[\[|\]\])'
 extra_phrase_splitter = re.compile(extra_phrase_pattern, re.UNICODE).findall
 
+
+extra_phrase_removal_pattern = re.compile(r'\[\[\d+\]\]')
+
 REQUIRED_PHRASE_OPEN = '{{'
 REQUIRED_PHRASE_CLOSE = '}}'
 
@@ -348,6 +351,8 @@ def index_tokenizer_with_stopwords(text, stopwords=STOPWORDS):
     """
     if not text:
         return [], {}
+    
+    text = extra_phrase_removal_pattern.sub('', text)
 
     tokens = []
     tokens_append = tokens.append