canonical_senf_pln

.dockerignore

@@ -0,0 +1,4 @@
+.git
+.env
+__pycache__
+*.pyc

.env.example

@@ -1,60 +1,60 @@
-#  LLM 
-OPENAI_API_KEY=sk-...
-OPENAI_MODEL=openai/gpt-4o-mini
-
-#  Parser 
-# Options: nl2pln | canonical_pln | manhin | langextract
-PARSER=canonical_pln
-PLNRAG_PARSER=canonical_pln
-NL2PLN_MODULE_PATH=data/simba_all.json
-CANONICAL_PLN_NL2PLN_MODULE_PATH=data/simba_canonical_pln.json
-
-# LangExtract parser (GPT/OpenAI-backed)
-LANGEXTRACT_API_KEY=
-LANGEXTRACT_MODEL_ID=gpt-4o-mini
-LANGEXTRACT_MODEL_URL=
-LANGEXTRACT_EXAMPLES_PATH=data/langextract_examples.json
-LANGEXTRACT_EXTRACTION_PASSES=1
-LANGEXTRACT_MAX_WORKERS=1
-LANGEXTRACT_SKIP_FUZZY=true
-
-#  Vector store 
-QDRANT_URL=http://localhost:6333
-QDRANT_COLLECTION=pln_rag
-OLLAMA_URL=http://localhost:11434/api/embeddings
-OLLAMA_MODEL=nomic-embed-text
-
-#  Atomspace 
-ATOMSPACE_PATH=data/atomspace/kb.metta
-
-#  FAISS (used by Manhin parser) 
-FAISS_PATH=data/faiss
-
-#  Processing 
-CHUNK_SIZE=512
-CHUNK_OVERLAP=64
-CONTEXT_TOP_K=10
-PARSER_BATCH_SENTENCES=4
-PARSER_BATCH_MAX_CHARS=2000
-
-#  Reasoning 
-CHAINING_TIMEOUT=180
-CHAINING_MAX_STEPS=100
-
-#  Query execution 
-QUERY_FALLBACK_ENABLED=true
-
-# ConceptNet background knowledge
-CONCEPTNET_ENABLED=false
-CONCEPTNET_AUTOLOAD=true
-CONCEPTNET_INPUT_FILE=data/conceptnet/conceptnet-assertions-5.7.0.csv.gz
-CONCEPTNET_ATOMSPACE_PATH=data/conceptnet/conceptnet_background.metta
-CONCEPTNET_VECTOR_PAYLOAD_PATH=data/conceptnet/conceptnet_background.jsonl
-CONCEPTNET_MANIFEST_PATH=data/conceptnet/conceptnet_manifest.json
-CONCEPTNET_INDEX_ON_STARTUP=true
-CONCEPTNET_MIN_WEIGHT=2.0
-CONCEPTNET_COVERAGE_PERCENT=100.0
-CONCEPTNET_SAMPLE_SEED=42
-CONCEPTNET_AUTO_REBUILD_ON_CHANGE=true
-CONCEPTNET_REINDEX_ON_RESET=true
-CONCEPTNET_STARTUP_FAIL_OPEN=true
+#  LLM 
+OPENAI_API_KEY=sk-...
+OPENAI_MODEL=openai/gpt-4o-mini
+
+#  Parser 
+# Options: nl2pln | canonical_pln | manhin | langextract
+PARSER=canonical_pln
+PLNRAG_PARSER=canonical_pln
+NL2PLN_MODULE_PATH=data/simba_all.json
+CANONICAL_PLN_NL2PLN_MODULE_PATH=data/simba_canonical_pln.json
+
+# LangExtract parser (GPT/OpenAI-backed)
+LANGEXTRACT_API_KEY=
+LANGEXTRACT_MODEL_ID=gpt-4o-mini
+LANGEXTRACT_MODEL_URL=
+LANGEXTRACT_EXAMPLES_PATH=data/langextract_examples.json
+LANGEXTRACT_EXTRACTION_PASSES=1
+LANGEXTRACT_MAX_WORKERS=1
+LANGEXTRACT_SKIP_FUZZY=true
+
+#  Vector store 
+QDRANT_URL=http://localhost:6333
+QDRANT_COLLECTION=pln_rag
+OLLAMA_URL=http://localhost:11434/api/embeddings
+OLLAMA_MODEL=nomic-embed-text
+
+#  Atomspace 
+ATOMSPACE_PATH=data/atomspace/kb.metta
+
+#  FAISS (used by Manhin parser) 
+FAISS_PATH=data/faiss
+
+#  Processing 
+CHUNK_SIZE=512
+CHUNK_OVERLAP=64
+CONTEXT_TOP_K=10
+PARSER_BATCH_SENTENCES=4
+PARSER_BATCH_MAX_CHARS=2000
+
+#  Reasoning 
+CHAINING_TIMEOUT=180
+CHAINING_MAX_STEPS=100
+
+#  Query execution 
+QUERY_FALLBACK_ENABLED=true
+
+# ConceptNet background knowledge
+CONCEPTNET_ENABLED=false
+CONCEPTNET_AUTOLOAD=true
+CONCEPTNET_INPUT_FILE=data/conceptnet/conceptnet-assertions-5.7.0.csv.gz
+CONCEPTNET_ATOMSPACE_PATH=data/conceptnet/conceptnet_background.metta
+CONCEPTNET_VECTOR_PAYLOAD_PATH=data/conceptnet/conceptnet_background.jsonl
+CONCEPTNET_MANIFEST_PATH=data/conceptnet/conceptnet_manifest.json
+CONCEPTNET_INDEX_ON_STARTUP=true
+CONCEPTNET_MIN_WEIGHT=2.0
+CONCEPTNET_COVERAGE_PERCENT=100.0
+CONCEPTNET_SAMPLE_SEED=42
+CONCEPTNET_AUTO_REBUILD_ON_CHANGE=true
+CONCEPTNET_REINDEX_ON_RESET=true
+CONCEPTNET_STARTUP_FAIL_OPEN=true

.gitignore

@@ -1,16 +1,16 @@
-.env
-__pycache__/
-*.pyc
-*.pyo
-data/atomspace/
-data/faiss/
-*.metta
-docs/reference/
-*matrix*.json
-*_sanity.json
-berekets_fallback_on_off.json
-.venv
-local-deps/
-data/benchmarks/*
-!data/benchmarks/stress25_v1.json
-data/conceptnet/conceptnet-assertions-5.7.0.csv.gz
+.env
+__pycache__/
+*.pyc
+*.pyo
+data/atomspace/
+data/faiss/
+*.metta
+docs/reference/
+*matrix*.json
+*_sanity.json
+berekets_fallback_on_off.json
+.venv
+local-deps/
+data/benchmarks/*
+!data/benchmarks/stress25_v1.json
+data/conceptnet/conceptnet-assertions-5.7.0.csv.gz

Dockerfile

@@ -25,8 +25,8 @@ FROM ubuntu:22.04
 ENV DEBIAN_FRONTEND=noninteractive
 ENV PYTHONUNBUFFERED=1
 ENV PYTHONDONTWRITEBYTECODE=1
-ENV PETTA_COMMIT=e1490899cefc67c128d5311ff4861f9997674957
-ENV PETTACHAINER_COMMIT=d21b93b5132a7fc8722f64d57b74fb7c3a8d1faa
+ENV PETTA_COMMIT=6b7f52f064bdbc82fabd0a0998404121fb01d52e
+ENV PETTACHAINER_COMMIT=9f44164dd3252ccf8e9c63a4b84caec59f56080e
 
 RUN apt-get update && apt-get install -y \
     software-properties-common \
@@ -65,6 +65,7 @@ WORKDIR /app
 COPY requirements.txt .
 RUN rm -rf /usr/lib/python3/dist-packages/blinker*
 RUN pip3 install --default-timeout=1000 -r requirements.txt
+RUN python3 -m nltk.downloader wordnet omw-1.4
 
 COPY . .
 

api/main.py

@@ -98,5 +98,9 @@ async def health():
         conceptnet_vectors_indexed=info["conceptnet_vectors_indexed"],
         conceptnet_vectors_expected=info["conceptnet_vectors_expected"],
         conceptnet_last_error=info["conceptnet_last_error"],
+        synonym_resolution_enabled=info["synonym_resolution_enabled"],
+        synonym_cached_pairs=info["synonym_cached_pairs"],
+        synonym_cached_synonyms=info["synonym_cached_synonyms"],
+        synonym_last_error=info["synonym_last_error"],
         uptime_seconds=round(time.time() - _start_time, 1),
     )

api/models.py

@@ -83,4 +83,8 @@ class HealthResponse(BaseModel):
     conceptnet_vectors_indexed: int
     conceptnet_vectors_expected: int
     conceptnet_last_error: str
+    synonym_resolution_enabled: bool
+    synonym_cached_pairs: int
+    synonym_cached_synonyms: int
+    synonym_last_error: str
     uptime_seconds: float

benchmark_parsers.py

@@ -136,10 +136,11 @@ def _is_truthy(value: Any) -> bool:
     text = str(value).strip().lower()
     return text in {"1", "true", "yes", "y"}
 
-ACTIVE_PARSERS = ("nl2pln", "canonical_pln")
+ACTIVE_PARSERS = ("nl2pln", "canonical_pln", "canonical_senf_pln")
 AVAILABLE_PARSERS = (
     "nl2pln",
     "canonical_pln",
+    "canonical_senf_pln",
     "langextract",
     "canonical_langextract",
     "canonical_pln_1686527",
@@ -251,6 +252,10 @@ def _get_parser_factory(name: str):
         from parsers.canonical_pln_parser import CanonicalPLNParser
 
         return CanonicalPLNParser
+    if name == "canonical_senf_pln":
+        from parsers.canonical_senf_pln_parser import CanonicalSenfPlnParser
+
+        return CanonicalSenfPlnParser
     if name == "langextract":
         from parsers.langextract_pln_parser import LangExtractPLNParser
 

compare_parsers.py

@@ -7,11 +7,13 @@
 
 def _load_parser_factories() -> dict[str, Callable[[], object]]:
     from parsers.canonical_pln_parser import CanonicalPLNParser
+    from parsers.canonical_senf_pln_parser import CanonicalSenfPlnParser
     from parsers.nl2pln_parser import NL2PLNParser
 
     factories: dict[str, Callable[[], object]] = {
         "nl2pln": NL2PLNParser,
         "canonical_pln": CanonicalPLNParser,
+        "canonical_senf_pln": CanonicalSenfPlnParser,
     }
 
     try:

config.py

@@ -9,7 +9,7 @@ class Settings(BaseSettings):
     openai_api_key: str
     openai_model: str = "openai/gpt-4o-mini"
 
-    # Options: "nl2pln" | "canonical_pln" | "manhin" | "langextract"
+    # Options: "nl2pln" | "canonical_pln" | "manhin" | "langextract" | "canonical_senf_pln"
     parser: str = "canonical_pln"
     nl2pln_module_path: str = "data/simba_all.json"
     canonical_pln_nl2pln_module_path: str = "data/simba_canonical_pln.json"
@@ -50,6 +50,21 @@ class Settings(BaseSettings):
     # Query execution
     query_fallback_enabled: bool = True
 
+    synonym_resolution_enabled: bool = True
+    synonym_cache_path: str = "data/synonyms/relations.json"
+    synonym_wordnet_enabled: bool = True
+    synonym_conceptnet_lookup_enabled: bool = True
+    synonym_conceptnet_url: str = "https://api.conceptnet.io"
+    synonym_conceptnet_limit: int = 50
+    synonym_embedding_enabled: bool = True
+    synonym_embedding_threshold: float = 0.68
+    synonym_embedding_top_k: int = 3
+    synonym_max_knowledge_terms: int = 64
+    synonym_max_verifications_per_query: int = 6
+    synonym_verifier_model: Optional[str] = None
+    synonym_verifier_min_confidence: float = 0.85
+    synonym_request_timeout: float = 10.0
+
     # Maximum number of query candidates to try before giving up.
     # Applies to all parsers when query_fallback_enabled is true.
     # Set to 0 to disable the cap.

core/exemplar_registry.py

@@ -0,0 +1,59 @@
+from typing import List, Dict, Tuple
+from core.senf import SENF, SENFExemplar, SENFEntity
+
+class ExemplarScorer:
+    """
+    For each entity-kind pair, score distances to a small exemplar set.
+    """
+
+    # Small exemplar registry for common and domain-relevant kinds:
+    REGISTRY: Dict[str, List[str]] = {
+        "Camera": ["professional_camera", "consumer_camera", "phone_camera", "security_camera"],
+        "Game": ["chess_game", "football_game", "childrens_game", "video_game"],
+        "Bird": ["robin", "eagle", "penguin", "ostrich"],
+        "Treatment": ["drug_treatment", "surgical_treatment", "behavioral_treatment"]
+    }
+
+    # Simple lexical cues for the MVP
+    CUES: Dict[str, Dict[str, float]] = {
+        "nikon": {"professional_camera": 0.12, "consumer_camera": 0.48},
+        "lens": {"professional_camera": 0.2, "consumer_camera": 0.5},
+        "strategy": {"chess_game": 0.05, "football_game": 0.80},
+        "exhausting": {"football_game": 0.06, "chess_game": 0.82}
+    }
+
+    def score(self, senf: SENF, context_text: str = "") -> None:
+        """
+        Populate exemplars in the SENF object based on entities and context.
+        """
+        context_lower = context_text.lower()
+
+        for ent_id, entity in senf.entities.items():
+            if not entity.kind:
+                continue
+
+            # If we don't have exemplars for this kind, skip
+            if entity.kind not in self.REGISTRY:
+                continue
+
+            prototypes = self.REGISTRY[entity.kind]
+
+            # Simple heuristic distance calculation for the MVP
+            for proto in prototypes:
+                distance = 0.5 # default moderate distance
+
+                # Check lexical cues in the context
+                for cue, distances in self.CUES.items():
+                    if cue in context_lower:
+                        if proto in distances:
+                            distance = distances[proto]
+                            break
+
+                senf.exemplars.append(
+                    SENFExemplar(
+                        entity_id=ent_id,
+                        kind=entity.kind,
+                        prototype=proto,
+                        distance=distance
+                    )
+                )