Graph overhaul p2

ai/graph/CommunityDetector.js

@@ -89,7 +89,7 @@ class CommunityDetector {
       db.exec('BEGIN');
       try {
         db.exec('DELETE FROM communities;');
-        const insertCommStmt = db.prepare('INSERT INTO communities (id, label, node_count, updated_at) VALUES (?, ?, ?, datetime("now"))');
+        const insertCommStmt = db.prepare("INSERT INTO communities (id, label, node_count, updated_at) VALUES (?, ?, ?, datetime('now'))");
         const updateEntStmt = db.prepare('UPDATE entities SET community_id = ? WHERE id = ?');
 
         let cIndex = 1;

ai/graph/DeterministicSemanticMiner.js

@@ -0,0 +1,118 @@
+/**
+ * DeterministicSemanticMiner - Pattern-based semantic relationship extraction engine
+ * Extracts rich domain relationships (USES, DEPENDS_ON, IMPLEMENTS, COMMUNICATES_WITH, WORKS_ON)
+ * directly from prose text to complement neural GLiNER2-Relex extraction.
+ */
+
+const PATTERNS = [
+  {
+    regex: /\b([A-Za-z0-9_-]{2,})\s+(?:uses|is built with|relies on|utilizes|powered by)\s+([A-Za-z0-9_-]{2,})\b/gi,
+    type: 'USES',
+    confidence: 0.90
+  },
+  {
+    regex: /\b([A-Za-z0-9_-]{2,})\s+(?:depends on|requires|needs)\s+([A-Za-z0-9_-]{2,})\b/gi,
+    type: 'DEPENDS_ON',
+    confidence: 0.92
+  },
+  {
+    regex: /\b([A-Za-z0-9_-]{2,})\s+(?:generates|produces|creates|renders)\s+([A-Za-z0-9_-]{2,})\b/gi,
+    type: 'GENERATES',
+    confidence: 0.90
+  },
+  {
+    regex: /\b([A-Za-z0-9_-]{2,})\s+(?:integrates with|connects to|calls|communicates with|sends data to)\s+([A-Za-z0-9_-]{2,})\b/gi,
+    type: 'INTEGRATES_WITH',
+    confidence: 0.88
+  },
+  {
+    regex: /\b([A-Za-z0-9_-]{2,})\s+(?:implements|extends|inherits from)\s+([A-Za-z0-9_-]{2,})\b/gi,
+    type: 'IMPLEMENTS',
+    confidence: 0.92
+  },
+  {
+    regex: /\b([A-Za-z0-9_-]{2,})\s+(?:enables|allows|empowers|unlocks)\s+([A-Za-z0-9_-]{2,}(?:\s+[A-Za-z0-9_-]{2,})?)\b/gi,
+    type: 'ENABLES',
+    confidence: 0.90
+  },
+  {
+    regex: /\b([A-Za-z0-9_-]{2,})\s+(?:supports|handles|accommodates)\s+([A-Za-z0-9_-]{2,})\b/gi,
+    type: 'SUPPORTS',
+    confidence: 0.90
+  },
+  {
+    regex: /\b(?:Connect|Use|Integrate)\s+([A-Za-z0-9_-]+(?:\s+or\s+[A-Za-z0-9_-]+)?)\s+(?:API|keys|service|model)\b/gi,
+    type: 'USES_TECHNOLOGY',
+    confidence: 0.92
+  },
+  {
+    regex: /\b([A-Z][a-zA-Z0-9_-]{2,})\s+(?:Integration|Support|Search|Chat|Graph|Engine|Adapter|Parser)\b/g,
+    type: 'FEATURE_CONCEPT',
+    confidence: 0.88
+  },
+  {
+    regex: /\b([A-Za-z0-9_-]{2,})\s+(?:works on|maintains|manages|leads)\s+([A-Za-z0-9_-]{2,})\b/gi,
+    type: 'WORKS_ON',
+    confidence: 0.88
+  },
+  {
+    regex: /\b([A-Z][a-zA-Z0-9_-]{2,})\s+(?:tool|framework|library|database|api|engine|protocol|diagram|service|module)\b/gi,
+    type: 'USES_TECHNOLOGY',
+    confidence: 0.92
+  }
+];
+
+class DeterministicSemanticMiner {
+  /**
+   * Mine text for semantic relationships
+   * @param {string} text Raw or cleansed text
+   * @returns {Array<{ sourceText: string, targetText: string, type: string, confidence: number, rawSentence: string }>}
+   */
+  mine(text = '') {
+    if (!text || typeof text !== 'string') return [];
+    const results = [];
+    const seen = new Set();
+
+    // Split text into sentences
+    const sentences = text.split(/(?<=[.!?])\s+/);
+
+    for (const sentence of sentences) {
+      const cleanSent = sentence.trim();
+      if (cleanSent.length < 10) continue;
+
+      for (const pattern of PATTERNS) {
+        pattern.regex.lastIndex = 0;
+        let match;
+        while ((match = pattern.regex.exec(cleanSent)) !== null) {
+          const src = match[1] ? match[1].trim() : '';
+          const tgt = match[2] ? match[2].trim() : '';
+
+          const STOP_WORDS = new Set(['this', 'that', 'these', 'those', 'it', 'they', 'we', 'you', 'he', 'she', 'what', 'which', 'who', 'where', 'when', 'why', 'how', 'a', 'an', 'the', 'and', 'or', 'but', 'if', 'else', 'for', 'not', 'workspace', 'column', 'value', 'again', 'test', 'diagram', 'screenshot', 'image', 'interactive', 'search', 'connect', 'visualize']);
+          if (STOP_WORDS.has(src.toLowerCase()) || (tgt && STOP_WORDS.has(tgt.toLowerCase()))) continue;
+
+          // Reject noise strings, HTML/Markdown attributes, long terms > 25 chars
+          if (src.length > 25 || (tgt && tgt.length > 25)) continue;
+          if (/[{}=|]|\bdata-|\bvalue \d|\bcolumn \d/i.test(src) || (tgt && /[{}=|]|\bdata-|\bvalue \d|\bcolumn \d/i.test(tgt))) continue;
+
+          if (src && (tgt ? src.toLowerCase() !== tgt.toLowerCase() : true)) {
+            const key = `${src}:${pattern.type}:${tgt || 'Concept'}`;
+            if (!seen.has(key)) {
+              seen.add(key);
+              results.push({
+                sourceText: src,
+                targetText: tgt || src,
+                type: pattern.type,
+                confidence: pattern.confidence,
+                rawSentence: cleanSent
+              });
+            }
+          }
+        }
+      }
+    }
+
+    return results;
+  }
+}
+
+module.exports = DeterministicSemanticMiner;

ai/graph/EntityResolver.js

@@ -8,16 +8,194 @@ const { createLogger } = require('../core/logger');
 const log = createLogger('EntityResolver');
 
 class EntityResolver {
-  constructor(graphDb) {
+  constructor(graphDb, embeddingService = null) {
     this.graphDb = graphDb;
+    this.embeddingService = embeddingService;
+  }
+
+  setEmbeddingService(service) {
+    this.embeddingService = service;
+  }
+
+  _cosineSimilarity(v1, v2) {
+    if (!v1 || !v2 || v1.length !== v2.length) return 0;
+    let dot = 0, norm1 = 0, norm2 = 0;
+    for (let i = 0; i < v1.length; i++) {
+      dot += v1[i] * v2[i];
+      norm1 += v1[i] * v1[i];
+      norm2 += v2[i] * v2[i];
+    }
+    if (norm1 === 0 || norm2 === 0) return 0;
+    return dot / (Math.sqrt(norm1) * Math.sqrt(norm2));
+  }
+
+  async resolveMentionVector(clean, _sanitizedType) {
+    if (!this.embeddingService || !this.graphDb?.db) return null;
+    try {
+      const candidateVector = await this.embeddingService.generateVector(clean);
+      if (!candidateVector) return null;
+
+      const storedVectors = this.graphDb.getAllEntityVectors();
+      if (storedVectors.length === 0) return null;
+
+      let bestMatch = null;
+      let highestSimilarity = 0.88; // Threshold for automatic vector concept merging
+
+      for (const item of storedVectors) {
+        const sim = this._cosineSimilarity(candidateVector, item.vector);
+        if (sim > highestSimilarity) {
+          highestSimilarity = sim;
+          bestMatch = item.entityId;
+        }
+      }
+
+      if (bestMatch) {
+        const existing = this.graphDb.db.prepare('SELECT id, name, canonical_name, type FROM entities WHERE id = ?').get(bestMatch);
+        if (existing) {
+          log.info(`Vector concept merged "${clean}" -> "${existing.canonical_name}" (similarity: ${highestSimilarity.toFixed(3)})`);
+          this.addAlias(clean, existing.id, parseFloat(highestSimilarity.toFixed(3)));
+          return {
+            id: existing.id,
+            name: existing.name,
+            canonical_name: existing.canonical_name,
+            type: existing.type,
+            isAlias: true
+          };
+        }
+      }
+    } catch (err) {
+      log.debug('Vector resolution skipped:', err.message);
+    }
+    return null;
+  }
+
+  /**
+   * Helper: Normalize string by decoding URI encoding and trimming whitespace
+   */
+  cleanName(str) {
+    if (!str || typeof str !== 'string') return '';
+    let s = str.trim();
+    try {
+      s = decodeURIComponent(s);
+    } catch { /* ignore URI decode error */ }
+    return s.replace(/\s+/g, ' ').trim();
+  }
+
+  /**
+   * Comprehensive Quality Gate: Checks if entity candidate string is valid knowledge term
+   */
+  /**
+   * Universal Linguistic Quality Gate (Zero hardcoded entity lists)
+   * Enforces grammatical boundary rules, character entropy, and syntax artifact filters.
+   */
+  isValidEntityName(name) {
+    if (!name || typeof name !== 'string') return false;
+    const clean = name.trim();
+    const norm = clean.toLowerCase();
+
+    // 1. Min/Max Length & Acronym Rule (2 & 3 char words must be uppercase acronyms like AI, UI, DB, API, SDK, CLI, SQL, APP, WEB)
+    if (clean.length < 2 || clean.length > 35 || clean.split(/\s+/).length > 4) return false;
+    const WHITELIST_SHORT = new Set(['AI', 'UI', 'UX', 'DB', 'JS', 'TS', 'IP', 'OS', 'ID', 'IT', 'API', 'SDK', 'CLI', 'SQL', 'APP', 'WEB', 'CPU', 'RAM', 'URL', 'SSH', 'SSL', 'CSV', 'XML', 'PNG', 'JPG', 'SVG', 'PDF']);
+    if (clean.length <= 3 && !WHITELIST_SHORT.has(clean.toUpperCase())) return false;
+
+    const words = norm.split(/\s+/);
+
+    // 2. Grammatical Boundary Rule: Cannot start or end with prepositions, articles, connectives, verbs, or UI tokens
+    const GRAMMAR_BOUNDARIES = new Set([
+      'a', 'an', 'the', 'and', 'or', 'but', 'for', 'in', 'on', 'at', 'to', 'from',
+      'by', 'with', 'of', 'as', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
+      'if', 'else', 'so', 'than', 'too', 'very', 'not', 'no', 'nor', 'it', 'its',
+      'create', 'update', 'delete', 'connect', 'build', 'run', 'make', 'use', 'get', 'set', 'help', 'test',
+      'again', 'teh', 'weh', 'value', 'column'
+    ]);
+    if (GRAMMAR_BOUNDARIES.has(words[0]) || GRAMMAR_BOUNDARIES.has(words[words.length - 1])) return false;
+
+    // 3. Sentence Clause & Aux Verb Rule: Reject clause fragments containing auxiliary verbs
+    if (/\b(will|would|could|should|have|has|had|help|helps|test)\b/i.test(clean)) return false;
+
+    // 4. Character Entropy & Phonetic Rule: Must contain vowels; reject repeated chars & invalid consonant clusters
+    if (!/[aeiouy]/i.test(norm)) return false; // Must contain at least one vowel
+    if (/(.)\1{3,}/.test(norm)) return false;  // Reject 4+ repeated chars (e.g. "dddde")
+    if (/[bcdfghjklmnpqrstvwxyz]{5,}/i.test(norm)) return false; // Reject 5+ consecutive consonants
+    if (/^\.[a-z]{1,2}\b/i.test(norm)) return false;
+
+    // 5. Markup & Syntax Artifact Rule: Reject editor markup, HTML attributes, numbers with decimals, table cells
+    if (/[{}=|]|\bdata-|\d+\.\d+|\bvalue \d|\bcolumn \d|^[-*+\s:#=]+$/i.test(norm)) return false;
+
+    return true;
+  }
+
+  /**
+   * Validate and sanitize entity type classification, coercing misclassifications to Concept.
+   * Pure algorithmic & pattern-based rules - ZERO hardcoded entity or person name lists.
+   */
+  sanitizeEntityType(name, proposedType = 'Concept') {
+    if (!this.isValidEntityName(name)) return null;
+
+    const clean = name.trim();
+    const norm = clean.toLowerCase();
+    const words = clean.split(/\s+/);
+    const CONNECTIVES = new Set(['and', 'or', 'for', 'in', 'on', 'at', 'to', 'from', 'with', 'by', 'of']);
+
+    // Rule 1: Multi-word Title-Cased Proper Name Pattern (e.g. "Bikash Panda", "Abhiram Panda", "Ada Lovelace")
+    const isMultiWordTitleCase = words.length >= 2 && words.every(w => /^[A-Z][a-z]+$/.test(w));
+    if (isMultiWordTitleCase && (proposedType === 'Person' || proposedType === 'Organization' || proposedType === 'Concept')) {
+      return 'Person';
+    }
+
+    // Rule 2: Single-word capitalized proper name (e.g. "Abhiram", "Bikash") proposed as Person or Organization
+    const isSingleTitleCase = words.length === 1 && /^[A-Z][a-z]{2,}$/.test(clean);
+    const COMMON_NON_PERSONS = new Set(['workspace', 'system', 'search', 'note', 'document', 'screenshot', 'diagram', 'project', 'settings', 'connect', 'api', 'column', 'value', 'test', 'again', 'create', 'teh']);
+    if (isSingleTitleCase && (proposedType === 'Person' || proposedType === 'Organization')) {
+      if (COMMON_NON_PERSONS.has(norm)) return 'Concept';
+      return 'Person';
+    }
+
+    // Rule 3: Common Non-Person Nouns cannot be Person
+    if (proposedType === 'Person' && COMMON_NON_PERSONS.has(norm)) {
+      return 'Concept';
+    }
+
+    // Rule 4: Structural media terms (screenshot, diagram, image) cannot be Event, Location, or Task
+    if ((proposedType === 'Event' || proposedType === 'Location' || proposedType === 'Task') && /^(screenshot|diagram|image|photo|drawing|picture|file|note)$/i.test(norm)) {
+      return 'Concept';
+    }
+
+    // Rule 5: Non-Task clauses ending in conjunctions or connectives
+    if (proposedType === 'Task' && CONNECTIVES.has(words[words.length - 1].toLowerCase())) {
+      return 'Concept';
+    }
+
+    // Rule 6: Strict Organization Typing
+    // Organization MUST contain explicit org indicators (Corp, Inc, Ltd, Company, Technologies, Labs, Group, Foundation, Team)
+    // Coerce misclassifications ("Interactive", "Support You", "Abhiram", "Integration") from Organization to Concept or Person.
+    if (proposedType === 'Organization') {
+      const HAS_ORG_SUFFIX = /\b(corp|inc|ltd|company|technologies|labs|group|foundation|team|studio|org|agency|institute)\b/i.test(clean);
+      if (!HAS_ORG_SUFFIX && !isMultiWordTitleCase) {
+        return 'Concept';
+      }
+    }
+
+    // Rule 7: Strict Location, Project & Task Coercion
+    // Generic UI terms, action phrases, or short greetings ("Interactive", "Hello World", "Integration", "Note Graph: Visualize") coerce to Concept
+    if (proposedType === 'Location' || proposedType === 'Project' || proposedType === 'Task') {
+      const GENERIC_PHRASES = new Set(['hello world', 'integration', 'interactive', 'support you', 'visualize', 'note graph', 'note graph: visualize']);
+      if (GENERIC_PHRASES.has(norm) || norm.includes(':')) {
+        return 'Concept';
+      }
+    }
+
+    return proposedType || 'Concept';
   }
 
   /**
    * Deterministic entity ID generation via SHA-256 (supports international non-ASCII characters)
    */
   generateEntityId(name, type = 'Entity') {
-    const normName = String(name || '').trim().toLowerCase();
-    const hash = crypto.createHash('sha256').update(`${type.toLowerCase()}:${normName}`).digest('hex').slice(0, 16);
+    const valid = this.cleanName(name);
+    const sanitizedType = this.sanitizeEntityType(valid, type) || 'Concept';
+    const normName = valid.toLowerCase();
+    const hash = crypto.createHash('sha256').update(`${sanitizedType.toLowerCase()}:${normName}`).digest('hex').slice(0, 16);
     return `ent-${hash}`;
   }
 
@@ -26,7 +204,7 @@ class EntityResolver {
    */
   resolveMention(mentionName, type = 'Entity') {
     if (!mentionName || typeof mentionName !== 'string') return null;
-    const clean = mentionName.trim();
+    const clean = this.cleanName(mentionName);
     if (clean.length === 0) return null;
 
     const aliasMatch = this.findAlias(clean);
@@ -40,14 +218,17 @@ class EntityResolver {
       };
     }
 
+    const sanitizedType = this.sanitizeEntityType(clean, type);
+    if (!sanitizedType) return null;
+
     // Reuse existing canonical entity ID if present in database to prevent type fragmentation
     if (this.graphDb?.db) {
       try {
         const existing = this.graphDb.db.prepare(
           'SELECT id, name, canonical_name, type FROM entities WHERE LOWER(name) = LOWER(?) OR LOWER(canonical_name) = LOWER(?) LIMIT 1'
         ).get(clean, clean);
         if (existing) {
-          const resolvedType = (existing.type && existing.type !== 'Concept') ? existing.type : type;
+          const resolvedType = this.sanitizeEntityType(clean, existing.type) || sanitizedType;
           return {
             id: existing.id,
             name: existing.name || clean,
@@ -59,12 +240,12 @@ class EntityResolver {
       } catch { /* ignore DB lookup error */ }
     }
 
-    const defaultId = this.generateEntityId(clean, type);
+    const defaultId = this.generateEntityId(clean, sanitizedType);
     return {
       id: defaultId,
       name: clean,
       canonical_name: clean,
-      type,
+      type: sanitizedType,
       isAlias: false
     };
   }