From 08659e450b2f185368e7afbfcbdf7c8002a15e90 Mon Sep 17 00:00:00 2001 From: Bikash Panda Date: Wed, 5 Aug 2026 19:48:11 +0530 Subject: [PATCH 1/6] feat(graph): overhaul semantic extraction quality and enterprise resilience Add DeterministicSemanticMiner for domain pattern relationship extraction. Calibrate GLiNER2 ONNX span logits & fix subword boundary indexing. Implement universal linguistic quality gate & type coercion rules. Eliminate hardcoded name lists in favor of POS & phonetic entropy rules. Add background ingestion queue & SQLite WAL concurrency handling. --- ai/graph/CommunityDetector.js | 2 +- ai/graph/DeterministicSemanticMiner.js | 118 ++++ ai/graph/EntityResolver.js | 117 +++- ai/graph/EvidenceFusionEngine.js | 17 +- ai/graph/GraphBuilder.js | 68 +- ai/graph/GraphDB.js | 308 ++++++++- ai/graph/GraphMaintenance.js | 11 +- ai/graph/GraphSchema.js | 2 + ai/graph/GraphService.js | 595 ++++++++++++------ ai/graph/GraphValidationEngine.js | 25 +- ai/graph/MarkdownASTParser.js | 66 +- .../semantic/adapters/GLiNER2RelexAdapter.js | 216 ++++--- .../WorkspaceMetadataKnowledgeSource.js | 41 +- ai/utils/ipcProtocol.js | 2 + electron/ai/aiHandlers.cjs | 28 + electron/preload.cjs | 2 + src/ai/utils/ipcProtocol.js | 2 + src/components/KnowledgeGraph.jsx | 63 +- src/services/electronService.js | 16 + tests/golden_workspace.test.js | 82 +++ tests/graph.test.js | 45 ++ tests/graph_enterprise.test.js | 582 +++++++++++++++++ tests/semantic_quality_precision.test.js | 206 ++++++ 23 files changed, 2225 insertions(+), 389 deletions(-) create mode 100644 ai/graph/DeterministicSemanticMiner.js create mode 100644 tests/golden_workspace.test.js create mode 100644 tests/graph_enterprise.test.js create mode 100644 tests/semantic_quality_precision.test.js diff --git a/ai/graph/CommunityDetector.js b/ai/graph/CommunityDetector.js index b6c8d31d..644cc026 100644 --- a/ai/graph/CommunityDetector.js +++ b/ai/graph/CommunityDetector.js @@ -89,7 +89,7 @@ class CommunityDetector { db.exec('BEGIN'); try { db.exec('DELETE FROM communities;'); - const insertCommStmt = db.prepare('INSERT INTO communities (id, label, node_count, updated_at) VALUES (?, ?, ?, datetime("now"))'); + const insertCommStmt = db.prepare("INSERT INTO communities (id, label, node_count, updated_at) VALUES (?, ?, ?, datetime('now'))"); const updateEntStmt = db.prepare('UPDATE entities SET community_id = ? WHERE id = ?'); let cIndex = 1; diff --git a/ai/graph/DeterministicSemanticMiner.js b/ai/graph/DeterministicSemanticMiner.js new file mode 100644 index 00000000..fd620e56 --- /dev/null +++ b/ai/graph/DeterministicSemanticMiner.js @@ -0,0 +1,118 @@ +/** + * DeterministicSemanticMiner - Pattern-based semantic relationship extraction engine + * Extracts rich domain relationships (USES, DEPENDS_ON, IMPLEMENTS, COMMUNICATES_WITH, WORKS_ON) + * directly from prose text to complement neural GLiNER2-Relex extraction. + */ + +const PATTERNS = [ + { + regex: /\b([A-Za-z0-9_-]{2,})\s+(?:uses|is built with|relies on|utilizes|powered by)\s+([A-Za-z0-9_-]{2,})\b/gi, + type: 'USES', + confidence: 0.90 + }, + { + regex: /\b([A-Za-z0-9_-]{2,})\s+(?:depends on|requires|needs)\s+([A-Za-z0-9_-]{2,})\b/gi, + type: 'DEPENDS_ON', + confidence: 0.92 + }, + { + regex: /\b([A-Za-z0-9_-]{2,})\s+(?:generates|produces|creates|renders)\s+([A-Za-z0-9_-]{2,})\b/gi, + type: 'GENERATES', + confidence: 0.90 + }, + { + regex: /\b([A-Za-z0-9_-]{2,})\s+(?:integrates with|connects to|calls|communicates with|sends data to)\s+([A-Za-z0-9_-]{2,})\b/gi, + type: 'INTEGRATES_WITH', + confidence: 0.88 + }, + { + regex: /\b([A-Za-z0-9_-]{2,})\s+(?:implements|extends|inherits from)\s+([A-Za-z0-9_-]{2,})\b/gi, + type: 'IMPLEMENTS', + confidence: 0.92 + }, + { + regex: /\b([A-Za-z0-9_-]{2,})\s+(?:enables|allows|empowers|unlocks)\s+([A-Za-z0-9_-]{2,}(?:\s+[A-Za-z0-9_-]{2,})?)\b/gi, + type: 'ENABLES', + confidence: 0.90 + }, + { + regex: /\b([A-Za-z0-9_-]{2,})\s+(?:supports|handles|accommodates)\s+([A-Za-z0-9_-]{2,})\b/gi, + type: 'SUPPORTS', + confidence: 0.90 + }, + { + regex: /\b(?:Connect|Use|Integrate)\s+([A-Za-z0-9_-]+(?:\s+or\s+[A-Za-z0-9_-]+)?)\s+(?:API|keys|service|model)\b/gi, + type: 'USES_TECHNOLOGY', + confidence: 0.92 + }, + { + regex: /\b([A-Z][a-zA-Z0-9_-]{2,})\s+(?:Integration|Support|Search|Chat|Graph|Engine|Adapter|Parser)\b/g, + type: 'FEATURE_CONCEPT', + confidence: 0.88 + }, + { + regex: /\b([A-Za-z0-9_-]{2,})\s+(?:works on|maintains|manages|leads)\s+([A-Za-z0-9_-]{2,})\b/gi, + type: 'WORKS_ON', + confidence: 0.88 + }, + { + regex: /\b([A-Z][a-zA-Z0-9_-]{2,})\s+(?:tool|framework|library|database|api|engine|protocol|diagram|service|module)\b/gi, + type: 'USES_TECHNOLOGY', + confidence: 0.92 + } +]; + +class DeterministicSemanticMiner { + /** + * Mine text for semantic relationships + * @param {string} text Raw or cleansed text + * @returns {Array<{ sourceText: string, targetText: string, type: string, confidence: number, rawSentence: string }>} + */ + mine(text = '') { + if (!text || typeof text !== 'string') return []; + const results = []; + const seen = new Set(); + + // Split text into sentences + const sentences = text.split(/(?<=[.!?])\s+/); + + for (const sentence of sentences) { + const cleanSent = sentence.trim(); + if (cleanSent.length < 10) continue; + + for (const pattern of PATTERNS) { + pattern.regex.lastIndex = 0; + let match; + while ((match = pattern.regex.exec(cleanSent)) !== null) { + const src = match[1] ? match[1].trim() : ''; + const tgt = match[2] ? match[2].trim() : ''; + + const STOP_WORDS = new Set(['this', 'that', 'these', 'those', 'it', 'they', 'we', 'you', 'he', 'she', 'what', 'which', 'who', 'where', 'when', 'why', 'how', 'a', 'an', 'the', 'and', 'or', 'but', 'if', 'else', 'for', 'not', 'workspace', 'column', 'value', 'again', 'test', 'diagram', 'screenshot', 'image', 'interactive', 'search', 'connect', 'visualize']); + if (STOP_WORDS.has(src.toLowerCase()) || (tgt && STOP_WORDS.has(tgt.toLowerCase()))) continue; + + // Reject noise strings, HTML/Markdown attributes, long terms > 25 chars + if (src.length > 25 || (tgt && tgt.length > 25)) continue; + if (/[{}=|]|\bdata-|\bvalue \d|\bcolumn \d/i.test(src) || (tgt && /[{}=|]|\bdata-|\bvalue \d|\bcolumn \d/i.test(tgt))) continue; + + if (src && (tgt ? src.toLowerCase() !== tgt.toLowerCase() : true)) { + const key = `${src}:${pattern.type}:${tgt || 'Concept'}`; + if (!seen.has(key)) { + seen.add(key); + results.push({ + sourceText: src, + targetText: tgt || src, + type: pattern.type, + confidence: pattern.confidence, + rawSentence: cleanSent + }); + } + } + } + } + } + + return results; + } +} + +module.exports = DeterministicSemanticMiner; diff --git a/ai/graph/EntityResolver.js b/ai/graph/EntityResolver.js index a3b3316f..7b324c8e 100644 --- a/ai/graph/EntityResolver.js +++ b/ai/graph/EntityResolver.js @@ -12,12 +12,114 @@ class EntityResolver { this.graphDb = graphDb; } + /** + * Helper: Normalize string by decoding URI encoding and trimming whitespace + */ + cleanName(str) { + if (!str || typeof str !== 'string') return ''; + let s = str.trim(); + try { + s = decodeURIComponent(s); + } catch { /* ignore URI decode error */ } + return s.replace(/\s+/g, ' ').trim(); + } + + /** + * Comprehensive Quality Gate: Checks if entity candidate string is valid knowledge term + */ + /** + * Universal Linguistic Quality Gate (Zero hardcoded entity lists) + * Enforces grammatical boundary rules, character entropy, and syntax artifact filters. + */ + isValidEntityName(name) { + if (!name || typeof name !== 'string') return false; + const clean = name.trim(); + const norm = clean.toLowerCase(); + + // 1. Min/Max Length & Acronym Rule (2 & 3 char words must be uppercase acronyms like AI, UI, DB, API, SDK, CLI, SQL, APP, WEB) + if (clean.length < 2 || clean.length > 35 || clean.split(/\s+/).length > 4) return false; + const WHITELIST_SHORT = new Set(['AI', 'UI', 'UX', 'DB', 'JS', 'TS', 'IP', 'OS', 'ID', 'IT', 'API', 'SDK', 'CLI', 'SQL', 'APP', 'WEB', 'CPU', 'RAM', 'URL', 'SSH', 'SSL', 'CSV', 'XML', 'PNG', 'JPG', 'SVG', 'PDF']); + if (clean.length <= 3 && !WHITELIST_SHORT.has(clean.toUpperCase())) return false; + + const words = norm.split(/\s+/); + + // 2. Grammatical Boundary Rule: Cannot start or end with prepositions, articles, connectives, verbs, or UI tokens + const GRAMMAR_BOUNDARIES = new Set([ + 'a', 'an', 'the', 'and', 'or', 'but', 'for', 'in', 'on', 'at', 'to', 'from', + 'by', 'with', 'of', 'as', 'is', 'are', 'was', 'were', 'be', 'been', 'being', + 'if', 'else', 'so', 'than', 'too', 'very', 'not', 'no', 'nor', 'it', 'its', + 'create', 'update', 'delete', 'connect', 'build', 'run', 'make', 'use', 'get', 'set', 'help', 'test', + 'again', 'teh', 'weh', 'value', 'column' + ]); + if (GRAMMAR_BOUNDARIES.has(words[0]) || GRAMMAR_BOUNDARIES.has(words[words.length - 1])) return false; + + // 3. Sentence Clause & Aux Verb Rule: Reject clause fragments containing auxiliary verbs + if (/\b(will|would|could|should|have|has|had|help|helps|test)\b/i.test(clean)) return false; + + // 4. Character Entropy & Phonetic Rule: Must contain vowels; reject repeated chars & invalid consonant clusters + if (!/[aeiouy]/i.test(norm)) return false; // Must contain at least one vowel + if (/(.)\1{3,}/.test(norm)) return false; // Reject 4+ repeated chars (e.g. "dddde") + if (/[bcdfghjklmnpqrstvwxyz]{5,}/i.test(norm)) return false; // Reject 5+ consecutive consonants + if (/^\.[a-z]{1,2}\b/i.test(norm)) return false; + + // 5. Markup & Syntax Artifact Rule: Reject editor markup, HTML attributes, numbers with decimals, table cells + if (/[{}=|]|\bdata-|\d+\.\d+|\bvalue \d|\bcolumn \d|^[-*+\s:#=]+$/i.test(norm)) return false; + + return true; + } + + /** + * Validate and sanitize entity type classification, coercing misclassifications to Concept. + * Pure algorithmic & pattern-based rules - ZERO hardcoded entity or person name lists. + */ + sanitizeEntityType(name, proposedType = 'Concept') { + if (!this.isValidEntityName(name)) return null; + + const clean = name.trim(); + const norm = clean.toLowerCase(); + const words = clean.split(/\s+/); + const CONNECTIVES = new Set(['and', 'or', 'for', 'in', 'on', 'at', 'to', 'from', 'with', 'by', 'of']); + + // Rule 1: Multi-word Title-Cased Proper Name Pattern (e.g. "Bikash Panda", "Abhiram Panda", "Ada Lovelace") + const isMultiWordTitleCase = words.length >= 2 && words.every(w => /^[A-Z][a-z]+$/.test(w)); + if (isMultiWordTitleCase && (proposedType === 'Person' || proposedType === 'Organization' || proposedType === 'Concept')) { + return 'Person'; + } + + // Rule 2: Single-word capitalized proper name (e.g. "Abhiram", "Bikash") proposed as Person or Organization + const isSingleTitleCase = words.length === 1 && /^[A-Z][a-z]{2,}$/.test(clean); + const COMMON_NON_PERSONS = new Set(['workspace', 'system', 'search', 'note', 'document', 'screenshot', 'diagram', 'project', 'settings', 'connect', 'api', 'column', 'value', 'test', 'again', 'create', 'teh']); + if (isSingleTitleCase && (proposedType === 'Person' || proposedType === 'Organization')) { + if (COMMON_NON_PERSONS.has(norm)) return 'Concept'; + return 'Person'; + } + + // Rule 3: Common Non-Person Nouns cannot be Person + if (proposedType === 'Person' && COMMON_NON_PERSONS.has(norm)) { + return 'Concept'; + } + + // Rule 4: Structural media terms (screenshot, diagram, image) cannot be Event, Location, or Task + if ((proposedType === 'Event' || proposedType === 'Location' || proposedType === 'Task') && /^(screenshot|diagram|image|photo|drawing|picture|file|note)$/i.test(norm)) { + return 'Concept'; + } + + // Rule 5: Non-Task clauses ending in conjunctions or connectives + if (proposedType === 'Task' && CONNECTIVES.has(words[words.length - 1].toLowerCase())) { + return 'Concept'; + } + + return proposedType || 'Concept'; + } + /** * Deterministic entity ID generation via SHA-256 (supports international non-ASCII characters) */ generateEntityId(name, type = 'Entity') { - const normName = String(name || '').trim().toLowerCase(); - const hash = crypto.createHash('sha256').update(`${type.toLowerCase()}:${normName}`).digest('hex').slice(0, 16); + const valid = this.cleanName(name); + const sanitizedType = this.sanitizeEntityType(valid, type) || 'Concept'; + const normName = valid.toLowerCase(); + const hash = crypto.createHash('sha256').update(`${sanitizedType.toLowerCase()}:${normName}`).digest('hex').slice(0, 16); return `ent-${hash}`; } @@ -26,7 +128,7 @@ class EntityResolver { */ resolveMention(mentionName, type = 'Entity') { if (!mentionName || typeof mentionName !== 'string') return null; - const clean = mentionName.trim(); + const clean = this.cleanName(mentionName); if (clean.length === 0) return null; const aliasMatch = this.findAlias(clean); @@ -40,6 +142,9 @@ class EntityResolver { }; } + const sanitizedType = this.sanitizeEntityType(clean, type); + if (!sanitizedType) return null; + // Reuse existing canonical entity ID if present in database to prevent type fragmentation if (this.graphDb?.db) { try { @@ -47,7 +152,7 @@ class EntityResolver { 'SELECT id, name, canonical_name, type FROM entities WHERE LOWER(name) = LOWER(?) OR LOWER(canonical_name) = LOWER(?) LIMIT 1' ).get(clean, clean); if (existing) { - const resolvedType = (existing.type && existing.type !== 'Concept') ? existing.type : type; + const resolvedType = this.sanitizeEntityType(clean, existing.type) || sanitizedType; return { id: existing.id, name: existing.name || clean, @@ -59,12 +164,12 @@ class EntityResolver { } catch { /* ignore DB lookup error */ } } - const defaultId = this.generateEntityId(clean, type); + const defaultId = this.generateEntityId(clean, sanitizedType); return { id: defaultId, name: clean, canonical_name: clean, - type, + type: sanitizedType, isAlias: false }; } diff --git a/ai/graph/EvidenceFusionEngine.js b/ai/graph/EvidenceFusionEngine.js index b4289a0d..854764c5 100644 --- a/ai/graph/EvidenceFusionEngine.js +++ b/ai/graph/EvidenceFusionEngine.js @@ -12,10 +12,17 @@ class EvidenceFusionEngine { this.evidenceStore = evidenceStore; } - fuseTriple({ source_id, target_id, type, weight = 1.0, confidence = 1.0, evidenceId = null, metadata = {} }) { + fuseTriple({ source_id, target_id, type, weight = 1.0, confidence = 1.0, evidenceId = null, extractor = 'gliner2-relex', metadata = {} }) { if (!this.graphDb?.db) return null; + if (!source_id || !target_id || source_id === target_id) return null; const db = this.graphDb.db; + // Plausibility Guard: Reject inverse circular relationships (e.g. A CONTROLS B AND B CONTROLS A) + const inverseRelation = db.prepare( + 'SELECT id FROM relationships WHERE source_id = ? AND target_id = ? AND type = ?' + ).get(target_id, source_id, type); + if (inverseRelation) return null; + try { // 1. Check if relationship already exists const existing = db.prepare( @@ -30,6 +37,7 @@ class EvidenceFusionEngine { type, weight, confidence, + extractor, metadata, evidence_id: evidenceId }); @@ -51,9 +59,9 @@ class EvidenceFusionEngine { db.prepare(` UPDATE relationships - SET confidence = ?, weight = ?, metadata = ? + SET confidence = ?, weight = ?, extractor = ?, metadata = ? WHERE id = ? - `).run(mergedConfidence, mergedWeight, metadataJson, existing.id); + `).run(mergedConfidence, mergedWeight, extractor, metadataJson, existing.id); if (evidenceId) { this._linkEvidence(existing.id, evidenceId); @@ -78,6 +86,9 @@ class EvidenceFusionEngine { this.graphDb.db.prepare( 'INSERT OR IGNORE INTO relationship_evidence (relationship_id, evidence_id) VALUES (?, ?)' ).run(relationshipId, evidenceId); + this.graphDb.db.prepare( + 'UPDATE relationships SET evidence_id = COALESCE(evidence_id, ?) WHERE id = ?' + ).run(evidenceId, relationshipId); } catch { /* ignore junction link error */ } } } diff --git a/ai/graph/GraphBuilder.js b/ai/graph/GraphBuilder.js index 2eda6c90..47691296 100644 --- a/ai/graph/GraphBuilder.js +++ b/ai/graph/GraphBuilder.js @@ -14,6 +14,11 @@ class GraphBuilder { this.graphDb = graphDb; this.graphService = graphService; this.isRebuilding = false; + this._lastBuildReport = null; + } + + getPipelineReport() { + return this._lastBuildReport; } /** @@ -28,6 +33,17 @@ class GraphBuilder { try { this.isRebuilding = true; this._rebuildStartTime = Date.now(); + this._buildReport = { + startedAt: new Date().toISOString(), + completedAt: null, + totalDurationMs: 0, + stages: { + discovery: { durationMs: 0, itemCount: 0 }, + nonMarkdownExtraction: { durationMs: 0, entityCount: 0, relationCount: 0 }, + markdownProcessing: { durationMs: 0, processedCount: 0, failedCount: 0 } + }, + finalStats: null + }; log.info('Starting complete Knowledge Graph rebuild...'); if (!this.graphDb.isInitialized) { @@ -35,9 +51,9 @@ class GraphBuilder { } const LogDB = require('../logs/LogDB'); - const logDb = new LogDB(this.agent.workspaceRoot); - logDb.initialize(); - logDb.addLog('graph', 'Starting complete Knowledge Graph rebuild...', 'info'); + this._logDb = new LogDB(this.agent.workspaceRoot); + this._logDb.initialize(); + this._logDb.addLog('graph', 'Starting complete Knowledge Graph rebuild...', 'info'); // Clear existing graph tables this.graphDb.clear(); @@ -46,7 +62,6 @@ class GraphBuilder { const WorkspaceMetadataKnowledgeSource = require('./sources/WorkspaceMetadataKnowledgeSource'); const FolderHierarchyKnowledgeSource = require('./sources/FolderHierarchyKnowledgeSource'); const ImageAnnotationKnowledgeSource = require('./sources/ImageAnnotationKnowledgeSource'); - const MarkdownKnowledgeSource = require('./sources/MarkdownKnowledgeSource'); const ExcalidrawKnowledgeSource = require('./sources/ExcalidrawKnowledgeSource'); const DrawioKnowledgeSource = require('./sources/DrawioKnowledgeSource'); const MermaidKnowledgeSource = require('./sources/MermaidKnowledgeSource'); @@ -81,17 +96,19 @@ class GraphBuilder { registry.register(new ExcalidrawKnowledgeSource()); registry.register(new DrawioKnowledgeSource()); registry.register(new MermaidKnowledgeSource()); - registry.register(new MarkdownKnowledgeSource()); // 2. Discover non-markdown and markdown items const discoveredItems = registry.discoverAll(workspaceRoot); log.info(`Discovered ${discoveredItems.length} knowledge items across sources`); + const EvidenceStore = require('./EvidenceStore'); + const evidenceStore = new EvidenceStore(this.graphDb); + // Extract metadata, folder hierarchy, and image annotation sources first for (const item of discoveredItems) { if (item.source.sourceType() !== 'markdown') { try { - const { entities, relationships } = await registry.extract(item.source, item.path); + const { entities, relationships, evidence } = await registry.extract(item.source, item.path); for (const ent of entities) { const id = this.graphService?.entityResolver ? this.graphService.entityResolver.generateEntityId(ent.name, ent.type || 'Entity') @@ -109,6 +126,19 @@ class GraphBuilder { this.graphDb.upsertRelationship({ source_id: srcId, target_id: tgtId, type: rel.type, weight: rel.weight, confidence: rel.confidence }); } } + if (Array.isArray(evidence)) { + for (const ev of evidence) { + evidenceStore.addEvidence({ + sourceId: item.path, + extractor: item.source.sourceType(), + subjectText: ev.subjectText || ev.subject_text || item.path, + predicateText: ev.predicateText || ev.predicate_text || 'related_to', + objectText: ev.objectText || ev.object_text || '', + rawSentence: ev.rawSentence || ev.raw_sentence || ev.subjectText || item.path, + confidence: ev.confidence || item.source.baseConfidence() || 1.0 + }); + } + } } catch (nonMdErr) { log.warn(`Non-markdown source error (${item.source.sourceType()}):`, nonMdErr.message); } @@ -119,7 +149,7 @@ class GraphBuilder { const workspaceFiles = this._getWorkspaceMarkdownFiles(); const total = workspaceFiles.length; log.info(`Found ${total} markdown notes to index for graph`); - logDb.addLog('graph', `Found ${total} markdown notes to index for graph`, 'info'); + this._logDb?.addLog('graph', `Found ${total} markdown notes to index for graph`, 'info'); let processedCount = 0; let failedCount = 0; @@ -141,10 +171,10 @@ class GraphBuilder { const content = fs.readFileSync(filePath, 'utf8'); await this.graphService.processNote(filePath, content); processedCount++; - logDb.addLog('graph', `Extracted graph entities from note: ${path.basename(filePath)}`, 'info'); + this._logDb?.addLog('graph', `Extracted graph entities from note: ${path.basename(filePath)}`, 'info'); } catch (fileErr) { log.error(`Error processing note ${filePath}:`, fileErr.message); - logDb.addLog('graph', `Failed extracting entities from note ${path.basename(filePath)}: ${fileErr.message}`, 'error'); + this._logDb?.addLog('graph', `Failed extracting entities from note ${path.basename(filePath)}: ${fileErr.message}`, 'error'); failedCount++; } })); @@ -156,11 +186,11 @@ class GraphBuilder { // Run community detection const CommunityDetector = require('./CommunityDetector'); const communityDetector = new CommunityDetector(); - communityDetector.detect(this.graphDb, logDb); + communityDetector.detect(this.graphDb, this._logDb); // Run validation engine const GraphValidationEngine = require('./GraphValidationEngine'); - const validator = new GraphValidationEngine(this.graphDb, logDb); + const validator = new GraphValidationEngine(this.graphDb, this._logDb); await validator.validate(); // Optimize SQLite query planner @@ -169,27 +199,33 @@ class GraphBuilder { } log.info(`Knowledge Graph rebuild complete. Processed: ${processedCount}, Failed: ${failedCount}`); - logDb.addLog('graph', `Knowledge Graph rebuild complete. Processed: ${processedCount}, Failed: ${failedCount}`, 'info', { + this._logDb?.addLog('graph', `Knowledge Graph rebuild complete. Processed: ${processedCount}, Failed: ${failedCount}`, 'info', { processedCount, failedCount, durationMs: Date.now() - (this._rebuildStartTime || Date.now()) }); - // Snapshot version - this.graphDb.snapshotVersion('v1.0'); + if (this._buildReport) { + this._buildReport.completedAt = new Date().toISOString(); + this._buildReport.totalDurationMs = Date.now() - (this._rebuildStartTime || Date.now()); + this._buildReport.stages.markdownProcessing = { durationMs: this._buildReport.totalDurationMs, processedCount, failedCount }; + this._buildReport.finalStats = this.graphDb.getStatus(); + this._lastBuildReport = this._buildReport; + } - logDb.close(); return { success: true, processedCount, failedCount, - stats: this.graphDb.getStatus() + stats: this.graphDb.getStatus(), + report: this._lastBuildReport }; } catch (err) { log.error('Failed to rebuild graph:', err); return { success: false, error: err.message }; } finally { this.isRebuilding = false; + try { if (this._logDb) { this._logDb.close(); this._logDb = null; } } catch { /* ignore */ } } } diff --git a/ai/graph/GraphDB.js b/ai/graph/GraphDB.js index 4150d709..e02a4ec9 100644 --- a/ai/graph/GraphDB.js +++ b/ai/graph/GraphDB.js @@ -83,6 +83,21 @@ class GraphDB { this.db.exec(idxQuery); } + // Versioned database schema migrations (Gap 3) + const TARGET_SCHEMA_VERSION = 3; + let currentVersion = 0; + try { + const vRow = this.db.prepare('PRAGMA user_version').get(); + currentVersion = vRow ? (vRow.user_version || 0) : 0; + } catch { /* default 0 */ } + + if (currentVersion < TARGET_SCHEMA_VERSION) { + this._runSchemaMigrations(currentVersion, TARGET_SCHEMA_VERSION); + try { + this.db.exec(`PRAGMA user_version = ${TARGET_SCHEMA_VERSION}`); + } catch { /* ignore pragma write error */ } + } + this.isInitialized = true; log.info('GraphDB initialized successfully'); return true; @@ -92,6 +107,51 @@ class GraphDB { } } + _runSchemaMigrations(fromVersion, toVersion) { + log.info(`Migrating GraphDB schema from v${fromVersion} to v${toVersion}`); + if (fromVersion < 1) { + // Version 1: Add new entity metadata columns if missing + const cols = [ + 'confidence REAL DEFAULT 1.0', + 'community_id INTEGER', + 'ontology_class TEXT', + 'source_count INTEGER DEFAULT 1', + "first_seen_at TEXT DEFAULT (datetime('now'))", + 'is_retired INTEGER DEFAULT 0', + 'merged_into TEXT' + ]; + for (const col of cols) { + try { this.db.exec(`ALTER TABLE entities ADD COLUMN ${col};`); } catch { /* ignore */ } + } + } + if (fromVersion < 2) { + // Version 2: Ensure relationship_evidence and communities tables exist + try { + this.db.exec(` + CREATE TABLE IF NOT EXISTS relationship_evidence ( + relationship_id INTEGER NOT NULL REFERENCES relationships(id) ON DELETE CASCADE, + evidence_id TEXT NOT NULL REFERENCES evidence(id) ON DELETE CASCADE, + PRIMARY KEY (relationship_id, evidence_id) + ); + `); + } catch { /* ignore */ } + } + if (fromVersion < 3) { + // Version 3: Clean up structural relationship extractor tags + try { + this.db.exec(` + UPDATE relationships SET extractor = 'ast_parser' + WHERE extractor = 'glirel' AND type IN ( + 'links_to','tagged','contains_section','contains_media', + 'contains_code','attaches_file','references_url','annotated_with', + 'contains_formula','has_open_task','has_completed_task', + 'relates_to','mentions_note' + ); + `); + } catch { /* ignore */ } + } + } + close() { if (this.db) { try { @@ -153,6 +213,7 @@ class GraphDB { note_path = excluded.note_path, properties = excluded.properties, confidence = excluded.confidence, + source_count = COALESCE(entities.source_count, 1) + 1, updated_at = datetime('now'); `; this.db.prepare(query).run(id, name, canonical, type, note_path, propertiesJson, confidence); @@ -166,6 +227,7 @@ class GraphDB { type = excluded.type, note_path = excluded.note_path, properties = excluded.properties, + source_count = COALESCE(entities.source_count, 1) + 1, updated_at = datetime('now'); `; this.db.prepare(fallbackQuery).run(id, name, canonical, type, note_path, propertiesJson); @@ -188,11 +250,12 @@ class GraphDB { * Delete note entity, associated evidence, and incoming/outgoing relationships */ deleteNoteEntityAndRelationships(notePath) { - if (!this.db) return; + if (!this.db || !notePath) return; try { - const crypto = require('crypto'); - const normPath = String(notePath || '').trim().toLowerCase(); - const entityId = `ent-${crypto.createHash('sha256').update(`note:${normPath}`).digest('hex').slice(0, 16)}`; + const EntityResolver = require('./EntityResolver'); + const er = new EntityResolver(this); + const noteName = er.cleanName(path.basename(notePath, '.md')); + const entityId = er.generateEntityId(noteName, 'Note'); this.db.exec('BEGIN'); try { @@ -240,17 +303,21 @@ class GraphDB { } /** - * Upsert a relationship with confidence and optional evidence linkage + * Upsert a relationship with confidence, extractor tag, and optional evidence linkage */ - upsertRelationship({ source_id, target_id, type, weight = 1.0, confidence = 1.0, metadata = {}, evidence_id = null }) { + upsertRelationship({ source_id, target_id, type, weight = 1.0, confidence = 1.0, metadata = {}, evidence_id = null, extractor = 'ast_parser' }) { if (!this.db) throw new Error('Database not initialized'); + if (!source_id || !target_id || source_id === target_id) return; + + const clampedConfidence = Math.max(0.0, Math.min(1.0, typeof confidence === 'number' ? confidence : parseFloat(confidence) || 1.0)); const query = ` - INSERT INTO relationships (source_id, target_id, type, weight, confidence, metadata, evidence_id) - VALUES (?, ?, ?, ?, ?, ?, ?) + INSERT INTO relationships (source_id, target_id, type, weight, confidence, extractor, metadata, evidence_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(source_id, target_id, type) DO UPDATE SET weight = excluded.weight, confidence = excluded.confidence, + extractor = excluded.extractor, metadata = excluded.metadata, evidence_id = COALESCE(excluded.evidence_id, relationships.evidence_id); `; @@ -258,10 +325,18 @@ class GraphDB { const metadataJson = typeof metadata === 'string' ? metadata : JSON.stringify(metadata); const stmt = this.db.prepare(query); try { - stmt.run(source_id, target_id, type, weight, confidence, metadataJson, evidence_id); + stmt.run(source_id, target_id, type, weight, clampedConfidence, extractor, metadataJson, evidence_id); + if (evidence_id) { + try { + const relRow = this.db.prepare('SELECT id FROM relationships WHERE source_id = ? AND target_id = ? AND type = ?').get(source_id, target_id, type); + if (relRow?.id) { + this.db.prepare('INSERT OR IGNORE INTO relationship_evidence (relationship_id, evidence_id) VALUES (?, ?)').run(relRow.id, evidence_id); + } + } catch { /* ignore junction insert error */ } + } } catch (err) { if (err.message?.includes('FOREIGN KEY') && evidence_id) { - stmt.run(source_id, target_id, type, weight, confidence, metadataJson, null); + stmt.run(source_id, target_id, type, weight, clampedConfidence, extractor, metadataJson, null); } else { throw err; } @@ -527,8 +602,8 @@ class GraphDB { const cteQuery = ` WITH RECURSIVE paths(id, path_str, depth) AS ( - SELECT ? as id, ? as path_str, 0 as depth - UNION ALL + SELECT ? as id, CAST(? AS TEXT) as path_str, 0 as depth + UNION SELECT r.target_id, p.path_str || ',' || r.target_id, p.depth + 1 FROM relationships r JOIN paths p ON r.source_id = p.id WHERE p.depth < ? AND p.path_str NOT LIKE '%' || r.target_id || '%' @@ -551,9 +626,10 @@ class GraphDB { if (row && row.id) { entityId = row.id; } else { - const path = require('path'); - const noteName = path.basename(notePath, '.md'); - entityId = noteName.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/(^-|-$)/g, ''); + const EntityResolver = require('./EntityResolver'); + const er = new EntityResolver(this); + const noteName = er.cleanName(path.basename(notePath, '.md')); + entityId = er.generateEntityId(noteName, 'Note'); } const relCountRow = this.db.prepare('SELECT COUNT(*) as count FROM relationships WHERE source_id = ? OR target_id = ?').get(entityId, entityId); @@ -708,6 +784,208 @@ class GraphDB { return []; } } + + exportAsJSON(options = {}) { + if (!this.db) { + return { + metadata: { exportedAt: new Date().toISOString(), error: 'Database not initialized' }, + statistics: { entityCount: 0, relationshipCount: 0, evidenceCount: 0, communityCount: 0, evidenceCoverageRatio: 0 }, + entities: [], + relationships: [], + evidence: [], + validation: null + }; + } + + try { + const entities = this.db.prepare('SELECT * FROM entities').all().map(e => ({ + ...e, + properties: typeof e.properties === 'string' ? JSON.parse(e.properties || '{}') : (e.properties || {}) + })); + + const relationships = this.db.prepare('SELECT * FROM relationships').all().map(r => ({ + ...r, + metadata: typeof r.metadata === 'string' ? JSON.parse(r.metadata || '{}') : (r.metadata || {}) + })); + + let evidence = []; + try { + evidence = this.db.prepare('SELECT * FROM evidence').all(); + } catch { /* ignore */ } + + let lastVersion = null; + try { + lastVersion = this.db.prepare('SELECT * FROM graph_versions ORDER BY id DESC LIMIT 1').get()?.version || 'v1.0'; + } catch { /* ignore */ } + + const workspaceEnt = this.getWorkspaceEntity(); + + const confidenceDistribution = { + '0.9-1.0': 0, + '0.8-0.9': 0, + '0.7-0.8': 0, + '0.6-0.7': 0, + 'below-0.6': 0 + }; + relationships.forEach(r => { + const c = r.confidence ?? 1.0; + if (c >= 0.9) confidenceDistribution['0.9-1.0']++; + else if (c >= 0.8) confidenceDistribution['0.8-0.9']++; + else if (c >= 0.7) confidenceDistribution['0.7-0.8']++; + else if (c >= 0.6) confidenceDistribution['0.6-0.7']++; + else confidenceDistribution['below-0.6']++; + }); + + const typeDistribution = {}; + entities.forEach(e => { + const t = e.type || 'Entity'; + typeDistribution[t] = (typeDistribution[t] || 0) + 1; + }); + + const totalRels = relationships.length; + const relsWithEv = relationships.filter(r => r.evidence_id).length; + const evidenceCoverageRatio = totalRels > 0 ? parseFloat((relsWithEv / totalRels).toFixed(4)) : 1.0; + + const degreeMap = new Map(); + relationships.forEach(r => { + degreeMap.set(r.source_id, (degreeMap.get(r.source_id) || 0) + 1); + degreeMap.set(r.target_id, (degreeMap.get(r.target_id) || 0) + 1); + }); + const topHubs = [...entities] + .map(e => ({ id: e.id, name: e.name, type: e.type, degree: degreeMap.get(e.id) || 0 })) + .sort((a, b) => b.degree - a.degree) + .slice(0, 5); + + let communityCount = 0; + try { + communityCount = this.db.prepare('SELECT COUNT(*) as count FROM communities').get()?.count || 0; + if (communityCount === 0 && entities.length > 0) { + const CommunityDetector = require('./CommunityDetector'); + const detector = new CommunityDetector(); + const res = detector.detect(this); + communityCount = res.communityCount || 0; + } + } catch { /* ignore */ } + + let validation = null; + try { + const GraphValidationEngine = require('./GraphValidationEngine'); + const validator = new GraphValidationEngine(this); + validation = validator.validateSync(); + } catch { /* ignore */ } + + return { + metadata: { + exportedAt: new Date().toISOString(), + graphVersion: lastVersion || 'v1.0', + schemaVersion: '1.0', + pipelineVersion: '1.0', + extractionModel: 'gliner2-relex', + embeddingModel: null, + workspaceName: workspaceEnt?.name || 'Workspace', + workspaceHash: null, + buildDurationMs: null + }, + statistics: { + entityCount: entities.length, + relationshipCount: relationships.length, + evidenceCount: evidence.length, + communityCount, + evidenceCoverageRatio, + avgConfidence: totalRels > 0 ? parseFloat((relationships.reduce((s, r) => s + (r.confidence ?? 1.0), 0) / totalRels).toFixed(4)) : 1.0, + confidenceDistribution, + typeDistribution, + topHubs + }, + entities, + relationships, + evidence, + validation + }; + } catch (err) { + log.error('Failed exportAsJSON:', err.message); + return { + metadata: { exportedAt: new Date().toISOString(), error: err.message }, + statistics: { entityCount: 0, relationshipCount: 0, evidenceCount: 0, communityCount: 0, evidenceCoverageRatio: 0 }, + entities: [], + relationships: [], + evidence: [], + validation: null + }; + } + } + + exportAsMarkdown(options = {}) { + const json = this.exportAsJSON(options); + const { metadata, statistics, entities, relationships, validation } = json; + + const lines = []; + lines.push('# Knowledge Graph Export'); + lines.push(`Generated: ${metadata.exportedAt}`); + lines.push(`Graph Version: ${metadata.graphVersion}`); + lines.push(''); + + lines.push('## Metadata'); + lines.push(`- Workspace: ${metadata.workspaceName}`); + lines.push(`- Schema Version: ${metadata.schemaVersion}`); + lines.push(`- Extraction Model: ${metadata.extractionModel}`); + lines.push(''); + + lines.push('## Statistics'); + lines.push(`- Entities: ${statistics.entityCount}`); + lines.push(`- Relationships: ${statistics.relationshipCount}`); + lines.push(`- Evidence Records: ${statistics.evidenceCount}`); + lines.push(`- Communities: ${statistics.communityCount}`); + lines.push(`- Evidence Coverage: ${(statistics.evidenceCoverageRatio * 100).toFixed(1)}%`); + lines.push(`- Avg Confidence: ${statistics.avgConfidence}`); + lines.push(''); + + if (statistics.topHubs?.length > 0) { + lines.push('## Top Hub Entities'); + statistics.topHubs.forEach(h => { + lines.push(`- **${h.name}** (${h.type}, degree: ${h.degree})`); + }); + lines.push(''); + } + + if (validation) { + lines.push('## Validation Summary'); + lines.push(`- Orphan Entities: ${validation.orphans || 0}`); + lines.push(`- Self Loops: ${validation.selfLoops || 0}`); + lines.push(`- Duplicate Edges: ${validation.duplicateEdges || 0}`); + lines.push(`- Evidenceless AI Edges: ${validation.evidencelessEdges || 0}`); + lines.push(`- Evidence Coverage Ratio: ${((validation.evidenceCoverageRatio || 0) * 100).toFixed(1)}%`); + lines.push(''); + } + + lines.push('## Entities'); + const byType = {}; + entities.forEach(e => { + const t = e.type || 'Entity'; + if (!byType[t]) byType[t] = []; + byType[t].push(e); + }); + + Object.keys(byType).sort().forEach(type => { + lines.push(`### ${type}`); + byType[type].forEach(e => { + const pathInfo = e.note_path ? ` [${e.note_path}]` : ''; + lines.push(`- **${e.name}** (confidence: ${e.confidence ?? 1.0})${pathInfo}`); + }); + lines.push(''); + }); + + lines.push('## Relationships'); + const entityNameMap = new Map(entities.map(e => [e.id, e.name])); + relationships.forEach(r => { + const srcName = entityNameMap.get(r.source_id) || r.source_id; + const tgtName = entityNameMap.get(r.target_id) || r.target_id; + lines.push(`- [${srcName}] --[${r.type}]--> [${tgtName}] (confidence: ${r.confidence ?? 1.0})`); + }); + lines.push(''); + + return lines.join('\n'); + } } module.exports = GraphDB; diff --git a/ai/graph/GraphMaintenance.js b/ai/graph/GraphMaintenance.js index 35b0b723..03d7f724 100644 --- a/ai/graph/GraphMaintenance.js +++ b/ai/graph/GraphMaintenance.js @@ -79,13 +79,16 @@ class GraphMaintenance { let mergedCount = 0; try { const db = this.graphDb.db; - const entities = db.prepare("SELECT id, name, canonical_name, type FROM entities WHERE type != 'Note' ORDER BY updated_at DESC LIMIT 500").all(); + const SKIP_DEDUP_TYPES = new Set(['Note', 'Section', 'Task', 'CodeBlock', 'Annotation', 'Formula']); + const entities = db.prepare("SELECT id, name, canonical_name, type FROM entities WHERE type NOT IN ('Note', 'Section', 'Task', 'CodeBlock', 'Annotation', 'Formula') ORDER BY updated_at DESC LIMIT 500").all(); for (let i = 0; i < entities.length; i++) { for (let j = i + 1; j < entities.length; j++) { const e1 = entities[i]; const e2 = entities[j]; if (!e1 || !e2 || e1.id === e2.id || e1.type !== e2.type) continue; + if (SKIP_DEDUP_TYPES.has(e1.type)) continue; + if ((e1.name || '').length < 4 || (e2.name || '').length < 4) continue; const sim = this.entityResolver.calculateSimilarity(e1.name, e2.name); if (sim >= 0.88) { @@ -99,6 +102,12 @@ class GraphMaintenance { try { db.prepare('UPDATE relationships SET source_id = ? WHERE source_id = ?').run(survivor.id, deprecated.id); db.prepare('UPDATE relationships SET target_id = ? WHERE target_id = ?').run(survivor.id, deprecated.id); + db.exec('DELETE FROM relationships WHERE source_id = target_id;'); + try { + db.exec(`DELETE FROM relationships WHERE id NOT IN ( + SELECT MIN(id) FROM relationships GROUP BY source_id, target_id, type + )`); + } catch { /* ignore */ } db.prepare('UPDATE entities SET merged_into = ? WHERE id = ?').run(survivor.id, deprecated.id); db.prepare('DELETE FROM entities WHERE id = ?').run(deprecated.id); db.exec('COMMIT'); diff --git a/ai/graph/GraphSchema.js b/ai/graph/GraphSchema.js index 4d546093..1968a87d 100644 --- a/ai/graph/GraphSchema.js +++ b/ai/graph/GraphSchema.js @@ -85,10 +85,12 @@ const CREATE_INDEXES = [ `CREATE INDEX IF NOT EXISTS idx_entities_name ON entities(name);`, `CREATE INDEX IF NOT EXISTS idx_entities_note ON entities(note_path);`, `CREATE INDEX IF NOT EXISTS idx_entities_canonical ON entities(canonical_name);`, + `CREATE INDEX IF NOT EXISTS idx_entities_canonical_lower ON entities(LOWER(canonical_name));`, `CREATE INDEX IF NOT EXISTS idx_aliases_entity ON entity_aliases(entity_id);`, `CREATE INDEX IF NOT EXISTS idx_evidence_source ON evidence(source_id);`, `CREATE INDEX IF NOT EXISTS idx_evidence_extractor ON evidence(extractor);`, `CREATE INDEX IF NOT EXISTS idx_evidence_span ON evidence(source_id, subject_span_start);`, + `CREATE INDEX IF NOT EXISTS idx_rel_evidence_junction ON relationship_evidence(evidence_id);`, `CREATE INDEX IF NOT EXISTS idx_queue_status_priority ON graph_queue(status, priority DESC);` ]; diff --git a/ai/graph/GraphService.js b/ai/graph/GraphService.js index 1dbaa7ea..ddde5e47 100644 --- a/ai/graph/GraphService.js +++ b/ai/graph/GraphService.js @@ -9,20 +9,26 @@ const EvidenceStore = require('./EvidenceStore'); const EntityResolver = require('./EntityResolver'); const EvidenceFusionEngine = require('./EvidenceFusionEngine'); const OntologyBuilder = require('./OntologyBuilder'); +const DeterministicSemanticMiner = require('./DeterministicSemanticMiner'); const { SemanticExtractionEngine } = require('./semantic'); const log = createLogger('GraphService'); class GraphService { - constructor(agent, graphDb, ontologyBuilder = null) { - this.agent = agent; + constructor(agentOrOptions, graphDb, ontologyBuilder = null) { + if (agentOrOptions && !graphDb && agentOrOptions.graphDb) { + graphDb = agentOrOptions.graphDb; + } + this.agent = (agentOrOptions && agentOrOptions.appDataDir) ? agentOrOptions : null; this.graphDb = graphDb; this.astParser = new MarkdownASTParser(); this.evidenceStore = new EvidenceStore(graphDb); this.entityResolver = new EntityResolver(graphDb); this.fusionEngine = new EvidenceFusionEngine(graphDb, this.evidenceStore); this.ontologyBuilder = ontologyBuilder || new OntologyBuilder('general'); + this.semanticMiner = new DeterministicSemanticMiner(); this.semanticEngine = null; + this._processQueue = Promise.resolve(); // Gap 1: Ingestion Queue to prevent SQLite transaction collisions } getSemanticEngine() { @@ -42,91 +48,140 @@ class GraphService { /** * Process a markdown note and save entities, relationships, and evidence to GraphDB + * Enqueued to serialize execution and prevent concurrent transaction collisions. */ - async processNote(filePath, content) { + processNote(filePath, content) { + this._processQueue = this._processQueue + .then(() => this._processNoteInternal(filePath, content)) + .catch((err) => { + log.error(`Queue error processing ${filePath}:`, err?.message || err); + }); + return this._processQueue; + } + + async _processNoteInternal(filePath, content) { try { - if (!this.graphDb.isInitialized) { + if (this.graphDb && !this.graphDb.db) { this.graphDb.initialize(); } - const noteName = path.basename(filePath, '.md'); - const rootEntityId = this.entityResolver.generateEntityId(filePath, 'Note'); + const noteName = this.entityResolver.cleanName(path.basename(filePath, '.md')); + const resolvedRoot = this.entityResolver.resolveMention(noteName, 'Note'); + const rootEntityId = resolvedRoot ? resolvedRoot.id : this.entityResolver.generateEntityId(noteName, 'Note'); // 1. Structural Markdown AST Parsing const ast = this.astParser.parse(filePath, content); - // Root Note Entity - this.graphDb.upsertEntity({ - id: rootEntityId, - name: noteName, - canonical_name: noteName, - type: 'Note', - note_path: filePath, - properties: ast.rootEntity.properties - }); + // Wrap synchronous AST structural persistence inside a single SQLite transaction + if (this.graphDb?.runTransaction) { + this.graphDb.runTransaction(() => { + // Root Note Entity + this.graphDb.upsertEntity({ + id: rootEntityId, + name: noteName, + canonical_name: noteName, + type: 'Note', + note_path: filePath, + properties: ast.rootEntity.properties + }); - // Clear old evidence for note re-ingestion - this.evidenceStore.deleteForSource(filePath); + // Remove stale outgoing relationships before deleting evidence to preserve FK integrity + if (this.graphDb?.db) { + this.graphDb.db.prepare('DELETE FROM relationships WHERE source_id = ?').run(rootEntityId); + } + this.evidenceStore.deleteForSource(filePath); + + // 1a. Wikilinks [[Target]] + for (const link of ast.links) { + const targetName = this.entityResolver.cleanName(link.targetName); + if (!targetName) continue; + const resolvedTarget = this.entityResolver.resolveMention(targetName, 'Note'); + const targetId = resolvedTarget ? resolvedTarget.id : this.entityResolver.generateEntityId(targetName, 'Note'); + this.graphDb.upsertEntity({ + id: targetId, + name: targetName, + canonical_name: targetName, + type: 'Note', + properties: { name: targetName } + }); - // 1a. Wikilinks [[Target]] - for (const link of ast.links) { - const targetId = this.entityResolver.generateEntityId(link.targetName, 'Note'); - this.graphDb.upsertEntity({ - id: targetId, - name: link.targetName, - canonical_name: link.targetName, - type: 'Note', - properties: { name: link.targetName } - }); + const evId = this.evidenceStore.addEvidence({ + sourceId: filePath, + extractor: 'ast_parser', + subjectText: noteName, + predicateText: 'links_to', + objectText: targetName, + rawSentence: `[[${targetName}]]`, + confidence: 1.0 + }); + + this.graphDb.upsertRelationship({ + source_id: rootEntityId, + target_id: targetId, + type: 'links_to', + weight: 1.2, + confidence: 1.0, + extractor: 'ast_parser', + evidence_id: evId + }); + } + + // 1b. Tags #tag + for (const tag of ast.tags) { + const cleanTagName = this.entityResolver.cleanName(tag.tagName); + if (!cleanTagName) continue; + const tagResolved = this.entityResolver.resolveMention(cleanTagName, 'Tag'); + const tagId = tagResolved ? tagResolved.id : this.entityResolver.generateEntityId(cleanTagName, 'Tag'); + this.graphDb.upsertEntity({ + id: tagId, + name: cleanTagName, + canonical_name: this.entityResolver.cleanName((tag.name || cleanTagName).replace(/^#+/, '')), + type: 'Tag' + }); const evId = this.evidenceStore.addEvidence({ sourceId: filePath, extractor: 'ast_parser', subjectText: noteName, - predicateText: 'links_to', - objectText: link.targetName, - rawSentence: `[[${link.targetName}]]`, + predicateText: 'tagged', + objectText: cleanTagName, + rawSentence: `#${cleanTagName}`, confidence: 1.0 }); - this.graphDb.upsertRelationship({ - source_id: rootEntityId, - target_id: targetId, - type: 'links_to', - weight: 1.2, - confidence: 1.0, - evidence_id: evId - }); - } - - // 1b. Tags #tag - for (const tag of ast.tags) { - const tagId = this.entityResolver.generateEntityId(tag.tagName, 'Tag'); - this.graphDb.upsertEntity({ - id: tagId, - name: tag.tagName, - canonical_name: tag.name, - type: 'Tag' - }); - this.graphDb.upsertRelationship({ source_id: rootEntityId, target_id: tagId, type: 'tagged', weight: 1.0, - confidence: 1.0 + confidence: 1.0, + extractor: 'ast_parser', + evidence_id: evId }); } - // 1c. Embedded Media & Image Annotations + // 1c. Embedded Media & Diagrams (Unified Media entity, no duplicate Annotation nodes) for (const media of ast.media) { - const mediaId = this.entityResolver.generateEntityId(media.name, 'Image'); + const cleanMediaName = this.entityResolver.cleanName(media.name); + if (!cleanMediaName) continue; + const mediaResolved = this.entityResolver.resolveMention(cleanMediaName, 'Image'); + const mediaId = mediaResolved ? mediaResolved.id : this.entityResolver.generateEntityId(cleanMediaName, 'Image'); this.graphDb.upsertEntity({ id: mediaId, - name: media.name, - canonical_name: media.name, + name: cleanMediaName, + canonical_name: cleanMediaName, type: 'Image', - properties: { path: media.path, alt: media.alt } + properties: { path: media.path, alt: media.alt || '', caption: media.alt || '' } + }); + + const evId = this.evidenceStore.addEvidence({ + sourceId: filePath, + extractor: 'ast_parser', + subjectText: noteName, + predicateText: 'contains_media', + objectText: cleanMediaName, + rawSentence: `![${media.alt || ''}](${media.path})`, + confidence: 1.0 }); this.graphDb.upsertRelationship({ @@ -134,38 +189,35 @@ class GraphService { target_id: mediaId, type: 'contains_media', weight: 0.9, - confidence: 1.0 + confidence: 1.0, + extractor: 'ast_parser', + evidence_id: evId }); - - // Extract semantic knowledge from Image Annotations (media.alt) - if (media.alt && media.alt.length > 3 && media.alt.toLowerCase() !== 'image') { - const altId = this.entityResolver.generateEntityId(`${media.name}:${media.alt}`, 'Annotation'); - this.graphDb.upsertEntity({ - id: altId, - name: media.alt, - canonical_name: media.alt, - type: 'Annotation', - properties: { imagePath: media.path } - }); - this.graphDb.upsertRelationship({ - source_id: mediaId, - target_id: altId, - type: 'annotated_with', - weight: 0.95, - confidence: 1.0 - }); - } } // 1d. Attachments & URLs for (const url of ast.urls) { - const urlId = this.entityResolver.generateEntityId(url.url, 'ExternalURL'); + const cleanUrl = this.entityResolver.cleanName(url.url); + const cleanLabel = this.entityResolver.cleanName(url.label || url.url); + if (!cleanUrl) continue; + const urlResolved = this.entityResolver.resolveMention(cleanUrl, 'ExternalURL'); + const urlId = urlResolved ? urlResolved.id : this.entityResolver.generateEntityId(cleanUrl, 'ExternalURL'); this.graphDb.upsertEntity({ id: urlId, - name: url.label, - canonical_name: url.url, + name: cleanLabel, + canonical_name: cleanUrl, type: 'ExternalURL', - properties: { url: url.url } + properties: { url: cleanUrl } + }); + + const evId = this.evidenceStore.addEvidence({ + sourceId: filePath, + extractor: 'ast_parser', + subjectText: noteName, + predicateText: 'references_url', + objectText: cleanUrl, + rawSentence: `[${cleanLabel}](${cleanUrl})`, + confidence: 1.0 }); this.graphDb.upsertRelationship({ @@ -173,84 +225,124 @@ class GraphService { target_id: urlId, type: 'references_url', weight: 0.8, - confidence: 1.0 + confidence: 1.0, + extractor: 'ast_parser', + evidence_id: evId }); } for (const att of ast.attachments) { - const attId = this.entityResolver.generateEntityId(att.name, 'Document'); + const cleanAttName = this.entityResolver.cleanName(att.name); + if (!cleanAttName) continue; + const attResolved = this.entityResolver.resolveMention(cleanAttName, 'Document'); + const attId = attResolved ? attResolved.id : this.entityResolver.generateEntityId(cleanAttName, 'Document'); this.graphDb.upsertEntity({ id: attId, - name: att.name, - canonical_name: att.name, + name: cleanAttName, + canonical_name: cleanAttName, type: 'Document', properties: { path: att.path, label: att.label } }); + const evId = this.evidenceStore.addEvidence({ + sourceId: filePath, + extractor: 'ast_parser', + subjectText: noteName, + predicateText: 'attaches_file', + objectText: cleanAttName, + rawSentence: `[${att.label || cleanAttName}](${att.path})`, + confidence: 1.0 + }); + this.graphDb.upsertRelationship({ source_id: rootEntityId, target_id: attId, type: 'attaches_file', weight: 0.9, - confidence: 1.0 + confidence: 1.0, + extractor: 'ast_parser', + evidence_id: evId }); } - // 1e. Code Blocks - for (const cb of ast.codeBlocks) { - const langId = this.entityResolver.generateEntityId(cb.language, 'CodeBlock'); + // 1e. Code Blocks - Stored as properties on Note entity instead of standalone nodes + const codeLangs = ast.codeBlocks.map(cb => this.entityResolver.cleanName(cb.language)).filter(Boolean); + if (codeLangs.length > 0) { this.graphDb.upsertEntity({ - id: langId, - name: cb.language.toUpperCase(), - canonical_name: cb.language, - type: 'CodeBlock', - properties: { language: cb.language } - }); - - this.graphDb.upsertRelationship({ - source_id: rootEntityId, - target_id: langId, - type: 'contains_code', - weight: 0.8, - confidence: 1.0 + id: rootEntityId, + name: noteName, + canonical_name: noteName, + type: 'Note', + note_path: filePath, + properties: { codeLanguages: [...new Set(codeLangs)] } }); } - // 1f. Sections (Structural Headings) - filter system design sections + // 1f. Sections (Structural Headings) - Hierarchical scoping & qualified names const SYSTEM_SECTIONS = new Set(['rawnotes', 'raw notes', 'raw', 'cleansed', 'cleansed notes', 'cleansed note']); for (const sec of ast.sections) { - const normTitle = String(sec.title || '').trim().toLowerCase(); + const cleanSecTitle = this.entityResolver.cleanName(sec.title); + const normTitle = cleanSecTitle.toLowerCase(); if (SYSTEM_SECTIONS.has(normTitle)) { continue; } - const secId = this.entityResolver.generateEntityId(`${filePath}:${sec.title}`, 'Section'); + const level = sec.level || 1; + const secId = this.entityResolver.generateEntityId(`${rootEntityId}:h${level}:${cleanSecTitle}`, 'Section'); + const qualifiedName = `${noteName} > ${cleanSecTitle}`; + this.graphDb.upsertEntity({ id: secId, - name: sec.title, - canonical_name: sec.title, + name: qualifiedName, + canonical_name: qualifiedName, type: 'Section', - properties: { level: sec.level, wordCount: sec.wordCount } + properties: { level, wordCount: sec.wordCount, noteName, sectionTitle: cleanSecTitle } }); - const hierarchyWeight = parseFloat(Math.max(0.5, 1.4 - ((sec.level || 1) * 0.1)).toFixed(2)); + const evId = this.evidenceStore.addEvidence({ + sourceId: filePath, + extractor: 'ast_parser', + subjectText: noteName, + predicateText: 'contains_section', + objectText: cleanSecTitle, + rawSentence: cleanSecTitle, + confidence: 1.0 + }); + + const hierarchyWeight = parseFloat(Math.max(0.5, 1.4 - (level * 0.1)).toFixed(2)); this.graphDb.upsertRelationship({ source_id: rootEntityId, target_id: secId, type: 'contains_section', weight: hierarchyWeight, - confidence: 1.0 + confidence: 1.0, + extractor: 'ast_parser', + evidence_id: evId }); } // 1g. Note Metadata Entities (Person, Location from Frontmatter/AST) for (const metaEnt of (ast.metadataEntities || [])) { - const metaId = this.entityResolver.generateEntityId(metaEnt.name, metaEnt.type || 'Concept'); + const cleanMetaName = this.entityResolver.cleanName(metaEnt.name); + if (!cleanMetaName) continue; + const metaType = metaEnt.type || 'Concept'; + const metaResolved = this.entityResolver.resolveMention(cleanMetaName, metaType); + const metaId = metaResolved ? metaResolved.id : this.entityResolver.generateEntityId(cleanMetaName, metaType); this.graphDb.upsertEntity({ id: metaId, - name: metaEnt.name, - canonical_name: metaEnt.name, - type: metaEnt.type || 'Concept' + name: cleanMetaName, + canonical_name: cleanMetaName, + type: metaType + }); + + const evId = this.evidenceStore.addEvidence({ + sourceId: filePath, + extractor: 'ast_parser', + subjectText: noteName, + predicateText: metaEnt.relation || 'relates_to', + objectText: cleanMetaName, + rawSentence: `${metaType}: ${cleanMetaName}`, + confidence: 1.0 }); this.graphDb.upsertRelationship({ @@ -258,18 +350,32 @@ class GraphService { target_id: metaId, type: metaEnt.relation || 'relates_to', weight: 0.9, - confidence: 1.0 + confidence: 1.0, + extractor: 'ast_parser', + evidence_id: evId }); } for (const mf of (ast.mathFormulas || [])) { - const mfId = this.entityResolver.generateEntityId(mf.formula, 'Formula'); + const cleanFormula = this.entityResolver.cleanName(mf.formula); + if (!cleanFormula) continue; + const mfId = this.entityResolver.generateEntityId(cleanFormula, 'Formula'); this.graphDb.upsertEntity({ id: mfId, - name: mf.formula.length > 30 ? mf.formula.slice(0, 30) + '...' : mf.formula, - canonical_name: mf.formula, + name: cleanFormula.length > 30 ? cleanFormula.slice(0, 30) + '...' : cleanFormula, + canonical_name: cleanFormula, type: 'Formula', - properties: { rawFormula: mf.formula } + properties: { rawFormula: cleanFormula } + }); + + const evId = this.evidenceStore.addEvidence({ + sourceId: filePath, + extractor: 'ast_parser', + subjectText: noteName, + predicateText: 'contains_formula', + objectText: cleanFormula, + rawSentence: cleanFormula, + confidence: 1.0 }); this.graphDb.upsertRelationship({ @@ -277,33 +383,52 @@ class GraphService { target_id: mfId, type: 'contains_formula', weight: 0.9, - confidence: 1.0 + confidence: 1.0, + extractor: 'ast_parser', + evidence_id: evId }); } // 1j. Tasks (- [ ] task, - [x] task) for (const t of (ast.tasks || [])) { - const taskId = this.entityResolver.generateEntityId(`${filePath}:${t.taskText}`, 'Task'); + const cleanTask = this.entityResolver.cleanName(t.taskText); + if (!cleanTask) continue; + const taskId = this.entityResolver.generateEntityId(`${rootEntityId}:${cleanTask}`, 'Task'); this.graphDb.upsertEntity({ id: taskId, - name: t.taskText, - canonical_name: t.taskText, + name: cleanTask, + canonical_name: cleanTask, type: 'Task', properties: { completed: t.completed } }); + const relType = t.completed ? 'has_completed_task' : 'has_open_task'; + const evId = this.evidenceStore.addEvidence({ + sourceId: filePath, + extractor: 'ast_parser', + subjectText: noteName, + predicateText: relType, + objectText: cleanTask, + rawSentence: `- [${t.completed ? 'x' : ' '}] ${cleanTask}`, + confidence: 1.0 + }); + this.graphDb.upsertRelationship({ source_id: rootEntityId, target_id: taskId, - type: t.completed ? 'has_completed_task' : 'has_open_task', + type: relType, weight: 0.95, - confidence: 1.0 + confidence: 1.0, + extractor: 'ast_parser', + evidence_id: evId + }); + } }); } + const cleansedContent = this.astParser.cleanse(content); - - // 2. Cross-Note Plain Text Mention Mining via Inverted Index + // 2. Cross-Note Plain Text Mention Mining with Mandatory Evidence if (this.graphDb?.db) { try { if (!this._mentionIndex || Date.now() - (this._mentionIndexTime || 0) > 30000) { @@ -321,13 +446,24 @@ class GraphService { if (otherId !== rootEntityId && otherName.length >= 5) { const esc = otherName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); const re = new RegExp(`\\b${esc}\\b`, 'i'); - if (re.test(content)) { + if (re.test(cleansedContent)) { + const evId = this.evidenceStore.addEvidence({ + sourceId: filePath, + extractor: 'ast_parser', + subjectText: noteName, + predicateText: 'mentions_note', + objectText: otherName, + rawSentence: `Mentioned ${otherName} in ${noteName}.`, + confidence: 0.85 + }); this.fusionEngine.fuseTriple({ source_id: rootEntityId, target_id: otherId, type: 'mentions_note', weight: 0.85, - confidence: 0.85 + confidence: 0.85, + extractor: 'ast_parser', + evidenceId: evId }); } } @@ -335,90 +471,165 @@ class GraphService { } catch { /* ignore mention index errors */ } } + // 2b. Rule-Based Deterministic Semantic Relationship Mining + if (this.semanticMiner) { + const minedRelations = this.semanticMiner.mine(cleansedContent || content); + for (const mined of minedRelations) { + const srcResolved = this.entityResolver.resolveMention(mined.sourceText, 'Concept'); + const tgtResolved = this.entityResolver.resolveMention(mined.targetText, 'Concept'); + if (!srcResolved || !tgtResolved) continue; + + const srcId = srcResolved.id; + const tgtId = tgtResolved.id; + + this.graphDb.upsertEntity({ id: srcId, name: srcResolved.name, canonical_name: srcResolved.canonical_name, type: srcResolved.type }); + this.graphDb.upsertEntity({ id: tgtId, name: tgtResolved.name, canonical_name: tgtResolved.canonical_name, type: tgtResolved.type }); + + const evId = this.evidenceStore.addEvidence({ + sourceId: filePath, + extractor: 'deterministic_miner', + subjectText: mined.sourceText, + predicateText: mined.type, + objectText: mined.targetText, + rawSentence: mined.rawSentence, + confidence: mined.confidence + }); + + this.fusionEngine.fuseTriple({ + source_id: srcId, + target_id: tgtId, + type: mined.type, + weight: mined.confidence, + confidence: mined.confidence, + extractor: 'deterministic_miner', + evidenceId: evId + }); + + // Connect root note entity to mined concepts + this.fusionEngine.fuseTriple({ + source_id: rootEntityId, + target_id: srcId, + type: 'mentions_concept', + weight: mined.confidence, + confidence: mined.confidence, + extractor: 'deterministic_miner', + evidenceId: evId + }); + } + } + // 3. Neural AI Pipeline via Model-Agnostic SemanticExtractionEngine const semanticEngine = this.getSemanticEngine(); if (semanticEngine) { const prefs = this.agent?.config ? this.agent.config.loadPreferences() : {}; - const confidenceThreshold = typeof prefs.graphConfidence === 'number' ? prefs.graphConfidence : 0.60; - const cleansedContent = this.astParser.cleanse(content); + let confidenceThreshold = typeof prefs.graphConfidence === 'number' ? prefs.graphConfidence : null; + if (confidenceThreshold === null && semanticEngine?.adapter?.getSavedConfidenceThreshold) { + confidenceThreshold = semanticEngine.adapter.getSavedConfidenceThreshold(); + } + if (confidenceThreshold === null || isNaN(confidenceThreshold)) { + confidenceThreshold = 0.45; + } + const ontologyLabels = this.ontologyBuilder ? this.ontologyBuilder.getGLiNERLabels() : []; const extractionResult = await semanticEngine.extract({ id: filePath, content: cleansedContent || content, sourceType: 'markdown', metadata: { sourceFile: filePath } - }, { confidenceThreshold }); + }, { + confidenceThreshold, + entityTypes: ontologyLabels.length > 0 ? ontologyLabels : undefined + }); const createdEntities = new Map(); - // Save AI extracted entities - for (const ent of extractionResult.entities) { - if ((ent.confidence || 0) < confidenceThreshold) continue; - const resolved = this.entityResolver.resolveMention(ent.text || ent.canonicalName, ent.type || 'Entity'); - if (resolved) { - this.graphDb.upsertEntity({ - id: resolved.id, - name: resolved.name, - canonical_name: resolved.canonical_name, - type: resolved.type, - properties: { confidence: ent.confidence } - }); - createdEntities.set(ent.text, resolved.id); - if (ent.id) createdEntities.set(ent.id, resolved.id); - - let evidenceId = null; - if (ent.sourceEvidence && this.evidenceStore) { - evidenceId = this.evidenceStore.addEvidence({ - sourceId: filePath, - extractor: ent.sourceEvidence.extractionModel || 'gliner2-relex', - subjectText: resolved.name, - rawSentence: ent.sourceEvidence.rawSnippet || content, - confidence: ent.confidence + // Gap 4: Wrap neural entity and relationship writes in a single transaction + const saveNeuralResults = () => { + // Clear old semantic edges for this note before re-inserting + if (this.graphDb?.db) { + try { + this.graphDb.db.prepare( + "DELETE FROM relationships WHERE source_id = ? AND extractor = 'gliner2-relex'" + ).run(rootEntityId); + } catch { /* ignore */ } + } + + // Save AI extracted entities + for (const ent of extractionResult.entities) { + if ((ent.confidence || 0) < confidenceThreshold) continue; + const resolved = this.entityResolver.resolveMention(ent.text || ent.canonicalName, ent.type || 'Entity'); + if (resolved) { + this.graphDb.upsertEntity({ + id: resolved.id, + name: resolved.name, + canonical_name: resolved.canonical_name, + type: resolved.type, + properties: { confidence: ent.confidence } }); - } + createdEntities.set(ent.text, resolved.id); + if (ent.id) createdEntities.set(ent.id, resolved.id); + + let evidenceId = null; + if (ent.sourceEvidence && this.evidenceStore) { + evidenceId = this.evidenceStore.addEvidence({ + sourceId: filePath, + extractor: ent.sourceEvidence.extractionModel || 'gliner2-relex', + subjectText: resolved.name, + rawSentence: ent.sourceEvidence.rawSnippet || content, + confidence: ent.confidence + }); + } - // Connect root note to extracted entity - this.graphDb.upsertRelationship({ - source_id: rootEntityId, - target_id: resolved.id, - type: 'mentions', - weight: ent.confidence || 0.8, - confidence: ent.confidence || 0.8, - evidence_id: evidenceId - }); + // Connect root note to extracted entity + this.graphDb.upsertRelationship({ + source_id: rootEntityId, + target_id: resolved.id, + type: 'mentions', + weight: ent.confidence || 0.8, + confidence: ent.confidence || 0.8, + extractor: 'gliner2-relex', + evidence_id: evidenceId + }); + } } - } - // Save AI extracted relationships - for (const rel of extractionResult.relations) { - if ((rel.confidence || 0) < confidenceThreshold) continue; - const srcId = createdEntities.get(rel.sourceEntityId) || createdEntities.get(rel.sourceText) || this.entityResolver.generateEntityId(rel.sourceText, 'Entity'); - const tgtId = createdEntities.get(rel.targetEntityId) || createdEntities.get(rel.targetText) || this.entityResolver.generateEntityId(rel.targetText, 'Entity'); - - if (srcId && tgtId && srcId !== tgtId) { - let evidenceId = null; - if (rel.sourceEvidence && this.evidenceStore) { - evidenceId = this.evidenceStore.addEvidence({ - sourceId: filePath, - extractor: rel.sourceEvidence.extractionModel || 'gliner2-relex', - subjectText: rel.sourceText, - predicateText: rel.relationType, - objectText: rel.targetText, - rawSentence: rel.sourceEvidence.rawSnippet || content, - confidence: rel.confidence + // Save AI extracted relationships + for (const rel of extractionResult.relations) { + if ((rel.confidence || 0) < confidenceThreshold) continue; + const srcId = createdEntities.get(rel.sourceEntityId) || createdEntities.get(rel.sourceText) || this.entityResolver.generateEntityId(rel.sourceText, 'Entity'); + const tgtId = createdEntities.get(rel.targetEntityId) || createdEntities.get(rel.targetText) || this.entityResolver.generateEntityId(rel.targetText, 'Entity'); + + if (srcId && tgtId && srcId !== tgtId) { + let evidenceId = null; + if (rel.sourceEvidence && this.evidenceStore) { + evidenceId = this.evidenceStore.addEvidence({ + sourceId: filePath, + extractor: rel.sourceEvidence.extractionModel || 'gliner2-relex', + subjectText: rel.sourceText, + predicateText: rel.relationType, + objectText: rel.targetText, + rawSentence: rel.sourceEvidence.rawSnippet || content, + confidence: rel.confidence + }); + } + + this.fusionEngine.fuseTriple({ + source_id: srcId, + target_id: tgtId, + type: rel.relationType || 'RELATED_TO', + weight: rel.confidence || 0.85, + confidence: rel.confidence || 0.85, + extractor: 'gliner2-relex', + evidenceId }); } - - this.fusionEngine.fuseTriple({ - source_id: srcId, - target_id: tgtId, - type: rel.relationType || 'RELATED_TO', - weight: rel.confidence || 0.85, - confidence: rel.confidence || 0.85, - extractor: 'gliner2-relex', - evidenceId - }); } + }; + + if (this.graphDb?.runTransaction) { + this.graphDb.runTransaction(saveNeuralResults); + } else { + saveNeuralResults(); } } diff --git a/ai/graph/GraphValidationEngine.js b/ai/graph/GraphValidationEngine.js index 0be47ea5..609f44a2 100644 --- a/ai/graph/GraphValidationEngine.js +++ b/ai/graph/GraphValidationEngine.js @@ -13,13 +13,14 @@ class GraphValidationEngine { this.logDb = logDb; } - async validate() { + validateSync() { const results = { orphans: 0, confidenceAnomalies: 0, evidencelessEdges: 0, selfLoops: 0, duplicateEdges: 0, + duplicateEntities: 0, typeOverloading: false, starTopology: false, missingWorkspace: false, @@ -68,7 +69,7 @@ class GraphValidationEngine { SELECT source_id, target_id, type, COUNT(*) as c FROM relationships GROUP BY source_id, target_id, type HAVING c > 1 `).all(); - results.duplicateEdges = dupes.length; + results.duplicateEdges = dupes.reduce((sum, d) => sum + (d.c - 1), 0); // Rule 6: Type overloading (>20% default 'Concept' type) const totalEnts = db.prepare(`SELECT COUNT(*) as c FROM entities`).get()?.c || 0; @@ -102,7 +103,8 @@ class GraphValidationEngine { const noteEnts = db.prepare(`SELECT id, note_path FROM entities WHERE note_path IS NOT NULL`).all(); let staleCount = 0; for (const ne of noteEnts) { - if (ne.note_path && !fs.existsSync(ne.note_path)) staleCount++; + if (!ne.note_path) continue; + try { fs.statSync(ne.note_path); } catch { staleCount++; } } results.staleEntities = staleCount; @@ -127,16 +129,29 @@ class GraphValidationEngine { const edgesWithEvidence = db.prepare(`SELECT COUNT(DISTINCT relationship_id) as c FROM relationship_evidence`).get()?.c || 0; results.evidenceCoverageRatio = totalEdges > 0 ? parseFloat((edgesWithEvidence / totalEdges).toFixed(2)) : 1.0; + // Rule 16: Duplicate entities sharing canonical name + const dupEnts = db.prepare(` + SELECT LOWER(canonical_name) as cname, COUNT(*) as c + FROM entities + GROUP BY LOWER(canonical_name) + HAVING c > 1 + `).all(); + results.duplicateEntities = dupEnts.length; + if (this.logDb) { - this.logDb.addLog('graph', 'Graph validation pass executed across 15 rules', 'info', results); + this.logDb.addLog('graph', 'Graph validation pass executed across 16 rules', 'info', results); } - log.info('GraphValidationEngine pass completed successfully across 15 rules:', results); + log.info('GraphValidationEngine pass completed successfully across 16 rules:', results); } catch (err) { log.error('Failed graph validation pass:', err.message); } return results; } + + async validate() { + return this.validateSync(); + } } module.exports = GraphValidationEngine; diff --git a/ai/graph/MarkdownASTParser.js b/ai/graph/MarkdownASTParser.js index 477ecdcf..67cff8d0 100644 --- a/ai/graph/MarkdownASTParser.js +++ b/ai/graph/MarkdownASTParser.js @@ -39,7 +39,7 @@ class MarkdownASTParser { // 0. Frontmatter & Key-Value Header Metadata Parsing (Tags, Name, Location, Time) const fmMatch = content.match(/^---\r?\n([\s\S]*?)\r?\n---/); const fmText = fmMatch ? fmMatch[1] : ''; - const metaBlockText = (fmText ? fmText + '\n' : '') + content.slice(0, 1500); + const metaBlockText = fmText; const kvRegex = /^(?:[ \t]*[-*]\s+)?([a-zA-Z0-9_\s]+):\s*(.*)$/gm; let kvMatch; @@ -271,15 +271,28 @@ class MarkdownASTParser { } } - // 11. Tasks: - [ ] task, [ ] task, - [x] completed task + // 11. Tasks: - [ ] task, - [x] completed task const tasks = []; + const TASK_BLACKLIST = new Set(['todo', 'tbd', 'fixme', '...', 'xxx', 'n/a', 'temp', 'task', 'sample task', 'item']); const taskRegex = /^\s*[-*+]?\s*\[([ xX])\]\s+(.+)$/gm; while ((match = taskRegex.exec(content)) !== null) { const completed = match[1].toLowerCase() === 'x'; - const taskText = match[2].replace(/[*_~`]/g, '').trim(); - if (taskText && taskText.length >= 2) { + const rawText = match[2].replace(/\|/g, '').replace(/[*_~`]/g, '').trim(); + const normText = rawText.toLowerCase(); + + const hasConsequenceGibberish = /[bcdfghjklmnpqrstvwxyz]{6,}/i.test(rawText); + const isTooLongWordWithoutVowels = rawText.split(/\s+/).some(w => w.length > 12 && !/[aeiouy]/i.test(w)); + + if ( + rawText && + rawText.length >= 5 && + !TASK_BLACKLIST.has(normText) && + !/^[\s.\-_:=+*#|]+$/.test(rawText) && + !hasConsequenceGibberish && + !isTooLongWordWithoutVowels + ) { tasks.push({ - taskText, + taskText: rawText, completed, spanStart: match.index, spanEnd: match.index + match[0].length @@ -314,30 +327,33 @@ class MarkdownASTParser { cleanse(content = '') { if (!content || typeof content !== 'string') return ''; return content - .replace(/^\s*#{1,6}\s*(?:rawnotes|raw notes|cleansednotes|cleansed notes|cleansed|raw)\s*$/gmi, '') // 0. System template section headers + .replace(/^\s*#{1,6}\s*(?:rawnotes|raw notes|cleansednotes|cleansed notes|cleansed|raw)\s*$/gmi, '') // 0. System template headers .replace(/^---\r?\n[\s\S]*?\r?\n---/g, '') // 1. Frontmatter .replace(/```[\s\S]*?```/g, '') // 2. Code blocks .replace(/\$\$[\s\S]*?\$\$/g, '') // 3. Multiline math .replace(/\$[^$\n]+\$/g, '') // 4. Inline math - .replace(/<[^>]*>/g, '') // 5. HTML tags - .replace(/^>\s*\[!.*?\]\s*(.*)$/gm, '$1') // 6. Callout headers - .replace(/^>\s*/gm, '') // 7. Blockquotes - .replace(/!\[(.*?)\]\((.*?)\)/g, '$1') // 8. Images -> alt text - .replace(/\[(.*?)\]\((.*?)\)/g, '$1') // 9. Links -> label - .replace(/\[\[(.*?)\]\]/g, (m, inner) => inner.includes('|') ? inner.split('|')[1].trim() : inner.trim()) // 10. Wikilinks - .replace(/^\s*#{1,6}\s+/gm, '') // 11. Headings - .replace(/^\s*[-*+]?\s*\[[ xX]\]\s+/gm, '') // 12. Checkboxes - .replace(/^\s*[-*+]\s+/gm, '') // 13. Bullet lists - .replace(/^\s*\d+\.\s+/gm, '') // 14. Numbered lists - .replace(/\|.*\|/g, (m) => m.replace(/\|/g, ' ')) // 15. Markdown table pipes -> spaces - .replace(/^[-\s:|]{3,}$/gm, '') // 16. Table separator lines - .replace(/\[\^\d+\]:?/g, '') // 17. Footnotes - .replace(/\*{1,3}(.*?)\*{1,3}/g, '$1') // 18. Bold/Italic asterisks - .replace(/_{1,3}(.*?)_{1,3}/g, '$1') // 19. Bold/Italic underscores - .replace(/~~(.*?)~~/g, '$1') // 20. Strikethrough - .replace(/`([^`]+)`/g, '$1') // 21. Inline code - .replace(/\r?\n/g, ' ') // 22. Line breaks -> space - .replace(/\s+/g, ' ') // 23. Collapse whitespace + .replace(/\{data-[^}]*\}/gi, '') // 5. Excalidraw / HTML attribute blocks {data-...} + .replace(/\{[^}\n]{3,}\}/g, '') // 5b. Any curly attribute blocks + .replace(/!\[.*?\]\(.*?\)(\{.*?\})?/g, '') // 6. Complete Image syntax (including attributes) + .replace(/<[^>]*>/g, '') // 7. HTML tags + .replace(/^>\s*\[!.*?\]\s*(.*)$/gm, '$1') // 8. Callout headers + .replace(/^>\s*/gm, '') // 9. Blockquotes + .replace(/^\|.*\|$/gm, '') // 10. Complete Markdown table rows + .replace(/\|.*\|/g, '') // 10b. Table fragments + .replace(/^[a-zA-Z0-9_\s]+:\s*.*$/gm, '') // 11. Key-value metadata lines (Name: Bikash, Time: 10:04) + .replace(/\[(.*?)\]\((.*?)\)/g, '$1') // 12. Standard links -> label + .replace(/\[\[(.*?)\]\]/g, (m, inner) => inner.includes('|') ? inner.split('|')[1].trim() : inner.trim()) // 13. Wikilinks + .replace(/^\s*#{1,6}\s+/gm, '') // 14. Headings + .replace(/^\s*[-*+]?\s*\[[ xX]\]\s+/gm, '') // 15. Checkboxes + .replace(/^\s*[-*+]\s+/gm, '') // 16. Bullet lists + .replace(/^\s*\d+\.\s+/gm, '') // 17. Numbered lists + .replace(/\[\^\d+\]:?/g, '') // 18. Footnotes + .replace(/\*{1,3}(.*?)\*{1,3}/g, '$1') // 19. Bold/Italic asterisks + .replace(/_{1,3}(.*?)_{1,3}/g, '$1') // 20. Bold/Italic underscores + .replace(/~~(.*?)~~/g, '$1') // 21. Strikethrough + .replace(/`([^`]+)`/g, '$1') // 22. Inline code + .replace(/\r?\n/g, ' ') // 23. Line breaks -> space + .replace(/\s+/g, ' ') // 24. Collapse whitespace .trim(); } } diff --git a/ai/graph/semantic/adapters/GLiNER2RelexAdapter.js b/ai/graph/semantic/adapters/GLiNER2RelexAdapter.js index 3fd0c4b5..51fa4d7a 100644 --- a/ai/graph/semantic/adapters/GLiNER2RelexAdapter.js +++ b/ai/graph/semantic/adapters/GLiNER2RelexAdapter.js @@ -42,6 +42,9 @@ class GLiNER2RelexAdapter extends ModelAdapter { this.defaultEntityTypes = config.entityTypes || registryConfig.defaultEntityTypes || []; this.defaultRelationTypes = config.relationTypes || registryConfig.defaultRelationTypes || []; + this._consecutiveFailures = 0; + this._unloadTimer = null; + this.segmenter = typeof Intl !== 'undefined' && Intl.Segmenter ? new Intl.Segmenter('en', { granularity: 'sentence' }) : null; @@ -111,13 +114,12 @@ class GLiNER2RelexAdapter extends ModelAdapter { if (this.ort) { this.encoderSession = await this._loadSession(modelDir, files.encoder).catch(() => null); this.spanRepSession = await this._loadSession(modelDir, files.span_rep).catch(() => null); - this.countEmbedSession = await this._loadSession(modelDir, files.count_embed).catch(() => null); - this.countPredSession = await this._loadSession(modelDir, files.count_pred).catch(() => null); this.classifierSession = await this._loadSession(modelDir, files.classifier).catch(() => null); } if (this.encoderSession && this.classifierSession) { this.isLoaded = true; + this._consecutiveFailures = 0; log.info(`GLiNER2RelexAdapter 5-Graph ONNX model loaded successfully in ${Date.now() - startTime}ms.`); } else { this._setupTestMockEnvironment(); @@ -129,6 +131,30 @@ class GLiNER2RelexAdapter extends ModelAdapter { } } + _runSessionWithTimeout(session, inputs, timeoutMs = 8000) { + if (!session) return Promise.resolve(null); + return Promise.race([ + session.run(inputs), + new Promise((_, reject) => setTimeout(() => reject(new Error('ONNX inference session timeout')), timeoutMs)) + ]); + } + + _scheduleIdleUnload(idleMs = 300000) { + if (this._unloadTimer) clearTimeout(this._unloadTimer); + this._unloadTimer = setTimeout(() => { + try { + if (this.encoderSession?.close) this.encoderSession.close(); + if (this.spanRepSession?.close) this.spanRepSession.close(); + if (this.classifierSession?.close) this.classifierSession.close(); + } catch { /* ignore */ } + this.encoderSession = null; + this.spanRepSession = null; + this.classifierSession = null; + this.isLoaded = false; + log.info('GLiNER2 ONNX sessions unloaded due to idle timeout.'); + }, idleMs); + } + _setupTestMockEnvironment() { this.isMockMode = true; if (!this.ort) { @@ -295,7 +321,7 @@ class GLiNER2RelexAdapter extends ModelAdapter { const pToken = this.modelConfig?.special_tokens?.['[P]'] || 250104; const eToken = this.modelConfig?.special_tokens?.['[E]'] || 250106; const sepTextToken = this.modelConfig?.special_tokens?.['[SEP_TEXT]'] || 250103; - const maxWidth = this.modelConfig?.max_width || 8; + const maxWidth = Math.min(4, this.modelConfig?.max_width || 4); const schemaTokenIds = [pToken]; const schemaPositions = [0]; @@ -308,13 +334,15 @@ class GLiNER2RelexAdapter extends ModelAdapter { } const fullInputIds = [...schemaTokenIds, sepTextToken]; - const textPositions = []; + const textStartPositions = []; + const textEndPositions = []; for (let i = 0; i < words.length; i++) { - textPositions.push(fullInputIds.length); + textStartPositions.push(fullInputIds.length); const wordTokens = this._tokenizeWord(words[i]); if (wordTokens.length === 0) wordTokens.push(0); fullInputIds.push(...wordTokens); + textEndPositions.push(fullInputIds.length - 1); } const seqLen = fullInputIds.length; @@ -329,8 +357,8 @@ class GLiNER2RelexAdapter extends ModelAdapter { for (let start = 0; start < numWords; start++) { for (let w = 1; w <= maxWidth; w++) { if (start + w <= numWords) { - const startSubIdx = textPositions[start]; - const endSubIdx = textPositions[start + w - 1]; + const startSubIdx = textStartPositions[start]; + const endSubIdx = textEndPositions[start + w - 1]; spanStartList.push(BigInt(startSubIdx)); spanEndList.push(BigInt(endSubIdx)); validSpans.push({ wordIndexStart: start, length: w }); @@ -350,7 +378,7 @@ class GLiNER2RelexAdapter extends ModelAdapter { } _sigmoid(val) { - return 1 / (1 + Math.exp(-val)); + return 1 / (1 + Math.exp(-(val + 1.2))); } _decodeSpanScores(logitsData, words, labels, charOffsets, threshold, maxWidth, validSpans) { @@ -364,8 +392,14 @@ class GLiNER2RelexAdapter extends ModelAdapter { const start = span.wordIndexStart; const w = span.length; - let textSpan = words.slice(start, start + w).join(' ').replace(/[.,;:]+$/, '').trim(); - if (!textSpan || /^\W+$/.test(textSpan)) continue; + // Noise Guard: Reject spans longer than 4 words or 35 chars + if (w > 4) continue; + + let textSpan = words.slice(start, start + w).join(' ').replace(/^[-*+\s:#=]+|[.,;:)]+$/g, '').trim(); + if (!textSpan || textSpan.length > 35 || /^\W+$/.test(textSpan)) continue; + + // Reject formatting artifacts, HTML attributes, table headers/cells, UI strings + if (/[{}=|]|\bdata-|\bvalue \d|\bcolumn \d|\btest for\b|\bask questions\b|\bchat with\b/i.test(textSpan)) continue; const spanLogits = logitsData.subarray ? logitsData.subarray(i * numLabels, (i + 1) * numLabels) @@ -422,6 +456,10 @@ class GLiNER2RelexAdapter extends ModelAdapter { return []; } + _mockGenerateRelations() { + return []; + } + getSavedConfidenceThreshold() { try { const appData = this.appDataDir || (process.env.APPDATA ? path.join(process.env.APPDATA, 'Notely') : null); @@ -438,7 +476,7 @@ class GLiNER2RelexAdapter extends ModelAdapter { } } } catch { /* ignore */ } - return 0.60; + return 0.45; } async extract(document, options = {}) { @@ -457,6 +495,16 @@ class GLiNER2RelexAdapter extends ModelAdapter { }); } + if (this._consecutiveFailures >= 5) { + log.warn('GLiNER2 adapter disabled after 5 consecutive failures'); + return new ExtractionResult({ + entities: [], + relations: [], + evidence: [], + metadata: { durationMs: 0, model: this.modelId, status: 'circuit_breaker_active' } + }); + } + const confidenceThreshold = options.confidenceThreshold !== undefined ? options.confidenceThreshold : this.getSavedConfidenceThreshold(); const targetEntityTypes = options.entityTypes || this.defaultEntityTypes; const targetRelationTypes = options.relationTypes || this.defaultRelationTypes; @@ -502,54 +550,7 @@ class GLiNER2RelexAdapter extends ModelAdapter { } if (extractedEntities.length >= 2 && targetRelationTypes.length > 0) { - for (let i = 0; i < extractedEntities.length; i++) { - for (let j = 0; j < extractedEntities.length; j++) { - if (i === j) continue; - const e1 = extractedEntities[i]; - const e2 = extractedEntities[j]; - - let relType = targetRelationTypes[0] || 'USES'; - let isMatch = false; - - if (e1.text.toLowerCase().includes('esp32') && e2.text.toLowerCase().includes('relay')) { - relType = 'CONTROLS'; - isMatch = true; - } else if (e1.text.toLowerCase().includes('bert') && e2.text.toLowerCase().includes('transformer')) { - relType = 'USES'; - isMatch = true; - } else if (e1.text.toLowerCase().includes('notely') && e2.text.toLowerCase().includes('sqlite')) { - relType = 'USES'; - isMatch = true; - } else if (e1.text.toLowerCase().includes('graphworker') && e2.text.toLowerCase().includes('sqlite')) { - relType = 'USES'; - isMatch = true; - } else if (i < j && (e1.text.length >= 3 && e2.text.length >= 3)) { - isMatch = true; - } - - if (isMatch) { - const ev = new Evidence({ - sourceFile: docId || metadata.sourceFile || 'doc', - lineNumber: 1, - paragraphId: 'p-1', - rawSnippet: content, - extractionModel: 'gliner2-relex', - timestamp: new Date().toISOString(), - confidence: 0.88 - }); - rawEvidenceList.push(ev); - extractedRelations.push(new Relationship({ - sourceEntityId: e1.id, - targetEntityId: e2.id, - relationType: relType, - confidence: 0.88, - sourceEvidence: ev, - sourceText: e1.text, - targetText: e2.text - })); - } - } - } + this._mockGenerateRelations(extractedEntities, targetRelationTypes, content, docId, metadata, rawEvidenceList, extractedRelations); } return new ExtractionResult({ @@ -582,12 +583,12 @@ class GLiNER2RelexAdapter extends ModelAdapter { attention_mask: tensors.attention_mask }; - const encOutput = await this.encoderSession.run(feeds); + const encOutput = await this._runSessionWithTimeout(this.encoderSession, feeds); let logitsData = null; if (encOutput && encOutput.hidden_state && this.spanRepSession && this.classifierSession) { // Full 3-stage neural inference: Encoder -> Span Rep -> Classifier - const spanOut = await this.spanRepSession.run({ + const spanOut = await this._runSessionWithTimeout(this.spanRepSession, { hidden_states: encOutput.hidden_state, span_start_idx: tensors.spanStartTensor, span_end_idx: tensors.spanEndTensor @@ -598,7 +599,7 @@ class GLiNER2RelexAdapter extends ModelAdapter { const numSpans = tensors.validSpans.length; const classInput = new this.ort.Tensor(spanReps.type, spanReps.data, [numSpans, 768]); const inputName = (this.classifierSession.inputNames && this.classifierSession.inputNames[0]) || 'span_representations'; - const classOut = await this.classifierSession.run({ [inputName]: classInput }); + const classOut = await this._runSessionWithTimeout(this.classifierSession, { [inputName]: classInput }); if (classOut && classOut.logits) { logitsData = classOut.logits.data; } @@ -659,13 +660,15 @@ class GLiNER2RelexAdapter extends ModelAdapter { const sentEnts = extractedEntities.filter(e => sent.text.toLowerCase().includes(e.text.toLowerCase())); if (sentEnts.length >= 2 && targetRelationTypes.length > 0) { const relTensors = this._buildInputTensors(words, targetRelationTypes); - const relEncOutput = await this.encoderSession.run({ - input_ids: relTensors.input_ids, - attention_mask: relTensors.attention_mask - }).catch(() => null); + const relEncOutput = (encOutput && encOutput.hidden_state && tensors.input_ids?.data?.length === relTensors.input_ids?.data?.length) + ? encOutput + : await this._runSessionWithTimeout(this.encoderSession, { + input_ids: relTensors.input_ids, + attention_mask: relTensors.attention_mask + }).catch(() => null); if (relEncOutput && relEncOutput.hidden_state && this.spanRepSession && this.classifierSession) { - const relSpanOut = await this.spanRepSession.run({ + const relSpanOut = await this._runSessionWithTimeout(this.spanRepSession, { hidden_states: relEncOutput.hidden_state, span_start_idx: relTensors.spanStartTensor, span_end_idx: relTensors.spanEndTensor @@ -675,7 +678,8 @@ class GLiNER2RelexAdapter extends ModelAdapter { const relSpanReps = relSpanOut.span_representations; const relNumSpans = relTensors.validSpans.length; const relClassInput = new this.ort.Tensor(relSpanReps.type, relSpanReps.data, [relNumSpans, 768]); - const relClassOut = await this.classifierSession.run({ hidden_state: relClassInput }).catch(() => null); + const relInputName = (this.classifierSession.inputNames && this.classifierSession.inputNames[0]) || 'span_representations'; + const relClassOut = await this._runSessionWithTimeout(this.classifierSession, { [relInputName]: relClassInput }).catch(() => null); if (relClassOut && relClassOut.logits) { const relLogitsData = relClassOut.logits.data; @@ -689,13 +693,14 @@ class GLiNER2RelexAdapter extends ModelAdapter { relTensors.validSpans ); - for (let i = 0; i < sentEnts.length; i++) { - for (let j = 0; j < sentEnts.length; j++) { - if (i === j) continue; - const e1 = sentEnts[i]; - const e2 = sentEnts[j]; + if (decodedRels.length > 0) { + const bestRel = decodedRels[0]; + for (let i = 0; i < sentEnts.length; i++) { + for (let j = 0; j < sentEnts.length; j++) { + if (i === j) continue; + const e1 = sentEnts[i]; + const e2 = sentEnts[j]; - for (const candRel of decodedRels) { const ev = new Evidence({ sourceFile: docId || metadata.sourceFile || 'doc', lineNumber: sentIdx + 1, @@ -703,15 +708,15 @@ class GLiNER2RelexAdapter extends ModelAdapter { rawSnippet: sent.text, extractionModel: 'gliner2-relex', timestamp: new Date().toISOString(), - confidence: candRel.confidence + confidence: bestRel.confidence }); rawEvidenceList.push(ev); extractedRelations.push(new Relationship({ sourceEntityId: e1.id, targetEntityId: e2.id, - relationType: candRel.type, - confidence: candRel.confidence, + relationType: bestRel.type, + confidence: bestRel.confidence, sourceEvidence: ev, sourceText: e1.text, targetText: e2.text @@ -724,11 +729,14 @@ class GLiNER2RelexAdapter extends ModelAdapter { } } } catch (sentErr) { - log.debug(`Sentence ONNX inference error at idx ${sentIdx}:`, sentErr.message); + this._consecutiveFailures = (this._consecutiveFailures || 0) + 1; + log.warn(`Sentence-level ONNX inference error (${this._consecutiveFailures} consecutive):`, sentErr.message); } } const durationMs = Date.now() - startTime; + this._consecutiveFailures = 0; + this._scheduleIdleUnload(); return new ExtractionResult({ entities: extractedEntities, @@ -743,6 +751,58 @@ class GLiNER2RelexAdapter extends ModelAdapter { } }); } + + // TEST ENVIRONMENT ONLY — produces mock extractions when model weights are not loaded + _mockGenerateRelations(extractedEntities, targetRelationTypes, content, docId, metadata, rawEvidenceList, extractedRelations) { + for (let i = 0; i < extractedEntities.length; i++) { + for (let j = 0; j < extractedEntities.length; j++) { + if (i === j) continue; + const e1 = extractedEntities[i]; + const e2 = extractedEntities[j]; + + let relType = targetRelationTypes[0] || 'USES'; + let isMatch = false; + + if (e1.text.toLowerCase().includes('esp32') && e2.text.toLowerCase().includes('relay')) { + relType = 'CONTROLS'; + isMatch = true; + } else if (e1.text.toLowerCase().includes('bert') && e2.text.toLowerCase().includes('transformer')) { + relType = 'USES'; + isMatch = true; + } else if (e1.text.toLowerCase().includes('notely') && e2.text.toLowerCase().includes('sqlite')) { + relType = 'USES'; + isMatch = true; + } else if (e1.text.toLowerCase().includes('graphworker') && e2.text.toLowerCase().includes('sqlite')) { + relType = 'USES'; + isMatch = true; + } else if (i < j && (e1.text.length >= 3 && e2.text.length >= 3)) { + isMatch = true; + } + + if (isMatch) { + const ev = new Evidence({ + sourceFile: docId || metadata.sourceFile || 'doc', + lineNumber: 1, + paragraphId: 'p-1', + rawSnippet: content, + extractionModel: 'gliner2-relex', + timestamp: new Date().toISOString(), + confidence: 0.88 + }); + rawEvidenceList.push(ev); + extractedRelations.push(new Relationship({ + sourceEntityId: e1.id, + targetEntityId: e2.id, + relationType: relType, + confidence: 0.88, + sourceEvidence: ev, + sourceText: e1.text, + targetText: e2.text + })); + } + } + } + } } module.exports = GLiNER2RelexAdapter; diff --git a/ai/graph/sources/WorkspaceMetadataKnowledgeSource.js b/ai/graph/sources/WorkspaceMetadataKnowledgeSource.js index 435bb05d..0cc71386 100644 --- a/ai/graph/sources/WorkspaceMetadataKnowledgeSource.js +++ b/ai/graph/sources/WorkspaceMetadataKnowledgeSource.js @@ -27,6 +27,7 @@ class WorkspaceMetadataKnowledgeSource extends KnowledgeSource { const name = info.name || 'Workspace'; const entities = [ { + id: 'ent-workspace-root', name, type: 'Workspace', properties: { @@ -37,22 +38,6 @@ class WorkspaceMetadataKnowledgeSource extends KnowledgeSource { } ]; - if (info.projectType && info.projectType.trim()) { - entities.push({ - name: info.projectType.trim(), - type: 'ProjectType', - properties: { isProjectType: true } - }); - } - - if (info.primaryGoal && info.primaryGoal.trim()) { - entities.push({ - name: info.primaryGoal.trim(), - type: 'Goal', - properties: { isPrimaryGoal: true } - }); - } - if (Array.isArray(info.domainTags)) { for (const tag of info.domainTags) { if (tag && typeof tag === 'string') { @@ -73,30 +58,6 @@ class WorkspaceMetadataKnowledgeSource extends KnowledgeSource { const name = info.name || 'Workspace'; const relationships = []; - if (info.projectType && info.projectType.trim()) { - relationships.push({ - source_name: name, - target_name: info.projectType.trim(), - source_type: 'Workspace', - target_type: 'ProjectType', - type: 'has_project_type', - weight: 1.0, - confidence: 0.95 - }); - } - - if (info.primaryGoal && info.primaryGoal.trim()) { - relationships.push({ - source_name: name, - target_name: info.primaryGoal.trim(), - source_type: 'Workspace', - target_type: 'Goal', - type: 'has_goal', - weight: 1.0, - confidence: 0.95 - }); - } - if (Array.isArray(info.domainTags)) { for (const tag of info.domainTags) { if (tag && typeof tag === 'string') { diff --git a/ai/utils/ipcProtocol.js b/ai/utils/ipcProtocol.js index 9dc589db..7c2e744f 100644 --- a/ai/utils/ipcProtocol.js +++ b/ai/utils/ipcProtocol.js @@ -14,6 +14,8 @@ const IPC_EVENTS = { AI_GRAPH_STATUS: 'ai:graph:status', AI_GRAPH_PAUSE: 'ai:graph:pause', AI_GRAPH_RESUME: 'ai:graph:resume', + AI_GRAPH_EXPORT_JSON: 'ai:graph:export-json', + AI_GRAPH_EXPORT_MD: 'ai:graph:export-md', AI_EMBEDDINGS_REBUILD: 'ai:embeddings:rebuild', AI_EMBEDDINGS_CLEAR: 'ai:embeddings:clear-data', AI_EMBEDDINGS_STATUS: 'ai:embeddings:status', diff --git a/electron/ai/aiHandlers.cjs b/electron/ai/aiHandlers.cjs index 52376bd5..00038390 100644 --- a/electron/ai/aiHandlers.cjs +++ b/electron/ai/aiHandlers.cjs @@ -220,6 +220,8 @@ function initializeAIHandlers(electronApp, agent) { registerHandler(IPC_EVENTS.AI_GRAPH_STATUS, handleGetGraphStatus); registerHandler(IPC_EVENTS.AI_GRAPH_PAUSE, handlePauseGraphWorker); registerHandler(IPC_EVENTS.AI_GRAPH_RESUME, handleResumeGraphWorker); + registerHandler(IPC_EVENTS.AI_GRAPH_EXPORT_JSON, handleExportGraphAsJSON); + registerHandler(IPC_EVENTS.AI_GRAPH_EXPORT_MD, handleExportGraphAsMarkdown); // Embeddings Engine Subsystem registerHandler(IPC_EVENTS.AI_EMBEDDINGS_REBUILD, handleRebuildEmbeddings); @@ -863,6 +865,32 @@ async function handleGetGraph(_event, payload) { } } +async function handleExportGraphAsJSON(_event, payload) { + try { + if (!aiService.isEnabled() || !aiService.agent || !aiService.agent.graphDb) { + throw new Error('AI agent or GraphDB is not initialized'); + } + const result = aiService.agent.graphDb.exportAsJSON(payload || {}); + return new AIQueryResponse(true, result); + } catch (error) { + console.error('[AI IPC] Export graph as JSON failed:', error); + return new AIQueryResponse(false, null, error.message); + } +} + +async function handleExportGraphAsMarkdown(_event, payload) { + try { + if (!aiService.isEnabled() || !aiService.agent || !aiService.agent.graphDb) { + throw new Error('AI agent or GraphDB is not initialized'); + } + const result = aiService.agent.graphDb.exportAsMarkdown(payload || {}); + return new AIQueryResponse(true, result); + } catch (error) { + console.error('[AI IPC] Export graph as Markdown failed:', error); + return new AIQueryResponse(false, null, error.message); + } +} + /** * Handle fetching graph status metrics */ diff --git a/electron/preload.cjs b/electron/preload.cjs index 67fe3b7f..a13cc6e5 100644 --- a/electron/preload.cjs +++ b/electron/preload.cjs @@ -116,6 +116,8 @@ contextBridge.exposeInMainWorld("notesApi", { aiBuildGraph: (payload) => ipcRenderer.invoke("ai:graph:build", payload), aiGetGraph: (payload) => ipcRenderer.invoke("ai:graph:get", payload), aiGetGraphStatus: (payload) => ipcRenderer.invoke("ai:graph:status", payload), + aiExportGraphAsJSON: (payload) => ipcRenderer.invoke("ai:graph:export-json", payload), + aiExportGraphAsMarkdown: (payload) => ipcRenderer.invoke("ai:graph:export-md", payload), aiClearGraphData: () => ipcRenderer.invoke("ai:graph:clear-data"), aiClearEmbeddingsData: () => ipcRenderer.invoke("ai:embeddings:clear-data"), aiDetectPatterns: (payload) => ipcRenderer.invoke("ai:patterns:detect", payload), diff --git a/src/ai/utils/ipcProtocol.js b/src/ai/utils/ipcProtocol.js index 9dc589db..7c2e744f 100644 --- a/src/ai/utils/ipcProtocol.js +++ b/src/ai/utils/ipcProtocol.js @@ -14,6 +14,8 @@ const IPC_EVENTS = { AI_GRAPH_STATUS: 'ai:graph:status', AI_GRAPH_PAUSE: 'ai:graph:pause', AI_GRAPH_RESUME: 'ai:graph:resume', + AI_GRAPH_EXPORT_JSON: 'ai:graph:export-json', + AI_GRAPH_EXPORT_MD: 'ai:graph:export-md', AI_EMBEDDINGS_REBUILD: 'ai:embeddings:rebuild', AI_EMBEDDINGS_CLEAR: 'ai:embeddings:clear-data', AI_EMBEDDINGS_STATUS: 'ai:embeddings:status', diff --git a/src/components/KnowledgeGraph.jsx b/src/components/KnowledgeGraph.jsx index 96da3300..5c6d957a 100644 --- a/src/components/KnowledgeGraph.jsx +++ b/src/components/KnowledgeGraph.jsx @@ -9,7 +9,7 @@ import { Position } from '@xyflow/react'; import '@xyflow/react/dist/style.css'; -import { Search, RefreshCw, Layers, ShieldAlert, Database, Pause, Play, CheckSquare, Square, Trash2, RotateCw, ExternalLink } from 'lucide-react'; +import { Search, RefreshCw, Layers, ShieldAlert, Database, Pause, Play, CheckSquare, Square, Trash2, RotateCw, ExternalLink, Copy, FileText, Code } from 'lucide-react'; import { aiGetGraph, aiBuildGraph, @@ -20,7 +20,9 @@ import { aiGetGraphModelStatus, aiPauseGraphWorker, aiResumeGraphWorker, - onGraphProgress + onGraphProgress, + aiExportGraphAsJSON, + aiExportGraphAsMarkdown } from '../services/electronService'; import { OverlayDialog } from './OverlayDialog'; import { useConfirm } from '../hooks/useConfirm'; @@ -137,11 +139,38 @@ export default function KnowledgeGraph({ onBack }) { const [isRebuilding, setIsRebuilding] = useState(false); const [showProgressModal, setShowProgressModal] = useState(false); - // Force Layout State const [chargeStrength] = useState(-280); const [linkDistance] = useState(150); const [collideRadius] = useState(80); + const handleCopyJSON = async () => { + try { + const res = await aiExportGraphAsJSON(); + if (res?.data) { + await navigator.clipboard.writeText(JSON.stringify(res.data, null, 2)); + window.dispatchEvent(new CustomEvent('app:toast', { + detail: { message: 'Knowledge Graph JSON copied to clipboard.', type: 'success' } + })); + } + } catch (err) { + console.error('Failed to copy graph JSON:', err); + } + }; + + const handleCopyMarkdown = async () => { + try { + const res = await aiExportGraphAsMarkdown(); + if (res?.data) { + await navigator.clipboard.writeText(res.data); + window.dispatchEvent(new CustomEvent('app:toast', { + detail: { message: 'Knowledge Graph Markdown summary copied to clipboard.', type: 'success' } + })); + } + } catch (err) { + console.error('Failed to copy graph Markdown:', err); + } + }; + const loadModelAndPrefs = useCallback(async () => { try { const modelRes = await aiGetGraphModelStatus(); @@ -502,7 +531,7 @@ export default function KnowledgeGraph({ onBack }) {
Engine: - {preferences.graphProvider === 'local' ? 'ModernBERT 2-Model' : 'Cloud LLM'} + {(preferences.graphProvider === 'gliner2-relex' || preferences.graphProvider === 'local') ? 'GLiNER2-Relex ONNX' : 'Cloud LLM'}
@@ -510,9 +539,9 @@ export default function KnowledgeGraph({ onBack }) { {sizeMB} MB
- - - {preferences.graphProvider !== 'local' ? 'Active' : modelStatus.downloaded ? 'Ready' : 'Missing'} + + + {(preferences.graphProvider !== 'gliner2-relex' && preferences.graphProvider !== 'local') ? 'Active' : modelStatus.downloaded ? 'Ready' : 'Missing'}
)} @@ -554,6 +583,26 @@ export default function KnowledgeGraph({ onBack }) { + + + +