Policy to Knowledge

graph TD
    Browser["🌐 Browser"]

    subgraph FE["Frontend Layer"]
        Shell["Suite Shell
React + Vite · :4000"]
        PipeUI["Pipeline UI
React + Vite · :5173"]
        ExplorerUI["KG Explorer
D3.js · :5000"]
    end

    subgraph BE["Backend Layer"]
        PipeAPI["Pipeline API
FastAPI · :8000"]
        AssistantAPI["Assistant
Flask · :5000"]
    end

    subgraph AI["AI Inference"]
        OpenAI["OpenAI GPT"]
    end

    subgraph Data["Data Layer"]
        JanusGraph["JanusGraph · :8182
Graph Database"]
        Cassandra["Apache Cassandra · :9042
Storage Backend"]
        OpenSearch["OpenSearch · :9200
Full-text + Vector k-NN"]
        Redis["Redis · :6379
Query Cache LRU"]
    end

    Browser --> Shell
    Shell -->|"iframe"| PipeUI
    Shell -->|"iframe"| ExplorerUI
    PipeUI --> PipeAPI
    ExplorerUI --> AssistantAPI
    PipeAPI --> OpenAI
    AssistantAPI --> Redis
    AssistantAPI --> JanusGraph
    JanusGraph --> Cassandra
    JanusGraph --> OpenSearch
    

graph TB
    subgraph Presentation["Presentation Layer"]
        UI["Web Client — D3.js + EventSource API
Visualization · User Interaction · SSE"]
    end

    subgraph Application["Application Layer"]
        API["REST API Gateway — Flask + CORS
Request Routing · Auth · Rate Limiting"]
        ConnMgr["Connection Manager — Singleton Pools
Resource Lifecycle · Circuit Breaking"]
    end

    subgraph Services["Service Layer"]
        AIOrch["AI Orchestration — GPT-4o + Tool Calling
Query Understanding · Tool Selection"]
        GraphSvc["Graph Query Service — Gremlin
Traversals · Pattern Matching"]
        VectorSvc["Vector Search — Embeddings + k-NN
Semantic Similarity · Ranking"]
    end

    subgraph DataLayer["Data Layer"]
        GraphDB[("JanusGraph
Multi-Graph")]
        VectorDB[("OpenSearch
k-NN Index")]
        Cache[("Redis
LRU Cache")]
    end

    UI --> API
    API --> ConnMgr
    API --> AIOrch
    AIOrch --> GraphSvc
    AIOrch --> VectorSvc
    ConnMgr --> GraphDB
    ConnMgr --> VectorDB
    GraphSvc --> ConnMgr
    VectorSvc --> ConnMgr
    Cache -.-> GraphSvc
    Cache -.-> VectorSvc
    

sequenceDiagram
    participant S as start.sh
    participant C as Cassandra :9042
    participant O as OpenSearch :9200
    participant R as Redis :6379
    participant J as JanusGraph :8182
    participant P as Pipeline API :8000
    participant U as Pipeline UI :5173
    participant K as Assistant :5000
    participant F as Suite Shell :4000

    S->>C: start + health check
    S->>O: start + health check
    S->>R: start + health check
    S->>J: start (depends on C + O)
    J-->>S: ready
    S->>P: start FastAPI
    S->>U: start Vite dev server
    S->>K: load KGs into JanusGraph
    K-->>S: graphs loaded
    S->>K: start Flask server
    S->>F: start Vite dev server
    F-->>S: Suite ready at localhost:4000
    

flowchart TB
    Docs["📄 Compliance Documents\nPDF · DOCX · MD · CSV · XLSX"]

    subgraph Extraction["Phase 1 — Extraction Pipeline"]
        direction TB

        A1["🗂️ Agent 1 — Document Organizer\nTOC-based hierarchical chunking\nSplits documents into structured sections"]
        A1out["📁 Organized Chunks\nknowledge-files-organized/"]

        A2["🔍 Agent 2 — Entity Extractor\nMeta-agent prompt optimization\nExtracts domain entities & relationships"]
        A2out["📋 Entity Definitions\nentity_types_and_relationships.json"]

        A3["⚖️ Agent 3 — Rules Extractor\nParallel batch processing\n10-category rule taxonomy"]
        A3out["📜 Business Rules\ncompliance_rules_with_entities.json"]

        A35["✅ Agent 3.5 — Rule Validator\nSource verification · Numeric consistency\nContradiction detection · Confidence scoring"]
        A35out["📊 Validation Report\nvalidation_report.json"]

        A4["🔗 Agent 4 — Rules + Entities Merger\nEnriches rules with entity context\nAssembles complete knowledge graph"]
        A4out["🗃️ Complete Knowledge Graph\ncompliance_knowledge_graph.json"]

        A5["⚡ Agent 5 — KG Optimizer\nRule deduplication · 7 dependency types\nImpact analysis & confidence scoring"]
        A5out["💎 Optimized Knowledge Graph\noptimized_compliance_knowledge_graph.json"]

        A6["🎨 Agent 6 — Visualization Generator\nvis.js network graphs\nSearchable rule tables & metrics"]
        A6out["🌐 Interactive HTML Report\n{name}_knowledge_graph.html"]

        A1 --> A1out --> A2
        A2 --> A2out --> A3
        A3 --> A3out --> A35
        A35 --> A35out -.->|non-blocking| A4
        A2out -->|entities| A4
        A3out -->|rules| A4
        A4 --> A4out --> A5
        A5 --> A5out --> A6
        A6 --> A6out
    end

    subgraph Merging["Phase 2 — Merge Pipeline (Multi-Graph Comparison)"]
        direction TB

        KG_A["📊 Knowledge Graph A"]
        KG_B["📊 Knowledge Graph B"]

        A7["📦 Agent 7 — Rule Type Clusterer\nGroups by behavior type\nformula · threshold · sequence\nmethod · mandate · prohibition\nclassification · timing"]
        A7out["🏷️ Rule Clusters\nrule_clusters.json"]

        A8["🧠 Agent 8 — Semantic Rule Matcher\nLLM-powered pairwise comparison\nBatch parallelism for scale\nIDENTICAL · EQUIVALENT\nCONTRADICTORY · UNRELATED"]
        A8out["🔎 Match Results\nmatch_results.json"]

        A9["🔀 Agent 9 — Set Operations\nComputes 5 set operations"]
        A9i["∩ Intersection"]
        A9u["∪ Union"]
        A9d1["A \\ B Difference"]
        A9d2["B \\ A Difference"]
        A9c["⚠️ Contradictions"]

        A10["📈 Agent 10 — Set Visualization\nVenn diagrams · Dashboard\nComparison HTML reports"]
        A10out["🌐 6 HTML Reports\nindex · intersection · union\ndifferences · contradictions"]

        KG_A --> A7
        KG_B --> A7
        A7 --> A7out --> A8
        A8 --> A8out --> A9
        A9 --> A9i
        A9 --> A9u
        A9 --> A9d1
        A9 --> A9d2
        A9 --> A9c
        A9i --> A10
        A9u --> A10
        A9d1 --> A10
        A9d2 --> A10
        A9c --> A10
        A10 --> A10out
    end

    subgraph Infra["AI Infrastructure"]
        LLM["🤖 LLM Provider\nOpenAI GPT\nvia the official OpenAI SDK"]
    end

    Docs --> A1
    A6out -->|"Per-document KG"| KG_A
    A6out -->|"Per-document KG"| KG_B
    A3 -.->|API calls| LLM
    A2 -.->|API calls| LLM
    A5 -.->|API calls| LLM
    A8 -.->|API calls| LLM
    

flowchart LR
    A["Agent requests prompt\n(e.g. entity_extraction)"] --> B{"domain-prompts/\n{domain}/{prompt}.txt\nexists?"}
    B -- Yes --> C["Load domain-specific\nprompt"]
    B -- No --> D["Load shared fallback\nprompts/{prompt}.txt"]
    C --> E["Substitute parameters\n(entity_context, batch_num, etc.)"]
    D --> E
    E --> F["Send to LLM\n(OpenAI GPT)"]
    

erDiagram
    business_rule ||--o{ business_rule : "depends_on"
    business_rule }o--|| entity_category : "belongs_to_category"
    business_rule ||--o{ business_rule : "relates_to"

    business_rule {
        string rule_id PK
        string rule_name
        string rule_type
        string description
        string conditions
        string consequences
        string exceptions
        string reference
        boolean mandatory
        double confidence_score
        boolean requires_review
        string review_reason
        string node_type
        string vertex_uuid UK
        string source_reference
        string effective_date
        string expiration_date
        string superseded_by
        string jurisdiction
        string risk_level
        string enforcement_action
        string applicability_scope
        string data_points_required
        string audit_frequency
        boolean reference_verified
        string reference_verification_note
        string confidence_breakdown
        string deduplication_info
        string related_rules
    }

    entity_category {
        string name
        string entity_type
        string entity_or_relationship
        string description
        string extraction_notes
        string content
        string category
    }