1. [Retrieval Phase] Chunks are fed into vector DB
             |-------------------- A. Retrieval Phase (Offline) --------------|
             |                                                                |
Raw          |  |--- Chunker ---|                                             |
documents →  |- | break docs in |--chunks →[Embedding]-vectors → [vectorDB]   |
logs         |  |smaller pieces |          [  Model  ]                 |      |
             |  |---------------|                                      \/     |
             |                                                       index    |
             |----------------------------------------------------------------|

             [Node1 (score 0.92), Node2 (score 0.87), Node3 (score 0.76)]

2. [Augmentation Phase] User asks a query & information retrieved from Vector DB
User's Query: Show firewall policies blocking outbound traffic?

 index from vectordb
     \/
   search vector index
     \/
   get top-k chunk texts [Node1][Node2][Node3]..[Nodek]
     |
     -----------------------→ augumented_prompt <------ User's Query
                                                      (Why is john.doe unable to connect to VPN?)

augmented_prompt=
"Context: 
[Node1][Node2][Node3] 
Question: Why is john.doe unable to connect to VPN
Answer:"

3. [Generation Phase] Feed augmented_prompt into LLM.
With (user_query + vector), LLM hallucinations reduces drastically

                     |-- LLM --|
augmented_prompt --→ | GPT5.0  | --→ Reponse (less hallucinations)
                     |---------|
      

./logs/vpn.log
2025-05-01 VPN_LOGIN_FAILED user=john.doe ip=185.22.11.4
2025-05-01 VPN_LOGIN_FAILED user=john.doe ip=185.22.11.4

./logs/firewall.log
2025-05-01 FIREWALL_DENY src=10.1.1.5 dst=8.8.8.8 policy=OUTBOUND_BLOCK
2025-05-01 FIREWALL_DENY src=10.1.1.6 dst=1.1.1.1 policy=OUTBOUND_BLOCK

┌─────────────┐     ┌──────────────────┐     ┌────────────────┐
│  RETRIEVAL  │ ──► │   AUGMENTATION   │ ──► │  GENERATION    │
│  (indexing  │     │  (build prompt   │     │  (LLM writes   │
│   + search) │     │   with context)  │     │   the answer)  │
└─────────────┘     └──────────────────┘     └────────────────┘
     setup               as_query_engine           .query()

$ cat rag_pipeline.py
import os
import dotenv
from llama_index.embeddings.openai import OpenAIEmbedding
from llama_index.core import VectorStoreIndex, SimpleDirectoryReader
from llama_index.llms.openai import OpenAI		#Import LLM
from llama_index.core import Settings

# Load GitHub Token and set env
dotenv.load_dotenv()
if not os.getenv("GITHUB_TOKEN"):
    raise ValueError("GITHUB_TOKEN is not set")
os.environ["OPENAI_API_KEY"] = os.getenv("GITHUB_TOKEN")
os.environ["OPENAI_BASE_URL"] = "https://models.inference.ai.azure.com/"

############## 1. Retrieval Phase Start #################
#                            |------ Chunking ------------------|
# documents (2 log files) -> |Split sentence into small pieces  |
#                            | (1 doc = 1 Node)                 |
#                            |----------------------------------|
#                                            \/
#                            |-------- Embedding ---------------|
#                            |For each chunk (batch of 150):    |
#                            | Create float vector/chunk        |
#                            |----------------------------------|
#                                           \/
#                            |-- vector db (in memory default) -| 
#                            |Stores: (embedding, text, meta)   |
#                            |stored in RAM(default) not disk   |
#                            |----------------------------------|
#                                           \/
#                                         index  ← searchable index object
#VectorStoreIndex (variable: index)
#    │
#    ├── docstore          → stores Node objects (chunk text + metadata)
#    │     e.g. Node 0: "2025-05-01 08:01:05 VPN_LOGIN_FAILED user=john.doe..."
#    │     e.g. Node 1: "2025-05-01 08:02:11 FIREWALL_DENY src=10.1.1.5..."
#    │
#    ├── vector_store      → stores embedding vectors for each node
#    │     e.g. [0.012, -0.034, 0.891, ...]  (1536 floats per chunk)
#    │
#    └── index_struct      → maps node IDs → vector store locations
#
## A. Setup Embedding Model. This is Neural Network	
embed_model = OpenAIEmbedding(
    model="text-embedding-3-small",
    api_key=os.getenv("OPENAI_API_KEY"),
    api_base=os.getenv("OPENAI_BASE_URL"),
)
Settings.embed_model = embed_model

## B. Break documents into Chunks
documents = SimpleDirectoryReader("./logs").load_data()

## C. Chunking, embedding, local vector store
# def from_documents(documents, insert_batch_size=150): 
#   nodes = self._chunk_documents(documents) #chunks the documents into Nodes
#   for batch in batches(nodes, batch_size=insert_batch_size):
#       texts = [node.text for node in batch]
#       embeddings = Settings.embed_model.get_text_embedding_batch(texts)
#       self._vector_store.add(embeddings, metadata=batch.metadata) #Store Tensors into the vector DB
index = VectorStoreIndex.from_documents(documents, insert_batch_size=150)
############## Retrieval Phase End #####################

# Create LLM
llm = OpenAI(
    model="gpt-4o-mini",
    api_key=os.getenv("OPENAI_API_KEY"),
    api_base=os.getenv("OPENAI_BASE_URL"),
)

############## 2. Augumentation Phase Start #################
# index from vectordb
#     \/
#   search vector index
#     \/
#   get top-k chunk texts [Node1][Node2][Node3]..[Nodek]
#     |
#     |
#     -----------------------→ LLM <------ User's Query
#                                             (Why is john.doe unable to connect to VPN?)
#   
# venv/lib/python3.10/site-packages/llama_index/core/indices/base.py
# def as_query_engine(self, llm: Optional[LLMType] = None, **kwargs: Any)
#     RetrieverQueryEngine(kwargs) 

query_engine = index.as_query_engine(
  llm=llm
)

############## 3. Generation Phase Start #################
response = query_engine.query("Show firewall policies blocking outbound traffic")
print(response)
Response=
The firewall policies blocking outbound traffic are as follows:

1. Policy: OUTBOUND_BLOCK
   - Source: 10.1.1.5
   - Destination: 8.8.8.8

2. Policy: OUTBOUND_BLOCK
   - Source: 10.1.1.6
   - Destination: 1.1.1.1

response = query_engine.query("Why is john.doe unable to connect to VPN?")
print(response)
Response=
john.doe is unable to connect to the VPN due to repeated login failures, as 
indicated by the log entries showing two instances of VPN_LOGIN_FAILED for the user.
############## Augumentation & Generation Phase End #################
      

import os
import dotenv
from llama_index.core import Settings, SimpleDirectoryReader, VectorStoreIndex
from llama_index.embeddings.openai import OpenAIEmbedding
from llama_index.llms.openai import OpenAI

def configure_environment() -> None:
    """Load credentials and configure OpenAI-compatible API access."""
    dotenv.load_dotenv()
    if not os.getenv("GITHUB_TOKEN"):
        raise ValueError("GITHUB_TOKEN is not set")
    os.environ["OPENAI_API_KEY"] = os.getenv("GITHUB_TOKEN")
    os.environ["OPENAI_BASE_URL"] = "https://models.inference.ai.azure.com/"

def main() -> None:
    configure_environment()

    ######## 1. Retrieval Phase ############
    embed_model = OpenAIEmbedding(
        model="text-embedding-3-small",
        api_key=os.getenv("OPENAI_API_KEY"),
        api_base=os.getenv("OPENAI_BASE_URL"),
    )
    Settings.embed_model = embed_model

    documents = SimpleDirectoryReader(logs_dir).load_data()
    index = VectorStoreIndex.from_documents(documents, insert_batch_size=150)
    ######## Retrieval Phase End ############

    llm = OpenAI(
        model="gpt-4o-mini",
        api_key=os.getenv("OPENAI_API_KEY"),
        api_base=os.getenv("OPENAI_BASE_URL"),
    )

    ######## 2. Augumentation Phase ############
    query_engine = index.as_query_engine(llm=llm, similarity_top_k=3)
    ######## Augumentation Phase End ############

    demo_questions = [
        "Show firewall policies blocking outbound traffic",
        "Why is john.doe unable to connect to VPN?",
    ]

    ######## 3. Generation Phase ############
    for q in demo_questions:
        response = query_engine.query(query_engine, q)
        contexts = [node.node.get_content() for node in response.source_nodes]
        print(f"\nQuestion: {response['question']}")
        print(f"Answer: {response['answer']}")
        print(f"Retrieved {len(response['contexts'])} context chunk(s)")

if __name__ == "__main__":
    main()