Advanced RAG Implemented

.python-version

@@ -0,0 +1 @@
+3.12

.streamlit/config.toml

@@ -0,0 +1,7 @@
+[theme]
+base="dark"
+primaryColor="#8B5CF6"
+backgroundColor="#0F172A"
+secondaryBackgroundColor="#1E293B"
+textColor="#F8FAFC"
+font="sans serif"

app.py

@@ -1,215 +1,509 @@
-import shutil
-
+import uuid
+import requests
 import streamlit as st
-import os
+import time
+
+# -----------------------------------
+# CONFIG
+# -----------------------------------
+
+API_BASE_URL = "http://127.0.0.1:8000/api"
+
+st.set_page_config(
+    page_title="Syllabus RAG",
+    layout="wide",
+)
+
 
-from src.ingest import load_documents
-from src.chunk import chunk_documents
-from src.embed import Embedder
-from src.vector_store import VectorStore
-from src.llm import generate_answer
+# -----------------------------------
+# CUSTOM CSS
+# -----------------------------------
+
+st.markdown(
+    """
+    <style>
+
+    html, body, [class*="css"] {
+        font-family: 'Inter', sans-serif;
+    }
+
+    .main {
+        background-color: #0F172A;
+    }
+
+    .block-container {
+        padding-top: 1.2rem;
+        padding-bottom: 2rem;
+        max-width: 1100px;
+    }
+
+    section[data-testid="stSidebar"] {
+        background-color: #0B1220;
+        border-right: 1px solid #334155;
+    }
+
+    h1, h2, h3 {
+        color: #F8FAFC;
+        font-weight: 700;
+    }
+
+    p, span, label {
+        color: #E2E8F0;
+    }
+
+    .stChatMessage {
+        border-radius: 16px;
+        padding: 0.8rem;
+    }
+
+    [data-testid="stChatMessageContent"] {
+        font-size: 1rem;
+        line-height: 1.7;
+    }
+
+    .stButton button {
+        border-radius: 12px;
+        border: none;
+        background: linear-gradient(
+            135deg,
+            #7C3AED,
+            #8B5CF6
+        );
+        color: white;
+        font-weight: 600;
+        padding: 0.65rem 1rem;
+        width: auto;
+    }
+
+    .stButton button:hover {
+        opacity: 0.92;
+    }
+
+    .upload-box {
+        background: #1E293B;
+        padding: 1rem;
+        border-radius: 18px;
+        border: 1px solid #334155;
+        margin-bottom: 1rem;
+    }
+
+    .answer-box {
+        background: #111827;
+        padding: 1rem;
+        border-radius: 16px;
+        border: 1px solid #334155;
+        margin-top: 1rem;
+    }
+
+    .rewrite-box {
+        background: rgba(139, 92, 246, 0.12);
+        padding: 0.8rem;
+        border-radius: 12px;
+        border-left: 4px solid #8B5CF6;
+        margin-bottom: 1rem;
+    }
+
+    .source-box {
+        background: #0B1220;
+        padding: 0.55rem 0.8rem;
+        border-radius: 10px;
+        margin-bottom: 0.45rem;
+        border: 1px solid #243244;
+        font-size: 0.9rem;
+        line-height: 1.4;
+    }
+
+    .hero-box {
+        padding: 1.5rem;
+        border-radius: 20px;
+        background: linear-gradient(
+            135deg,
+            #111827,
+            #1E293B
+        );
+        border: 1px solid #334155;
+        margin-bottom: 2rem;
+    }
+
+    .hero-title {
+        font-size: 2.2rem;
+        font-weight: 800;
+        color: white;
+    }
+
+    .hero-subtitle {
+        color: #CBD5E1;
+        margin-top: 0.5rem;
+        font-size: 1rem;
+    }
+
+    </style>
+    """,
+    unsafe_allow_html=True,
+)
+
+
+# -----------------------------------
+# STREAMING EFFECT
+# -----------------------------------
 
-@st.cache_resource
-def get_embedder():
-    return Embedder()
 
+def stream_text(text):
+    for word in text.split():
+        yield word + " "
+        time.sleep(0.04)
 
-UPLOAD_DIR = "data/uploads"
-INDEX_DIR = "data/index"
 
-os.makedirs(UPLOAD_DIR, exist_ok=True)
-os.makedirs(INDEX_DIR, exist_ok=True)
+# -----------------------------------
+# SESSION STATE
+# -----------------------------------
 
+if "session_id" not in st.session_state:
+    st.session_state.session_id = str(uuid.uuid4())
 
 
+if "messages" not in st.session_state:
+    st.session_state.messages = []
 
-st.set_page_config(page_title="RAG", layout="wide")
 
+# -----------------------------------
+# SIDEBAR
+# -----------------------------------
 
 st.sidebar.title("Navigation")
 
 page = st.sidebar.radio(
     "Go to",
-    ["Manage Data", "Ask Questions"]
+    ["Manage Data", "Ask Questions"],
 )
 
-def rebuild_index():
-    existing_files = os.listdir(UPLOAD_DIR)
+st.sidebar.divider()
 
-    if not existing_files:
-        # Remove index if no documents remain
-        if os.path.exists(INDEX_DIR):
-            shutil.rmtree(INDEX_DIR)
-        return
+st.sidebar.caption(f"Session ID: {st.session_state.session_id[:8]}")
 
-    progress = st.progress(0)
-    status = st.empty()
+# RESET BUTTON
+if st.sidebar.button("🗑 Reset Session"):
 
-    try:
-        status.text("Loading documents...")
-        records = load_documents([
-            os.path.join(UPLOAD_DIR, f)
-            for f in existing_files
-        ])
-        progress.progress(20)
+    with st.spinner("Resetting session..."):
+
+        response = requests.delete(
+            f"{API_BASE_URL}/reset-session",
+            params={"session_id": st.session_state.session_id},
+            timeout=30,
+        )
+
+    if response.status_code == 200:
+
+        st.session_state.messages = []
+
+        st.session_state.session_id = str(uuid.uuid4())
 
-        status.text("Chunking documents...")
-        chunks = chunk_documents(records)
-        progress.progress(40)
+        st.sidebar.success("Session reset successfully!")
 
-        status.text("Generating embeddings...")
-        embedder = Embedder()
-        embeddings = embedder.embed_texts([c["text"] for c in chunks])
-        progress.progress(70)
+        st.rerun()
 
-        status.text("Building vector index...")
-        store = VectorStore(embeddings.shape[1])
-        store.add(embeddings, chunks)
-        store.save(INDEX_DIR)
-        progress.progress(100)
+    else:
 
-        status.success("Index rebuilt successfully")
+        st.sidebar.error("Failed to reset session.")
 
-    except Exception as e:
-        status.error("Indexing failed")
-        raise e
 
+# -----------------------------------
+# DOCUMENT PAGE
+# -----------------------------------
 
 
-# -----------------------
-# EXISTING FILES
-# -----------------------
 def render_data_page():
-    st.header("Manage Documents")
 
-    left_col, right_col = st.columns([3, 1])
+    st.markdown(
+        """<div class="hero-box">
+        <div class="hero-title">
+            Document Management
+        </div>
+        <div class="hero-subtitle">
+            Upload PDF and PPTX syllabus documents for retrieval-based question answering.
+        </div>
+    </div>
+    """,
+        unsafe_allow_html=True,
+    )
 
-    # -------------------------------
-    # LEFT: FILE UPLOAD
-    # -------------------------------
-    with left_col:
-        st.subheader("Upload files")
+    uploaded_files = st.file_uploader(
+        "Upload PDF/PPTX files",
+        type=["pdf", "pptx"],
+        accept_multiple_files=True,
+    )
 
-        uploaded_files = st.file_uploader(
-            "Upload PDFs / PPTX",
-            type=["pdf", "pptx"],
-            accept_multiple_files=True
-        )
+    if uploaded_files:
+
+        st.subheader("Selected Files")
+
+        for file in uploaded_files:
+
+            st.markdown(f"{file.name}")
+
+    if st.button("Process Documents"):
+
+        if not uploaded_files:
+
+            st.warning("Please upload at least one file.")
 
-        if uploaded_files:
-            for file in uploaded_files:
-                path = os.path.join(UPLOAD_DIR, file.name)
-                with open(path, "wb") as f:
-                    f.write(file.read())
-            
-            rebuild_index()
+            return
 
-            st.success(f"{len(uploaded_files)} files uploaded successfully")
+        files = []
 
-    # -------------------------------
-    # RIGHT: EXISTING FILES
-    # -------------------------------
-    with right_col:
-        st.subheader("Uploaded files")
+        for file in uploaded_files:
 
-        existing_files = os.listdir(UPLOAD_DIR)
+            files.append(
+                (
+                    "files",
+                    (
+                        file.name,
+                        file,
+                        file.type,
+                    ),
+                )
+            )
+
+        with st.spinner("Processing documents..."):
+            try:
+                response = requests.post(
+                    f"{API_BASE_URL}/ingest",
+                    params={"session_id": st.session_state.session_id},
+                    files=files,
+                    timeout=120,
+                )
+            except requests.RequestException as e:
+                st.error("Failed to process documents, please try again.")
+                return
+
+        if response.status_code == 200:
+
+            data = response.json()
+
+            st.success("Documents processed successfully!")
+
+            col1, col2, col3 = st.columns(3)
+
+            col1.metric(
+                "Documents",
+                data["documents_processed"],
+            )
+
+            col2.metric(
+                "Chunks",
+                data["chunks_created"],
+            )
+
+            col3.metric(
+                "Vectors",
+                data["vectors_stored"],
+            )
 
-        if not existing_files:
-            st.info("No files uploaded")
         else:
-            for file in existing_files:
-                col1, col2 = st.columns([4, 1])
-                col1.write(file)
 
-                if col2.button("❌", key=f"delete_{file}"):
-                    os.remove(os.path.join(UPLOAD_DIR, file))
-                    st.warning(f"Deleted {file}")
-                    rebuild_index()
-                    st.rerun()
+            st.error("Failed to process documents, please try again.")
+
 
+# -----------------------------------
+# CHAT PAGE
+# -----------------------------------
 
-    
-# -------------------------------
-# CHAT BOT
-# -------------------------------
 
 def render_chat_page():
-    st.title("Text-Bounded RAG System ChatBot")
 
-    if "messages" not in st.session_state:
-        st.session_state.messages = []
-        
+    st.markdown(
+        """
+<div class="hero-box">
+    <div class="hero-title">
+        Syllabus RAG Assistant
+    </div>
+    <div class="hero-subtitle">
+        Ask questions grounded strictly in your uploaded syllabus documents.
+    </div>
+</div>
+""",
+        unsafe_allow_html=True,
+    )
+
+    # -----------------------------------
+    # FETCH DOCUMENTS
+    # -----------------------------------
+
+    available_documents = []
+
+    try:
+
+        documents_response = requests.get(
+            f"{API_BASE_URL}/documents",
+            params={"session_id": st.session_state.session_id},
+            timeout=30,
+        )
+
+        if documents_response.status_code == 200:
+
+            available_documents = documents_response.json().get("documents", [])
+
+    except Exception:
+
+        available_documents = []
+
+    # -----------------------------------
+    # SIDEBAR FILTERING
+    # -----------------------------------
+
+    st.sidebar.subheader("🔎 Search Filters")
+
+    selected_documents = st.sidebar.multiselect(
+        "Search Specific Documents",
+        available_documents,
+    )
+
+    if not selected_documents:
+        st.sidebar.caption("Searching all uploaded documents")
+    else:
+        st.sidebar.caption(f"Searching {len(selected_documents)} selected documents")
+    # -----------------------------------
+    # EMPTY STATE
+    # -----------------------------------
+
+    if not available_documents:
+
+        st.info("Upload documents first before asking questions.")
+
+        return
+
+    # -----------------------------------
+    # CHAT HISTORY
+    # -----------------------------------
+
     for message in st.session_state.messages:
+
         with st.chat_message(message["role"]):
-            st.markdown(message["content"]) 
-            
-    prompt = st.chat_input("Ask Question")
-    
+
+            st.markdown(message["content"])
+
+    prompt = st.chat_input("Ask a question...")
+
     if prompt:
-        
+
+        # -----------------------------------
+        # USER MESSAGE
+        # -----------------------------------
+
+        st.session_state.messages.append(
+            {
+                "role": "user",
+                "content": prompt,
+            }
+        )
+
         with st.chat_message("user"):
+
             st.markdown(prompt)
-            
-        st.session_state.messages.append({"role": "user", "content": prompt})
-        
-        response = ""
-        
-        upload_files = os.listdir(UPLOAD_DIR)
-        assistant_placeholder = st.empty()
-        assistant_placeholder.empty()
-        
-        if not upload_files:
-            response = "No documents uploaded. Please upload files first."
-
-        elif not os.path.exists(os.path.join(INDEX_DIR, "index.faiss")):
-            response = "Please build the index first"
-        else:
 
-            with st.spinner("..."):
-                store = VectorStore.load(INDEX_DIR)
-                embedder = Embedder()
-
-                q_emb = embedder.embed_texts([prompt])
-                results = store.search(q_emb, top_k=4)
-
-                MAX_SCORE = max(r["score"] for r in results)
-                print("MAX_SCORE", MAX_SCORE)
-                if MAX_SCORE < 0.4:
-                    response = "The uploaded material does not cover this topic."
-                else:
-                    context_chunks = [
-                        f"[{r['doc_name']} | Page {r['page']}]\n{r['text']}"
-                        for r in results
-                    ]
-                    response = generate_answer(context_chunks, prompt)
-            
-        with assistant_placeholder.container():
+        # -----------------------------------
+        # QUERY API
+        # -----------------------------------
+
+        with st.spinner("Thinking..."):
+            try:
+                response = requests.post(
+                    f"{API_BASE_URL}/query",
+                    json={
+                        "session_id": st.session_state.session_id,
+                        "query": prompt,
+                        "documents": selected_documents,
+                    },
+                    timeout=120,
+                )
+            except requests.RequestException as e:
+
                 with st.chat_message("assistant"):
-                    st.markdown(response)
-                    if (
-                            "The uploaded material does not cover this topic." not in response
-                            and "Please build the index first" not in response
-                            and "No documents uploaded. Please upload files first." not in response
-                        ):
-
-                        st.markdown("Sources")
-                        for i in range(0,2):
-                            with st.expander(f"{results[i]['doc_name']} — Page {results[i]['page']}"):
-                                st.write(results[i]["text"])
-                    
-                    
-
-                st.session_state.messages.append({"role": "assistant", "content": response})
-            # -------------------------------
-            # OUTPUT
-            # -------------------------------
-        
-
-
-    
+
+                    st.error("Query failed. Please try again.")
+
+                return
+
+        # -----------------------------------
+        # REQUEST FAILURE
+        # -----------------------------------
+
+        data = response.json()
+
+        # -----------------------------------
+        # RESPONSE
+        # -----------------------------------
+
+        if "message" in data:
+
+            assistant_response = data["message"]
+
+        else:
+
+            assistant_response = data["answer"]
+
+        # -----------------------------------
+        # ASSISTANT RESPONSE
+        # -----------------------------------
+
+        with st.chat_message("assistant"):
+
+            # rewritten query
+            if "rewritten_query" in data:
+
+                rewritten_query = data["rewritten_query"]
+
+                if rewritten_query.lower().strip() != prompt.lower().strip():
+
+                    st.markdown(
+                        f"""
+                        <div class="rewrite-box">
+                        <b>Rewritten Query</b><br>
+                        {rewritten_query}
+                        </div>
+                        """,
+                        unsafe_allow_html=True,
+                    )
+
+            # streamed answer
+            st.write_stream(stream_text(assistant_response))
+
+            # citations
+            if "answer" in data and "citations" in data:
+
+                st.subheader("Sources")
+
+                for citation in data["citations"][:3]:
+
+                    st.markdown(
+                        f"""
+                        <div class="source-box">
+                         <b>{citation['document']}, Page {citation['page']}</b>
+                        </div>
+                        """,
+                        unsafe_allow_html=True,
+                    )
+
+        # -----------------------------------
+        # SAVE CHAT HISTORY
+        # -----------------------------------
+
+        st.session_state.messages.append(
+            {
+                "role": "assistant",
+                "content": assistant_response,
+            }
+        )
+
+
+# -----------------------------------
+# ROUTING
+# -----------------------------------
+
 if page == "Manage Data":
+
     render_data_page()
 
 elif page == "Ask Questions":
-    render_chat_page()
 
+    render_chat_page()

main.py

@@ -0,0 +1,6 @@
+def main():
+    print("Hello from rag!")
+
+
+if __name__ == "__main__":
+    main()

pyproject.toml

@@ -0,0 +1,23 @@
+[project]
+name = "rag"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "faiss-cpu>=1.13.2",
+    "fastapi>=0.136.1",
+    "huggingface-hub>=1.14.0",
+    "langchain>=1.3.0",
+    "langchain-huggingface>=1.2.2",
+    "nltk>=3.9.4",
+    "numpy>=2.4.4",
+    "pypdf>=6.11.0",
+    "python-dotenv>=1.2.2",
+    "python-multipart>=0.0.28",
+    "python-pptx>=1.0.2",
+    "requests>=2.34.1",
+    "sentence-transformers>=5.5.0",
+    "streamlit>=1.57.0",
+    "uvicorn>=0.46.0",
+]

src/api/document.py

@@ -0,0 +1,21 @@
+from fastapi import APIRouter
+
+from src.core.store_manager import load_or_create_store
+
+router = APIRouter()
+
+
+@router.get("/documents")
+async def get_documents(session_id: str):
+    """
+    Return all uploaded document names
+    for the current session.
+    """
+
+    vector_store = load_or_create_store(session_id)
+
+    documents = list({record.doc_name for record in vector_store.records})
+
+    documents.sort()
+
+    return {"documents": documents}

src/api/ingest.py

@@ -0,0 +1,88 @@
+from typing import Annotated
+from fastapi import APIRouter, UploadFile, File, HTTPException
+
+import tempfile
+import os
+
+from src.services.ingestion import load_documents
+from src.services.chunking import chunk_documents
+from src.services.embedding import Embedder
+
+from typing import List
+
+from src.core.store_manager import load_or_create_store, get_session_store_path
+
+router = APIRouter()
+
+embedder = Embedder()
+
+ALLOWED_EXTENSIONS = [".pdf", ".pptx"]
+MAX_FILE_SIZE = 10 * 1024 * 1024  # 10MB
+
+
+@router.post("/ingest")
+async def ingest_documents(
+    session_id: str, files: Annotated[List[UploadFile], File(...)]
+):
+
+    temp_files = []
+
+    try:
+
+        for file in files:
+
+            ext = os.path.splitext(file.filename)[1].lower()
+
+            if ext not in ALLOWED_EXTENSIONS:
+
+                raise HTTPException(
+                    status_code=400, detail=f"Unsupported file type: {file.filename}"
+                )
+
+            contents = await file.read()
+
+            if len(contents) > MAX_FILE_SIZE:
+
+                raise HTTPException(
+                    status_code=400, detail=f"{file.filename} exceeds size limit"
+                )
+
+            with tempfile.NamedTemporaryFile(delete=False, suffix=ext) as temp_file:
+
+                temp_file.write(contents)
+
+                temp_files.append(
+                    {"temp_path": temp_file.name, "original_name": file.filename}
+                )
+
+        # extraction
+        records = load_documents(temp_files)
+
+        # chunking
+        chunks = chunk_documents(records)
+
+        # embeddings
+        texts = [chunk.text for chunk in chunks]
+
+        embeddings = embedder.embed_texts(texts)
+
+        # vector store
+        vector_store = load_or_create_store(session_id)
+        vector_store.add(embeddings=embeddings, records=chunks)
+        store_path = get_session_store_path(session_id)
+        vector_store.save(store_path)
+
+        return {
+            "status": "success",
+            "documents_processed": len(files),
+            "records_extracted": len(records),
+            "chunks_created": len(chunks),
+            "vectors_stored": len(chunks),
+        }
+
+    finally:
+
+        for file_info in temp_files:
+
+            if os.path.exists(file_info["temp_path"]):
+                os.remove(file_info["temp_path"])

src/api/query.py

@@ -0,0 +1,88 @@
+from fastapi import APIRouter
+
+from src.schemas.models import QueryRequest
+
+from src.services.retrieval import Retriever
+
+from src.services.generator import generate_answer
+
+from src.services.query_rewriter import rewrite_query
+
+router = APIRouter()
+
+retriever = Retriever()
+
+
+@router.post("/query")
+async def query_documents(request: QueryRequest):
+
+    original_query = request.query
+
+    session_id = request.session_id
+
+    # -----------------------------
+    # QUERY REWRITING
+    # -----------------------------
+
+    rewritten_query = rewrite_query(original_query)
+
+    # -----------------------------
+    # RETRIEVAL
+    # -----------------------------
+
+    results = retriever.retrieve(
+        session_id=session_id,
+        query=rewritten_query,
+        top_k=5,
+        documents=request.documents,
+    )
+
+    # -----------------------------
+    # OUT-OF-SYLLABUS REJECTION
+    # -----------------------------
+
+    if not results:
+
+        return {
+            "query": original_query,
+            "rewritten_query": rewritten_query,
+            "message": ("The uploaded material " "does not cover this topic."),
+        }
+
+    # -----------------------------
+    # GENERATION
+    # -----------------------------
+
+    answer = generate_answer(context_chunks=results, question=rewritten_query)
+
+    # -----------------------------
+    # CITATIONS
+    # -----------------------------
+
+    citations = []
+
+    seen = set()
+
+    for result in results:
+
+        chunk = result["chunk"]
+
+        key = (chunk.doc_name, chunk.page)
+
+        if key in seen:
+            continue
+
+        seen.add(key)
+
+        citations.append({"document": chunk.doc_name, "page": chunk.page})
+
+    # -----------------------------
+    # FINAL RESPONSE
+    # -----------------------------
+
+    return {
+        "query": original_query,
+        "rewritten_query": rewritten_query,
+        "answer": answer,
+        "citations": citations,
+    }

src/api/reset.py

@@ -0,0 +1,18 @@
+import shutil
+import os
+
+from fastapi import APIRouter
+
+router = APIRouter()
+
+
+@router.delete("/reset-session")
+async def reset_session(session_id: str):
+
+    session_path = f"storage/sessions/{session_id}"
+
+    if os.path.exists(session_path):
+
+        shutil.rmtree(session_path)
+
+    return {"status": "success", "message": "Session reset successfully."}

src/chunk.py

@@ -1,34 +0,0 @@
-def chunk_documents(
-    records: list[dict],
-    chunk_size: int = 500,
-    overlap: int = 100
-) -> list[dict]:
-    """
-    Page-bounded sliding window chunking.
-    Chunks NEVER cross page boundaries.
-    """
-
-    chunks = []
-
-    for record in records:
-        text = record["text"]
-        doc_name = record["doc_name"]
-        page = record["page"]
-
-        start = 0
-        text_len = len(text)
-
-        while start < text_len:
-            end = start + chunk_size
-            chunk_text = text[start:end].strip()
-
-            if chunk_text:
-                chunks.append({
-                    "text": chunk_text,
-                    "doc_name": doc_name,
-                    "page": page
-                })
-
-            start += chunk_size - overlap
-
-    return chunks

src/core/store_manager.py

@@ -0,0 +1,28 @@
+import os
+
+from src.services.vector_store import VectorStore
+from src.utils.cleanup import cleanup_old_sessions
+
+BASE_STORAGE_PATH = "storage/sessions"
+
+
+def get_session_store_path(session_id: str) -> str:
+
+    return os.path.join(BASE_STORAGE_PATH, session_id)
+
+
+def load_or_create_store(session_id: str) -> VectorStore:
+    cleanup_old_sessions()
+    store_path = get_session_store_path(session_id)
+
+    index_path = os.path.join(store_path, "index.faiss")
+
+    # existing store
+    if os.path.exists(index_path):
+
+        return VectorStore.load(store_path)
+
+    # new store
+    os.makedirs(store_path, exist_ok=True)
+
+    return VectorStore(embedding_dim=384)

src/main.py

@@ -1,33 +1,20 @@
-from ingest import load_documents
-from chunk import chunk_documents
-from embed import Embedder
-from vector_store import VectorStore
-from llm import generate_answer
-
-# Build index (run once)
-records = load_documents(["data/AI & ML DIGITAL NOTES.pdf"])
-chunks = chunk_documents(records)
-
-embedder = Embedder()
-embeddings = embedder.embed_texts([c["text"] for c in chunks])
-
-store = VectorStore(embeddings.shape[1])
-store.add(embeddings, chunks)
-store.save("data/index")
-
-# Query time
-store = VectorStore.load("data/index")
-
-question = "What is deep learning?"
-
-q_emb = embedder.embed_texts([question])
-results = store.search(q_emb, top_k=3)
-
-context_chunks = [
-    f"[{r['doc_name']} | Page {r['page']}]\n{r['text']}"
-    for r in results
-]
-
-answer = generate_answer(context_chunks, question)
-
-print(answer)
+from fastapi import FastAPI
+from src.api.ingest import router as ingest_router
+from src.api.query import router as query_router
+from src.api.reset import router as reset_router
+from src.api.document import router as doc_router
+from fastapi.middleware.cors import CORSMiddleware
+
+app = FastAPI()
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+app.include_router(ingest_router, prefix="/api", tags=["ingestion"])
+app.include_router(query_router, prefix="/api", tags=["query"])
+app.include_router(reset_router, prefix="/api", tags=["reset"])
+app.include_router(doc_router, prefix="/api", tags=["documents"])

src/schemas/models.py

@@ -0,0 +1,31 @@
+from pydantic import BaseModel
+from typing import Optional
+
+
+class DocumentRecord(BaseModel):
+    text: str
+    doc_name: str
+    page: int
+    source: str
+
+    document_id: Optional[str] = None
+    chunk_id: Optional[str] = None
+    section_title: Optional[str] = None
+    metadata: Optional[dict] = None
+
+
+class ChunkRecord(BaseModel):
+    chunk_id: str
+    document_id: str
+    text: str
+    doc_name: str
+    page: int
+    chunk_index: int
+
+    metadata: Optional[dict] = None
+
+
+class QueryRequest(BaseModel):
+    session_id: str
+    query: str
+    documents: list[str] | None = None

src/services/chunking.py

@@ -0,0 +1,100 @@
+import uuid
+from typing import Optional
+
+from nltk.tokenize import sent_tokenize
+from pydantic import BaseModel
+
+from src.schemas.models import DocumentRecord
+from src.schemas.models import ChunkRecord
+
+
+def chunk_documents(
+    records: list[DocumentRecord],
+    chunk_size: int = 500,
+    overlap: int = 100,
+) -> list[ChunkRecord]:
+    """
+    Sentence-aware page-bounded chunking.
+
+    - Chunks NEVER cross page boundaries
+    - Preserves semantic meaning better than character slicing
+    - Maintains overlap between chunks
+    """
+
+    chunks = []
+
+    for record in records:
+
+        sentences = sent_tokenize(record.text)
+
+        current_chunk = []
+        current_length = 0
+
+        chunk_index = 0
+
+        for sentence in sentences:
+
+            sentence_length = len(sentence)
+
+            # if adding this sentence exceeds chunk size,
+            # finalize current chunk
+            if current_length + sentence_length > chunk_size and current_chunk:
+
+                chunk_text = " ".join(current_chunk).strip()
+                word_count = len(chunk_text.split())
+                if word_count < 5:
+                    continue
+                chunks.append(
+                    ChunkRecord(
+                        chunk_id=str(uuid.uuid4()),
+                        document_id=record.document_id,
+                        text=chunk_text,
+                        doc_name=record.doc_name,
+                        page=record.page,
+                        chunk_index=chunk_index,
+                        metadata=record.metadata,
+                    )
+                )
+
+                chunk_index += 1
+
+                # overlap logic
+                overlap_sentences = []
+                overlap_length = 0
+
+                for s in reversed(current_chunk):
+
+                    overlap_length += len(s)
+
+                    if overlap_length > overlap:
+                        break
+
+                    overlap_sentences.insert(0, s)
+
+                current_chunk = overlap_sentences
+                current_length = sum(len(s) for s in current_chunk)
+
+            current_chunk.append(sentence)
+            current_length += sentence_length
+
+        # final chunk
+        if current_chunk:
+
+            chunk_text = " ".join(current_chunk).strip()
+            word_count = len(chunk_text.split())
+
+            if word_count < 5:
+                continue
+            chunks.append(
+                ChunkRecord(
+                    chunk_id=str(uuid.uuid4()),
+                    document_id=record.document_id,
+                    text=chunk_text,
+                    doc_name=record.doc_name,
+                    page=record.page,
+                    chunk_index=chunk_index,
+                    metadata=record.metadata,
+                )
+            )
+
+    return chunks

src/{embed.py → services/embedding.py}

@@ -1,17 +1,21 @@
 import numpy as np
+
 from sentence_transformers import SentenceTransformer
+
+
 class Embedder:
-    def __init__(self, model_name: str = "all-MiniLM-L6-v2"):
+
+    def __init__(self, model_name: str = "BAAI/bge-small-en-v1.5"):
+
         self.model = SentenceTransformer(model_name)
 
-    def embed_texts(self, texts):
+    def embed_texts(self, texts: list[str]) -> np.ndarray:
         """
-        Convert a list of texts into embedding vectors.
+        Convert texts into normalized embedding vectors.
         """
+
         embeddings = self.model.encode(
-            texts,
-            show_progress_bar=True,
-            normalize_embeddings=True
+            texts, normalize_embeddings=True, convert_to_numpy=True
         )
 
-        return np.array(embeddings, dtype="float32")
+        return embeddings.astype("float32")

src/{llm.py → services/generator.py}

@@ -1,51 +1,79 @@
 import os
-from google import genai
+
 from dotenv import load_dotenv
 
-load_dotenv()
+from langchain_huggingface import (
+    ChatHuggingFace,
+    HuggingFaceEndpoint,
+)
 
+load_dotenv()
 
-from langchain_huggingface import ChatHuggingFace, HuggingFaceEndpoint
 
 llm = HuggingFaceEndpoint(
     repo_id="Qwen/Qwen3-4B-Instruct-2507",
-    huggingfacehub_api_token=f"{os.getenv('HUGGINGFACE_API_KEY')}",
+    huggingfacehub_api_token=os.getenv("HUGGINGFACE_API_KEY"),
     task="text-generation",
     max_new_tokens=256,
-    temperature=0.7,
+    temperature=0.1,
+    do_sample=False,
 )
 
-chat_model = ChatHuggingFace(llm=llm)
 
+chat_model = ChatHuggingFace(llm=llm)
 
 
 def generate_answer(context_chunks, question: str) -> str:
     """
-    Generate a syllabus-grounded answer using retrieved chunks.
+    Generate a syllabus-grounded answer
+    using retrieved chunks.
     """
 
-    context = "\n\n".join(context_chunks)
+    # use only top reranked chunks
+    top_chunks = context_chunks[:3]
+
+    context_parts = []
+
+    for result in top_chunks:
+
+        chunk = result["chunk"]
+
+        context_parts.append(f"""
+Document: {chunk.doc_name}
+Page: {chunk.page}
+
+Content:
+{chunk.text}
+""")
+
+    context = "\n\n".join(context_parts)
 
     prompt = f"""
-You are an academic assistant.
+You are a syllabus-bounded academic assistant.
 
 STRICT RULES:
 - Use ONLY the provided context
 - Do NOT use outside knowledge
+- Check if the question is academic or casual, if its casual conversate and direct towards academics. IF its academic, answer using the context.  
 - If the answer is fully present, answer clearly
-- If partially present, say "The uploaded material partially cover this topic." and also give the answer(s) based on the context
-- If not present, say "The uploaded material does not cover this topic."
-- Always cite page numbers in the answer
-
-Context:
-{context}
+- If partially present, say:
+  "The uploaded material partially covers this topic."
+  Then provide the available answer
+- If the answer is not present, say:
+  "The uploaded material does not cover this topic."
+- Keep answers concise and factual
+- Cite sources in this format:
+  (Page <page>, <document>)
 
 Question:
 {question}
 
+Context:
+{context}
+
 Answer:
 """
 
     response = chat_model.invoke(prompt)
 
-    return response.text.strip()
+    return response.content.strip()

src/{ingest.py → services/ingestion.py}

@@ -1,69 +1,87 @@
-from importlib.resources import path
 import os
 from pypdf import PdfReader
-from pdf2image import convert_from_path
+from src.schemas.models import DocumentRecord
+import re
+import uuid
+from pptx import Presentation
+
 
+def clean_text(text: str) -> str:
+    text = re.sub(r"[ \t]+", " ", text)
+    text = re.sub(r"\n{2,}", "\n", text)
+    return text.strip()
 
-def extract_pages_from_pdf(file_path: str) -> list[dict]:
-    if(not os.path.exists(file_path)):
+
+def extract_pages_from_pdf(file_path: str, original_name: str) -> list[DocumentRecord]:
+    if not os.path.exists(file_path):
         raise FileNotFoundError("file not found")
-    
+
     reader = PdfReader(file_path)
     records = []
-    for pageno,page in enumerate(reader.pages,start = 1):
-        text = (page.extract_text() or "").strip()
-        
-        records.append({
-                "text": text,
-                "doc_name": os.path.basename(file_path),
-                "page": pageno,
-                "source": "pdf"
-            })
-            
-    return records
-
-
+    document_id = str(uuid.uuid4())
+    for pageno, page in enumerate(reader.pages, start=1):
+        text = clean_text(page.extract_text() or "")
+
+        records.append(
+            DocumentRecord(
+                text=text,
+                doc_name=original_name,
+                page=pageno,
+                source="pdf",
+                document_id=document_id,
+                metadata={"file_type": "pdf", "file_name": original_name},
+            )
+        )
 
+    return records
 
-from pptx import Presentation
 
-def extract_slides_from_ppt(file_path: str):
+def extract_slides_from_ppt(file_path: str, original_name: str) -> list[DocumentRecord]:
     prs = Presentation(file_path)
     records = []
-
+    document_id = str(uuid.uuid4())
     for slide_num, slide in enumerate(prs.slides, start=1):
         slide_text = []
 
         for shape in slide.shapes:
-            if hasattr(shape, "text"):
+            if hasattr(shape, "text") and shape.text:
                 text = shape.text.strip()
                 if text:
                     slide_text.append(text)
 
-        full_text = "\n".join(slide_text).strip()
+        full_text = clean_text("\n".join(slide_text))
 
         if full_text:
-            records.append({
-                "text": full_text,
-                "doc_name": os.path.basename(file_path),
-                "page": slide_num,     # slide number
-                "source": "ppt"
-            })
+            records.append(
+                DocumentRecord(
+                    text=full_text,
+                    doc_name=original_name,
+                    page=slide_num,
+                    source="ppt",
+                    document_id=document_id,
+                    metadata={
+                        "file_type": "ppt",
+                        "file_name": original_name,
+                    },
+                )
+            )
 
     return records
 
 
-def load_documents(file_paths):
+def load_documents(files: list[dict]) -> list[DocumentRecord]:
     all_records = []
 
-    for path in file_paths:
+    for file_info in files:
+        path = file_info["temp_path"]
+        original_name = file_info["original_name"]
         ext = path.lower()
 
         if ext.endswith(".pdf"):
-            all_records.extend(extract_pages_from_pdf(path))
+            all_records.extend(extract_pages_from_pdf(path, original_name))
 
         elif ext.endswith(".pptx"):
-            all_records.extend(extract_slides_from_ppt(path))
+            all_records.extend(extract_slides_from_ppt(path, original_name))
 
         elif ext.endswith(".ppt"):
             raise ValueError(
@@ -73,7 +91,4 @@ def load_documents(file_paths):
         else:
             raise ValueError(f"Unsupported file type: {path}")
 
-    return [r for r in all_records if r["text"].strip()]
-
-
-        
+    return [r for r in all_records if r.text.strip()]

src/services/query_rewriter.py

@@ -0,0 +1,77 @@
+import os
+
+from dotenv import load_dotenv
+
+from langchain_huggingface import (
+    ChatHuggingFace,
+    HuggingFaceEndpoint,
+)
+
+load_dotenv()
+
+
+llm = HuggingFaceEndpoint(
+    repo_id="Qwen/Qwen3-4B-Instruct-2507",
+    huggingfacehub_api_token=os.getenv("HUGGINGFACE_API_KEY"),
+    task="text-generation",
+    max_new_tokens=64,
+    temperature=0.1,
+    do_sample=False,
+)
+
+
+chat_model = ChatHuggingFace(llm=llm)
+
+
+def rewrite_query(query: str) -> str:
+    """
+    Rewrite student queries into
+    clearer retrieval-friendly queries.
+    """
+
+    prompt = f"""
+You are a query rewriting assistant
+for a syllabus-based RAG system.
+
+Your job:
+- If the query is casual conversation,
+    greeting, or unrelated to academics,
+    return the original query unchanged.
+-Else, rewrite the query to be more specific and clear for retrieval.
+- Expand abbreviations
+- Improve clarity for retrieval
+- Preserve original meaning
+- Keep concise
+- Return ONLY a single string which is the rewritten query
+
+Examples:
+
+Student Query:
+what is util
+
+Rewritten Query:
+What is utilitarian theory in ethics?
+
+Student Query:
+oop pillars
+
+Rewritten Query:
+What are the four pillars of object-oriented programming?
+
+Student Query:
+db normalization explain 
+
+Rewritten Query:
+Explain database normalization.
+
+Student Query:
+{query}
+
+Rewritten Query:
+"""
+
+    response = chat_model.invoke(prompt)
+
+    rewritten_query = response.content.strip()
+
+    return rewritten_query

src/services/reranker.py

@@ -0,0 +1,24 @@
+from sentence_transformers import CrossEncoder
+
+
+class Reranker:
+
+    def __init__(self):
+
+        self.model = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2")
+
+    def rerank(self, query: str, results: list):
+
+        pairs = [(query, result["chunk"].text) for result in results]
+
+        scores = self.model.predict(pairs)
+
+        reranked = []
+
+        for score, result in zip(scores, results):
+
+            reranked.append({"score": float(score), "chunk": result["chunk"]})
+
+        reranked.sort(key=lambda x: x["score"], reverse=True)
+
+        return reranked

src/services/retrieval.py

@@ -0,0 +1,70 @@
+from src.core.store_manager import load_or_create_store
+
+from src.services.embedding import Embedder
+
+from src.services.reranker import Reranker
+
+
+class Retriever:
+
+    def __init__(self):
+
+        self.embedder = Embedder()
+
+        self.reranker = Reranker()
+
+        self.similarity_threshold = 0.60
+
+    def retrieve(self, session_id: str, query: str, top_k: int = 10, documents=None):
+
+        vector_store = load_or_create_store(session_id)
+
+        # --------------------------------
+        # QUERY EMBEDDING
+        # --------------------------------
+
+        query_embedding = self.embedder.embed_texts([query])[0]
+
+        # --------------------------------
+        # VECTOR RETRIEVAL
+        # --------------------------------
+
+        retrieved_results = vector_store.search(
+            query_embedding=query_embedding, top_k=top_k
+        )
+
+        # --------------------------------
+        # DOCUMENT FILTERING
+        # --------------------------------
+
+        if documents:
+
+            retrieved_results = [
+                r for r in retrieved_results if (r["chunk"].doc_name in documents)
+            ]
+
+        # --------------------------------
+        # NO RESULTS
+        # --------------------------------
+
+        if not retrieved_results:
+
+            return []
+
+        # --------------------------------
+        # THRESHOLD REJECTION
+        # --------------------------------
+
+        top_score = retrieved_results[0]["score"]
+
+        if top_score < self.similarity_threshold:
+
+            return []
+
+        # --------------------------------
+        # RERANKING
+        # --------------------------------
+
+        reranked_results = self.reranker.rerank(query=query, results=retrieved_results)
+
+        return reranked_results[:5]

src/{vector_store.py → services/vector_store.py}

@@ -2,15 +2,16 @@ import faiss
 import pickle
 import os
 import numpy as np
+from src.schemas.models import DocumentRecord, ChunkRecord
 
 
 class VectorStore:
     def __init__(self, embedding_dim: int):
         # Inner Product index (cosine similarity with normalized vectors)
         self.index = faiss.IndexFlatIP(embedding_dim)
-        self.records: list[dict] = []
+        self.records: list[ChunkRecord] = []
 
-    def add(self, embeddings: np.ndarray, records: list[dict]):
+    def add(self, embeddings: np.ndarray, records: list[ChunkRecord]):
         """
         Add embeddings and their corresponding metadata records.
         Order MUST be preserved.
@@ -22,17 +23,20 @@ class VectorStore:
         """
         Search the index and return top_k matching chunks with metadata.
         """
+        query_embedding = np.asarray(query_embedding, dtype="float32")
+
+        if query_embedding.ndim == 1:
+            query_embedding = np.expand_dims(query_embedding, axis=0)
+        if self.index.ntotal == 0:
+            return []
         scores, indices = self.index.search(query_embedding, top_k)
 
         results = []
         for score, idx in zip(scores[0], indices[0]):
+            if idx == -1:
+                continue
             record = self.records[idx]
-            results.append({
-                "score": float(score),
-                "text": record["text"],
-                "doc_name": record["doc_name"],
-                "page": record["page"]
-            })
+            results.append({"score": float(score), "chunk": record})
 
         return results
 

src/utils/cleanup.py

@@ -0,0 +1,32 @@
+import os
+import shutil
+import time
+
+BASE_PATH = "storage/sessions"
+
+MAX_AGE_HOURS = 48
+
+
+def cleanup_old_sessions():
+
+    now = time.time()
+
+    max_age_seconds = MAX_AGE_HOURS * 3600
+
+    if not os.path.exists(BASE_PATH):
+        return
+
+    for session_id in os.listdir(BASE_PATH):
+
+        session_path = os.path.join(BASE_PATH, session_id)
+
+        if not os.path.isdir(session_path):
+            continue
+
+        modified_time = os.path.getmtime(session_path)
+
+        age = now - modified_time
+
+        if age > max_age_seconds:
+
+            shutil.rmtree(session_path, ignore_errors=True)