Merge pull request #17 from ittia-research/dev

change to multi-sources mode

Author: etwk

Dockerfile.local

@@ -1,9 +1,15 @@
 FROM pytorch/pytorch:2.4.0-cuda12.4-cudnn9-runtime
+
 WORKDIR /app
+
 COPY requirements.*.txt /app
 RUN pip install --no-cache-dir -r requirements.base.txt
 RUN pip install --no-cache-dir -r requirements.local.txt
-COPY . /app
-EXPOSE 8000
+
 WORKDIR /app/src
+
+COPY ./src .
+
+EXPOSE 8000
+
 CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

Dockerfile.remote

@@ -1,8 +1,14 @@
 FROM python:3.11-slim-bookworm
+
 WORKDIR /app
+
 COPY requirements.base.txt /app
 RUN pip install --no-cache-dir -r requirements.base.txt
-COPY . /app
-EXPOSE 8000
+
 WORKDIR /app/src
+
+COPY ./src .
+
+EXPOSE 8000
+
 CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

README.md

@@ -55,7 +55,6 @@ LLM
 
 Embedding:
 - [ ] chunk size optimize
-- [ ] Ollama embedding performance
 
 Contexts
 - [ ] Filter out non-related contexts before send for verdict
@@ -66,9 +65,15 @@ Retrieval
 ### pipeline
 DSPy:
 - [ ] make dspy.settings apply to sessions only in order to support multiple retrieve index
+- [ ] choose the right LLM temperature
+- [ ] better training datasets
 
 ### Retrival
 - [ ] Better retrival solution: high performance, concurrency, multiple index, index editable.
+- [ ] Getting more sources when needed.
+
+### Verdict
+- [ ] Set final verdict standards.
 
 ### Toolchain
 - [ ] Evaluate MLOps pipeline

datasets/HotPotQA/HotPotQA_statement_verdict.ipynb

@@ -2,6 +2,8 @@ services:
   check:
     image: ittia/check:remote
     container_name: check
+    volumes:
+      - /data/cache:/data/cache
     env_file:
       - ./infra/env.d/check
     ports:

docker-compose.yml

@@ -3,5 +3,7 @@
   - Change from AutoGen to plain OpenAI, since AutoGen AssistantAgent adds system role which are not compateble with Gemma 2 + vllm.
 
 ## pipeline
+2024/8/26:
+  - Changed to multi-sources mode (divide sources based on hostname), instead of use all web search results as one single source.
 2024/8/13:
   - Introduce DSPy to replace the get verdict part, with multi-step reasoning.

docs/changelog.md

@@ -1,13 +1,22 @@
+CONCURRENCY_VERDICT=8
+
+DSP_CACHEBOOL=True
+DSP_CACHEDIR=/data/cache
+
 EMBEDDING_API_KEY=<CHANGE_ME>
-EMBEDDING_BASE_URL=http://ollama:11434
+EMBEDDING_BASE_URL=http://infinity:7997
+EMBEDDING_BATCH_SIZE=1024
 EMBEDDING_MODEL_DEPLOY=api
-EMBEDDING_MODEL_NAME=jina/jina-embeddings-v2-base-en
+EMBEDDING_MODEL_NAME=jinaai/jina-embeddings-v2-base-en
+EMBEDDING_SERVER_TYPE=infinity
 INDEX_CHUNK_SIZES=[2048, 512, 128]
 
-LLM_MODEL_NAME=google/gemma-2-27b-it
+LLM_MODEL_NAME=mistralai/Mistral-Nemo-Instruct-2407
 OPENAI_API_KEY=<CHANGE_ME>
 OPENAI_BASE_URL=http://localhost:8000/v1
 
+OPTIMIZER_FILE_NAME=verdict_MIPROv2.json
+
 RERANK_API_KEY=<CHANGE_ME>
 RERANK_BASE_URL=http://infinity:7997
 RERANK_MODEL_DEPLOY=api

infra/env.d/check

@@ -57,7 +57,7 @@ async def fact_check(input):
             verdict = await run_in_threadpool(pipeline.get_verdict, search_json=search, statement=statement)
             logger.info(f"Verdict: {verdict}")
         except Exception as e:
-            logger.error(f"Getting verdict for statement {statement} failed: {e}")
+            logger.error(f"Getting verdict for statement '{statement}' failed: {e}")
             continue
 
         verdicts.append(verdict)

src/main.py

@@ -10,8 +10,8 @@
 llm_long = dspy.OpenAI(model=settings.LLM_MODEL_NAME, api_base=f"{settings.OPENAI_BASE_URL}/", max_tokens=500, stop='\n\n')
 
 from .citation import Citation
+from .context_verdict import ContextVerdict
 from .retrieve import LlamaIndexRM
 from .search import Search
 from .search_query import SearchQuery
-from .statements import Statements
-from .verdict import Verdict
+from .statements import Statements

src/modules/__init__.py

@@ -1,13 +1,12 @@
 import dspy
 
-# TODO: citation needs higher token limits
 class GenerateCitedParagraph(dspy.Signature):
     """Generate a paragraph with citations."""
-    context = dspy.InputField(desc="may contain relevant facts")
+    context = dspy.InputField(desc="May contain relevant facts.")
     statement = dspy.InputField()
     verdict = dspy.InputField()
-    paragraph = dspy.OutputField(desc="includes citations")
-
+    paragraph = dspy.OutputField(desc="Includes citations.")
+    
 """Generate citation from context and verdict"""
 class Citation(dspy.Module):
     def __init__(self):