[add] Drone CI/CD

.drone.yml

@@ -0,0 +1,17 @@
+kind: pipeline
+type: kubernetes
+name: default
+
+steps:
+  # Publish Dev Docker Image
+  - name: publish_docker
+    image: plugins/docker
+    settings:
+      repo: gitea.va.reichard.io/evan/minyma
+      registry: gitea.va.reichard.io
+      tags:
+        - dev
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password

README.md

@@ -15,6 +15,8 @@
 
 AI Chat Bot with Vector / Embedding DB Context
 
+[![Build Status](https://drone.va.reichard.io/api/badges/evan/minyma/status.svg)](https://drone.va.reichard.io/evan/minyma)
+
 ## Running Server
 
 ```bash
@@ -23,13 +25,12 @@ export OPENAI_API_KEY=`cat openai_key`
 minyma server run
 
 # Docker Quick Start
-make docker_build_local
 docker run \
     -p 5000:5000 \
     -e OPENAI_API_KEY=`cat openai_key` \
     -e DATA_PATH=/data \
     -v ./data:/data \
-    minyma:latest
+    gitea.va.reichard.io/evan/minyma:latest
 ```
 
 The server will now be accessible at `http://localhost:5000`
@@ -44,7 +45,11 @@ and the only supported normalizer is the `pubmed` normalizer.
 To normalize data, you can use Minyma's `normalize` CLI command:
 
 ```bash
-minyma normalize --filename ./pubmed_manuscripts.jsonl --normalizer pubmed --database chroma --datapath ./chroma
+minyma normalize \
+    --filename ./pubmed_manuscripts.jsonl \
+    --normalizer pubmed \
+    --database chroma \
+    --datapath ./chroma
 ```
 
 The above example does the following:
@@ -76,6 +81,9 @@ pip install -e .
 
 # Creds
 export OPENAI_API_KEY=`cat openai_key`
+
+# Docker
+make docker_build_local
 ```
 
 # Notes

minyma/__init__.py

@@ -1,8 +1,8 @@
+from os import path
 import click
 import signal
 import sys
 from importlib.metadata import version
-from minyma.config import Config
 from minyma.oai import OpenAIConnector
 from minyma.vdb import ChromaDB
 from flask import Flask
@@ -15,14 +15,15 @@ def signal_handler(sig, frame):
 
 
 def create_app():
-    global oai, cdb
+    global oai, vdb
 
+    from minyma.config import Config
     import minyma.api.common as api_common
     import minyma.api.v1 as api_v1
 
     app = Flask(__name__)
-    cdb = ChromaDB(Config.DATA_PATH)
+    vdb = ChromaDB(path.join(Config.DATA_PATH, "chroma"))
-    oai = OpenAIConnector(Config.OPENAI_API_KEY, cdb)
+    oai = OpenAIConnector(Config.OPENAI_API_KEY, vdb)
 
     app.register_blueprint(api_common.bp)
     app.register_blueprint(api_v1.bp)

minyma/api/v1.py

@@ -17,8 +17,28 @@ def get_response():
     if message == "":
         return {"error": "Empty Message"}
 
-    oai_response = minyma.oai.query(message)
+    resp = minyma.oai.query(message)
-    return oai_response
+
+    # Derive LLM Data
+    llm_resp = resp.get("llm", {})
+    llm_choices = llm_resp.get("choices", [])
+
+    # Derive VDB Data
+    vdb_resp = resp.get("vdb", {})
+    combined_context  = [{
+            "id": vdb_resp.get("ids")[i],
+            "distance": vdb_resp.get("distances")[i],
+            "doc": vdb_resp.get("docs")[i],
+            "metadata": vdb_resp.get("metadatas")[i],
+    } for i, _ in enumerate(vdb_resp.get("docs", []))]
+
+    # Return Data
+    return {
+        "response": None if len(llm_choices) == 0 else llm_choices[0].get("message", {}).get("content"),
+        "context": combined_context,
+        "usage": llm_resp.get("usage"),
+    }
+
 
 
 """
@@ -34,5 +54,5 @@ def get_related():
     if message == "":
         return {"error": "Empty Message"}
 
-    related_documents = minyma.cdb.get_related(message)
+    related_documents = minyma.vdb.get_related(message)
     return related_documents

minyma/normalizer.py

@@ -1,12 +1,16 @@
 from io import TextIOWrapper
 import json
 
-class DataNormalizer:
+class DataNormalizer():
     def __init__(self, file: TextIOWrapper):
-        pass
+        self.file = file
+
+    def __len__(self) -> int:
+        return 0
 
     def __iter__(self):
-        pass
+        yield None
+
 
 class PubMedNormalizer(DataNormalizer):
     """
@@ -15,6 +19,13 @@ class PubMedNormalizer(DataNormalizer):
     """
     def __init__(self, file: TextIOWrapper):
         self.file = file
+        self.length = 0
+
+    def __len__(self):
+        last_pos = self.file.tell()
+        self.length = sum(1 for _ in self.file)
+        self.file.seek(last_pos)
+        return self.length
 
     def __iter__(self):
         count = 0
@@ -42,4 +53,10 @@ class PubMedNormalizer(DataNormalizer):
             count += 1
 
             # ID = Line Number
-            yield { "doc": norm_text, "id": str(count - 1) }
+            yield {
+               "id": str(count - 1),
+               "doc": norm_text,
+               "metadata": {
+                   "file": l.get("file")
+               },
+           }

minyma/oai.py

@@ -19,6 +19,7 @@ class OpenAIConnector:
     def __init__(self, api_key: str, vdb: VectorDB):
         self.vdb = vdb
         self.model = "gpt-3.5-turbo"
+        self.word_cap = 1000
         openai.api_key = api_key
 
     def query(self, question: str) -> Any:
@@ -30,8 +31,9 @@ class OpenAIConnector:
         if len(all_docs) == 0:
             return { "error": "No Context Found" }
 
-        # Join on new line, generate main prompt
+        # Join on new line (cap @ word limit), generate main prompt
-        context = '\n'.join(all_docs)
+        reduced_docs = list(map(lambda x: " ".join(x.split()[:self.word_cap]), all_docs))
+        context = '\n'.join(reduced_docs)
         prompt = PROMPT_TEMPLATE.format(context = context, question = question)
 
         # Query OpenAI ChatCompletion
@@ -41,4 +43,4 @@ class OpenAIConnector:
         )
 
         # Return Response
-        return response
+        return { "llm": response, "vdb": related }

minyma/templates/index.html

@@ -2,10 +2,14 @@
 <html lang="en">
   <head>
     <meta charset="utf-8" />
+    <meta
+      name="viewport"
+      content="width=device-width, initial-scale=0.9, user-scalable=no, viewport-fit=cover"
+    />
     <title>Minyma - Chat</title>
     <script src="https://cdn.tailwindcss.com"></script>
   </head>
-  <body class="bg-slate-900 h-screen p-5 flex flex-col justify-between">
+  <body class="bg-slate-900 h-[100dvh] p-5 flex flex-col justify-between">
     <header class="w-full">
       <svg
         preserveAspectRatio="xMidYMid meet"
@@ -55,7 +59,7 @@
       </svg>
     </header>
     <main
-      class="flex flex-col justify-between w-11/12 mx-auto bg-slate-700 text-gray-300 rounded p-2 gap-4 h-full"
+      class="flex flex-col justify-between w-11/12 mx-auto bg-slate-700 text-gray-300 rounded p-2 gap-4 h-full overflow-scroll"
     >
       <div
         id="messages"
@@ -154,7 +158,63 @@
         })
           .then((data) => {
             console.log("SUCCESS:", data);
-            content.innerText = data.choices[0].message.content;
+
+            // Create Response Element
+            let responseEl = document.createElement("p");
+            responseEl.setAttribute(
+              "class",
+              "whitespace-break-spaces border-b pb-3 mb-3"
+            );
+            responseEl.innerText = data.response;
+
+            // Create Context Element
+            let contextEl = document.createElement("div");
+            contextEl.innerHTML = `
+	      <h1 class="font-bold">Context:</h1>
+	      <ul class="list-disc ml-6"></ul>`;
+            let ulEl = contextEl.querySelector("ul");
+
+            // Create Context Links
+            data.context
+
+              // Capture PubMed ID & Distance
+              .map((item) => [
+                item.metadata.file.match("\/(.*)\.txt$"),
+                item.distance,
+              ])
+
+              // Filter Non-Matches
+              .filter(([match]) => match)
+
+              // Get Match Value & Round Distance (2)
+              .map(([match, distance]) => [
+                match[1],
+                Math.round(distance * 100) / 100,
+              ])
+
+              // Create Links
+              .forEach(([pmid, distance]) => {
+                let newEl = document.createElement("li");
+                let linkEl = document.createElement("a");
+                linkEl.setAttribute("target", "_blank");
+                linkEl.setAttribute(
+                  "class",
+                  "text-blue-500 hover:text-blue-600"
+                );
+                linkEl.setAttribute(
+                  "href",
+                  "https://www.ncbi.nlm.nih.gov/pmc/articles/" + pmid
+                );
+                linkEl.textContent = "[" + distance + "] " + pmid;
+                newEl.append(linkEl);
+                ulEl.append(newEl);
+              });
+
+            // Add to DOM
+            content.setAttribute("class", "w-full");
+            content.innerHTML = "";
+            content.append(responseEl);
+            content.append(contextEl);
           })
           .catch((e) => {
             console.log("ERROR:", e);

minyma/vdb.py

@@ -1,7 +1,6 @@
 from chromadb.api import API
 from itertools import islice
-from os import path
+from tqdm import tqdm
-from tqdm.auto import tqdm
 from typing import Any, cast
 import chromadb
 
@@ -29,47 +28,45 @@ class VectorDB:
 ChromaDV VectorDB Type
 """
 class ChromaDB(VectorDB):
-    def __init__(self, base_path: str):
+    def __init__(self, path: str):
-        chroma_path = path.join(base_path, "chroma")
+        self.client: API = chromadb.PersistentClient(path=path)
-        self.client: API = chromadb.PersistentClient(path=chroma_path)
+        self.word_cap = 2500
-        self.word_limit = 1000
         self.collection_name: str = "vdb"
         self.collection: chromadb.Collection = self.client.create_collection(name=self.collection_name, get_or_create=True)
 
-    def get_related(self, question) -> Any:
+    def get_related(self, question: str) -> Any:
         """Returns line separated related docs"""
         results = self.collection.query(
-            query_texts=[question],
+            query_texts=[question.lower()],
             n_results=2
         )
 
         all_docs: list = cast(list, results.get("documents", [[]]))[0]
+        all_metadata: list = cast(list, results.get("metadatas", [[]]))[0]
         all_distances: list = cast(list, results.get("distances", [[]]))[0]
         all_ids: list = cast(list, results.get("ids", [[]]))[0]
 
         return {
-            "distances":all_distances, 
+            "distances": all_distances,
+            "metadatas": all_metadata,
             "docs": all_docs,
             "ids": all_ids
         }
 
-    def load_documents(self, normalizer: DataNormalizer):
+    def load_documents(self, normalizer: DataNormalizer, chunk_size: int = 10):
-        # 10 Item Chunking
+        length = len(normalizer) / chunk_size
-        for items in tqdm(chunk(normalizer, 50)):
+        for items in tqdm(chunk(normalizer, chunk_size), total=length):
             ids = []
             documents = []
+            metadatas = []
 
-            # Limit words per document to accommodate context token limits
             for item in items:
-                doc = " ".join(item.get("doc").split()[:self.word_limit])
+                documents.append(" ".join(item.get("doc").split()[:self.word_cap]))
-                documents.append(doc)
                 ids.append(item.get("id"))
+                metadatas.append(item.get("metadata", {}))
 
-            # Ideally we parse out metadata from each document
-            # and pass to the metadata kwarg. However, each
-            # document appears to have a slightly different format,
-            # so it's difficult to parse out.
             self.collection.add(
+                ids=ids,
                 documents=documents,
-                ids=ids
+                metadatas=metadatas,
             )

test.html

@@ -1,43 +0,0 @@
-<svg
-  preserveAspectRatio="xMidYMid meet"
-  color-interpolation-filters="sRGB"
-  style="margin: auto"
-  height="80"
-  width="200"
-  viewBox="70 90 200 90"
->
-  <g fill="#ebb919" transform="translate(69.05000305175781,91.03400039672852)">
-    <g transform="translate(0,0)">
-      <g transform="scale(1)">
-        <g>
-          <path
-            d="M33.96-30.84L33.96-30.84Q36.48-30.84 38.37-29.88 40.26-28.92 41.46-27.24 42.66-25.56 43.26-23.34 43.86-21.12 43.86-18.54L43.86-18.54 43.86 0 36.66 0 36.66-18.54Q36.66-20.64 35.16-22.14L35.16-22.14Q33.72-23.64 31.56-23.64L31.56-23.64Q29.4-23.64 27.96-22.14L27.96-22.14Q26.46-20.64 26.46-18.54L26.46-18.54 26.46 0 19.26 0 19.26-18.54Q19.26-20.64 17.76-22.14L17.76-22.14Q17.04-22.92 16.11-23.28 15.18-23.64 14.16-23.64L14.16-23.64Q11.94-23.64 10.5-22.14L10.5-22.14Q9-20.64 9-18.54L9-18.54 9 0 1.8 0 1.8-30 9-30 9-27.36Q10.74-28.86 12.66-29.85 14.58-30.84 16.56-30.84L16.56-30.84Q19.26-30.84 21-29.76 22.74-28.68 24.12-26.76L24.12-26.76Q25.74-28.5 28.32-29.67 30.9-30.84 33.96-30.84ZM54.96 0L47.76 0 47.76-30 54.96-30 54.96 0ZM47.76-34.8L47.76-42 54.96-42 54.96-34.8 47.76-34.8ZM74.28-30.84L74.28-30.84Q77.22-30.84 79.62-29.73 82.02-28.62 83.73-26.67 85.44-24.72 86.37-22.14 87.3-19.56 87.3-16.62L87.3-16.62 87.3 0 80.1 0 80.1-16.62Q80.1-19.62 78-21.6L78-21.6Q75.96-23.64 73.08-23.64L73.08-23.64Q70.14-23.64 68.1-21.6L68.1-21.6Q66.06-19.56 66.06-16.62L66.06-16.62 66.06 0 58.86 0 58.86-30 66.06-30 66.06-27.72Q67.68-29.1 69.72-29.97 71.76-30.84 74.28-30.84ZM116.94-30L124.86-30 110.94 0 109.08 4.08Q107.4 7.74 104.04 9.9 100.68 12.06 96.6 12.06L96.6 12.06 93.42 12.06 95.22 4.86 96.96 4.86Q98.7 4.86 100.2 3.9 101.7 2.94 102.42 1.32L102.42 1.32 103.02 0 89.1-30 97.02-30 106.98-8.52 116.94-30ZM159.12-30.84L159.12-30.84Q161.64-30.84 163.53-29.88 165.42-28.92 166.62-27.24 167.82-25.56 168.42-23.34 169.02-21.12 169.02-18.54L169.02-18.54 169.02 0 161.82 0 161.82-18.54Q161.82-20.64 160.32-22.14L160.32-22.14Q158.88-23.64 156.72-23.64L156.72-23.64Q154.56-23.64 153.12-22.14L153.12-22.14Q151.62-20.64 151.62-18.54L151.62-18.54 151.62 0 144.42 0 144.42-18.54Q144.42-20.64 142.92-22.14L142.92-22.14Q142.2-22.92 141.27-23.28 140.34-23.64 139.32-23.64L139.32-23.64Q137.1-23.64 135.66-22.14L135.66-22.14Q134.16-20.64 134.16-18.54L134.16-18.54 134.16 0 126.96 0 126.96-30 134.16-30 134.16-27.36Q135.9-28.86 137.82-29.85 139.74-30.84 141.72-30.84L141.72-30.84Q144.42-30.84 146.16-29.76 147.9-28.68 149.28-26.76L149.28-26.76Q150.9-28.5 153.48-29.67 156.06-30.84 159.12-30.84ZM196.5-30.06L203.7-30.06 203.7 0 196.5 0 196.5-15Q196.5-18.6 193.98-21.12L193.98-21.12Q191.46-23.64 187.86-23.64L187.86-23.64Q186.12-23.64 184.53-22.98 182.94-22.32 181.74-21.12L181.74-21.12Q179.22-18.6 179.22-15L179.22-15Q179.22-11.46 181.74-8.94L181.74-8.94Q182.94-7.68 184.53-7.05 186.12-6.42 187.86-6.42L187.86-6.42Q189.66-6.42 191.1-7.02L191.1-7.02 193.68-0.6Q190.92 0.78 187.26 0.78L187.26 0.78Q183.96 0.78 181.17-0.45 178.38-1.68 176.34-3.84 174.3-6 173.16-8.88 172.02-11.76 172.02-15L172.02-15Q172.02-18.3 173.16-21.18 174.3-24.06 176.34-26.22 178.38-28.38 181.17-29.61 183.96-30.84 187.26-30.84L187.26-30.84Q190.2-30.84 192.48-29.94 194.76-29.04 196.5-27.66L196.5-27.66 196.5-30.06Z"
-            transform="translate(-1.7999999523162842, 42)"
-          ></path>
-        </g>
-      </g>
-    </g>
-    <g fill="#ebb919" transform="translate(5,60.060001373291016)">
-      <rect
-        x="0"
-        height="1"
-        y="3.434999942779541"
-        width="88.66999673843384"
-      ></rect>
-      <rect
-        height="1"
-        y="3.434999942779541"
-        width="88.66999673843384"
-        x="103.22999715805054"
-      ></rect>
-      <g transform="translate(91.66999673843384,0)">
-        <g transform="scale(1)">
-          <path
-            d="M4.43-3.20L2.06-3.20L2.44-4.40C2.58-4.84 2.72-5.28 2.84-5.72C2.97-6.15 3.10-6.60 3.22-7.06L3.26-7.06C3.39-6.60 3.52-6.15 3.65-5.72C3.78-5.28 3.91-4.84 4.06-4.40ZM4.68-2.40L5.42 0L6.49 0L3.83-7.87L2.70-7.87L0.04 0L1.06 0L1.81-2.40ZM7.61-7.87L7.61 0L8.60 0L8.60-7.87Z"
-            transform="translate(-0.036000000000000004, 7.872)"
-          ></path>
-        </g>
-      </g>
-    </g>
-  </g>
-</svg>