.gitignore

@@ -4,4 +4,6 @@ __pycache__
 data
 venv
 openai_key
+ha_key
 minyma.egg-info/
+NOTES.md

README.md

@@ -13,15 +13,56 @@
 
 ---
 
-AI Chat Bot with Vector / Embedding DB Context
+AI Chat Bot with Plugins (Home Assistant, Vehicle Lookup, DuckDuckGo Search)
 
 [![Build Status](https://drone.va.reichard.io/api/badges/evan/minyma/status.svg)](https://drone.va.reichard.io/evan/minyma)
 
+## Plugins
+
+### Vehicle Lookup API
+
+This utilizes Carvana's undocumented API to lookup details on a vehicle.
+
+```
+User:      What vehicle is NY plate HELLO?
+Assistant: The vehicle corresponding to NY plate HELLO is a 2016 MAZDA CX-5
+           Grand Touring Sport Utility 4D with VIN JM3KE4DY6G0672552.
+```
+
+### Home Assistant API
+
+This utilizes Home Assistants [Conversational API](https://developers.home-assistant.io/docs/intent_conversation_api/).
+
+```
+User:      Turn off the living room lights
+Assistant: The living room lights have been turned off. Is there anything else I can assist you with?
+
+User:      Turn on the living room lights
+Assistant: The living room lights have been turned on successfully.
+```
+
+### DuckDuckGo
+
+This utilizes DuckDuckGo Search by scraping the top 5 results.
+
+```
+User:      Tell me about Evan Reichard
+Assistant: Evan Reichard is a Principal Detection and Response Engineer based
+           in the Washington DC-Baltimore Area. He has been in this role since
+           August 2022. Evan has created a browser extension that helps SOC
+           analysts and saves them over 300 hours per month. Additionally,
+           there are three professionals named Evan Reichard on LinkedIn and
+           there are also profiles of people named Evan Reichard on Facebook.
+```
+
 ## Running Server
 
 ```bash
 # Locally (See "Development" Section)
 export OPENAI_API_KEY=`cat openai_key`
+export HOME_ASSISTANT_API_KEY=`cat ha_key`
+export HOME_ASSISTANT_URL=https://some-url.com
+
 minyma server run
 
 # Docker Quick Start

minyma/__init__.py

@@ -25,7 +25,7 @@ def create_app():
     app = Flask(__name__)
     vdb = ChromaDB(path.join(Config.DATA_PATH, "chroma"))
     oai = OpenAIConnector(Config.OPENAI_API_KEY, vdb)
-    plugins = PluginLoader()
+    plugins = PluginLoader(Config)
 
     app.register_blueprint(api_common.bp)
     app.register_blueprint(api_v1.bp)

minyma/api/v1.py

@@ -20,24 +20,20 @@ def get_response():
     resp = minyma.oai.query(message)
 
     # Derive LLM Data
-    llm_resp = resp.get("llm", {})
+    # llm_resp = resp.get("llm", {})
-    llm_choices = llm_resp.get("choices", [])
+    # llm_choices = llm_resp.get("choices", [])
 
     # Derive VDB Data
-    vdb_resp = resp.get("vdb", {})
+    # vdb_resp = resp.get("vdb", {})
-    combined_context  = [{
+    # combined_context  = [{
-            "id": vdb_resp.get("ids")[i],
+    #         "id": vdb_resp.get("ids")[i],
-            "distance": vdb_resp.get("distances")[i],
+    #         "distance": vdb_resp.get("distances")[i],
-            "doc": vdb_resp.get("docs")[i],
+    #         "doc": vdb_resp.get("docs")[i],
-            "metadata": vdb_resp.get("metadatas")[i],
+    #         "metadata": vdb_resp.get("metadatas")[i],
-    } for i, _ in enumerate(vdb_resp.get("docs", []))]
+    # } for i, _ in enumerate(vdb_resp.get("docs", []))]
 
     # Return Data
-    return {
+    return resp
-        "response": None if len(llm_choices) == 0 else llm_choices[0].get("message", {}).get("content"),
-        "context": combined_context,
-        "usage": llm_resp.get("usage"),
-    }
 
 
 

minyma/config.py

@@ -21,3 +21,5 @@ class Config:
 
     DATA_PATH: str = get_env("DATA_PATH", default="./data")
     OPENAI_API_KEY: str = get_env("OPENAI_API_KEY", required=True)
+    HOME_ASSISTANT_API_KEY: str = get_env("HOME_ASSISTANT_API_KEY", required=False)
+    HOME_ASSISTANT_URL: str = get_env("HOME_ASSISTANT_URL", required=False)

minyma/oai.py

@@ -54,6 +54,11 @@ class OpenAIConnector:
         openai.api_key = api_key
 
     def query(self, question: str) -> Any:
+        # Track Usage
+        prompt_tokens = 0
+        completion_tokens = 0
+        total_tokens = 0
+
         # Get Available Functions
         functions = "\n".join(list(map(lambda x: "- %s" % x["def"], minyma.plugins.plugin_defs().values())))
 
@@ -82,9 +87,14 @@ class OpenAIConnector:
             )
         )
 
+        # Update Usage
+        prompt_tokens += response.usage.get("prompt_tokens", 0)
+        completion_tokens += response.usage.get("completion_tokens", 0)
+        total_tokens += response.usage.get("prompt_tokens", 0)
+
         print("[OpenAIConnector] Completed Initial OAI Query:\n", indent(json.dumps({ "usage": response.usage, "function_calls": all_funcs }, indent=2), ' ' * 2))
 
-        # Execute Functions
+        # Execute Requested Functions
         func_responses = {}
         for func in all_funcs:
             func_responses[func] = minyma.plugins.execute(func)
@@ -107,10 +117,26 @@ class OpenAIConnector:
           messages=messages
         )
 
+        # Update Usage
+        prompt_tokens += response.usage.get("prompt_tokens", 0)
+        completion_tokens += response.usage.get("completion_tokens", 0)
+        total_tokens += response.usage.get("prompt_tokens", 0)
+
         print("[OpenAIConnector] Completed Follup Up OAI Query:\n", indent(json.dumps({ "usage": response.usage }, indent=2), ' ' * 2))
 
+        # Get Content
+        content = response.choices[0]["message"]["content"]
+
         # Return Response
-        return { "llm": response }
+        return {
+            "response": content,
+            "functions": func_responses,
+            "usage": {
+                "prompt_tokens": prompt_tokens,
+                "completion_tokens": completion_tokens,
+                "total_tokens": total_tokens
+            }
+        }
 
     def old_query(self, question: str) -> Any:
         # Get related documents from vector db

minyma/plugin.py

@@ -7,7 +7,8 @@ class MinymaPlugin:
     pass
 
 class PluginLoader:
-    def __init__(self):
+    def __init__(self, config):
+        self.config = config
         self.plugins = self.get_plugins()
         self.definitions = self.plugin_defs()
 
@@ -52,38 +53,6 @@ class PluginLoader:
 
         return defs
 
-    def derive_function_definitions(self):
-        """Dynamically generate function definitions"""
-        function_definitions = []
-
-        for plugin in self.plugins:
-            plugin_name = plugin.name
-
-            for method_obj in plugin.functions:
-                method_name = method_obj.__name__
-                signature = inspect.signature(method_obj)
-                parameters = list(signature.parameters.values())
-
-                # Extract Parameter Information
-                params_info = {}
-                for param in parameters:
-                    param_name = param.name
-                    param_type = param.annotation
-                    params_info[param_name] = {"type": TYPE_DEFS[param_type]}
-
-                # Store Function Information
-                method_info = {
-                    "name": "%s_%s" %(plugin_name, method_name),
-                    "description": inspect.getdoc(method_obj),
-                    "parameters": {
-                        "type": "object",
-                        "properties": params_info
-                    }
-                }
-                function_definitions.append(method_info)
-
-        return function_definitions
-
 
     def get_plugins(self):
         """Dynamically load plugins"""
@@ -120,5 +89,7 @@ class PluginLoader:
         # Instantiate Plugins
         plugins = []
         for cls in plugin_classes:
-            plugins.append(cls())
+            instance = cls(self.config)
+            print("[PluginLoader] %s - Loaded: %d Feature(s)" % (cls.__name__, len(instance.functions)))
+            plugins.append(instance)
         return plugins

minyma/plugins/duckduckgo.py

@@ -11,7 +11,8 @@ HEADERS = {
 class DuckDuckGoPlugin(MinymaPlugin):
     """Search DuckDuckGo"""
 
-    def __init__(self):
+    def __init__(self, config):
+        self.config = config
         self.name = "duck_duck_go"
         self.functions = [self.search]
 

minyma/plugins/home_assistant.py

@@ -0,0 +1,39 @@
+import json
+import urllib.parse
+import requests
+from minyma.plugin import MinymaPlugin
+
+
+class HomeAssistantPlugin(MinymaPlugin):
+    """Perform Home Assistant Command"""
+
+    def __init__(self, config):
+        self.config = config
+        self.name = "home_automation"
+
+
+        if not config.HOME_ASSISTANT_API_KEY or not config.HOME_ASSISTANT_URL:
+            if not config.HOME_ASSISTANT_API_KEY:
+                print("[HomeAssistantPlugin] Missing HOME_ASSISTANT_API_KEY")
+            if not config.HOME_ASSISTANT_URL:
+                print("[HomeAssistantPlugin] Missing HOME_ASSISTANT_URL")
+
+            self.functions = []
+        else:
+            self.functions = [self.command]
+
+    def command(self, natural_language_command: str):
+        url = urllib.parse.urljoin(self.config.HOME_ASSISTANT_URL, "/api/conversation/process")
+        headers = {
+            "Authorization": "Bearer %s" % self.config.HOME_ASSISTANT_API_KEY,
+            "Content-Type": "application/json",
+        }
+
+        data = {"text": natural_language_command, "language": "en"}
+        resp = requests.post(url, json=data, headers=headers)
+
+        # Parse JSON
+        try:
+            return json.dumps(resp.json())
+        except requests.JSONDecodeError:
+            return json.dumps({ "error": "Command Failed" })

minyma/plugins/vehicle_lookup.py

@@ -11,7 +11,8 @@ HEADERS = {
 class VehicleLookupPlugin(MinymaPlugin):
     """Search Vehicle Information"""
 
-    def __init__(self):
+    def __init__(self, config):
+        self.config = config
         self.name = "vehicle_state_plate"
         self.functions = [self.lookup]
 
@@ -49,13 +50,12 @@ class VehicleLookupPlugin(MinymaPlugin):
 
         # Invalid JSON
         if json_resp is None:
-            return {
+            return json.dumps({
                 "error": error,
                 "response": text_resp,
-            }
+            })
 
         try:
-
             # Check Result
             status_resp = json_resp.get("status", "Unknown")
             if status_resp != "Succeeded":
@@ -74,10 +74,10 @@ class VehicleLookupPlugin(MinymaPlugin):
             trim = vehicle_info.get("vehicles")[0].get("trim")
 
         except Exception as e:
-            return {
+            return json.dumps({
                 "error": "Unknown Error: %s" % e,
                 "response": text_resp,
-            }
+            })
 
         return json.dumps({
             "result": {

minyma/templates/index.html

@@ -175,6 +175,7 @@
 	      <ul class="list-disc ml-6"></ul>`;
             let ulEl = contextEl.querySelector("ul");
 
+            /*
             // Create Context Links
             data.context
 
@@ -210,6 +211,7 @@
                 newEl.append(linkEl);
                 ulEl.append(newEl);
               });
+	    */
 
             // Add to DOM
             content.setAttribute("class", "w-full");