From cf8e023b8251cd484aa9d8f3266023ef77febbad Mon Sep 17 00:00:00 2001 From: Evan Reichard Date: Tue, 7 Nov 2023 20:34:18 -0500 Subject: [PATCH] [add] home assistant plugin --- .gitignore | 2 ++ README.md | 43 +++++++++++++++++++++++++++++++- minyma/__init__.py | 2 +- minyma/api/v1.py | 24 ++++++++---------- minyma/config.py | 2 ++ minyma/oai.py | 30 ++++++++++++++++++++-- minyma/plugin.py | 39 ++++------------------------- minyma/plugins/duckduckgo.py | 3 ++- minyma/plugins/home_assistant.py | 39 +++++++++++++++++++++++++++++ minyma/plugins/vehicle_lookup.py | 12 ++++----- minyma/templates/index.html | 2 ++ 11 files changed, 139 insertions(+), 59 deletions(-) create mode 100644 minyma/plugins/home_assistant.py diff --git a/.gitignore b/.gitignore index 0fc361c..01671fc 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,6 @@ __pycache__ data venv openai_key +ha_key minyma.egg-info/ +NOTES.md diff --git a/README.md b/README.md index d9060a5..8e7f4e3 100644 --- a/README.md +++ b/README.md @@ -13,15 +13,56 @@ --- -AI Chat Bot with Vector / Embedding DB Context +AI Chat Bot with Plugins (Home Assistant, Vehicle Lookup, DuckDuckGo Search) [![Build Status](https://drone.va.reichard.io/api/badges/evan/minyma/status.svg)](https://drone.va.reichard.io/evan/minyma) +## Plugins + +### Vehicle Lookup API + +This utilizes Carvana's undocumented API to lookup details on a vehicle. + +``` +User: What vehicle is NY plate HELLO? +Assistant: The vehicle corresponding to NY plate HELLO is a 2016 MAZDA CX-5 + Grand Touring Sport Utility 4D with VIN JM3KE4DY6G0672552. +``` + +### Home Assistant API + +This utilizes Home Assistants [Conversational API](https://developers.home-assistant.io/docs/intent_conversation_api/). + +``` +User: Turn off the living room lights +Assistant: The living room lights have been turned off. Is there anything else I can assist you with? + +User: Turn on the living room lights +Assistant: The living room lights have been turned on successfully. +``` + +### DuckDuckGo + +This utilizes DuckDuckGo Search by scraping the top 5 results. + +``` +User: Tell me about Evan Reichard +Assistant: Evan Reichard is a Principal Detection and Response Engineer based + in the Washington DC-Baltimore Area. He has been in this role since + August 2022. Evan has created a browser extension that helps SOC + analysts and saves them over 300 hours per month. Additionally, + there are three professionals named Evan Reichard on LinkedIn and + there are also profiles of people named Evan Reichard on Facebook. +``` + ## Running Server ```bash # Locally (See "Development" Section) export OPENAI_API_KEY=`cat openai_key` +export HOME_ASSISTANT_API_KEY=`cat ha_key` +export HOME_ASSISTANT_URL=https://some-url.com + minyma server run # Docker Quick Start diff --git a/minyma/__init__.py b/minyma/__init__.py index 1158152..c6dc393 100644 --- a/minyma/__init__.py +++ b/minyma/__init__.py @@ -25,7 +25,7 @@ def create_app(): app = Flask(__name__) vdb = ChromaDB(path.join(Config.DATA_PATH, "chroma")) oai = OpenAIConnector(Config.OPENAI_API_KEY, vdb) - plugins = PluginLoader() + plugins = PluginLoader(Config) app.register_blueprint(api_common.bp) app.register_blueprint(api_v1.bp) diff --git a/minyma/api/v1.py b/minyma/api/v1.py index 18a1eb9..6017620 100644 --- a/minyma/api/v1.py +++ b/minyma/api/v1.py @@ -20,24 +20,20 @@ def get_response(): resp = minyma.oai.query(message) # Derive LLM Data - llm_resp = resp.get("llm", {}) - llm_choices = llm_resp.get("choices", []) + # llm_resp = resp.get("llm", {}) + # llm_choices = llm_resp.get("choices", []) # Derive VDB Data - vdb_resp = resp.get("vdb", {}) - combined_context = [{ - "id": vdb_resp.get("ids")[i], - "distance": vdb_resp.get("distances")[i], - "doc": vdb_resp.get("docs")[i], - "metadata": vdb_resp.get("metadatas")[i], - } for i, _ in enumerate(vdb_resp.get("docs", []))] + # vdb_resp = resp.get("vdb", {}) + # combined_context = [{ + # "id": vdb_resp.get("ids")[i], + # "distance": vdb_resp.get("distances")[i], + # "doc": vdb_resp.get("docs")[i], + # "metadata": vdb_resp.get("metadatas")[i], + # } for i, _ in enumerate(vdb_resp.get("docs", []))] # Return Data - return { - "response": None if len(llm_choices) == 0 else llm_choices[0].get("message", {}).get("content"), - "context": combined_context, - "usage": llm_resp.get("usage"), - } + return resp diff --git a/minyma/config.py b/minyma/config.py index 15d8b58..9fb1594 100644 --- a/minyma/config.py +++ b/minyma/config.py @@ -21,3 +21,5 @@ class Config: DATA_PATH: str = get_env("DATA_PATH", default="./data") OPENAI_API_KEY: str = get_env("OPENAI_API_KEY", required=True) + HOME_ASSISTANT_API_KEY: str = get_env("HOME_ASSISTANT_API_KEY", required=False) + HOME_ASSISTANT_URL: str = get_env("HOME_ASSISTANT_URL", required=False) diff --git a/minyma/oai.py b/minyma/oai.py index dc447cf..2d2976c 100644 --- a/minyma/oai.py +++ b/minyma/oai.py @@ -54,6 +54,11 @@ class OpenAIConnector: openai.api_key = api_key def query(self, question: str) -> Any: + # Track Usage + prompt_tokens = 0 + completion_tokens = 0 + total_tokens = 0 + # Get Available Functions functions = "\n".join(list(map(lambda x: "- %s" % x["def"], minyma.plugins.plugin_defs().values()))) @@ -82,9 +87,14 @@ class OpenAIConnector: ) ) + # Update Usage + prompt_tokens += response.usage.get("prompt_tokens", 0) + completion_tokens += response.usage.get("completion_tokens", 0) + total_tokens += response.usage.get("prompt_tokens", 0) + print("[OpenAIConnector] Completed Initial OAI Query:\n", indent(json.dumps({ "usage": response.usage, "function_calls": all_funcs }, indent=2), ' ' * 2)) - # Execute Functions + # Execute Requested Functions func_responses = {} for func in all_funcs: func_responses[func] = minyma.plugins.execute(func) @@ -107,10 +117,26 @@ class OpenAIConnector: messages=messages ) + # Update Usage + prompt_tokens += response.usage.get("prompt_tokens", 0) + completion_tokens += response.usage.get("completion_tokens", 0) + total_tokens += response.usage.get("prompt_tokens", 0) + print("[OpenAIConnector] Completed Follup Up OAI Query:\n", indent(json.dumps({ "usage": response.usage }, indent=2), ' ' * 2)) + # Get Content + content = response.choices[0]["message"]["content"] + # Return Response - return { "llm": response } + return { + "response": content, + "functions": func_responses, + "usage": { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": total_tokens + } + } def old_query(self, question: str) -> Any: # Get related documents from vector db diff --git a/minyma/plugin.py b/minyma/plugin.py index f67e18a..d331fb6 100644 --- a/minyma/plugin.py +++ b/minyma/plugin.py @@ -7,7 +7,8 @@ class MinymaPlugin: pass class PluginLoader: - def __init__(self): + def __init__(self, config): + self.config = config self.plugins = self.get_plugins() self.definitions = self.plugin_defs() @@ -52,38 +53,6 @@ class PluginLoader: return defs - def derive_function_definitions(self): - """Dynamically generate function definitions""" - function_definitions = [] - - for plugin in self.plugins: - plugin_name = plugin.name - - for method_obj in plugin.functions: - method_name = method_obj.__name__ - signature = inspect.signature(method_obj) - parameters = list(signature.parameters.values()) - - # Extract Parameter Information - params_info = {} - for param in parameters: - param_name = param.name - param_type = param.annotation - params_info[param_name] = {"type": TYPE_DEFS[param_type]} - - # Store Function Information - method_info = { - "name": "%s_%s" %(plugin_name, method_name), - "description": inspect.getdoc(method_obj), - "parameters": { - "type": "object", - "properties": params_info - } - } - function_definitions.append(method_info) - - return function_definitions - def get_plugins(self): """Dynamically load plugins""" @@ -120,5 +89,7 @@ class PluginLoader: # Instantiate Plugins plugins = [] for cls in plugin_classes: - plugins.append(cls()) + instance = cls(self.config) + print("[PluginLoader] %s - Loaded: %d Feature(s)" % (cls.__name__, len(instance.functions))) + plugins.append(instance) return plugins diff --git a/minyma/plugins/duckduckgo.py b/minyma/plugins/duckduckgo.py index ab9c05b..7cf6167 100644 --- a/minyma/plugins/duckduckgo.py +++ b/minyma/plugins/duckduckgo.py @@ -11,7 +11,8 @@ HEADERS = { class DuckDuckGoPlugin(MinymaPlugin): """Search DuckDuckGo""" - def __init__(self): + def __init__(self, config): + self.config = config self.name = "duck_duck_go" self.functions = [self.search] diff --git a/minyma/plugins/home_assistant.py b/minyma/plugins/home_assistant.py new file mode 100644 index 0000000..5ef0b8b --- /dev/null +++ b/minyma/plugins/home_assistant.py @@ -0,0 +1,39 @@ +import json +import urllib.parse +import requests +from minyma.plugin import MinymaPlugin + + +class HomeAssistantPlugin(MinymaPlugin): + """Perform Home Assistant Command""" + + def __init__(self, config): + self.config = config + self.name = "home_automation" + + + if not config.HOME_ASSISTANT_API_KEY or not config.HOME_ASSISTANT_URL: + if not config.HOME_ASSISTANT_API_KEY: + print("[HomeAssistantPlugin] Missing HOME_ASSISTANT_API_KEY") + if not config.HOME_ASSISTANT_URL: + print("[HomeAssistantPlugin] Missing HOME_ASSISTANT_URL") + + self.functions = [] + else: + self.functions = [self.command] + + def command(self, natural_language_command: str): + url = urllib.parse.urljoin(self.config.HOME_ASSISTANT_URL, "/api/conversation/process") + headers = { + "Authorization": "Bearer %s" % self.config.HOME_ASSISTANT_API_KEY, + "Content-Type": "application/json", + } + + data = {"text": natural_language_command, "language": "en"} + resp = requests.post(url, json=data, headers=headers) + + # Parse JSON + try: + return json.dumps(resp.json()) + except requests.JSONDecodeError: + return json.dumps({ "error": "Command Failed" }) diff --git a/minyma/plugins/vehicle_lookup.py b/minyma/plugins/vehicle_lookup.py index 8b67ca5..599cbf8 100644 --- a/minyma/plugins/vehicle_lookup.py +++ b/minyma/plugins/vehicle_lookup.py @@ -11,7 +11,8 @@ HEADERS = { class VehicleLookupPlugin(MinymaPlugin): """Search Vehicle Information""" - def __init__(self): + def __init__(self, config): + self.config = config self.name = "vehicle_state_plate" self.functions = [self.lookup] @@ -49,13 +50,12 @@ class VehicleLookupPlugin(MinymaPlugin): # Invalid JSON if json_resp is None: - return { + return json.dumps({ "error": error, "response": text_resp, - } + }) try: - # Check Result status_resp = json_resp.get("status", "Unknown") if status_resp != "Succeeded": @@ -74,10 +74,10 @@ class VehicleLookupPlugin(MinymaPlugin): trim = vehicle_info.get("vehicles")[0].get("trim") except Exception as e: - return { + return json.dumps({ "error": "Unknown Error: %s" % e, "response": text_resp, - } + }) return json.dumps({ "result": { diff --git a/minyma/templates/index.html b/minyma/templates/index.html index dfac7da..5b8c0d5 100644 --- a/minyma/templates/index.html +++ b/minyma/templates/index.html @@ -175,6 +175,7 @@ `; let ulEl = contextEl.querySelector("ul"); + /* // Create Context Links data.context @@ -210,6 +211,7 @@ newEl.append(linkEl); ulEl.append(newEl); }); + */ // Add to DOM content.setAttribute("class", "w-full");