1.0.1

README.md

@@ -79,7 +79,10 @@ Options include [OpenAI Conversation](https://www.home-assistant.io/integrations
     - `execute_service`
       - `domain`(string): domain to be passed to `hass.services.async_call`
       - `service`(string): service to be passed to `hass.services.async_call`
-      - `service_data`(string): service_data to be passed to `hass.services.async_call`
+      - `service_data`(object): service_data to be passed to `hass.services.async_call`.
+        - `entity_id`(string): target entity
+        - `device_id`(string): target device
+        - `area_id`(string): target area
     - `add_automation`
       - `automation_config`(string): An automation configuration in a yaml format
     - `get_history`
@@ -142,6 +145,9 @@ Then you will be able to let OpenAI call your function.
 ### 1. template
 #### 1-1. Get current weather
 
+For real world example, see [weather](https://github.com/jekalmin/extended_openai_conversation/tree/main/examples/function/weather).<br/>
+This is just an example from [OpenAI documentation](https://platform.openai.com/docs/guides/function-calling/common-use-cases)
+
 ```yaml
 - spec:
     name: get_current_weather

custom_components/extended_openai_conversation/__init__.py

@@ -17,14 +17,14 @@
 from homeassistant.components import conversation
 from homeassistant.config_entries import ConfigEntry
 from homeassistant.const import CONF_API_KEY, MATCH_ALL, ATTR_NAME
-from homeassistant.core import HomeAssistant, ServiceCall, SupportsResponse
+from homeassistant.core import HomeAssistant
+from homeassistant.helpers.typing import ConfigType
 from homeassistant.util import ulid
 from homeassistant.components.homeassistant.exposed_entities import async_should_expose
 from homeassistant.exceptions import (
     ConfigEntryNotReady,
     HomeAssistantError,
     TemplateError,
-    ServiceNotFound,
 )
 
 from homeassistant.helpers import (
@@ -59,42 +59,36 @@
 )
 
 from .exceptions import (
-    EntityNotFound,
-    EntityNotExposed,
-    CallServiceError,
     FunctionNotFound,
-    NativeNotFound,
     FunctionLoadFailed,
     ParseArgumentsFailed,
     InvalidFunction,
 )
 
 from .helpers import (
-    FUNCTION_EXECUTORS,
-    FunctionExecutor,
-    NativeFunctionExecutor,
-    ScriptFunctionExecutor,
-    TemplateFunctionExecutor,
-    RestFunctionExecutor,
-    ScrapeFunctionExecutor,
-    CompositeFunctionExecutor,
-    convert_to_template,
     validate_authentication,
     get_function_executor,
     is_azure,
 )
 
+from .services import async_setup_services
+
 
 _LOGGER = logging.getLogger(__name__)
 
 CONFIG_SCHEMA = cv.config_entry_only_config_schema(DOMAIN)
-AZURE_DOMAIN_PATTERN = r"\.openai\.azure\.com"
 
 
 # hass.data key for agent.
 DATA_AGENT = "agent"
 
 
+async def async_setup(hass: HomeAssistant, config: ConfigType) -> bool:
+    """Set up OpenAI Conversation."""
+    await async_setup_services(hass, config)
+    return True
+
+
 async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool:
     """Set up OpenAI Conversation from a config entry."""
 
@@ -141,9 +135,15 @@ def __init__(self, hass: HomeAssistant, entry: ConfigEntry) -> None:
         self.history: dict[str, list[dict]] = {}
         base_url = entry.data.get(CONF_BASE_URL)
         if is_azure(base_url):
-            self.client = AsyncAzureOpenAI(api_key=entry.data[CONF_API_KEY], azure_endpoint=base_url, api_version=entry.data.get(CONF_API_VERSION))
+            self.client = AsyncAzureOpenAI(
+                api_key=entry.data[CONF_API_KEY],
+                azure_endpoint=base_url,
+                api_version=entry.data.get(CONF_API_VERSION),
+            )
         else:
-            self.client = AsyncOpenAI(api_key=entry.data[CONF_API_KEY], base_url=base_url)
+            self.client = AsyncOpenAI(
+                api_key=entry.data[CONF_API_KEY], base_url=base_url
+            )
 
     @property
     def supported_languages(self) -> list[str] | Literal["*"]:
@@ -305,8 +305,7 @@ async def query(
             function_call=function_call,
         )
 
-
-        _LOGGER.info("Response %s", response)
+        _LOGGER.info("Response %s", response.model_dump(exclude_none=True))
         choice: Choice = response.choices[0]
         message = choice.message
         if choice.finish_reason == "function_call":

custom_components/extended_openai_conversation/const.py

@@ -84,3 +84,5 @@
 ]
 CONF_ATTACH_USERNAME = "attach_username"
 DEFAULT_ATTACH_USERNAME = False
+
+SERVICE_QUERY_IMAGE = "query_image"

custom_components/extended_openai_conversation/exceptions.py

@@ -35,15 +35,15 @@ def __init__(self, domain: str, service: str, data: object) -> None:
         """Initialize error."""
         super().__init__(
             self,
-            f"unable to call service {domain}.{service} with data {data}. 'entity_id' is required",
+            f"unable to call service {domain}.{service} with data {data}. One of 'entity_id', 'area_id', or 'device_id' is required",
         )
         self.domain = domain
         self.service = service
         self.data = data
 
     def __str__(self) -> str:
         """Return string representation."""
-        return f"unable to call service {self.domain}.{self.service} with data {self.data}. 'entity_id' is required"
+        return f"unable to call service {self.domain}.{self.service} with data {self.data}. One of 'entity_id', 'area_id', or 'device_id' is required"
 
 
 class FunctionNotFound(HomeAssistantError):

custom_components/extended_openai_conversation/helpers.py

@@ -18,7 +18,6 @@
     automation,
     rest,
     scrape,
-    history,
     conversation,
     recorder,
 )
@@ -38,13 +37,7 @@
 from homeassistant.core import HomeAssistant, State
 from homeassistant.helpers import config_validation as cv
 from homeassistant.helpers.template import Template
-from homeassistant.helpers.script import (
-    Script,
-    SCRIPT_MODE_SINGLE,
-    SCRIPT_MODE_PARALLEL,
-    DEFAULT_MAX,
-    DEFAULT_MAX_EXCEEDED,
-)
+from homeassistant.helpers.script import Script
 from homeassistant.exceptions import HomeAssistantError, ServiceNotFound
 
 
@@ -231,16 +224,18 @@ async def execute_service(
                 "service_data", service_argument.get("data", {})
             )
             entity_id = service_data.get("entity_id", service_argument.get("entity_id"))
+            area_id = service_data.get("area_id")
+            device_id = service_data.get("device_id")
 
             if isinstance(entity_id, str):
                 entity_id = [e.strip() for e in entity_id.split(",")]
             service_data["entity_id"] = entity_id
 
-            if entity_id is None:
+            if entity_id is None and area_id is None and device_id is None:
                 raise CallServiceError(domain, service, service_data)
             if not hass.services.has_service(domain, service):
                 raise ServiceNotFound(domain, service)
-            self.validate_entity_ids(hass, entity_id, exposed_entities)
+            self.validate_entity_ids(hass, entity_id or [], exposed_entities)
 
             try:
                 await hass.services.async_call(
@@ -249,7 +244,7 @@ async def execute_service(
                     service_data=service_data,
                 )
                 result.append(True)
-            except HomeAssistantError:
+            except HomeAssistantError as e:
                 _LOGGER.error(e)
                 result.append(False)
 

custom_components/extended_openai_conversation/services.py

@@ -0,0 +1,76 @@
+import logging
+
+import voluptuous as vol
+from openai import AsyncOpenAI
+from openai._exceptions import OpenAIError
+
+from homeassistant.core import (
+    HomeAssistant,
+    ServiceCall,
+    ServiceResponse,
+    SupportsResponse,
+)
+from homeassistant.exceptions import HomeAssistantError
+from homeassistant.helpers.typing import ConfigType
+from homeassistant.helpers import selector, config_validation as cv
+
+from .const import DOMAIN, SERVICE_QUERY_IMAGE
+
+QUERY_IMAGE_SCHEMA = vol.Schema(
+    {
+        vol.Required("config_entry"): selector.ConfigEntrySelector(
+            {
+                "integration": DOMAIN,
+            }
+        ),
+        vol.Required("model", default="gpt-4-vision-preview"): cv.string,
+        vol.Required("prompt"): cv.string,
+        vol.Required("images"): vol.All(cv.ensure_list, [{"url": cv.url}]),
+        vol.Optional("max_tokens", default=300): cv.positive_int,
+    }
+)
+
+_LOGGER = logging.getLogger(__package__)
+
+
+async def async_setup_services(hass: HomeAssistant, config: ConfigType) -> None:
+    """Set up services for the extended openai conversation component."""
+
+    async def query_image(call: ServiceCall) -> ServiceResponse:
+        """Query an image."""
+        try:
+            model = call.data["model"]
+            images = [
+                {"type": "image_url", "image_url": image}
+                for image in call.data["images"]
+            ]
+
+            messages = [
+                {
+                    "role": "user",
+                    "content": [{"type": "text", "text": call.data["prompt"]}] + images,
+                }
+            ]
+            _LOGGER.info("Prompt for %s: %s", model, messages)
+
+            response = await AsyncOpenAI(
+                api_key=hass.data[DOMAIN][call.data["config_entry"]]["api_key"]
+            ).chat.completions.create(
+                model=model,
+                messages=messages,
+                max_tokens=call.data["max_tokens"],
+            )
+            response_dict = response.model_dump()
+            _LOGGER.info("Response %s", response_dict)
+        except OpenAIError as err:
+            raise HomeAssistantError(f"Error generating image: {err}") from err
+
+        return response_dict
+
+    hass.services.async_register(
+        DOMAIN,
+        SERVICE_QUERY_IMAGE,
+        query_image,
+        schema=QUERY_IMAGE_SCHEMA,
+        supports_response=SupportsResponse.ONLY,
+    )

custom_components/extended_openai_conversation/services.yaml

@@ -0,0 +1,30 @@
+query_image:
+  fields:
+    config_entry:
+      required: true
+      selector:
+        config_entry:
+          integration: extended_openai_conversation
+    model:
+      example: gpt-4-vision-preview
+      selector:
+        text:
+    prompt:
+      example: "What’s in this image?"
+      required: true
+      selector:
+        text:
+          multiline: true
+    images:
+      example: '{"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"}'
+      required: true
+      default: []
+      selector:
+        object:
+    max_tokens:
+      example: 300
+      default: 300
+      selector:
+        number:
+          min: 1
+          mode: box

custom_components/extended_openai_conversation/strings.json

@@ -33,5 +33,37 @@
         }
       }
     }
+  },
+  "services": {
+    "query_image": {
+      "name": "Query image",
+      "description": "Take in images and answer questions about them",
+      "fields": {
+        "config_entry": {
+          "name": "Config Entry",
+          "description": "The config entry to use for this service"
+        },
+        "model": {
+          "name": "Model",
+          "description": "The model",
+          "example": "gpt-4-vision-preview"
+        },
+        "prompt": {
+          "name": "Prompt",
+          "description": "The text to ask about image",
+          "example": "What’s in this image?"
+        },
+        "images": {
+          "name": "Images",
+          "description": "A list of images that would be asked",
+          "example": "{\"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg\"}"
+        },
+        "max_tokens": {
+          "name": "Max Tokens",
+          "description": "The maximum tokens",
+          "example": "300"
+        }
+      }
+    }
   }
 }