[#43] add "query_image" service

custom_components/extended_openai_conversation/__init__.py

@@ -5,6 +5,7 @@
 from typing import Literal
 import json
 import yaml
+import voluptuous as vol
 
 from openai import AsyncOpenAI, AsyncAzureOpenAI
 from openai.types.chat.chat_completion import (
@@ -17,7 +18,13 @@
 from homeassistant.components import conversation
 from homeassistant.config_entries import ConfigEntry
 from homeassistant.const import CONF_API_KEY, MATCH_ALL, ATTR_NAME
-from homeassistant.core import HomeAssistant, ServiceCall, SupportsResponse
+from homeassistant.core import (
+    HomeAssistant,
+    ServiceCall,
+    ServiceResponse,
+    SupportsResponse,
+)
+from homeassistant.helpers.typing import ConfigType
 from homeassistant.util import ulid
 from homeassistant.components.homeassistant.exposed_entities import async_should_expose
 from homeassistant.exceptions import (
@@ -31,6 +38,7 @@
     intent,
     template,
     entity_registry as er,
+    selector,
 )
 
 from .const import (
@@ -78,6 +86,61 @@
 
 # hass.data key for agent.
 DATA_AGENT = "agent"
+SERVICE_QUERY_IMAGE = "query_image"
+
+
+async def async_setup(hass: HomeAssistant, config: ConfigType) -> bool:
+    """Set up OpenAI Conversation."""
+
+    async def query_image(call: ServiceCall) -> ServiceResponse:
+        """Query an image."""
+        try:
+            model = call.data["model"]
+            images = [
+                {"type": "image_url", "image_url": image}
+                for image in call.data["images"]
+            ]
+
+            messages = [
+                {
+                    "role": "user",
+                    "content": [{"type": "text", "text": call.data["prompt"]}] + images,
+                }
+            ]
+            _LOGGER.info("Prompt for %s: %s", model, messages)
+
+            response = await openai.ChatCompletion.acreate(
+                api_key=hass.data[DOMAIN][call.data["config_entry"]]["api_key"],
+                model=model,
+                messages=messages,
+                max_tokens=call.data["max_tokens"],
+            )
+            _LOGGER.info("Response %s", response)
+        except error.OpenAIError as err:
+            raise HomeAssistantError(f"Error generating image: {err}") from err
+
+        return response
+
+    hass.services.async_register(
+        DOMAIN,
+        SERVICE_QUERY_IMAGE,
+        query_image,
+        schema=vol.Schema(
+            {
+                vol.Required("config_entry"): selector.ConfigEntrySelector(
+                    {
+                        "integration": DOMAIN,
+                    }
+                ),
+                vol.Required("model", default="gpt-4-vision-preview"): cv.string,
+                vol.Required("prompt"): cv.string,
+                vol.Required("images"): vol.All(cv.ensure_list, [{"url": cv.url}]),
+                vol.Optional("max_tokens", default=300): cv.positive_int,
+            }
+        ),
+        supports_response=SupportsResponse.ONLY,
+    )
+    return True
 
 
 

custom_components/extended_openai_conversation/services.yaml

@@ -0,0 +1,30 @@
+query_image:
+  fields:
+    config_entry:
+      required: true
+      selector:
+        config_entry:
+          integration: extended_openai_conversation
+    model:
+      example: gpt-4-vision-preview
+      selector:
+        text:
+    prompt:
+      example: "What’s in this image?"
+      required: true
+      selector:
+        text:
+          multiline: true
+    images:
+      example: '{"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"}'
+      required: true
+      default: []
+      selector:
+        object:
+    max_tokens:
+      example: 300
+      default: 300
+      selector:
+        number:
+          min: 1
+          mode: box