[#43] add "query_image" service

custom_components/extended_openai_conversation/__init__.py

@@ -17,7 +17,8 @@
 from homeassistant.components import conversation
 from homeassistant.config_entries import ConfigEntry
 from homeassistant.const import CONF_API_KEY, MATCH_ALL, ATTR_NAME
-from homeassistant.core import HomeAssistant, ServiceCall, SupportsResponse
+from homeassistant.core import HomeAssistant
+from homeassistant.helpers.typing import ConfigType
 from homeassistant.util import ulid
 from homeassistant.components.homeassistant.exposed_entities import async_should_expose
 from homeassistant.exceptions import (
@@ -70,6 +71,8 @@
     is_azure,
 )
 
+from .services import async_setup_services
+
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -80,6 +83,11 @@
 DATA_AGENT = "agent"
 
 
+async def async_setup(hass: HomeAssistant, config: ConfigType) -> bool:
+    """Set up OpenAI Conversation."""
+    await async_setup_services(hass, config)
+    return True
+
 
 async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool:
     """Set up OpenAI Conversation from a config entry."""
@@ -127,9 +135,15 @@ def __init__(self, hass: HomeAssistant, entry: ConfigEntry) -> None:
         self.history: dict[str, list[dict]] = {}
         base_url = entry.data.get(CONF_BASE_URL)
         if is_azure(base_url):
-            self.client = AsyncAzureOpenAI(api_key=entry.data[CONF_API_KEY], azure_endpoint=base_url, api_version=entry.data.get(CONF_API_VERSION))
+            self.client = AsyncAzureOpenAI(
+                api_key=entry.data[CONF_API_KEY],
+                azure_endpoint=base_url,
+                api_version=entry.data.get(CONF_API_VERSION),
+            )
         else:
-            self.client = AsyncOpenAI(api_key=entry.data[CONF_API_KEY], base_url=base_url)
+            self.client = AsyncOpenAI(
+                api_key=entry.data[CONF_API_KEY], base_url=base_url
+            )
 
     @property
     def supported_languages(self) -> list[str] | Literal["*"]:
@@ -291,7 +305,6 @@ async def query(
             function_call=function_call,
         )
 
-
         _LOGGER.info("Response %s", response.model_dump(exclude_none=True))
         choice: Choice = response.choices[0]
         message = choice.message

custom_components/extended_openai_conversation/const.py

@@ -84,3 +84,5 @@
 ]
 CONF_ATTACH_USERNAME = "attach_username"
 DEFAULT_ATTACH_USERNAME = False
+
+SERVICE_QUERY_IMAGE = "query_image"

custom_components/extended_openai_conversation/services.py

@@ -0,0 +1,76 @@
+import logging
+
+import voluptuous as vol
+from openai import AsyncOpenAI
+from openai._exceptions import OpenAIError
+
+from homeassistant.core import (
+    HomeAssistant,
+    ServiceCall,
+    ServiceResponse,
+    SupportsResponse,
+)
+from homeassistant.exceptions import HomeAssistantError
+from homeassistant.helpers.typing import ConfigType
+from homeassistant.helpers import selector, config_validation as cv
+
+from .const import DOMAIN, SERVICE_QUERY_IMAGE
+
+QUERY_IMAGE_SCHEMA = vol.Schema(
+    {
+        vol.Required("config_entry"): selector.ConfigEntrySelector(
+            {
+                "integration": DOMAIN,
+            }
+        ),
+        vol.Required("model", default="gpt-4-vision-preview"): cv.string,
+        vol.Required("prompt"): cv.string,
+        vol.Required("images"): vol.All(cv.ensure_list, [{"url": cv.url}]),
+        vol.Optional("max_tokens", default=300): cv.positive_int,
+    }
+)
+
+_LOGGER = logging.getLogger(__package__)
+
+
+async def async_setup_services(hass: HomeAssistant, config: ConfigType) -> None:
+    """Set up services for the extended openai conversation component."""
+
+    async def query_image(call: ServiceCall) -> ServiceResponse:
+        """Query an image."""
+        try:
+            model = call.data["model"]
+            images = [
+                {"type": "image_url", "image_url": image}
+                for image in call.data["images"]
+            ]
+
+            messages = [
+                {
+                    "role": "user",
+                    "content": [{"type": "text", "text": call.data["prompt"]}] + images,
+                }
+            ]
+            _LOGGER.info("Prompt for %s: %s", model, messages)
+
+            response = await AsyncOpenAI(
+                api_key=hass.data[DOMAIN][call.data["config_entry"]]["api_key"]
+            ).chat.completions.create(
+                model=model,
+                messages=messages,
+                max_tokens=call.data["max_tokens"],
+            )
+            response_dict = response.model_dump()
+            _LOGGER.info("Response %s", response_dict)
+        except OpenAIError as err:
+            raise HomeAssistantError(f"Error generating image: {err}") from err
+
+        return response_dict
+
+    hass.services.async_register(
+        DOMAIN,
+        SERVICE_QUERY_IMAGE,
+        query_image,
+        schema=QUERY_IMAGE_SCHEMA,
+        supports_response=SupportsResponse.ONLY,
+    )

custom_components/extended_openai_conversation/services.yaml

@@ -0,0 +1,30 @@
+query_image:
+  fields:
+    config_entry:
+      required: true
+      selector:
+        config_entry:
+          integration: extended_openai_conversation
+    model:
+      example: gpt-4-vision-preview
+      selector:
+        text:
+    prompt:
+      example: "What’s in this image?"
+      required: true
+      selector:
+        text:
+          multiline: true
+    images:
+      example: '{"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"}'
+      required: true
+      default: []
+      selector:
+        object:
+    max_tokens:
+      example: 300
+      default: 300
+      selector:
+        number:
+          min: 1
+          mode: box

custom_components/extended_openai_conversation/strings.json

@@ -33,5 +33,37 @@
         }
       }
     }
+  },
+  "services": {
+    "query_image": {
+      "name": "Query image",
+      "description": "Take in images and answer questions about them",
+      "fields": {
+        "config_entry": {
+          "name": "Config Entry",
+          "description": "The config entry to use for this service"
+        },
+        "model": {
+          "name": "Model",
+          "description": "The model",
+          "example": "gpt-4-vision-preview"
+        },
+        "prompt": {
+          "name": "Prompt",
+          "description": "The text to ask about image",
+          "example": "What’s in this image?"
+        },
+        "images": {
+          "name": "Images",
+          "description": "A list of images that would be asked",
+          "example": "{\"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg\"}"
+        },
+        "max_tokens": {
+          "name": "Max Tokens",
+          "description": "The maximum tokens",
+          "example": "300"
+        }
+      }
+    }
   }
 }

custom_components/extended_openai_conversation/translations/de.json

@@ -33,5 +33,37 @@
                 }
             }
         }
+    },
+    "services": {
+        "query_image": {
+            "name": "Query image",
+            "description": "Take in images and answer questions about them",
+            "fields": {
+                "config_entry": {
+                    "name": "Config Entry",
+                    "description": "The config entry to use for this service"
+                },
+                "model": {
+                    "name": "Model",
+                    "description": "The model",
+                    "example": "gpt-4-vision-preview"
+                },
+                "prompt": {
+                    "name": "Prompt",
+                    "description": "The text to ask about image",
+                    "example": "What’s in this image?"
+                },
+                "images": {
+                    "name": "Images",
+                    "description": "A list of images that would be asked",
+                    "example": "{\"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg\"}"
+                },
+                "max_tokens": {
+                    "name": "Max Tokens",
+                    "description": "The maximum tokens",
+                    "example": "300"
+                }
+            }
+        }
     }
 }

custom_components/extended_openai_conversation/translations/en.json

@@ -33,5 +33,37 @@
                 }
             }
         }
+    },
+    "services": {
+        "query_image": {
+            "name": "Query image",
+            "description": "Take in images and answer questions about them",
+            "fields": {
+                "config_entry": {
+                    "name": "Config Entry",
+                    "description": "The config entry to use for this service"
+                },
+                "model": {
+                    "name": "Model",
+                    "description": "The model",
+                    "example": "gpt-4-vision-preview"
+                },
+                "prompt": {
+                    "name": "Prompt",
+                    "description": "The text to ask about image",
+                    "example": "What’s in this image?"
+                },
+                "images": {
+                    "name": "Images",
+                    "description": "A list of images that would be asked",
+                    "example": "{\"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg\"}"
+                },
+                "max_tokens": {
+                    "name": "Max Tokens",
+                    "description": "The maximum tokens",
+                    "example": "300"
+                }
+            }
+        }
     }
 }

custom_components/extended_openai_conversation/translations/hu.json

@@ -33,5 +33,37 @@
                 }
             }
         }
+    },
+    "services": {
+        "query_image": {
+            "name": "Query image",
+            "description": "Take in images and answer questions about them",
+            "fields": {
+                "config_entry": {
+                    "name": "Config Entry",
+                    "description": "The config entry to use for this service"
+                },
+                "model": {
+                    "name": "Model",
+                    "description": "The model",
+                    "example": "gpt-4-vision-preview"
+                },
+                "prompt": {
+                    "name": "Prompt",
+                    "description": "The text to ask about image",
+                    "example": "What’s in this image?"
+                },
+                "images": {
+                    "name": "Images",
+                    "description": "A list of images that would be asked",
+                    "example": "{\"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg\"}"
+                },
+                "max_tokens": {
+                    "name": "Max Tokens",
+                    "description": "The maximum tokens",
+                    "example": "300"
+                }
+            }
+        }
     }
-}
+}

custom_components/extended_openai_conversation/translations/ko.json

@@ -33,5 +33,37 @@
                 }
             }
         }
+    },
+    "services": {
+        "query_image": {
+            "name": "Query image",
+            "description": "Take in images and answer questions about them",
+            "fields": {
+                "config_entry": {
+                    "name": "Config Entry",
+                    "description": "The config entry to use for this service"
+                },
+                "model": {
+                    "name": "Model",
+                    "description": "The model",
+                    "example": "gpt-4-vision-preview"
+                },
+                "prompt": {
+                    "name": "Prompt",
+                    "description": "The text to ask about image",
+                    "example": "What’s in this image?"
+                },
+                "images": {
+                    "name": "Images",
+                    "description": "A list of images that would be asked",
+                    "example": "{\"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg\"}"
+                },
+                "max_tokens": {
+                    "name": "Max Tokens",
+                    "description": "The maximum tokens",
+                    "example": "300"
+                }
+            }
+        }
     }
 }