feat: add image grid output for vision enabled LLMs (#1057)

* feat: add image grid output for vision enabled LLMs

* update tests

* linting

* update tests

* dynamically size tile output

* add path sensor and update tests

* fix tests

* formatting

custom_components/mail_and_packages/config_flow.py

@@ -24,6 +24,7 @@
     CONF_CUSTOM_IMG_FILE,
     CONF_DURATION,
     CONF_FOLDER,
+    CONF_GENERATE_GRID,
     CONF_GENERATE_MP4,
     CONF_IMAGE_SECURITY,
     CONF_IMAP_SECURITY,
@@ -230,6 +231,9 @@ def _get_default(key: str, fallback_default: Any = None) -> None:
             vol.Optional(
                 CONF_DURATION, default=_get_default(CONF_DURATION)
             ): vol.Coerce(int),
+            vol.Optional(
+                CONF_GENERATE_GRID, default=_get_default(CONF_GENERATE_GRID)
+            ): cv.boolean,
             vol.Optional(
                 CONF_GENERATE_MP4, default=_get_default(CONF_GENERATE_MP4)
             ): cv.boolean,
@@ -384,6 +388,7 @@ async def _show_config_2(self, user_input):
             CONF_DURATION: DEFAULT_GIF_DURATION,
             CONF_IMAGE_SECURITY: DEFAULT_IMAGE_SECURITY,
             CONF_IMAP_TIMEOUT: DEFAULT_IMAP_TIMEOUT,
+            CONF_GENERATE_GRID: False,
             CONF_GENERATE_MP4: False,
             CONF_ALLOW_EXTERNAL: DEFAULT_ALLOW_EXTERNAL,
             CONF_CUSTOM_IMG: DEFAULT_CUSTOM_IMG,

custom_components/mail_and_packages/const.py

@@ -27,6 +27,7 @@
 ATTR_AMAZON_IMAGE = "amazon_image"
 ATTR_COUNT = "count"
 ATTR_CODE = "code"
+ATTR_GRID_IMAGE_NAME = "grid_image"
 ATTR_ORDER = "order"
 ATTR_TRACKING = "tracking"
 ATTR_TRACKING_NUM = "tracking_#"
@@ -52,6 +53,7 @@
 CONF_SCAN_INTERVAL = "scan_interval"
 CONF_IMAGE_SECURITY = "image_security"
 CONF_IMAP_TIMEOUT = "imap_timeout"
+CONF_GENERATE_GRID = "generate_grid"
 CONF_GENERATE_MP4 = "generate_mp4"
 CONF_AMAZON_FWDS = "amazon_fwds"
 CONF_AMAZON_DAYS = "amazon_days"
@@ -1213,6 +1215,13 @@
         key="usps_mail_image_url",
         entity_category=EntityCategory.DIAGNOSTIC,
     ),
+    "usps_mail_grid_image_path": SensorEntityDescription(
+        name="Mail Grid Image Path",
+        icon="mdi:folder-multiple-image",
+        key="usps_mail_grid_image_path",
+        entity_category=EntityCategory.DIAGNOSTIC,
+        entity_registry_enabled_default=False,
+    ),
 }
 
 BINARY_SENSORS: Final[dict[str, MailandPackagesBinarySensorEntityDescription]] = {
@@ -1221,12 +1230,14 @@
         key="usps_update",
         device_class=BinarySensorDeviceClass.UPDATE,
         selectable=False,
+        entity_registry_enabled_default=False,
     ),
     "amazon_update": MailandPackagesBinarySensorEntityDescription(
         name="Amazon Image Updated",
         key="amazon_update",
         device_class=BinarySensorDeviceClass.UPDATE,
         selectable=False,
+        entity_registry_enabled_default=False,
     ),
     "usps_mail_delivered": MailandPackagesBinarySensorEntityDescription(
         name="USPS Mail Delivered",

custom_components/mail_and_packages/helpers.py

@@ -61,6 +61,7 @@
     ATTR_CODE,
     ATTR_COUNT,
     ATTR_EMAIL,
+    ATTR_GRID_IMAGE_NAME,
     ATTR_IMAGE_NAME,
     ATTR_IMAGE_PATH,
     ATTR_ORDER,
@@ -77,6 +78,7 @@
     CONF_CUSTOM_IMG_FILE,
     CONF_DURATION,
     CONF_FOLDER,
+    CONF_GENERATE_GRID,
     CONF_GENERATE_MP4,
     CONF_IMAP_SECURITY,
     CONF_STORAGE,
@@ -194,6 +196,7 @@ def process_emails(hass: HomeAssistant, config: ConfigEntry) -> dict:
     resources = config.get(CONF_RESOURCES)
     imap_security = config.get(CONF_IMAP_SECURITY)
     verify_ssl = config.get(CONF_VERIFY_SSL)
+    generate_grid = config.get(CONF_GENERATE_GRID)
 
     # Create the dict container
     data = {}
@@ -217,6 +220,11 @@ def process_emails(hass: HomeAssistant, config: ConfigEntry) -> dict:
     _LOGGER.debug("Image name: %s", image_name)
     _image[ATTR_IMAGE_NAME] = image_name
 
+    if generate_grid:
+        png_file = image_name.replace(".gif", "_grid.png")
+        _LOGGER.debug("Grid image name: %s", png_file)
+        _image[ATTR_GRID_IMAGE_NAME] = png_file
+
     # Amazon delivery image name
     image_name = image_file_name(hass, config, True)
     _LOGGER.debug("Amazon Image Name: %s", image_name)
@@ -273,7 +281,9 @@ def copy_images(hass: HomeAssistant, config: ConfigEntry) -> None:
 
 
 def image_file_name(
-    hass: HomeAssistant, config: ConfigEntry, amazon: bool = False
+    hass: HomeAssistant,
+    config: ConfigEntry,
+    amazon: bool = False,
 ) -> str:
     """Determine if filename is to be changed or not.
 
@@ -381,6 +391,7 @@ def fetch(
     img_out_path = f"{hass.config.path()}/{default_image_path(hass, config)}"
     gif_duration = config.get(CONF_DURATION)
     generate_mp4 = config.get(CONF_GENERATE_MP4)
+    generate_grid = config.get(CONF_GENERATE_GRID)
     amazon_fwds = cv.ensure_list_csv(config.get(CONF_AMAZON_FWDS))
     image_name = data[ATTR_IMAGE_NAME]
     amazon_image_name = data[ATTR_AMAZON_IMAGE]
@@ -404,6 +415,7 @@ def fetch(
             image_name,
             generate_mp4,
             nomail,
+            generate_grid,
         )
     elif sensor == AMAZON_PACKAGES:
         count[sensor] = get_items(
@@ -483,15 +495,15 @@ def login(
     try:
         if security == "SSL":
             if not verify:
-                context = ssl.client_context_no_verify()
+                context = ssl.get_default_no_verify_context()
             else:
-                context = ssl.client_context()
+                context = ssl.get_default_context()
             account = imaplib.IMAP4_SSL(host=host, port=port, ssl_context=context)
         elif security == "startTLS":
             if not verify:
-                context = ssl.client_context_no_verify()
+                context = ssl.get_default_no_verify_context()
             else:
-                context = ssl.client_context()
+                context = ssl.get_default_context()
             account = imaplib.IMAP4(host=host, port=port)
             account.starttls(context)
         else:
@@ -661,6 +673,7 @@ def get_mails(
     image_name: str,
     gen_mp4: bool = False,
     custom_img: str = None,
+    gen_grid: bool = False,
 ) -> int:
     """Create GIF image based on the attachments in the inbox."""
     image_count = 0
@@ -837,6 +850,8 @@ def get_mails(
 
         if gen_mp4:
             _generate_mp4(image_output_path, image_name)
+        if gen_grid:
+            generate_grid_img(image_output_path, image_name, image_count)
 
     return image_count
 
@@ -875,6 +890,43 @@ def _generate_mp4(path: str, image_file: str) -> None:
     )
 
 
+def generate_grid_img(path: str, image_file: str, count: int) -> None:
+    """Generate png grid from gif.
+
+    use a subprocess so we don't lock up the thread
+    comamnd: ffmpeg -f gif -i infile.gif outfile.mp4
+    """
+    count = max(count, 1)
+    if count % 2 == 0:
+        length = int(count / 2)
+    else:
+        length = int(count / 2) + count % 2
+
+    gif_image = os.path.join(path, image_file)
+    png_file = os.path.join(path, image_file.replace(".gif", "_grid.png"))
+    filecheck = os.path.isfile(png_file)
+    _LOGGER.debug("Generating png image grid: %s", png_file)
+    if filecheck:
+        cleanup_images(os.path.split(png_file))
+        _LOGGER.debug("Removing old png grid: %s", png_file)
+
+    # TODO: find a way to call ffmpeg the right way from HA
+    subprocess.call(
+        [
+            "ffmpeg",
+            "-i",
+            gif_image,
+            "-r",
+            "0.20",
+            "-filter_complex",
+            f"tile=2x{length}:padding=10:color=black",
+            png_file,
+        ],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
+
+
 def resize_images(images: list, width: int, height: int) -> list:
     """Resize images.
 
@@ -933,6 +985,9 @@ def cleanup_images(path: str, image: Optional[str] = None) -> None:
 
     Only supose to delete .gif, .mp4, and .jpg files
     """
+    if isinstance(path, tuple):
+        path = path[0]
+        image = path[1]
     if image is not None:
         try:
             os.remove(path + image)
@@ -941,7 +996,12 @@ def cleanup_images(path: str, image: Optional[str] = None) -> None:
         return
 
     for file in os.listdir(path):
-        if file.endswith(".gif") or file.endswith(".mp4") or file.endswith(".jpg"):
+        if (
+            file.endswith(".gif")
+            or file.endswith(".mp4")
+            or file.endswith(".jpg")
+            or file.endswith(".png")
+        ):
             try:
                 os.remove(path + file)
             except Exception as err:

custom_components/mail_and_packages/sensor.py

@@ -20,6 +20,7 @@
     AMAZON_EXCEPTION,
     AMAZON_EXCEPTION_ORDER,
     AMAZON_ORDER,
+    ATTR_GRID_IMAGE_NAME,
     ATTR_IMAGE,
     ATTR_IMAGE_NAME,
     ATTR_IMAGE_PATH,
@@ -186,17 +187,18 @@ def native_value(self) -> Optional[str]:
         image = ""
         the_path = None
 
-        if ATTR_IMAGE_NAME in self.coordinator.data.keys():
-            image = self.coordinator.data[ATTR_IMAGE_NAME]
+        image = self.coordinator.data.get(ATTR_IMAGE_NAME)
 
-        if ATTR_IMAGE_PATH in self.coordinator.data.keys():
-            path = self.coordinator.data[ATTR_IMAGE_PATH]
-        else:
-            path = self._config.data[CONF_PATH]
+        grid_image = self.coordinator.data.get(ATTR_GRID_IMAGE_NAME)
+
+        path = self.coordinator.data.get(ATTR_IMAGE_PATH, self._config.data[CONF_PATH])
 
         if self.type == "usps_mail_image_system_path":
             _LOGGER.debug("Updating system image path to: %s", path)
             the_path = f"{self.hass.config.path()}/{path}{image}"
+        elif self.type == "usps_mail_grid_image_path":
+            _LOGGER.debug("Updating grid image path to: %s", path)
+            the_path = f"{self.hass.config.path()}/{path}{grid_image}"
         elif self.type == "usps_mail_image_url":
             if (
                 self.hass.config.external_url is None

custom_components/mail_and_packages/strings.json

@@ -35,6 +35,7 @@
                     "gif_duration": "Image Duration (seconds)",
                     "image_security": "Random Image Filename",
                     "imap_timeout": "Time in seconds before connection timeout (seconds, minimum 10)",
+                    "generate_grid": "Create image grid for LLM vision models",
                     "generate_mp4": "Create mp4 from images",
                     "allow_external": "Create image for notification apps",
                     "custom_img": "Use custom 'no image' image?"
@@ -77,6 +78,7 @@
                     "image_path": "Image Path",
                     "gif_duration": "Image Duration (seconds)",
                     "image_security": "Random Image Filename",
+                    "generate_grid": "Create image grid for LLM vision models",
                     "generate_mp4": "Create mp4 from images",
                     "resources": "Sensors List",
                     "imap_timeout": "Time in seconds before connection timeout (seconds, minimum 10)",

custom_components/mail_and_packages/translations/ca.json

@@ -37,7 +37,8 @@
                     "imap_timeout": "Temps en segons abans de l'expiració de la connexió (segons, mínim 10)",
                     "generate_mp4": "Crea mp4 a partir d’imatges",
                     "allow_external": "Crea una imatge per a aplicacions de notificacions",
-                    "custom_img": "Utilitza la imatge personalitzada 'no hi ha imatge'?"
+                    "custom_img": "Utilitza la imatge personalitzada 'no hi ha imatge'?",
+                    "generate_grid": "Crea una quadrícula d'imatges per a models de visió LLM"
                 },
                 "description": "Finalitzeu la configuració personalitzant la següent en funció de la vostra instal·lació de correu electrònic i la instal·lació d’assistència a casa. \n\n Per obtenir més informació sobre les opcions [Integració de paquets i correus] (https:\/\/github.com\/moralmunky\/Home-Assistant-Mail-And-Packages\/wiki\/Configuration-and-Email-Settings#configuration), reviseu les opcions [configuració, plantilles , secció i automatitzacions] (https:\/\/github.com\/moralmunky\/Home-Assistant-Mail-And-Packages\/wiki\/Configuration-and-Email-Settings#configuration) a GitHub.",
                 "title": "Correu i paquets (pas 2 de 2)"
@@ -81,7 +82,8 @@
                     "resources": "Llista de sensors",
                     "imap_timeout": "Temps en segons abans de l'expiració de la connexió (segons, mínim 10)",
                     "allow_external": "Crea una imatge per a aplicacions de notificacions",
-                    "custom_img": "Utilitza la imatge personalitzada 'no hi ha imatge'?"
+                    "custom_img": "Utilitza la imatge personalitzada 'no hi ha imatge'?",
+                    "generate_grid": "Crea una quadrícula d'imatges per a models de visió LLM"
                 },
                 "description": "Acabeu la configuració personalitzant el següent en funció de l'estructura del vostre correu electrònic i de la instal·lació de Home Assistant.\n\nPer obtenir més detalls sobre les opcions d'[integració de correu i paquets](https:\/\/github.com\/moralmunky\/Home-Assistant-Mail-And-Packages\/wiki\/Configuration-and-Email-Settings#configuration), consulteu la [secció de configuració, plantilles i automatitzacions](https:\/\/github.com\/moralmunky\/Home-Assistant-Mail-And-Packages\/wiki\/Configuration-and-Email-Settings#configuration) a GitHub.\n\nSi utilitzeu correus electrònics reenviats d'Amazon, separeu cada adreça amb una coma o introduïu (cap) per esborrar aquesta configuració.",
                 "title": "Correu i paquets (pas 2 de 2)"

custom_components/mail_and_packages/translations/de.json

@@ -37,7 +37,8 @@
                     "imap_timeout": "Zeit in Sekunden bis zum Timeout der Verbindung (Sekunden, mindestens 10)",
                     "generate_mp4": "Erstellen Sie mp4 aus Bildern",
                     "allow_external": "Bild für Benachrichtigungs-Apps erstellen",
-                    "custom_img": "Benutzerdefiniertes Bild 'kein Bild' verwenden?"
+                    "custom_img": "Benutzerdefiniertes Bild 'kein Bild' verwenden?",
+                    "generate_grid": "Erstellen Sie ein Bildraster für LLM-Vision-Modelle"
                 },
                 "description": "Beenden Sie die Konfiguration, indem Sie Folgendes basierend auf Ihrer E-Mail-Struktur und der Installation von Home Assistant anpassen. \n\n Weitere Informationen zu den Optionen [Mail- und Paketintegration] (https:\/\/github.com\/moralmunky\/Home-Assistant-Mail-And-Packages\/wiki\/Configuration-and-Email-Settings#configuration) finden Sie in den [Konfiguration, Vorlagen und Abschnitt Automatisierungen] (https:\/\/github.com\/moralmunky\/Home-Assistant-Mail-And-Packages\/wiki\/Configuration-and-Email-Settings#configuration) auf GitHub.",
                 "title": "Briefe und Pakete (Schritt 2 von 2)"
@@ -81,7 +82,8 @@
                     "resources": "Sensoren Liste",
                     "imap_timeout": "Zeit in Sekunden bis zum Timeout der Verbindung (Sekunden, mindestens 10)",
                     "allow_external": "Bild für Benachrichtigungs-Apps erstellen",
-                    "custom_img": "Benutzerdefiniertes Bild 'kein Bild' verwenden?"
+                    "custom_img": "Benutzerdefiniertes Bild 'kein Bild' verwenden?",
+                    "generate_grid": "Erstellen Sie ein Bildraster für LLM-Vision-Modelle"
                 },
                 "description": "Schließen Sie die Konfiguration ab, indem Sie das Folgende an Ihre E-Mail-Struktur und Home Assistant-Installation anpassen.\n\nWeitere Informationen zu den [Mail und Packages Integration](https:\/\/github.com\/moralmunky\/Home-Assistant-Mail-And-Packages\/wiki\/Configuration-and-Email-Settings#configuration) Optionen finden Sie im [Konfigurations-, Vorlagen- und Automatisierungsabschnitt](https:\/\/github.com\/moralmunky\/Home-Assistant-Mail-And-Packages\/wiki\/Configuration-and-Email-Settings#configuration) auf GitHub.\n\nWenn Sie weitergeleitete E-Mails von Amazon verwenden, trennen Sie bitte jede Adresse durch ein Komma oder geben Sie (keine) ein, um diese Einstellung zu löschen.",
                 "title": "Briefe und Pakete (Schritt 2 von 2)"

custom_components/mail_and_packages/translations/en.json

@@ -37,7 +37,8 @@
                     "imap_timeout": "Time in seconds before connection timeout (seconds, minimum 10)",
                     "generate_mp4": "Create mp4 from images",
                     "allow_external": "Create image for notification apps",
-                    "custom_img": "Use custom 'no image' image?"
+                    "custom_img": "Use custom 'no image' image?",
+                    "generate_grid": "Create image grid for LLM vision models"
                 },
                 "description": "Finish the configuration by customizing the following based on your email structure and Home Assistant installation.\n\nFor details on the [Mail and Packages integration](https:\/\/github.com\/moralmunky\/Home-Assistant-Mail-And-Packages\/wiki\/Configuration-and-Email-Settings#configuration) options review the [configuration, templates, and automations section](https:\/\/github.com\/moralmunky\/Home-Assistant-Mail-And-Packages\/wiki\/Configuration-and-Email-Settings#configuration) on GitHub.\n\nIf using Amazon forwarded emails please separate each address with a comma.",
                 "title": "Mail and Packages (Step 2 of 2)"
@@ -81,7 +82,8 @@
                     "resources": "Sensors List",
                     "imap_timeout": "Time in seconds before connection timeout (seconds, minimum 10)",
                     "allow_external": "Create image for notification apps",
-                    "custom_img": "Use custom 'no image' image?"
+                    "custom_img": "Use custom 'no image' image?",
+                    "generate_grid": "Create image grid for LLM vision models"
                 },
                 "description": "Finish the configuration by customizing the following based on your email structure and Home Assistant installation.\n\nFor details on the [Mail and Packages integration](https:\/\/github.com\/moralmunky\/Home-Assistant-Mail-And-Packages\/wiki\/Configuration-and-Email-Settings#configuration) options review the [configuration, templates, and automations section](https:\/\/github.com\/moralmunky\/Home-Assistant-Mail-And-Packages\/wiki\/Configuration-and-Email-Settings#configuration) on GitHub.\n\nIf using Amazon forwarded emails please separate each address with a comma or enter (none) to clear this setting.",
                 "title": "Mail and Packages (Step 2 of 2)"