gistfile1.txt

blueprint:
  name: AI Event Summary (Beta)
  author: valentinfrlch
  description: >
    AI-powered summaries for Frigate events or camera entities.
    Sends a notification with a preview to your phone that is updated dynamically when the AI summary is available.
  domain: automation
  source_url: https://github.com/valentinfrlch/ha-llmvision/blob/main/blueprints/event_summary_beta.yaml
  input:
    input_mode:
      name: Mode
      description: Select input mode
      selector:
        select:
          options:
            - Frigate
            - Camera
    important:
      name: Important (Beta)
      description: >
        Use AI to classify events as Critical, Normal or Low.
        Notifications are sent only for events classified as Normal or higher.
        Critical events override 'Do Not Disturb' settings.
        Use with caution: AI can make mistakes.
      default: false
      selector:
        boolean:
    remember:
      name: Remember
      description: Stores this event in Event Calendar so you can ask about it. If important is enabled, only events classified as Normal or Critical will be saved.
      default: false
      selector:
        boolean:
    notify_device:
      name: Notify Device
      description: The devices to send the notification to. Multiple devices may be used. Only works with Home Assistant mobile app.
      default: []
      selector:
        device:
          multiple: true
          filter:
            integration: mobile_app
    camera_entities:
      name: Camera Entities
      description: >-
        **(Camera and Frigate mode)**
  
        List of camera entities to monitor.
      default: []
      selector:
        entity:
          multiple: true
          filter:
            domain: camera
    required_zones:
      name: Required Zone(s)
      description: >-
        **(Frigate mode only)**
  
        Only run if the Frigate event occurs within the specified zones (e.g. Driveway, Entry, etc.)
      default: []
      selector:
        text:
          multiline: false
          multiple: true
    object_type:
      name: Included Object Type(s)
      description: >-
        **(Frigate mode only)**
  
        Only run if Frigate labels the object as one of these. (e.g. person, dog, bird, etc.)
      default: []
      selector:
        text:
          multiline: false
          multiple: true
    ignore_stationary:
      name: Ignore Stationary Objects
      description: >-
        **(Frigate mode only)**
  
        Do not alert if objects are detected as stationary by Frigate.
        (Set to false if you want to process all events.)
      default: true
      selector:
        boolean:
    trigger_state:
      name: Trigger State
      description: >-
        **(Camera mode only)**
  
        Trigger the automation when your cameras change to this state.
      default: 'recording'
      selector:
        text:
          multiline: false
    motion_sensors:
      name: Motion Sensor
      description: >-
        **(Camera mode only)**
  
        Set if your cameras don't change state. Use the same order as the camera entities.
      default: []
      selector:
        entity:
          multiple: true
          filter:
            domain: binary_sensor
    preview_mode:
      name: Preview Mode
      description: >-
        **(Camera mode only)**
  
        Choose between a live preview or a snapshot of the event.
      default: Live Preview
      selector:
        select:
          options:
            - Live Preview
            - Snapshot
    cooldown:
      name: Cooldown
      description: Time in minutes to wait before running again. Recommended for busy areas.
      default: 10
      selector:
        number:
          min: 0
          max: 60
    tap_navigate:
      name: Tap Navigate
      description: >-
        Path to navigate to when the notification is opened (e.g. /lovelace/cameras).
      default: /lovelace/0
      selector:
        text:
          multiline: false
    duration:
      name: Duration
      description: >-
        **(Camera mode only)**
  
        How long to record before analyzing (in seconds).
      default: 5
      selector:
        number:
          min: 1
          max: 60
    max_frames:
      name: Max Frames
      description: >-
        **(Camera and Frigate mode)**
  
        How many frames to analyze (choose the frames with the most movement).
      default: 3
      selector:
        number:
          min: 1
          max: 60
    provider:
      name: Provider
      description: Provider to use for analysis. See docs for additional information.
      selector:
        config_entry:
          integration: llmvision
    model:
      name: Model
      description: Model to use for the video_analyzer action. Leave blank to automatically detect the best model.
      default: "gpt-4o-mini"
      selector:
        text:
          multiline: false
    message:
      name: Prompt
      description: Model prompt for the video_analyzer action.
      default: "Summarize what's happening in the camera feed (one sentence max). Don't describe the scene! If there is a person, describe what they're doing and what they look like. If they look like a courier, mention that! If nothing is happening, say so."
      selector:
        text:
          multiline: true
    target_width:
      name: Target Width
      description: Downscale images (uses less tokens and speeds up processing).
      default: 1280
      selector:
        number:
          min: 512
          max: 3840
    max_tokens:
      name: Maximum Tokens
      description: Maximum number of tokens to generate (controls the summary length).
      default: 20
      selector:
        number:
          min: 1
          max: 100
    detail:
      name: Detail
      description: Detail parameter (OpenAI only).
      default: 'low'
      selector:
        select:
          options:
            - high
            - low
    temperature:
      name: Temperature
      description: Randomness. Lower is more accurate; higher is more creative.
      default: 0.1
      selector:
        number:
          min: 0.1
          max: 1.0
          step: 0.1

variables:
  important: !input important
  cooldown: !input cooldown
  input_mode: !input input_mode
  preview_mode: !input preview_mode
  notify_devices: !input notify_device
  device_name_map: >
    {% set ns = namespace(device_names=[]) %}
    {% for device_id in notify_devices %}
      {% set device_name = device_attr(device_id, "name") %}
      {% set sanitized_name = "mobile_app_" + device_name | slugify %}
      {% set ns.device_names = ns.device_names + [sanitized_name] %}
    {% endfor %}
    {{ ns.device_names }}
  camera_entities_list: !input camera_entities
  required_zones_list: !input required_zones
  object_types_list: !input object_type
  ignore_stationary: !input ignore_stationary
  motion_sensors_list: !input motion_sensors
  camera_entity: >
    {% if input_mode == 'Camera' %}
      {% if motion_sensors_list|length > 0 and trigger.entity_id in motion_sensors_list %}
        {% set idx = motion_sensors_list.index(trigger.entity_id) %}
        {% if idx < (camera_entities_list | length) %}
          {{ camera_entities_list[idx] }}
        {% else %}
          {{ trigger.entity_id }}
        {% endif %}
      {% else %}
        {{ trigger.entity_id }}
      {% endif %}
    {% else %}
      {{ trigger.payload_json['after']['camera'] }}
    {% endif %}
  tag: >
    {% if input_mode == 'Frigate' %}
      {{ trigger.payload_json['after']['id'] }}
    {% else %}
      {{ camera_entity }}
    {% endif %}
  group: >
    {% if input_mode == 'Frigate' %}
      {{ trigger.payload_json['after']['camera'] }}
    {% else %}
      {{ camera_entity }}
    {% endif %}
  label: >
    {% if input_mode == 'Frigate' %}
      {{ trigger.payload_json['after']['label']|capitalize }} seen
    {% else %}
      Motion detected
    {% endif %}
  camera: >
    {% if input_mode == 'Frigate' %}
      {{ trigger.payload_json['after']['camera'].replace('_', ' ')|capitalize }}
    {% else %}
      {{ camera_entity.replace("camera.", "").replace("_", " ")|capitalize }}
    {% endif %}
  video: >
    {% if input_mode == 'Frigate' %}
      /api/frigate/notifications/{{ trigger.payload_json['after']['id'] }}/clip.mp4
    {% else %} {% endif %}
  image: >
    {% if input_mode == 'Frigate' %}
      ''
    {% else %}
      {% if preview_mode == 'Live Preview' %}
        {{ '/api/camera_proxy/' + camera_entity }}
      {% else %}
        /local/llmvision/{{ camera_entity.replace("camera.", "") }}_0.jpg
      {% endif %}
    {% endif %}
  importance_prompt: >
    Your job is to classify security events based on cctv footage. Your options: "passive" if an event seems unimportant, "time-sensitive" if important and "critical" for suspicious events.
    Use "critical" only for possible burglaries and similar events. "time-sensitive" could be a courier at the front door or an event of similar importance.
    Reply with these replies exactly.

max_exceeded: silent
mode: single

trigger_variables:
  input_mode: !input input_mode
  listen_mqtt: "{{ input_mode == 'Frigate' }}"
  listen_state: "{{ input_mode == 'Camera' }}"

triggers:
  - trigger: mqtt
    topic: frigate/events
    id: frigate_trigger
    enabled: "{{ listen_mqtt }}"
  - trigger: state
    entity_id: !input camera_entities
    to: !input trigger_state
    id: camera_trigger
    enabled: "{{ listen_state }}"
  - trigger: state
    entity_id: !input motion_sensors
    to: 'on'
    id: motion_sensor_trigger
    enabled: "{{ listen_state }}"

condition:
  - condition: template
    value_template: >
      {% if input_mode == 'Frigate' %}
        {{ trigger.payload_json["type"] == "end"
           and ('camera.' + trigger.payload_json['after']['camera']|lower) in camera_entities_list
           and ((object_types_list|length) == 0 or ((trigger.payload_json['after']['label']|lower) in object_types_list))
           and (not required_zones_list or ((set(trigger.payload_json['after']['current_zones']).intersection(required_zones_list))))
           and not (ignore_stationary and trigger.payload_json['after']['stationary'])
        }}
      {% else %}
        true
      {% endif %}

action:
  - choose:
      - conditions:
          - condition: template
            value_template: "{{ important }}"
        sequence:
          - alias: "Decide Important"
            choose:
              - conditions:
                  - condition: template
                    value_template: "{{ input_mode == 'Frigate' }}"
                sequence:
                  - action: llmvision.image_analyzer
                    data:
                      image_entity: "{{ ['camera.' + trigger.payload_json['after']['camera']|lower] }}"
                      provider: !input provider
                      model: !input model
                      message: "{{ importance_prompt }}"
                      include_filename: true
                      target_width: 1280
                      detail: low
                      max_tokens: 3
                      temperature: 0.1
                    response_variable: importance
              - conditions:
                  - condition: template
                    value_template: "{{ input_mode == 'Camera' }}"
                sequence:
                  - action: llmvision.image_analyzer
                    data:
                      image_entity: "{{ [camera_entity] }}"
                      provider: !input provider
                      model: !input model
                      message: "{{ importance_prompt }}"
                      include_filename: true
                      target_width: 1280
                      detail: low
                      max_tokens: 3
                      temperature: 0.1
                    response_variable: importance

  - choose:
      - conditions:
          - condition: template
            value_template: "{{ importance is defined and importance.response_text|lower == 'passive' }}"
        sequence:
          - stop: "Event is not important"

  - choose:
      - conditions:
          - condition: template
            value_template: "{{ image != '' or video != '' }}"
        sequence:
          - alias: "Send instant notification to notify devices"
            repeat:
              for_each: "{{ device_name_map }}"
              sequence:
                - action: "notify.{{ repeat.item }}"
                  data:
                    title: "{{ label }}"
                    message: "{{ camera }}"
                    data:
                      video: "{{ video if video != '' else None }}"
                      image: "{{ image if image != '' else None }}"
                      entity_id: "{{ camera_entity if input_mode=='Camera' and preview_mode=='Live Preview' }}"
                      url: !input tap_navigate
                      clickAction: !input tap_navigate
                      tag: "{{ tag }}"
                      group: "{{ group }}"
                      alert_once: true
                      push:
                        interruption-level: "{{ importance.response_text|lower if importance is defined else 'active' }}"

  - alias: "Analyze event"
    choose:
      - conditions:
          - condition: template
            value_template: "{{ input_mode == 'Frigate' }}"
        sequence:
          - action: llmvision.video_analyzer
            data:
              event_id: "{{ trigger.payload_json['after']['id'] }}"
              provider: !input provider
              model: !input model
              message: !input message
              remember: !input remember
              generate_title: !input remember
              include_filename: true
              max_frames: !input max_frames
              target_width: !input target_width
              detail: !input detail
              max_tokens: !input max_tokens
              temperature: !input temperature
            response_variable: response
      - conditions:
          - condition: template
            value_template: "{{ input_mode == 'Camera' }}"
        sequence:
          - action: llmvision.stream_analyzer
            data:
              image_entity: "{{ [camera_entity] }}"
              duration: !input duration
              provider: !input provider
              model: !input model
              message: !input message
              remember: !input remember
              generate_title: !input remember
              include_filename: true
              max_frames: !input max_frames
              target_width: !input target_width
              detail: !input detail
              max_tokens: !input max_tokens
              temperature: !input temperature
              expose_images: "{{ true if preview_mode == 'Snapshot' }}"
            response_variable: response

  - delay: '00:00:05'
  
  - choose:
      - conditions:
          - condition: template
            value_template: "{{ image != '' or video != '' }}"
        sequence:
          - alias: "Send updated notification with analysis"
            repeat:
              for_each: "{{ device_name_map }}"
              sequence:
                - action: "notify.{{ repeat.item }}"
                  data:
                    title: "{{ label }}"
                    message: "Analysis: {{ response.response_text | default('No analysis available') }}"
                    data:
                      video: "{{ video if video != '' else None }}"
                      image: "{{ image if image != '' else None }}"
                      entity_id: "{{ camera_entity if input_mode=='Camera' and preview_mode=='Live Preview' }}"
                      url: !input tap_navigate
                      clickAction: !input tap_navigate
                      tag: "{{ tag }}"
                      group: "{{ group }}"
                      push:
                        interruption-level: passive

  - delay: '00:{{ cooldown|int }}:00'