blueprint

gistfile1.yaml

blueprint:
  name: AI Event Summary with NTFY (v1.4.0 Beta 1)
  author: valentinfrlch
  description: >
    AI-powered summaries for security camera events.
    Sends a notification with a preview to your phone that is updated dynamically when the AI summary is available.
  domain: automation
  source_url: https://github.com/valentinfrlch/ha-llmvision/blob/main/blueprints/event_summary_beta.yaml
  input:
    important:
      name: Important (Experimental)
      description: >
        Use AI to classify events as Critical, Normal or Low.
        Notifications are sent only for events classified as Normal or higher.
        Critical events override 'Do Not Disturb' settings.
        Use with caution: AI can make mistakes.
      default: false
      selector:
        boolean:
    remember:
      name: Remember
      description: Stores this event in the Timeline so you can ask about it. If important is enabled, only events classified as Normal or Critical will be saved.
      default: false
      selector:
        boolean:
    use_memory:
      name: Use Memory
      description: 'Use information stored in memory to provide additional context. Memory must be set up.'
      default: false
      selector:
        boolean:
    message:
      name: Prompt
      description: Model prompt for the video_analyzer action
      default: "Summarize what's happening in the camera feed (one sentence max). Don't describe the scene! If there is a person, describe what they're doing and what they look like. If they look like a courier mention that! If nothing is happening, say so."
      selector:
        text:
          multiline: true
    run_conditions:
      name: Run Conditions
      description: All conditions must be true in order for the blueprint to run.
      default: false
      selector:
        condition:
    notify_device:
      name: Notify Device
      description: The devices to send the notification to. Multiple devices may be used. Only works with Home Assistant mobile app.
      default: []
      selector:
        device:
          multiple: true
          filter:
            integration: mobile_app
    notification_delivery:
      name: Notification Delivery
      description: "Controls how notifications are delivered. \n \n **Dynamic** immediately notifies with a live preview and updates the notification silently with a summary once it is available. \n **Consolidated** Delays the notification until the event summary is generated. Use this if you're receiving multiple notifications for the same event."
      default: 'Dynamic'
      selector:
        select:
          options:
            - Dynamic
            - Consolidated
    camera_entities:
      name: Camera Entities
      description: List of camera entities to monitor
      default: []
      selector:
        entity:
          multiple: true
          filter:
            domain: camera
    trigger_state:
      name: Trigger State
      description: Trigger the automation when your cameras change to this state
      default: 'recording'
      selector:
        text:
          multiline: false
    motion_sensors:
      name: Motion Sensor
      description: Set if your cameras don't change state. Use the same order used for camera entities.
      default: []
      selector:
        entity:
          multiple: true
          filter:
            domain: binary_sensor
    preview_mode:
      name: Preview Mode
      description: Choose between a live preview or a snapshot of the event
      default: Live Preview
      selector:
        select:
          options:
            - Live Preview
            - Snapshot
    cooldown:
      name: Cooldown
      description: Time in minutes to wait before running again. Recommended for busy areas.
      default: 10
      selector:
        number:
          min: 0
          max: 60
    tap_navigate:
      name: Tap Navigate
      description: >-
        Path to navigate to when notification is opened (e.g. /lovelace/cameras).
      default: /lovelace/0
      selector:
        text:
          multiline: false
    duration:
      name: Duration
      description: How long to record before analyzing (in seconds)
      default: 5
      selector:
        number:
          min: 1
          max: 60
    max_frames:
      name: Max Frames
      description: How many frames to analyze. Picks frames with the most movement.
      default: 3
      selector:
        number:
          min: 1
          max: 60
    provider:
      name: Provider
      description: Provider to use for analysis. See docs for additional information.
      selector:
        config_entry:
          integration: llmvision
    model:
      name: Model
      description: Model to use for the video_analyzer action. Leave blank to automatically detect the best model.
      default: "gpt-4o-mini"
      selector:
        text:
          multiline: false
    target_width:
      name: Target Width
      description: Downscale images (uses less tokens and speeds up processing)
      default: 1280
      selector:
        number:
          min: 512
          max: 3840
    max_tokens:
      name: Maximum Tokens
      description: Maximum number of tokens to generate. Use this to control the length of the summaries.
      default: 20
      selector:
        number:
          min: 1
          max: 100
    temperature:
      name: Temperature
      description: Randomness. Lower is more accurate, higher is more creative.
      default: 0.1
      selector:
        number:
          min: 0.1
          max: 1.0
          step: 0.1

variables:
  important: !input important
  cooldown: !input cooldown
  preview_mode: !input preview_mode
  notify_devices: !input notify_device
  notification_delivery: !input notification_delivery
  device_name_map: >
    {% set ns = namespace(device_names=[]) %}
    {% for device_id in notify_devices %}
      {% set device_name = device_attr(device_id, "name") %}
      {% set sanitized_name = "mobile_app_" + device_name | slugify  %}
      {% set ns.device_names = ns.device_names + [sanitized_name] %}
    {% endfor %}
    {{ ns.device_names }}
  camera_entities_list: !input camera_entities
  motion_sensors_list: !input motion_sensors
  camera_entity: >
    {% if motion_sensors_list and not trigger.entity_id.startswith("camera") %}
      {% set index = motion_sensors_list.index(trigger.entity_id) %}
        {{ camera_entities_list[index] }}
    {% else %}
      {{ trigger.entity_id }}
    {% endif %}
  tag: >
    {{ camera_entity + int(as_timestamp(now()))|string }}
  group: >
    {{ camera_entity }}
  label: Motion detected
  camera: >
      {{ camera_entity.replace("camera.", "").replace("_", " ")|capitalize }}
  importance_prompt: >
    Your job is to classify security events based on home security footage. Your options: "passive" if an event seems unimportant, "time-sensitive" if important and "critical" for suspicious events.
    Use "critical" only for possible burglaries and similar events. "time-sensitive" could be a person at the front door or an event of similar importance. You must reply with one of these options exactly.

max_exceeded: silent
mode: single

triggers:
  - trigger: state
    entity_id: !input camera_entities
    to: !input trigger_state
    id: 'camera_trigger'
  - trigger: state
    entity_id: !input motion_sensors
    to: 'on'
    id: 'motion_sensor_trigger'

condition:
  - condition: and
    conditions: !input run_conditions
  

action:
  - choose:
      - conditions:
          - condition: template
            value_template: "{{ important }}"
        sequence:
          - action: llmvision.image_analyzer
            data:
              image_entity: "{{[camera_entity]}}"
              provider: !input provider
              model: !input model
              message: "{{importance_prompt}}"
              include_filename: true
              target_width: 1280
              max_tokens: 3
              temperature: 0.1
            response_variable: importance

  # Cancel automation if event not deemed important
  - choose:
      - conditions:
          - condition: template
            value_template: "{{ important and importance.response_text|lower == 'passive' }}"
        sequence:
          - stop: "Event is not important"

  - choose:
      - conditions:
          - condition: template
            value_template: "{{ notification_delivery == 'Dynamic' }}"
        sequence:
        - alias: "Send instant notification to notify devices"
          repeat:
            for_each: "{{device_name_map}}"
            sequence:
              - action: "notify.ntfy"
                data:
                  title: "{{ label }}"
                  message: "{{camera}}"
                  click: !input tap_navigate #Android
                  tag: "{{tag}}"

  - alias: "Analyze event"
    action: llmvision.stream_analyzer
    data:
      image_entity: "{{[camera_entity]}}"
      duration: !input duration
      provider: !input provider
      model: !input model
      message: !input message
      use_memory: !input use_memory
      remember: !input remember
      expose_images: "{{preview_mode == 'Snapshot' or remember}}"
      expose_images_persist: !input remember
      generate_title: !input remember
      include_filename: true
      max_frames: !input max_frames
      target_width: !input target_width
      max_tokens: !input max_tokens
      temperature: !input temperature  
    response_variable: response



  - choose:
      - conditions:
          - condition: template
            value_template: "{{ preview_mode=='Snapshot' }}"
        sequence:
        - alias: "(Snapshot) Update notification on notify devices"
          repeat:
            for_each: "{{device_name_map}}"
            sequence:
              - action: "notify.ntfy"
                data:
                  title: "{{ label }}"
                  message: "{{response.response_text}}"
                  data:
                    attach: "{{response.key_frame.replace('/config/www/','/local/') }}"
                    click: !input tap_navigate #Android
                    tag: "{{tag}}"
      - conditions:
          - condition: template
            value_template: "{{ preview_mode=='Live Preview' }}"
        sequence:
        - alias: "(Live Preview) Update notification on notify devices"
          repeat:
            for_each: "{{device_name_map}}"
            sequence:
              - action: "notify.{{ repeat.item }}"
                data:
                  title: "{{ label }}"
                  message: "{{response.response_text}}"
                  data:
                    entity_id: "{{camera_entity}}"
                    url: !input tap_navigate #iOS
                    clickAction: !input tap_navigate #Android
                    tag: "{{tag}}"
                    group: "{{group}}"
                    push:
                      interruption-level: "{{'passive' if notification_delivery=='Dynamic' else 'active'}}"
            

  - delay: '00:{{cooldown|int}}:00'