initial commit: Dawarich LXC role (CT 459 on pve02, .159)

Self-hosted location history. 4-container compose: Rails 8 app + Sidekiq + PostGIS 16-3.4 + Redis 7, plus watchtower. Authentik OIDC end-to-end. Image pinned at freikin/dawarich:1.7.11 (OIDC support requires >= 1.7.8). PostGIS DB lives in this LXC, not on the central DB VM (.172) — central image is postgres:16-alpine without postgis, swapping it carries broader blast radius than colocating here. Convention exception captured in homelab-docs project_dawarich memory. Roles: - dawarich: system + Docker + compose + weekly prune timer - alloy: logs+journald → Loki, node metrics → Prometheus Bring-up sequence proven 2026-06-01. README documents the 5-trap build chain (image version, entrypoint scripts, solid_cache SQLite bind mount, APPLICATION_HOSTS+localhost, force_ssl+healthcheck). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-06-01 21:24:09 -04:00
commit e7b8d4df17
20 changed files with 1126 additions and 0 deletions
@@ -0,0 +1,5 @@
+---
+- name: restart alloy
+  command: docker compose -f /opt/alloy/docker-compose.yml up -d --force-recreate
+  args:
+    chdir: /opt/alloy
@@ -0,0 +1,37 @@
+---
+- name: Create Alloy directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: cbalders
+    group: cbalders
+    mode: '0755'
+  loop:
+    - /opt/alloy
+    - /var/lib/alloy
+
+- name: Deploy Alloy compose
+  template:
+    src: docker-compose.yml.j2
+    dest: /opt/alloy/docker-compose.yml
+    owner: cbalders
+    group: cbalders
+    mode: '0644'
+  notify: restart alloy
+
+- name: Deploy Alloy config
+  template:
+    src: config.alloy.j2
+    dest: /opt/alloy/config.alloy
+    owner: cbalders
+    group: cbalders
+    mode: '0644'
+  notify: restart alloy
+
+# Bring Alloy up (idempotent — docker compose up -d is a no-op if running
+# and config hasn't changed). The handler force-recreates on config edit.
+- name: Ensure Alloy is running
+  command: docker compose -f /opt/alloy/docker-compose.yml up -d
+  args:
+    chdir: /opt/alloy
+  changed_when: false
@@ -0,0 +1,161 @@
+// Alloy — Docker container logs + journald → Loki. River syntax.
+// Per-LXC config; only the `host:` label changes via alloy_host_label var.
+
+// ============================================================
+// Docker — discover + scrape container logs
+// ============================================================
+
+discovery.docker "containers" {
+  host             = "unix:///var/run/docker.sock"
+  refresh_interval = "15s"
+}
+
+discovery.relabel "containers" {
+  targets = discovery.docker.containers.targets
+
+  rule {
+    source_labels = ["__meta_docker_container_name"]
+    regex         = "/(.+)"
+    target_label  = "container"
+  }
+  rule {
+    source_labels = ["__meta_docker_container_log_stream"]
+    target_label  = "stream"
+  }
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
+    target_label  = "service"
+  }
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_compose_project"]
+    target_label  = "compose_project"
+  }
+  rule {
+    target_label = "job"
+    replacement  = "docker"
+  }
+  rule {
+    target_label = "host"
+    replacement  = "{{ alloy_host_label }}"
+  }
+}
+
+loki.source.docker "containers" {
+  host       = "unix:///var/run/docker.sock"
+  targets    = discovery.relabel.containers.output
+  forward_to = [loki.process.docker.receiver]
+}
+
+loki.process "docker" {
+  // Replay protection on first boot — Loki rejects > 7d.
+  stage.drop {
+    older_than          = "24h"
+    drop_counter_reason = "older_than_24h"
+  }
+  // pulse-agent broken-pipe noise on hosts running pulse-agent --enable-host.
+  stage.drop {
+    expression          = ".*broken pipe.*"
+    drop_counter_reason = "broken_pipe_noise"
+  }
+  forward_to = [loki.write.default.receiver]
+}
+
+// ============================================================
+// Journald — LXC systemd units
+// ============================================================
+
+loki.source.journal "host" {
+  path       = "/var/log/journal"
+  max_age    = "1m"
+  forward_to = [loki.process.journal.receiver]
+
+  relabel_rules = loki.relabel.journal.rules
+  labels = {
+    job  = "journald",
+    host = "{{ alloy_host_label }}",
+  }
+}
+
+loki.relabel "journal" {
+  forward_to = []
+
+  rule {
+    source_labels = ["__journal__systemd_unit"]
+    target_label  = "unit"
+  }
+  rule {
+    source_labels = ["__journal__hostname"]
+    target_label  = "instance"
+  }
+  rule {
+    source_labels = ["__journal_priority_keyword"]
+    target_label  = "severity"
+  }
+}
+
+loki.process "journal" {
+  forward_to = [loki.write.default.receiver]
+}
+
+// ============================================================
+// Loki — push
+// ============================================================
+
+loki.write "default" {
+  endpoint {
+    url = "{{ alloy_loki_url }}"
+  }
+}
+
+// ============================================================
+// Prometheus — embedded node_exporter (replaces standalone)
+// ============================================================
+
+prometheus.exporter.unix "node" {
+  // Container view of host filesystems — bind-mounted in compose.
+  rootfs_path = "/host/rootfs"
+  procfs_path = "/host/proc"
+  sysfs_path  = "/host/sys"
+  // Default collectors match upstream node_exporter set.
+}
+
+prometheus.scrape "node" {
+  targets         = prometheus.exporter.unix.node.targets
+  forward_to      = [prometheus.relabel.node.receiver]
+  scrape_interval = "15s"
+  job_name        = "{{ alloy_prom_job | default('node_lxc') }}"
+}
+
+// Rewrite the instance + job labels to match the historical
+// node_exporter scrape, so existing dashboards continue to slice by
+// the same instance string. Alloy's exporter component injects
+// job="integrations/unix" on its targets — we have to overwrite it
+// here; `job_name` in prometheus.scrape only names the scrape pool,
+// it doesn't relabel the metrics.
+prometheus.relabel "node" {
+  forward_to = [prometheus.remote_write.observe.receiver]
+  rule {
+    target_label = "instance"
+    replacement  = "{{ alloy_prom_instance | default(ansible_default_ipv4.address + ':9100') }}"
+  }
+  rule {
+    target_label = "job"
+    replacement  = "{{ alloy_prom_job | default('node_lxc') }}"
+  }
+  // Match the static labels Prom previously injected at scrape time
+  // (see homelab-ansible-vm-observability/roles/configs/files/prometheus.yml).
+  rule {
+    target_label = "group"
+    replacement  = "{{ alloy_prom_group | default('lxc') }}"
+  }
+  rule {
+    target_label = "hostname"
+    replacement  = "{{ alloy_prom_hostname | default(alloy_host_label) }}"
+  }
+}
+
+prometheus.remote_write "observe" {
+  endpoint {
+    url = "{{ alloy_prom_remote_write | default('http://observe.lan.balders.ca:9090/api/v1/write') }}"
+  }
+}
@@ -0,0 +1,33 @@
+---
+# Self-contained Alloy stack on /opt/alloy. Runs independently from the
+# host's main service (kestra/infisical/mcp/etc) so a service-side compose
+# down doesn't take logging with it.
+services:
+  alloy:
+    image: grafana/alloy:latest
+    container_name: alloy
+    restart: unless-stopped
+    command:
+      - run
+      - /etc/alloy/config.alloy
+      - --storage.path=/var/lib/alloy/data
+      - --server.http.listen-addr=0.0.0.0:12345
+    ports:
+      - "12345:12345"
+    # Share the host's PID namespace so prometheus.exporter.unix reads
+    # /proc with the host kernel's cgroup view (cgroup-aware MemAvailable).
+    # Without this, /proc/meminfo returns hybrid values: MemTotal from the
+    # host cgroup but Cached/SReclaimable from the container, leading to
+    # a ~25% MemAvailable inflation. See docs/audit/alloy-consolidation-2026-05-21.md.
+    pid: host
+    volumes:
+      - /opt/alloy/config.alloy:/etc/alloy/config.alloy:ro
+      - /var/lib/alloy:/var/lib/alloy
+      - /var/log/journal:/var/log/journal:ro
+      - /run/log/journal:/run/log/journal:ro
+      - /etc/machine-id:/etc/machine-id:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      # Host metric collection for prometheus.exporter.unix (node_exporter replacement)
+      - /:/host/rootfs:ro,rslave
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
@@ -0,0 +1,8 @@
+[Unit]
+Description=Prune dangling docker images (Watchtower leaves them behind)
+After=docker.service
+Requires=docker.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/docker image prune -af
@@ -0,0 +1,9 @@
+[Unit]
+Description=Weekly docker image prune
+
+[Timer]
+OnCalendar=Sun 03:30
+Persistent=true
+
+[Install]
+WantedBy=timers.target
@@ -0,0 +1,4 @@
+---
+- name: reload systemd
+  systemd:
+    daemon_reload: true
@@ -0,0 +1,207 @@
+---
+# ---------- System setup ----------
+- name: Set timezone
+  copy:
+    content: "{{ timezone }}"
+    dest: /etc/timezone
+    mode: '0644'
+  register: tz_file
+
+- name: Apply timezone
+  command: dpkg-reconfigure -f noninteractive tzdata
+  when: tz_file.changed
+
+- name: Set hostname
+  copy:
+    content: "dawarich"
+    dest: /etc/hostname
+    mode: '0644'
+
+- name: Apply hostname
+  command: hostname dawarich
+  changed_when: false
+
+- name: Ensure hostname in /etc/hosts
+  lineinfile:
+    path: /etc/hosts
+    regexp: '^127\.0\.1\.1'
+    line: "127.0.1.1 dawarich"
+
+- name: Update apt cache
+  apt:
+    update_cache: true
+    cache_valid_time: 3600
+  register: apt_cache_result
+  failed_when: false
+  changed_when: apt_cache_result.changed | default(false)
+
+- name: Install base packages
+  apt:
+    name: "{{ packages }}"
+    state: present
+
+- name: Create user
+  user:
+    name: "{{ item.name }}"
+    groups: "{{ item.groups }}"
+    shell: "{{ item.shell }}"
+    append: true
+  loop: "{{ users }}"
+
+- name: Deploy SSH authorized keys
+  authorized_key:
+    user: cbalders
+    key: "{{ item }}"
+    state: present
+  loop: "{{ ssh_authorized_keys }}"
+
+# ---------- Docker ----------
+- name: Check if Docker is installed
+  command: docker --version
+  register: docker_check
+  changed_when: false
+  failed_when: false
+
+- name: Install Docker
+  when: docker_check.rc != 0
+  block:
+    - name: Download Docker install script
+      get_url:
+        url: https://get.docker.com
+        dest: /tmp/get-docker.sh
+        mode: '0755'
+
+    - name: Run Docker install script
+      command: /tmp/get-docker.sh
+
+    - name: Remove install script
+      file:
+        path: /tmp/get-docker.sh
+        state: absent
+
+- name: Ensure Docker service is running
+  systemd:
+    name: docker
+    state: started
+    enabled: true
+
+- name: Add cbalders to docker group
+  user:
+    name: cbalders
+    groups: docker
+    append: true
+
+# ---------- Docker registry mirror ----------
+- name: Ensure /etc/docker exists
+  file:
+    path: /etc/docker
+    state: directory
+    mode: '0755'
+
+- name: Read existing daemon.json
+  slurp:
+    src: /etc/docker/daemon.json
+  register: daemon_json_raw
+  failed_when: false
+
+- name: Configure Docker registry mirror
+  copy:
+    content: "{{ ((daemon_json_raw.content | b64decode | from_json) if daemon_json_raw.content is defined else {}) | combine({'registry-mirrors': ['http://registry.lan.balders.ca:5000']}) | to_nice_json }}"
+    dest: /etc/docker/daemon.json
+    owner: root
+    group: root
+    mode: '0644'
+  register: docker_daemon_config
+
+- name: Restart Docker if mirror config changed
+  systemd:
+    name: docker
+    state: restarted
+  when: docker_daemon_config.changed
+
+# ---------- Dawarich stack ----------
+- name: Create app directories
+  file:
+    path: "{{ item.path }}"
+    state: directory
+    owner: "{{ item.owner | default('cbalders') }}"
+    group: "{{ item.group | default('cbalders') }}"
+    mode: '0755'
+  loop:
+    - { path: /opt/dawarich }
+    # Postgis container runs as uid 70 (alpine 'postgres' user). Pre-chown
+    # so the first boot doesn't fail with "directory is not empty / wrong
+    # permissions" the way Nextcloud did (feedback_nextcloud_db_perms).
+    - { path: /opt/dawarich/postgres, owner: '70', group: '70' }
+    # Redis runs as uid 999 on bind volumes (feedback_redis_uid_999).
+    - { path: /opt/dawarich/redis, owner: '999', group: '999' }
+    # Dawarich Rails app writes uploads/cache here as uid 1000.
+    - { path: /opt/dawarich/public }
+    - { path: /opt/dawarich/watched }
+    - { path: /opt/dawarich/storage }
+    # solid_cache + solid_queue + solid_cable SQLite files (writable by the
+    # rails uid inside the container — image runs as uid 1000).
+    - { path: /opt/dawarich/db_data }
+
+- name: Deploy docker-compose
+  template:
+    src: docker-compose.yml.j2
+    dest: /opt/dawarich/docker-compose.yml
+    owner: cbalders
+    group: cbalders
+    mode: '0640'
+  register: compose_changed
+
+- name: Pull images
+  command: docker compose pull
+  args:
+    chdir: /opt/dawarich
+  changed_when: false
+
+- name: Recreate containers if config changed
+  command: docker compose up -d --force-recreate
+  args:
+    chdir: /opt/dawarich
+  when: compose_changed.changed
+  changed_when: true
+
+- name: Ensure containers are running
+  command: docker compose up -d
+  args:
+    chdir: /opt/dawarich
+  when: not compose_changed.changed
+  changed_when: false
+
+# Dawarich runs migrations on boot (entrypoint includes db:migrate). First
+# boot can take ~60-90s because the initial schema + PostGIS extension load
+# is heavy. Don't fail the play if it's not ready yet — Watchtower-driven
+# recreates will converge.
+- name: Wait for Dawarich to be healthy
+  uri:
+    url: "http://localhost:{{ dawarich_port }}/api/v1/health"
+    method: GET
+  register: dawarich_health
+  until: dawarich_health.status is defined and dawarich_health.status == 200
+  retries: 30
+  delay: 5
+  ignore_errors: true
+
+# ---------- Weekly docker prune (Watchtower side-effect cleanup) ----------
+- name: Deploy docker-prune systemd units
+  copy:
+    src: "{{ item }}"
+    dest: "/etc/systemd/system/{{ item }}"
+    owner: root
+    group: root
+    mode: '0644'
+  loop:
+    - docker-prune.service
+    - docker-prune.timer
+  notify: reload systemd
+
+- name: Enable docker-prune timer
+  systemd:
+    name: docker-prune.timer
+    enabled: true
+    state: started
+    daemon_reload: true
@@ -0,0 +1,187 @@
+# {{ ansible_managed }}
+# Dawarich — self-hosted location history. Four containers + watchtower:
+#   dawarich_db     postgis/postgis:16-3.4-alpine  (PostGIS-enabled Postgres)
+#   dawarich_redis  redis:7-alpine                  (cache + Sidekiq queue)
+#   dawarich_app    freikin/dawarich                (Rails web tier, port 3000)
+#   dawarich_sidekiq freikin/dawarich               (background jobs)
+# DB lives here (not the central DB VM) because the central image is plain
+# postgres:16-alpine without the postgis extension. See memory project_dawarich.
+
+services:
+  dawarich_db:
+    image: "{{ postgis_image }}"
+    container_name: dawarich_db
+    restart: unless-stopped
+    shm_size: 1g
+    environment:
+      POSTGRES_USER: "{{ dawarich_db_user }}"
+      POSTGRES_PASSWORD: "{{ dawarich_db_password }}"
+      POSTGRES_DB: "{{ dawarich_db_name }}"
+    volumes:
+      - /opt/dawarich/postgres:/var/lib/postgresql/data
+    # No published ports — only the app + sidekiq reach it over the compose net.
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U {{ dawarich_db_user }} -d {{ dawarich_db_name }}"]
+      interval: 15s
+      timeout: 5s
+      retries: 10
+      start_period: 30s
+
+  dawarich_redis:
+    image: "{{ redis_image }}"
+    container_name: dawarich_redis
+    restart: unless-stopped
+    volumes:
+      - /opt/dawarich/redis:/data
+    # No published ports — internal-only.
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 15s
+      timeout: 3s
+      retries: 5
+
+  dawarich_app:
+    image: "{{ dawarich_image }}"
+    container_name: dawarich_app
+    restart: unless-stopped
+    depends_on:
+      dawarich_db:
+        condition: service_healthy
+      dawarich_redis:
+        condition: service_healthy
+    ports:
+      - "{{ dawarich_port }}:3000"
+    volumes:
+      - /opt/dawarich/public:/var/app/public
+      - /opt/dawarich/watched:/var/app/tmp/imports/watched
+      - /opt/dawarich/storage:/var/app/storage
+      # Solid_cache + solid_queue + solid_cable SQLite files. Without this
+      # bind, the queue DB is ephemeral and in-flight Sidekiq jobs are lost
+      # on container recreate. Path matches the entrypoint's expectation.
+      - /opt/dawarich/db_data:/dawarich_db_data
+    # The image ships a `web-entrypoint.sh` that runs `db:prepare` (creates
+    # the solid_cache + solid_queue SQLite files in /var/app/storage, runs
+    # Postgres migrations) before exec'ing the command. We MUST use it —
+    # without it, Rails 8 boots fail with "No database file specified" on
+    # solid_cache's first lookup. The image default entrypoint is `bundle exec`
+    # alone, which skips the prep.
+    entrypoint: ["/usr/local/bin/web-entrypoint.sh"]
+    command: ["bin/rails", "server", "-p", "3000", "-b", "::"]
+    environment:
+      RAILS_ENV: production
+      SELF_HOSTED: "true"
+      # localhost + ::1 needed for Docker healthcheck (Rails 8 host
+      # authorization rejects any host not in this list, including loopback).
+      APPLICATION_HOSTS: "{{ dawarich_domain }},localhost,127.0.0.1,::1"
+      APPLICATION_PROTOCOL: "https"
+      DOMAIN: "{{ dawarich_domain }}"
+      TIME_ZONE: "{{ timezone }}"
+      DISTANCE_UNIT: "km"
+      # Database
+      DATABASE_HOST: dawarich_db
+      DATABASE_PORT: "5432"
+      DATABASE_USERNAME: "{{ dawarich_db_user }}"
+      DATABASE_PASSWORD: "{{ dawarich_db_password }}"
+      DATABASE_NAME: "{{ dawarich_db_name }}"
+      RAILS_MAX_THREADS: "5"
+      # Redis
+      REDIS_URL: "redis://dawarich_redis:6379"
+      # Secrets
+      SECRET_KEY_BASE: "{{ dawarich_secret_key_base }}"
+      OTP_ENCRYPTION_PRIMARY_KEY: "{{ dawarich_otp_primary_key }}"
+      OTP_ENCRYPTION_DETERMINISTIC_KEY: "{{ dawarich_otp_deterministic_key }}"
+      OTP_ENCRYPTION_KEY_DERIVATION_SALT: "{{ dawarich_otp_salt }}"
+      # Registration policy — OIDC-only when enabled. Anything else and a
+      # local user form silently appears on /users/sign_up.
+      ALLOW_EMAIL_PASSWORD_REGISTRATION: "{{ dawarich_allow_email_password_registration | string | lower }}"
+{% if dawarich_oidc_enabled %}
+      # ---- Auth: Authentik OIDC ----
+      OIDC_CLIENT_ID: "{{ dawarich_oidc_client_id }}"
+      OIDC_CLIENT_SECRET: "{{ dawarich_oidc_client_secret }}"
+      OIDC_ISSUER: "{{ dawarich_oidc_issuer }}"
+      OIDC_REDIRECT_URI: "{{ dawarich_oidc_redirect_uri }}"
+      OIDC_PROVIDER_NAME: "{{ dawarich_oidc_provider_name }}"
+      OIDC_AUTO_REGISTER: "{{ dawarich_oidc_auto_register | string | lower }}"
+      OIDC_PKCE_ENABLED: "true"
+{% endif %}
+    # TCP socket check, not HTTP. Rails 8 force_ssl (APPLICATION_PROTOCOL=https)
+    # redirects every plain-HTTP path to https — curl/wget can't satisfy a 2xx
+    # without TLS, and Puma is plain-HTTP (Caddy terminates TLS upstream). The
+    # socket check confirms Puma is listening, which is what we actually care
+    # about for orchestration.
+    healthcheck:
+      test: ["CMD-SHELL", "ruby -rsocket -e 'TCPSocket.new(\"localhost\",3000).close' || exit 1"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 120s
+
+  dawarich_sidekiq:
+    image: "{{ dawarich_image }}"
+    container_name: dawarich_sidekiq
+    restart: unless-stopped
+    depends_on:
+      dawarich_db:
+        condition: service_healthy
+      dawarich_redis:
+        condition: service_healthy
+      dawarich_app:
+        condition: service_started
+    volumes:
+      - /opt/dawarich/public:/var/app/public
+      - /opt/dawarich/watched:/var/app/tmp/imports/watched
+      - /opt/dawarich/storage:/var/app/storage
+      # Solid_cache + solid_queue + solid_cable SQLite files. Without this
+      # bind, the queue DB is ephemeral and in-flight Sidekiq jobs are lost
+      # on container recreate. Path matches the entrypoint's expectation.
+      - /opt/dawarich/db_data:/dawarich_db_data
+    # Same entrypoint pattern as the web tier — sidekiq's script ensures the
+    # DB is migrated before workers start consuming jobs.
+    entrypoint: ["/usr/local/bin/sidekiq-entrypoint.sh"]
+    command: ["sidekiq"]
+    environment:
+      RAILS_ENV: production
+      SELF_HOSTED: "true"
+      # localhost + ::1 needed for Docker healthcheck (Rails 8 host
+      # authorization rejects any host not in this list, including loopback).
+      APPLICATION_HOSTS: "{{ dawarich_domain }},localhost,127.0.0.1,::1"
+      APPLICATION_PROTOCOL: "https"
+      DOMAIN: "{{ dawarich_domain }}"
+      TIME_ZONE: "{{ timezone }}"
+      DATABASE_HOST: dawarich_db
+      DATABASE_PORT: "5432"
+      DATABASE_USERNAME: "{{ dawarich_db_user }}"
+      DATABASE_PASSWORD: "{{ dawarich_db_password }}"
+      DATABASE_NAME: "{{ dawarich_db_name }}"
+      REDIS_URL: "redis://dawarich_redis:6379"
+      SECRET_KEY_BASE: "{{ dawarich_secret_key_base }}"
+      OTP_ENCRYPTION_PRIMARY_KEY: "{{ dawarich_otp_primary_key }}"
+      OTP_ENCRYPTION_DETERMINISTIC_KEY: "{{ dawarich_otp_deterministic_key }}"
+      OTP_ENCRYPTION_KEY_DERIVATION_SALT: "{{ dawarich_otp_salt }}"
+      BACKGROUND_PROCESSING_CONCURRENCY: "{{ dawarich_sidekiq_concurrency }}"
+
+  watchtower:
+    container_name: dawarich_watchtower
+    image: containrrr/watchtower:latest
+    restart: unless-stopped
+    dns:
+      - 192.168.1.1
+    ports:
+      - "8088:8080"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    environment:
+      - DOCKER_API_VERSION=1.40
+      - WATCHTOWER_HTTP_API_UPDATE=true
+      - WATCHTOWER_HTTP_API_TOKEN={{ watchtower_api_token }}
+      - TZ={{ timezone }}
+      - WATCHTOWER_CLEANUP=true
+      - WATCHTOWER_INCLUDE_RESTARTING=true
+      - WATCHTOWER_NO_STARTUP_MESSAGE=true
+      - WATCHTOWER_NOTIFICATION_REPORT=true
+      - WATCHTOWER_NOTIFICATIONS=shoutrrr
+      - WATCHTOWER_NOTIFICATION_URL={{ watchtower_gotify_url }}
+      - WATCHTOWER_NOTIFICATIONS_HOSTNAME=DAWARICH
+      - WATCHTOWER_NOTIFICATION_TITLE_TAG=DAWARICH
+      # Render empty when nothing was updated/failed — see feedback_watchtower_pushover.
+      - 'WATCHTOWER_NOTIFICATION_TEMPLATE={% raw %}{{- if .Report -}}{{- with .Report -}}{{- if or .Updated .Failed -}}{{ len .Updated }} updated, {{ len .Failed }} failed{{- range .Updated}} | updated {{.Name}}{{- end -}}{{- range .Failed}} | FAILED {{.Name}}{{- end -}}{{- end -}}{{- end -}}{{- end -}}{% endraw %}'