alloy: bare-metal systemd shipper for journald → Loki

Meridian + LiteLLM both run as systemd services on this LXC (no docker)
so the Docker-container Alloy pattern from other repos doesn't apply.
Apt-install grafana/alloy via apt.grafana.com, journald-only scrape,
ships to Loki on observe.lan.balders.ca.

Side benefit: Meridian.service + LiteLLM.service logs (including the
gpt-* alias shadowing requests from paperless-ai) now searchable in
Loki, not just journalctl on the LXC.

roles/alloy/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+- name: restart alloy
+  systemd:
+    name: alloy
+    state: restarted
+    daemon_reload: yes

roles/alloy/tasks/main.yml

@@ -0,0 +1,62 @@
+---
+# Bare-metal Alloy on systemd. No docker on this LXC (Meridian + LiteLLM
+# both run as systemd services). Apt-installed for clean self-updates.
+
+- name: Install Alloy prereqs
+  apt:
+    name:
+      - gpg
+      - apt-transport-https
+    state: present
+    update_cache: false
+
+- name: Add Grafana apt signing key
+  get_url:
+    url: https://apt.grafana.com/gpg.key
+    dest: /etc/apt/keyrings/grafana.gpg.asc
+    mode: '0644'
+
+- name: Add Grafana apt repo
+  copy:
+    content: |
+      deb [signed-by=/etc/apt/keyrings/grafana.gpg.asc] https://apt.grafana.com stable main
+    dest: /etc/apt/sources.list.d/grafana.list
+    mode: '0644'
+  register: alloy_apt_repo
+
+- name: Apt update (if repo just added)
+  apt:
+    update_cache: yes
+  when: alloy_apt_repo.changed
+
+- name: Install Alloy
+  apt:
+    name: alloy
+    state: present
+    update_cache: yes
+    cache_valid_time: 3600
+
+# The package's default alloy user needs to read /var/log/journal. systemd-journal
+# group membership lets it read persistent journal without root.
+- name: Add alloy user to systemd-journal group
+  user:
+    name: alloy
+    groups: systemd-journal
+    append: yes
+  notify: restart alloy
+
+- name: Deploy Alloy config
+  template:
+    src: config.alloy.j2
+    dest: /etc/alloy/config.alloy
+    owner: alloy
+    group: alloy
+    mode: '0644'
+  notify: restart alloy
+
+- name: Enable + start Alloy
+  systemd:
+    name: alloy
+    enabled: yes
+    state: started
+    daemon_reload: yes

roles/alloy/templates/config.alloy.j2

@@ -0,0 +1,41 @@
+// Alloy — journald-only on this host (no Docker — Meridian + LiteLLM run
+// as systemd services). Ships to Loki on observe.lan.balders.ca.
+
+loki.source.journal "host" {
+  path       = "/var/log/journal"
+  max_age    = "12h"
+  forward_to = [loki.process.journal.receiver]
+
+  relabel_rules = loki.relabel.journal.rules
+  labels = {
+    job  = "journald",
+    host = "{{ alloy_host_label }}",
+  }
+}
+
+loki.relabel "journal" {
+  forward_to = []
+
+  rule {
+    source_labels = ["__journal__systemd_unit"]
+    target_label  = "unit"
+  }
+  rule {
+    source_labels = ["__journal__hostname"]
+    target_label  = "instance"
+  }
+  rule {
+    source_labels = ["__journal_priority_keyword"]
+    target_label  = "severity"
+  }
+}
+
+loki.process "journal" {
+  forward_to = [loki.write.default.receiver]
+}
+
+loki.write "default" {
+  endpoint {
+    url = "{{ alloy_loki_url }}"
+  }
+}

site.yml

@@ -45,3 +45,4 @@
     - { role: meridian,      tags: ['meridian'] }
     - { role: litellm,       tags: ['litellm'] }
     - { role: node_exporter, tags: ['node_exporter'] }
+    - { role: alloy,         tags: ['alloy'] }

vars/main.yml

@@ -27,6 +27,11 @@ ssh_authorized_keys:
   - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINALaic1jpoP6t1urbZqJLI1eU5NeTVD9k8AAMAvOvvk OfficeMini"
   - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGzTHdCiQjhIHsGB8oMpyKtr9TZXrXeIRKwcwe698zMW Generated By Termius"
 
+# Alloy ships journald to Loki on observe.lan.balders.ca. No docker on
+# this LXC — bare-metal systemd Alloy via Grafana apt repo.
+alloy_host_label: meridian
+alloy_loki_url: http://observe.lan.balders.ca:3100/loki/api/v1/push
+
 # Meridian
 meridian_user: meridian
 meridian_home: /opt/meridian