initial commit: Dawarich LXC role (CT 459 on pve02, .159)
Self-hosted location history. 4-container compose: Rails 8 app + Sidekiq + PostGIS 16-3.4 + Redis 7, plus watchtower. Authentik OIDC end-to-end. Image pinned at freikin/dawarich:1.7.11 (OIDC support requires >= 1.7.8). PostGIS DB lives in this LXC, not on the central DB VM (.172) — central image is postgres:16-alpine without postgis, swapping it carries broader blast radius than colocating here. Convention exception captured in homelab-docs project_dawarich memory. Roles: - dawarich: system + Docker + compose + weekly prune timer - alloy: logs+journald → Loki, node metrics → Prometheus Bring-up sequence proven 2026-06-01. README documents the 5-trap build chain (image version, entrypoint scripts, solid_cache SQLite bind mount, APPLICATION_HOSTS+localhost, force_ssl+healthcheck). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Your Name
@@ -0,0 +1,161 @@
|
||||
// Alloy — Docker container logs + journald → Loki. River syntax.
|
||||
// Per-LXC config; only the `host:` label changes via alloy_host_label var.
|
||||
|
||||
// ============================================================
|
||||
// Docker — discover + scrape container logs
|
||||
// ============================================================
|
||||
|
||||
discovery.docker "containers" {
|
||||
host = "unix:///var/run/docker.sock"
|
||||
refresh_interval = "15s"
|
||||
}
|
||||
|
||||
discovery.relabel "containers" {
|
||||
targets = discovery.docker.containers.targets
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_docker_container_name"]
|
||||
regex = "/(.+)"
|
||||
target_label = "container"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_docker_container_log_stream"]
|
||||
target_label = "stream"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
|
||||
target_label = "service"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_docker_container_label_com_docker_compose_project"]
|
||||
target_label = "compose_project"
|
||||
}
|
||||
rule {
|
||||
target_label = "job"
|
||||
replacement = "docker"
|
||||
}
|
||||
rule {
|
||||
target_label = "host"
|
||||
replacement = "{{ alloy_host_label }}"
|
||||
}
|
||||
}
|
||||
|
||||
loki.source.docker "containers" {
|
||||
host = "unix:///var/run/docker.sock"
|
||||
targets = discovery.relabel.containers.output
|
||||
forward_to = [loki.process.docker.receiver]
|
||||
}
|
||||
|
||||
loki.process "docker" {
|
||||
// Replay protection on first boot — Loki rejects > 7d.
|
||||
stage.drop {
|
||||
older_than = "24h"
|
||||
drop_counter_reason = "older_than_24h"
|
||||
}
|
||||
// pulse-agent broken-pipe noise on hosts running pulse-agent --enable-host.
|
||||
stage.drop {
|
||||
expression = ".*broken pipe.*"
|
||||
drop_counter_reason = "broken_pipe_noise"
|
||||
}
|
||||
forward_to = [loki.write.default.receiver]
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Journald — LXC systemd units
|
||||
// ============================================================
|
||||
|
||||
loki.source.journal "host" {
|
||||
path = "/var/log/journal"
|
||||
max_age = "1m"
|
||||
forward_to = [loki.process.journal.receiver]
|
||||
|
||||
relabel_rules = loki.relabel.journal.rules
|
||||
labels = {
|
||||
job = "journald",
|
||||
host = "{{ alloy_host_label }}",
|
||||
}
|
||||
}
|
||||
|
||||
loki.relabel "journal" {
|
||||
forward_to = []
|
||||
|
||||
rule {
|
||||
source_labels = ["__journal__systemd_unit"]
|
||||
target_label = "unit"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__journal__hostname"]
|
||||
target_label = "instance"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__journal_priority_keyword"]
|
||||
target_label = "severity"
|
||||
}
|
||||
}
|
||||
|
||||
loki.process "journal" {
|
||||
forward_to = [loki.write.default.receiver]
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Loki — push
|
||||
// ============================================================
|
||||
|
||||
loki.write "default" {
|
||||
endpoint {
|
||||
url = "{{ alloy_loki_url }}"
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Prometheus — embedded node_exporter (replaces standalone)
|
||||
// ============================================================
|
||||
|
||||
prometheus.exporter.unix "node" {
|
||||
// Container view of host filesystems — bind-mounted in compose.
|
||||
rootfs_path = "/host/rootfs"
|
||||
procfs_path = "/host/proc"
|
||||
sysfs_path = "/host/sys"
|
||||
// Default collectors match upstream node_exporter set.
|
||||
}
|
||||
|
||||
prometheus.scrape "node" {
|
||||
targets = prometheus.exporter.unix.node.targets
|
||||
forward_to = [prometheus.relabel.node.receiver]
|
||||
scrape_interval = "15s"
|
||||
job_name = "{{ alloy_prom_job | default('node_lxc') }}"
|
||||
}
|
||||
|
||||
// Rewrite the instance + job labels to match the historical
|
||||
// node_exporter scrape, so existing dashboards continue to slice by
|
||||
// the same instance string. Alloy's exporter component injects
|
||||
// job="integrations/unix" on its targets — we have to overwrite it
|
||||
// here; `job_name` in prometheus.scrape only names the scrape pool,
|
||||
// it doesn't relabel the metrics.
|
||||
prometheus.relabel "node" {
|
||||
forward_to = [prometheus.remote_write.observe.receiver]
|
||||
rule {
|
||||
target_label = "instance"
|
||||
replacement = "{{ alloy_prom_instance | default(ansible_default_ipv4.address + ':9100') }}"
|
||||
}
|
||||
rule {
|
||||
target_label = "job"
|
||||
replacement = "{{ alloy_prom_job | default('node_lxc') }}"
|
||||
}
|
||||
// Match the static labels Prom previously injected at scrape time
|
||||
// (see homelab-ansible-vm-observability/roles/configs/files/prometheus.yml).
|
||||
rule {
|
||||
target_label = "group"
|
||||
replacement = "{{ alloy_prom_group | default('lxc') }}"
|
||||
}
|
||||
rule {
|
||||
target_label = "hostname"
|
||||
replacement = "{{ alloy_prom_hostname | default(alloy_host_label) }}"
|
||||
}
|
||||
}
|
||||
|
||||
prometheus.remote_write "observe" {
|
||||
endpoint {
|
||||
url = "{{ alloy_prom_remote_write | default('http://observe.lan.balders.ca:9090/api/v1/write') }}"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
---
|
||||
# Self-contained Alloy stack on /opt/alloy. Runs independently from the
|
||||
# host's main service (kestra/infisical/mcp/etc) so a service-side compose
|
||||
# down doesn't take logging with it.
|
||||
services:
|
||||
alloy:
|
||||
image: grafana/alloy:latest
|
||||
container_name: alloy
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- run
|
||||
- /etc/alloy/config.alloy
|
||||
- --storage.path=/var/lib/alloy/data
|
||||
- --server.http.listen-addr=0.0.0.0:12345
|
||||
ports:
|
||||
- "12345:12345"
|
||||
# Share the host's PID namespace so prometheus.exporter.unix reads
|
||||
# /proc with the host kernel's cgroup view (cgroup-aware MemAvailable).
|
||||
# Without this, /proc/meminfo returns hybrid values: MemTotal from the
|
||||
# host cgroup but Cached/SReclaimable from the container, leading to
|
||||
# a ~25% MemAvailable inflation. See docs/audit/alloy-consolidation-2026-05-21.md.
|
||||
pid: host
|
||||
volumes:
|
||||
- /opt/alloy/config.alloy:/etc/alloy/config.alloy:ro
|
||||
- /var/lib/alloy:/var/lib/alloy
|
||||
- /var/log/journal:/var/log/journal:ro
|
||||
- /run/log/journal:/run/log/journal:ro
|
||||
- /etc/machine-id:/etc/machine-id:ro
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
# Host metric collection for prometheus.exporter.unix (node_exporter replacement)
|
||||
- /:/host/rootfs:ro,rslave
|
||||
- /proc:/host/proc:ro
|
||||
- /sys:/host/sys:ro
|
||||
Reference in New Issue
Block a user