eletrotupi / tcc/ commit / c26ff2e

deploy: collect container logs into a self-hosted VictoriaLogs

The API logged to stdout and nothing shipped it off the box, so history
died with every container recreate and the unbounded json-file driver
was a slow disk-fill risk. Vector now reads container logs through the
Docker API and forwards them to a VictoriaLogs instance on the same
host, with retention and rotation caps driven from the compose env

Vector parses each pino JSON line and lifts its fields before shipping,
and the app role installs its config alongside the compose file
Pedro Lucas Porcellis porcellis@eletrotupi.com · 14 days ago c26ff2e888265576d4b65d4b74de89f561690311 Browse files
Parents ee68bbc
Changed files
7
Additions
+146
Deletions
-2
ansible/roles/app/defaults/main.yml modified+1 -1
ansible/roles/app/defaults/main.yml
1 1 
---
2 2 
# roles/app/defaults/main.yml
3 3 
# Host-side application configuration
4 4
5 5 
app_env: production
6 6 
# ghcr.io/<owner>/<repo> must match the image the deploy workflow pushes
7 - 
app_github_repo: "your-org/nexo"
7 + 
app_github_repo: "pedrolucasp/nexo"
ansible/roles/app/files/docker-compose.production.yml modified+37 -1
ansible/roles/app/files/docker-compose.production.yml
1 1 
# /srv/app/production/docker-compose.yml
2 2 
# Secrets: cp .env.production.example .env.production and fill in values
3 3 
# Start:   docker compose up -d
4 - 
# Logs:    docker compose logs -f api
4 + 
# Logs UI: https://<host>/logs/ (nginx proxies it; loopback only otherwise)
5 5
6 6 
services:
7 7 
  db:
8 8 
    image: postgres:17-alpine
9 9 
    shm_size: 1g
19 19 
      POSTGRES_USER: ${POSTGRES_USER}
20 20 
      POSTGRES_DB: ${POSTGRES_DB}
21 21 
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
22 22 
    volumes:
23 23 
      - db_data:/var/lib/postgresql/data
24 + 
    logging: &default-logging
25 + 
      driver: json-file
26 + 
      options:
27 + 
        max-size: "${LOG_MAX_SIZE:-10m}"
28 + 
        max-file: "${LOG_MAX_FILE:-3}"
24 29
25 30 
  valkey:
26 31 
    image: valkey/valkey:9.0-alpine
27 32 
    restart: unless-stopped
28 33 
    # No ports exposed
29 34 
    volumes:
30 35 
      - valkey_data:/data
31 36 
      - ./valkey.conf:/etc/valkey/valkey.conf
32 37 
    command: valkey-server /etc/valkey/valkey.conf
38 + 
    logging: *default-logging
33 39
34 40 
  api:
35 41 
    image: ghcr.io/${GITHUB_REPOSITORY}:${IMAGE_TAG:-latest}
36 42 
    restart: unless-stopped
37 43 
    ports:
42 48 
    depends_on:
43 49 
      valkey:
44 50 
        condition: service_started
45 51 
      db:
46 52 
        condition: service_healthy
53 + 
    logging: *default-logging
54 +
55 + 
  # Logs store — UI at http://127.0.0.1:9428/logs/select/vmui/ (loopback only)
56 + 
  victorialogs:
57 + 
    image: victoriametrics/victoria-logs:${VICTORIALOGS_TAG:-v1.53.0}
58 + 
    command:
59 + 
      - "-storageDataPath=/vlogs"
60 + 
      - "-http.pathPrefix=/logs"
61 + 
      - "-retentionPeriod=${VICTORIALOGS_RETENTION:-14d}"
62 + 
      - "-retention.maxDiskSpaceUsageBytes=${VICTORIALOGS_MAX_DISK:-2GiB}"
63 + 
    volumes:
64 + 
      - vlogs_data:/vlogs
65 + 
    ports:
66 + 
      - "127.0.0.1:${VICTORIALOGS_PORT:-9428}:9428"
67 + 
    restart: unless-stopped
68 + 
    logging: *default-logging
69 +
70 + 
  # Reads container logs through the Docker API and ships them to VictoriaLogs
71 + 
  # `docker compose logs` keeps working: this does not replace the log driver
72 + 
  vector:
73 + 
    image: timberio/vector:${VECTOR_TAG:-0.59.X-distroless-libc}
74 + 
    user: root
75 + 
    volumes:
76 + 
      - /var/run/docker.sock:/var/run/docker.sock:ro
77 + 
      - ./vector.yaml:/etc/vector/vector.yaml:ro
78 + 
    depends_on:
79 + 
      - victorialogs
80 + 
    restart: unless-stopped
81 + 
    logging: *default-logging
47 82
48 83 
volumes:
49 84 
  db_data:
50 85 
  valkey_data:
86 + 
  vlogs_data:
ansible/roles/app/files/vector.yaml added+37
/dev/null → ansible/roles/app/files/vector.yaml
1 + 
# Collects Docker container logs and forwards them to VictoriaLogs
2 + 
# Bind-mounted by docker-compose.yml (dev) and installed into /srv/app/<env>/
3 + 
# by the Ansible `app` role (production)
4 + 
sources:
5 + 
  docker:
6 + 
    type: docker_logs
7 +
8 + 
transforms:
9 + 
  parse_json_message:
10 + 
    type: remap
11 + 
    inputs: [docker]
12 + 
    # pino writes one JSON object per line — when a line parses we lift its
13 + 
    # fields to the top level so level/msg/userId become queryable in
14 + 
    # VictoriaLogs, otherwise the raw line stays as the message
15 + 
    source: |
16 + 
      parsed, err = parse_json(.message)
17 + 
      if err == null && is_object(parsed) {
18 + 
        . = merge(., parsed) ?? .
19 + 
      }
20 +
21 + 
sinks:
22 + 
  victorialogs:
23 + 
    type: elasticsearch
24 + 
    inputs: [parse_json_message]
25 + 
    endpoints: ["http://victorialogs:9428/logs/insert/elasticsearch/"]
26 + 
    mode: bulk
27 + 
    api_version: v8
28 + 
    compression: gzip
29 + 
    healthcheck:
30 + 
      enabled: false
31 + 
    request:
32 + 
      headers:
33 + 
        VL-Stream-Fields: label.com.docker.compose.service
34 + 
        VL-Time-Field: timestamp
35 + 
        VL-Msg-Field: msg,message,_msg
36 + 
        AccountID: "0"
37 + 
        ProjectID: "0"
ansible/roles/app/tasks/main.yml modified+9
ansible/roles/app/tasks/main.yml
1 1 
---
2 2 
# roles/app/tasks/main.yml
3 3 
# Places the production app config in /srv/app/<env>/
4 4 
# Secrets are NOT created here: the .example files are dropped in and you fill
5 5 
# them in by hand (that part is deliberately manual, so get ready to SSH and copy it)
39 39 
    owner: "{{ deploy_user }}"
40 40 
    group: "{{ deploy_user }}"
41 41 
    mode: "0640"
42 42 
  tags: [app]
43 43
44 + 
- name: Install the Vector configuration
45 + 
  ansible.builtin.copy:
46 + 
    src: vector.yaml
47 + 
    dest: "/srv/app/{{ app_env }}/vector.yaml"
48 + 
    owner: "{{ deploy_user }}"
49 + 
    group: "{{ deploy_user }}"
50 + 
    mode: "0640"
51 + 
  tags: [app]
52 +
44 53 
- name: Install the environment example files
45 54 
  ansible.builtin.template:
46 55 
    src: "{{ item }}.j2"
47 56 
    dest: "/srv/app/{{ app_env }}/{{ item }}.example"
48 57 
    owner: "{{ deploy_user }}"
68 77 
      {{ item.item }} is missing: copy it from {{ item.item }}.example and fill
69 78 
      it in before the first `docker compose up -d`
70 79 
  loop: "{{ app_env_stat.results }}"
71 80 
  when: not item.stat.exists
72 81 
  tags: [app]
ansible/roles/app/templates/env.example.j2 modified+11
ansible/roles/app/templates/env.example.j2
1 1 
# Copy to `.env` in the same directory. `docker compose` loads it automatically
2 2 
# to interpolate the ${...} variables used in docker-compose.yml
3 3 
# NEVER commit the filled-in file!
4 4
5 5 
POSTGRES_USER=orbit
9 9 
# ghcr.io/<owner>/<repo> must match the image the deploy workflow pushes
10 10 
GITHUB_REPOSITORY={{ app_github_repo }}
11 11
12 12 
# CI rewrites IMAGE_TAG on every deploy
13 13 
IMAGE_TAG=latest
14 +
15 + 
# Logging stack (VictoriaLogs + Vector)
16 + 
VICTORIALOGS_TAG=v1.53.0
17 + 
VICTORIALOGS_PORT=9428
18 + 
VICTORIALOGS_RETENTION=14d
19 + 
VICTORIALOGS_MAX_DISK=2GiB
20 + 
VECTOR_TAG=0.59.X-distroless-libc
21 +
22 + 
# Container log rotation (json-file driver)
23 + 
LOG_MAX_SIZE=10m
24 + 
LOG_MAX_FILE=3
ansible/roles/app/templates/env.production.example.j2 modified+15
ansible/roles/app/templates/env.production.example.j2
1 1 
# Copy to `.env.production`. Loaded by the api container (env_file)
2 2 
# NEVER commit the filled-in file
3 3
4 4 
NODE_ENV=production
5 5 
PORT=3000
26 26 
S3_REGION=""
27 27 
S3_URL=""
28 28 
S3_ACCESS_KEY=""
29 29 
S3_ACCESS_SECRET=""
30 30 
S3_BUCKET=""
31 +
32 + 
# Image tag — set by CI, or pin manually
33 + 
GITHUB_REPOSITORY=pedrolucasp/nexo
34 + 
IMAGE_TAG=latest
35 +
36 + 
# Logging stack (VictoriaLogs + Vector)
37 + 
VICTORIALOGS_TAG=v1.53.0
38 + 
VICTORIALOGS_PORT=9428
39 + 
VICTORIALOGS_RETENTION=14d
40 + 
VICTORIALOGS_MAX_DISK=2GiB
41 + 
VECTOR_TAG=0.59.X-distroless-libc
42 +
43 + 
# Container log rotation (json-file driver)
44 + 
LOG_MAX_SIZE=10m
45 + 
LOG_MAX_FILE=3
docker-compose.yml modified+36
docker-compose.yml
1 1 
services:
2 2 
  db:
3 3 
    image: postgres:17-alpine
4 4 
    shm_size: 1g
5 5 
    user: postgres
15 15 
      POSTGRES_USER: user
16 16 
      POSTGRES_DB: orbit_dev
17 17 
      POSTGRES_PASSWORD: password
18 18 
    volumes:
19 19 
      - db_data:/var/lib/postgresql/data
20 + 
    logging: &default-logging
21 + 
      driver: json-file
22 + 
      options:
23 + 
        max-size: "${LOG_MAX_SIZE:-10m}"
24 + 
        max-file: "${LOG_MAX_FILE:-3}"
20 25
21 26 
  valkey:
22 27 
    image: valkey/valkey:9.0-alpine
23 28 
    ports:
24 29 
      - '6379:6379'
32 37 
      test: ["CMD", "valkey-cli", "ping"]
33 38 
      interval: 10s
34 39 
      timeout: 5s
35 40 
      retries: 5
36 41 
      start_period: 5s
42 + 
    logging: *default-logging
37 43
38 44 
  api:
39 45 
    build: ./api
40 46 
    volumes:
41 47 
      - ./api:/app
48 54 
      valkey:
49 55 
        condition: service_healthy
50 56 
      db:
51 57 
        condition: service_healthy
52 58 
    restart: always
59 + 
    logging: *default-logging
60 +
61 + 
  # Logs store — UI at http://localhost:9428/logs/select/vmui/ (loopback only)
62 + 
  victorialogs:
63 + 
    image: victoriametrics/victoria-logs:${VICTORIALOGS_TAG:-v1.53.0}
64 + 
    command:
65 + 
      - "-storageDataPath=/vlogs"
66 + 
      - "-http.pathPrefix=/logs"
67 + 
      - "-retentionPeriod=${VICTORIALOGS_RETENTION:-14d}"
68 + 
      - "-retention.maxDiskSpaceUsageBytes=${VICTORIALOGS_MAX_DISK:-2GiB}"
69 + 
    volumes:
70 + 
      - vlogs_data:/vlogs
71 + 
    ports:
72 + 
      - "127.0.0.1:${VICTORIALOGS_PORT:-9428}:9428"
73 + 
    restart: unless-stopped
74 + 
    logging: *default-logging
75 +
76 + 
  # Reads container logs through the Docker API and ships them to VictoriaLogs
77 + 
  # `docker compose logs` keeps working: this does not replace the log driver
78 + 
  vector:
79 + 
    image: timberio/vector:${VECTOR_TAG:-0.59.X-distroless-libc}
80 + 
    user: root
81 + 
    volumes:
82 + 
      - /var/run/docker.sock:/var/run/docker.sock:ro
83 + 
      - ./ansible/roles/app/files/vector.yaml:/etc/vector/vector.yaml:ro
84 + 
    depends_on:
85 + 
      - victorialogs
86 + 
    restart: unless-stopped
87 + 
    logging: *default-logging
53 88
54 89 
volumes:
55 90 
  db_data:
56 91 
  valkey_data:
92 + 
  vlogs_data: