deploy: manage the app secrets through an encrypted vault
The env files on the server were hand-filled from .example templates, so the production password and API keys lived outside the repo and only on one box. They now come from an encrypted group_vars vault that the app role renders into .env and .env.production, and scripts/import-vault.sh builds that vault from the values already live on the server The Makefile passes the vault password only when it exists, so the other targets keep working before the first import
Pedro Lucas Porcellis porcellis@eletrotupi.com · 14 days ago
5b0a5a6cf15017c93e1dd78af7d46e7eb1c3f262 Browse filesParents
c8bab44Changed files
12
Additions
+217
Deletions
-116
.gitignore
1 1
# Personal scratch area for local notes, not part of the project
2 2
/notes/
3 3
4 4
# Secrets
5 5
.env
6 6
.env.*
7 7
!.env.example
8 +
9 +
# Ansible: local inventory (server address, key path) and vault password
10 +
ansible/inventory/hosts.yml
11 +
ansible/.vault_pass
Makefile
1 1
# Deployment helpers
2 2
#
3 3
# Day-to-day app deploys happen in GitHub Actions
4 4
# (.github/workflows/deploy.yml) These targets are for the host-config work,
5 5
# which is done rarely
6 6
7 7
SHELL := bash
8 8
ANSIBLE_DIR := ansible
9 9
INVENTORY := inventory/hosts.yml
10 10
11 -
.PHONY: help deploy-diff deploy-check provision cert deploy-nginx
11 +
# Only pass the vault password when it exists, so these targets keep working
12 +
# before the vault has been imported (scripts/import-vault.sh creates it)
13 +
VAULT_ARGS := $(shell [ -f $(ANSIBLE_DIR)/.vault_pass ] && echo --vault-password-file $(CURDIR)/$(ANSIBLE_DIR)/.vault_pass)
14 +
15 +
.PHONY: help deploy-diff deploy-check provision cert deploy-nginx app vault-import
12 16
13 17
help:
14 18
@echo "make deploy-diff # pull live configs and diff them against the repo"
15 19
@echo "make deploy-check # ansible --check --diff (dry run, no changes)"
16 20
@echo "make provision # full host bootstrap (setup + deploy + cert + app)"
17 21
@echo "make deploy-nginx # publish the nginx vhost (deploy.yml)"
22 +
@echo "make app # render the app config from the vault (app.yml)"
18 23
@echo "make cert # install uacme / manage certificates (cert.yml)"
24 +
@echo "make vault-import # build the encrypted vault from the live env files"
19 25
20 26
deploy-diff:
21 27
$(ANSIBLE_DIR)/scripts/pull-live-configs.sh
28 +
29 +
vault-import:
30 +
$(ANSIBLE_DIR)/scripts/import-vault.sh
22 31
23 32
deploy-check:
24 -
cd $(ANSIBLE_DIR) && ansible-playbook -i $(INVENTORY) provision.yml --check --diff
33 +
cd $(ANSIBLE_DIR) && ansible-playbook -i $(INVENTORY) $(VAULT_ARGS) provision.yml --check --diff
25 34
26 35
provision:
27 -
cd $(ANSIBLE_DIR) && ansible-playbook -i $(INVENTORY) provision.yml --ask-become-pass
36 +
cd $(ANSIBLE_DIR) && ansible-playbook -i $(INVENTORY) $(VAULT_ARGS) provision.yml --ask-become-pass
28 37
29 38
cert:
30 -
cd $(ANSIBLE_DIR) && ansible-playbook -i $(INVENTORY) cert.yml --ask-become-pass
39 +
cd $(ANSIBLE_DIR) && ansible-playbook -i $(INVENTORY) $(VAULT_ARGS) cert.yml --ask-become-pass
31 40
32 41
deploy-nginx:
33 -
cd $(ANSIBLE_DIR) && ansible-playbook -i $(INVENTORY) deploy.yml --ask-become-pass
42 +
cd $(ANSIBLE_DIR) && ansible-playbook -i $(INVENTORY) $(VAULT_ARGS) deploy.yml --ask-become-pass
43 +
44 +
app:
45 +
cd $(ANSIBLE_DIR) && ansible-playbook -i $(INVENTORY) $(VAULT_ARGS) app.yml --ask-become-pass
ansible/inventory/group_vars/all.yml → /dev/null
1 -
# The deploy user to create
2 -
deploy_user: deploy
3 -
4 -
# The SSH public key that will be authorised for the deploy user
5 -
deploy_ssh_pubkey_file: ~/.ssh/nexo_deploy_key.pub
6 -
7 -
# App directories to create under /srv/app
8 -
app_environments:
9 -
- production
10 -
- staging
/dev/null → ansible/inventory/group_vars/all/main.yml
1 +
# Shared, non-secret configuration for every host
2 +
#
3 +
# Secrets live in the encrypted sibling file, vault.yml. It is produced by
4 +
# scripts/import-vault.sh and holds these variables:
5 +
#
6 +
# vault_postgres_password
7 +
# vault_jwt_secret
8 +
# vault_resend_api_key
9 +
# vault_s3_access_key
10 +
# vault_s3_access_secret
11 +
# vault_nginx_logs_htpasswd
12 +
13 +
# The deploy user to create
14 +
deploy_user: deploy
15 +
16 +
# The SSH public key that will be authorised for the deploy user
17 +
deploy_ssh_pubkey_file: ~/.ssh/nexo_deploy_key.pub
18 +
19 +
# App directories to create under /srv/app
20 +
app_environments:
21 +
- production
22 +
- staging
23 +
24 +
# Image pushed by the deploy workflow
25 +
app_github_repo: pedrolucasp/nexo
26 +
image_tag: latest
27 +
28 +
# API runtime
29 +
app_port: 3000
30 +
app_log_level: info
31 +
app_frontend_url: https://usenexo.xyz
32 +
33 +
# Postgres (the password is in the vault)
34 +
postgres_user: nexo
35 +
postgres_db: nexo_production
36 +
37 +
# Valkey
38 +
valkey_host: valkey
39 +
valkey_port: 6379
40 +
41 +
# Email (Resend); the API key is in the vault
42 +
# XXX: the display name still says "orbit"
43 +
resend_from: "orbit <naoresponda@updates.cabotia.dev>"
44 +
45 +
# Object storage (S3-compatible); the keys are in the vault
46 +
s3_region: br
47 +
s3_url: br-gru-1.linodeobjects.com
48 +
s3_bucket: nexo-stg
49 +
50 +
# Logging stack
51 +
victorialogs_tag: v1.53.0
52 +
victorialogs_port: 9428
53 +
victorialogs_retention: 14d
54 +
victorialogs_max_disk: 2GiB
55 +
vector_tag: 0.59.X-distroless-libc
56 +
log_max_size: 10m
57 +
log_max_file: 3
ansible/roles/app/defaults/main.yml
1 1
---
2 2
# roles/app/defaults/main.yml
3 -
# Host-side application configuration
3 +
# Host-side application configuration. Non-secret app config lives in
4 +
# inventory/group_vars/all/main.yml; secrets live in the encrypted vault.yml
4 5
5 6
app_env: production
6 -
# ghcr.io/<owner>/<repo> must match the image the deploy workflow pushes
7 -
app_github_repo: "pedrolucasp/nexo"
ansible/roles/app/tasks/main.yml
1 1
---
2 2
# roles/app/tasks/main.yml
3 3
# Places the production app config in /srv/app/<env>/
4 -
# Secrets are NOT created here: the .example files are dropped in and you fill
5 -
# them in by hand (that part is deliberately manual, so get ready to SSH and copy it)
4 +
# The env files are rendered from group_vars (secrets come from the encrypted
5 +
# vault.yml), so there is nothing to fill in by hand anymore
6 6
7 7
- name: Ensure /srv/app exists
8 8
ansible.builtin.file:
9 9
path: /srv/app
10 10
state: directory
48 48
owner: "{{ deploy_user }}"
49 49
group: "{{ deploy_user }}"
50 50
mode: "0640"
51 51
tags: [app]
52 52
53 -
- name: Install the environment example files
53 +
- name: Install the environment files
54 54
ansible.builtin.template:
55 -
src: "{{ item }}.j2"
56 -
dest: "/srv/app/{{ app_env }}/{{ item }}.example"
55 +
src: "{{ item.src }}"
56 +
dest: "/srv/app/{{ app_env }}/{{ item.dest }}"
57 57
owner: "{{ deploy_user }}"
58 58
group: "{{ deploy_user }}"
59 -
mode: "0640"
60 -
loop:
61 -
- env
62 -
- env.production
63 -
tags: [app]
64 -
65 -
- name: Check whether the real env files exist on the server
66 -
ansible.builtin.stat:
67 -
path: "/srv/app/{{ app_env }}/{{ item }}"
59 +
mode: "0600"
68 60
loop:
69 -
- .env
70 -
- .env.production
71 -
register: app_env_stat
72 -
tags: [app]
73 -
74 -
- name: Remind to fill in the env files
75 -
ansible.builtin.debug:
76 -
msg: >-
77 -
{{ item.item }} is missing: copy it from {{ item.item }}.example and fill
78 -
it in before the first `docker compose up -d`
79 -
loop: "{{ app_env_stat.results }}"
80 -
when: not item.stat.exists
61 +
- { src: env.j2, dest: .env }
62 +
- { src: env.production.j2, dest: .env.production }
63 +
loop_control:
64 +
label: "{{ item.dest }}"
81 65
tags: [app]
ansible/roles/app/templates/env.example.j2 → /dev/null
1 -
# Copy to `.env` in the same directory. `docker compose` loads it automatically
2 -
# to interpolate the ${...} variables used in docker-compose.yml
3 -
# NEVER commit the filled-in file!
4 -
5 -
POSTGRES_USER=orbit
6 -
POSTGRES_DB=orbit_production
7 -
POSTGRES_PASSWORD=change-me
8 -
9 -
# ghcr.io/<owner>/<repo> must match the image the deploy workflow pushes
10 -
GITHUB_REPOSITORY={{ app_github_repo }}
11 -
12 -
# CI rewrites IMAGE_TAG on every deploy
13 -
IMAGE_TAG=latest
14 -
15 -
# Logging stack (VictoriaLogs + Vector)
16 -
VICTORIALOGS_TAG=v1.53.0
17 -
VICTORIALOGS_PORT=9428
18 -
VICTORIALOGS_RETENTION=14d
19 -
VICTORIALOGS_MAX_DISK=2GiB
20 -
VECTOR_TAG=0.59.X-distroless-libc
21 -
22 -
# Container log rotation (json-file driver)
23 -
LOG_MAX_SIZE=10m
24 -
LOG_MAX_FILE=3
/dev/null → ansible/roles/app/templates/env.j2
1 +
# Rendered by Ansible (role: app). Do not edit on the server
2 +
POSTGRES_USER={{ postgres_user }}
3 +
POSTGRES_DB={{ postgres_db }}
4 +
POSTGRES_PASSWORD={{ vault_postgres_password }}
5 +
6 +
GITHUB_REPOSITORY={{ app_github_repo }}
7 +
IMAGE_TAG={{ image_tag }}
8 +
9 +
# Logging stack (VictoriaLogs + Vector)
10 +
VICTORIALOGS_TAG={{ victorialogs_tag }}
11 +
VICTORIALOGS_PORT={{ victorialogs_port }}
12 +
VICTORIALOGS_RETENTION={{ victorialogs_retention }}
13 +
VICTORIALOGS_MAX_DISK={{ victorialogs_max_disk }}
14 +
VECTOR_TAG={{ vector_tag }}
15 +
16 +
# Container log rotation (json-file driver)
17 +
LOG_MAX_SIZE={{ log_max_size }}
18 +
LOG_MAX_FILE={{ log_max_file }}
ansible/roles/app/templates/env.production.example.j2 → /dev/null
1 -
# Copy to `.env.production`. Loaded by the api container (env_file)
2 -
# NEVER commit the filled-in file
3 -
4 -
NODE_ENV=production
5 -
PORT=3000
6 -
LOG_LEVEL=info
7 -
8 -
# Database (must match POSTGRES_* in .env)
9 -
DATABASE_URL="postgres://orbit:change-me@db:5432/orbit_production"
10 -
11 -
# Queue / cache
12 -
VALKEY_HOST=valkey
13 -
VALKEY_PORT=6379
14 -
15 -
# Auth
16 -
JWT_SECRET="change-me"
17 -
18 -
# CORS / public URL
19 -
FRONTEND_URL="https://usenexo.xyz"
20 -
21 -
# Email (Resend)
22 -
RESEND_API_KEY="xxxxxx"
23 -
RESEND_FROM="orbit <noreply@example.com>"
24 -
25 -
# Object storage (avatars, S3-compatible)
26 -
S3_REGION=""
27 -
S3_URL=""
28 -
S3_ACCESS_KEY=""
29 -
S3_ACCESS_SECRET=""
30 -
S3_BUCKET=""
31 -
32 -
# Image tag — set by CI, or pin manually
33 -
GITHUB_REPOSITORY=pedrolucasp/nexo
34 -
IMAGE_TAG=latest
35 -
36 -
# Logging stack (VictoriaLogs + Vector)
37 -
VICTORIALOGS_TAG=v1.53.0
38 -
VICTORIALOGS_PORT=9428
39 -
VICTORIALOGS_RETENTION=14d
40 -
VICTORIALOGS_MAX_DISK=2GiB
41 -
VECTOR_TAG=0.59.X-distroless-libc
42 -
43 -
# Container log rotation (json-file driver)
44 -
LOG_MAX_SIZE=10m
45 -
LOG_MAX_FILE=3
/dev/null → ansible/roles/app/templates/env.production.j2
1 +
# Rendered by Ansible (role: app). Do not edit on the server
2 +
NODE_ENV=production
3 +
PORT={{ app_port }}
4 +
LOG_LEVEL={{ app_log_level }}
5 +
6 +
DATABASE_URL="postgres://{{ postgres_user }}:{{ vault_postgres_password }}@db:5432/{{ postgres_db }}"
7 +
8 +
VALKEY_HOST={{ valkey_host }}
9 +
VALKEY_PORT={{ valkey_port }}
10 +
11 +
JWT_SECRET="{{ vault_jwt_secret }}"
12 +
FRONTEND_URL="{{ app_frontend_url }}"
13 +
14 +
RESEND_API_KEY="{{ vault_resend_api_key }}"
15 +
RESEND_FROM="{{ resend_from }}"
16 +
17 +
S3_REGION="{{ s3_region }}"
18 +
S3_URL="{{ s3_url }}"
19 +
S3_ACCESS_KEY="{{ vault_s3_access_key }}"
20 +
S3_ACCESS_SECRET="{{ vault_s3_access_secret }}"
21 +
S3_BUCKET="{{ s3_bucket }}"
ansible/roles/nginx/defaults/main.yml
1 1
---
2 2
# roles/nginx/defaults/main.yml
3 3
4 4
nginx_conf_dir: /etc/nginx/http.d # Alpine nginx include dir
5 5
nginx_well_known_root: /var/www # served for ACME HTTP-01 challenges
10 10
11 11
# VictoriaLogs UI (vmui), proxied at https://<host>/logs/ behind basic auth
12 12
# Must match -http.pathPrefix on the victorialogs service
13 13
nginx_logs_proxy_pass: http://127.0.0.1:9428
14 14
nginx_logs_htpasswd_file: /etc/nginx/.htpasswd-logs
15 -
# CHANGE ME: a "user:hash" line, e.g. from `openssl passwd -apr1`
16 -
# The default is the throwaway "logs" user with password "nexo-logs"
17 -
nginx_logs_htpasswd: "logs:$apr1$Bb2qaSgH$TUfwUmDTRR/FpOLRu.Jnj/"
15 +
# "user:hash" line; the credential comes from the encrypted vault, with a
16 +
# throwaway fallback so the role still renders before the vault is imported
17 +
nginx_logs_htpasswd: "{{ vault_nginx_logs_htpasswd | default('logs:$apr1$Bb2qaSgH$TUfwUmDTRR/FpOLRu.Jnj/') }}"
/dev/null → ansible/scripts/import-vault.sh
1 +
#!/usr/bin/env bash
2 +
#
3 +
# Build the encrypted Ansible vault from the values that are live on the VPS
4 +
#
5 +
# Pulls /srv/app/production/.env and .env.production, extracts the secrets,
6 +
# generates a logs UI credential, and writes the encrypted group_vars vault.
7 +
# Read-only on the server
8 +
#
9 +
# ansible/scripts/import-vault.sh
10 +
#
11 +
# The vault password lives in ansible/.vault_pass (git-ignored); this script
12 +
# creates it on first run and prints it, so store it somewhere safe. Connection
13 +
# details come from ansible/inventory/hosts.yml, or the VPS_HOST / VPS_USER /
14 +
# VPS_KEY environment variables
15 +
16 +
set -euo pipefail
17 +
18 +
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
19 +
ansible_dir="$repo_root/ansible"
20 +
inventory="$ansible_dir/inventory/hosts.yml"
21 +
vault_file="$ansible_dir/inventory/group_vars/all/vault.yml"
22 +
vault_pass_file="$ansible_dir/.vault_pass"
23 +
24 +
field() { awk -F': *' -v k="^[[:space:]]*$1" '$0 ~ k {print $2; exit}' "$inventory"; }
25 +
26 +
HOST="${VPS_HOST:-}"
27 +
USER="${VPS_USER:-}"
28 +
KEY="${VPS_KEY:-}"
29 +
30 +
if [ -f "$inventory" ]; then
31 +
[ -n "$HOST" ] || HOST="$(field 'ansible_host:')"
32 +
[ -n "$USER" ] || USER="$(field 'ansible_user:')"
33 +
[ -n "$KEY" ] || KEY="$(field 'ansible_ssh_private_key_file:')"
34 +
fi
35 +
36 +
KEY="${KEY/#\~/$HOME}"
37 +
KEY="${KEY:-$HOME/.ssh/id_ed25519}"
38 +
[ -n "$HOST" ] || { echo "no host: set VPS_HOST or provide inventory/hosts.yml" >&2; exit 2; }
39 +
[ -n "$USER" ] || { echo "no user: set VPS_USER or provide inventory/hosts.yml" >&2; exit 2; }
40 +
41 +
tmp="$(mktemp -d)"
42 +
trap 'rm -rf "$tmp"' EXIT
43 +
ssh_opts=(-i "$KEY" -o StrictHostKeyChecking=accept-new)
44 +
45 +
echo "Pulling live env from ${USER}@${HOST} ..."
46 +
scp "${ssh_opts[@]}" "${USER}@${HOST}:/srv/app/production/.env" "$tmp/env" >/dev/null
47 +
scp "${ssh_opts[@]}" "${USER}@${HOST}:/srv/app/production/.env.production" "$tmp/env.production" >/dev/null
48 +
49 +
# The env files are plain KEY=value; sourcing them is the least fussy parse
50 +
set -a
51 +
# shellcheck disable=SC1091
52 +
. "$tmp/env"
53 +
# shellcheck disable=SC1091
54 +
. "$tmp/env.production"
55 +
set +a
56 +
57 +
if [ ! -f "$vault_pass_file" ]; then
58 +
umask 077
59 +
openssl rand -base64 32 >"$vault_pass_file"
60 +
echo "==> generated $vault_pass_file"
61 +
echo " vault password: $(cat "$vault_pass_file")"
62 +
fi
63 +
64 +
logs_pass="$(openssl rand -base64 18)"
65 +
logs_hash="$(openssl passwd -apr1 "$logs_pass")"
66 +
67 +
cat >"$vault_file.plain" <<EOF
68 +
vault_postgres_password: "${POSTGRES_PASSWORD:?missing POSTGRES_PASSWORD}"
69 +
vault_jwt_secret: "${JWT_SECRET:?missing JWT_SECRET}"
70 +
vault_resend_api_key: "${RESEND_API_KEY:?missing RESEND_API_KEY}"
71 +
vault_s3_access_key: "${S3_ACCESS_KEY:?missing S3_ACCESS_KEY}"
72 +
vault_s3_access_secret: "${S3_ACCESS_SECRET:?missing S3_ACCESS_SECRET}"
73 +
vault_nginx_logs_htpasswd: "logs:${logs_hash}"
74 +
EOF
75 +
76 +
ansible-vault encrypt \
77 +
--vault-password-file "$vault_pass_file" \
78 +
--output "$vault_file" \
79 +
"$vault_file.plain"
80 +
rm -f "$vault_file.plain"
81 +
82 +
echo
83 +
echo "Wrote $vault_file (encrypted)."
84 +
echo "Logs UI credential: user 'logs', password '${logs_pass}'"
85 +
echo "Back up both the vault password and this credential."