From 8e8aabd5e7c0c2809b2d38fc4cf37209f63557ae Mon Sep 17 00:00:00 2001 From: Joakim Date: Sun, 14 Dec 2025 17:16:21 +0100 Subject: [PATCH] Improve logging and infrastructure variable consistency Changes: - Migrate Authentik to journald logging (remove file-based logs) - Update Gitea to use infrastructure variables for PostgreSQL access - Add comprehensive logging documentation to deployment guide - Add infrastructure variable pattern guide to integration docs Authentik Logging: - Remove LogDriver=k8s-file from server and worker containers - Remove logs directory creation from user setup tasks - Update deployment guide with journald examples and JSON log patterns Gitea Infrastructure Variables: - Add infrastructure dependencies section to role defaults - Replace hardcoded paths with postgresql_unix_socket_directories variable - Replace hardcoded 'postgres' group with postgresql_client_group variable - Add infrastructure variable validation in tasks - Remove manual socket permission override (handled by infrastructure) Documentation: - Add journald logging best practices to service integration guide - Add infrastructure variable pattern documentation with Gitea example - Update Authentik deployment guide with journald commands and JSON filtering - Document benefits: centralized logging, single source of truth, maintainability Validated on arch-vps: - Authentik logs accessible via journalctl and podman logs (identical output) - Gitea user added to postgres-clients group (GID 962) - No PostgreSQL socket permission errors after service restart --- docs/authentik-deployment-guide.md | 57 ++++-- docs/service-integration-guide.md | 187 ++++++++++++++++++ roles/authentik/tasks/user.yml | 1 - .../templates/authentik-server.container | 5 - .../templates/authentik-worker.container | 5 - roles/gitea/defaults/main.yml | 15 +- roles/gitea/tasks/main.yml | 30 +-- 7 files changed, 259 insertions(+), 41 deletions(-) diff --git a/docs/authentik-deployment-guide.md b/docs/authentik-deployment-guide.md index 2f1b87e..b85a272 100644 --- a/docs/authentik-deployment-guide.md +++ b/docs/authentik-deployment-guide.md @@ -453,35 +453,64 @@ curl -v https://auth.jnss.me/ #### Key Log Locations -```bash -# Authentik application logs -ssh root@your-vps "cat /opt/authentik/logs/server.log" -ssh root@your-vps "cat /opt/authentik/logs/worker.log" +Authentik uses **journald** for centralized logging. Both `journalctl` and `podman logs` provide access to the same log stream: -# systemd service logs +```bash +# View Authentik logs via journalctl (system-wide logging) ssh root@your-vps "journalctl -u authentik-server -f" ssh root@your-vps "journalctl -u authentik-worker -f" +# View Authentik logs via podman (container-specific) +ssh root@your-vps "podman logs -f authentik-server" +ssh root@your-vps "podman logs -f authentik-worker" + +# View recent logs with timestamp (last 50 lines) +ssh root@your-vps "journalctl -u authentik-server --lines 50 --no-pager" + +# Filter logs by time +ssh root@your-vps "journalctl -u authentik-server --since '10 minutes ago'" +ssh root@your-vps "journalctl -u authentik-server --since '2025-12-14 16:00:00'" + +# Search logs for specific patterns +ssh root@your-vps "journalctl -u authentik-server | grep ERROR" +ssh root@your-vps "journalctl -u authentik-worker | grep 'database connection'" + # Caddy logs for reverse proxy issues ssh root@your-vps "journalctl -u caddy -f" ``` +**Note**: Logs are in JSON format with structured fields (timestamp, level, logger, event, etc.). + #### Common Log Patterns -**Successful startup**: -``` -INFO authentik.core.signals: authentik 2025.10.x starting -INFO authentik.core.models: Database version up-to-date +Authentik logs are in **JSON format** for structured analysis. Here are common patterns: + +**Successful API request**: +```json +{"auth_via": "secret_key", "domain_url": "0.0.0.0", "event": "/api/v3/outposts/proxy/", + "level": "info", "logger": "authentik.asgi", "method": "GET", "status": 200, + "timestamp": "2025-12-14T16:13:17.269312"} ``` -**Database connection success**: -``` -INFO authentik.core.db: Connected to database via unix socket +**Startup and initialization**: +```json +{"event": "updating brand certificates", "level": "info", + "logger": "authentik.router.brand_tls", "timestamp": "2025-12-14T16:13:17Z"} ``` -**Cache connection success**: +**Warning patterns**: +```json +{"event": "No providers assigned to this outpost, check outpost configuration", + "level": "warning", "logger": "authentik.outpost.proxyv2"} ``` -INFO authentik.core.cache: Connected to cache via unix socket + +**Filtering JSON logs by level**: +```bash +# Filter by error level +ssh root@your-vps "journalctl -u authentik-server --since today | grep '\"level\":\"error\"'" + +# Filter by specific event +ssh root@your-vps "journalctl -u authentik-server | grep '\"event\":\"database connection\"'" ``` ## Performance Monitoring diff --git a/docs/service-integration-guide.md b/docs/service-integration-guide.md index 32924e7..01673f4 100644 --- a/docs/service-integration-guide.md +++ b/docs/service-integration-guide.md @@ -343,6 +343,193 @@ If you get permission denied errors: - Verify container process has correct supplementary groups - Verify services are in `/system.slice/` +5. **Logging**: + - Use **journald** (Podman's default) for system-level containers + - Never use file-based logging (`LogDriver=k8s-file`) for system services + - Access logs via `journalctl -u service-name` or `podman logs container-name` + - Logs are automatically persistent and integrated with systemd + +6. **Infrastructure Variables**: + - Always use infrastructure variables (e.g., `{{ postgresql_unix_socket_directories }}`) + - Never hardcode infrastructure paths or group names + - Define infrastructure dependencies in application role defaults with read-only notice + - Validate infrastructure variables exist in tasks before use + +## Logging Best Practices + +System-level containers should **always use journald** for logging integration with systemd. + +### Why Journald for System Containers + +- **Centralized logging**: All service logs accessible via `journalctl` +- **Persistent by default**: Logs survive container restarts +- **No disk management**: systemd handles log rotation and retention +- **Unified interface**: Same logging interface as native systemd services +- **Query capabilities**: Filter by time, level, service, or pattern + +### Configuration + +**Do NOT use file-based logging** in system containers: + +```ini +# ❌ WRONG - File-based logging (rootless pattern) +[Container] +LogDriver=k8s-file +LogOpt=path=/opt/service/logs/app.log +Volume=/opt/service/logs:/opt/service/logs +``` + +**Use journald (Podman's default)** for system containers: + +```ini +# ✅ CORRECT - journald logging (system-level) +[Container] +ContainerName=myservice +Image=myservice:latest +# No LogDriver needed - journald is default +``` + +### Accessing Logs + +Both `journalctl` and `podman logs` provide access to the **same log stream**: + +```bash +# Via journalctl (system-wide) +journalctl -u myservice -f # Follow logs +journalctl -u myservice --lines 50 # Last 50 lines +journalctl -u myservice --since "10 min ago" # Time-filtered +journalctl -u myservice | grep ERROR # Pattern search + +# Via podman (container-specific) +podman logs -f myservice # Same output as journalctl +podman logs --tail 50 myservice +``` + +### JSON Structured Logging + +Many modern applications (like Authentik) output JSON-formatted logs: + +```bash +# Filter JSON logs by field +journalctl -u authentik-server | grep '"level":"error"' +journalctl -u authentik-server | grep '"event":"database connection"' + +# Pretty-print JSON logs with jq +journalctl -u authentik-server --output=cat | jq '.' +``` + +## Infrastructure Variable Pattern + +Application roles should **reference** infrastructure variables, never redefine them. + +### Pattern Overview + +**Infrastructure roles** (PostgreSQL, Valkey) define and export variables: +```yaml +# roles/postgresql/defaults/main.yml +postgresql_unix_socket_directories: "/var/run/postgresql" +postgresql_client_group: "postgres-clients" +postgresql_port: 5432 + +# roles/postgresql/tasks/main.yml +- name: Export PostgreSQL client group GID as fact + set_fact: + postgresql_client_group_gid: "{{ getent_group[postgresql_client_group].1 }}" +``` + +**Application roles** reference these variables (never redefine): +```yaml +# roles/myservice/defaults/main.yml +# ================================================================= +# Infrastructure Dependencies (Read-only) +# ================================================================= +# These variables reference infrastructure services defined by their roles +# Applications MUST NOT modify these values - they are provided by infrastructure + +postgresql_unix_socket_directories: "/var/run/postgresql" +postgresql_client_group: "postgres-clients" +postgresql_port: 5432 +``` + +### Using Infrastructure Variables + +**In application role defaults**: +```yaml +# Use infrastructure variables for configuration +myservice_db_host: "{{ postgresql_unix_socket_directories }}" +myservice_db_port: "{{ postgresql_port }}" +``` + +**In application role tasks**: +```yaml +# Validate infrastructure variables exist +- name: Validate infrastructure variables are defined + assert: + that: + - postgresql_unix_socket_directories is defined + - postgresql_client_group is defined + fail_msg: "Missing required infrastructure variables" + +# Use variables for group membership +- name: Add service user to PostgreSQL client group + user: + name: "{{ myservice_user }}" + groups: "{{ postgresql_client_group }}" + append: true + +# Use variables for database operations +- name: Create database via socket + postgresql_db: + name: "{{ myservice_db_name }}" + login_unix_socket: "{{ postgresql_unix_socket_directories }}" +``` + +### Example: Gitea Role + +See `roles/gitea/` for a complete example of infrastructure variable usage: + +**defaults/main.yml**: +```yaml +# Infrastructure Dependencies section +postgresql_unix_socket_directories: "/var/run/postgresql" +postgresql_client_group: "postgres-clients" +postgresql_port: 5432 + +# Database Configuration using infrastructure variables +gitea_db_host: "{{ postgresql_unix_socket_directories }}" +gitea_db_port: "{{ postgresql_port }}" +``` + +**tasks/main.yml**: +```yaml +# Validate infrastructure variables +- name: Validate infrastructure variables are defined + assert: + that: + - postgresql_unix_socket_directories is defined + - postgresql_client_group is defined + +# Use infrastructure variables +- name: Add git user to PostgreSQL client group + user: + name: git + groups: "{{ postgresql_client_group }}" + append: true + +- name: Create Gitea database + postgresql_db: + name: gitea + login_unix_socket: "{{ postgresql_unix_socket_directories }}" +``` + +### Benefits + +1. **Single Source of Truth**: Infrastructure paths defined once +2. **Consistency**: All services use same socket paths and groups +3. **Maintainability**: Change infrastructure path in one place +4. **Validation**: Catch missing dependencies early via assertions +5. **Documentation**: Clear separation between infrastructure and application + ## Authentication Integration with Authentik ### Overview diff --git a/roles/authentik/tasks/user.yml b/roles/authentik/tasks/user.yml index 8b0df05..ffbd43c 100644 --- a/roles/authentik/tasks/user.yml +++ b/roles/authentik/tasks/user.yml @@ -29,7 +29,6 @@ - "{{ authentik_home }}" - "{{ authentik_home }}/data" - "{{ authentik_home }}/media" - - "{{ authentik_home }}/logs" - name: Get authentik user UID and GID for container configuration shell: | diff --git a/roles/authentik/templates/authentik-server.container b/roles/authentik/templates/authentik-server.container index 8d73f93..0b889fa 100644 --- a/roles/authentik/templates/authentik-server.container +++ b/roles/authentik/templates/authentik-server.container @@ -11,11 +11,6 @@ EnvironmentFile={{ authentik_home }}/.env User={{ authentik_uid }}:{{ authentik_gid }} PodmanArgs=--group-add {{ postgresql_client_group_gid }} --group-add {{ valkey_client_group_gid }} -# Logging configuration -LogDriver=k8s-file -LogOpt=path={{ authentik_home }}/logs/server.log -Volume={{ authentik_home }}/logs:{{ authentik_home }}/logs - # Volume mounts for data and sockets Volume={{ authentik_media_dir }}:/media Volume={{ authentik_data_dir }}:/data diff --git a/roles/authentik/templates/authentik-worker.container b/roles/authentik/templates/authentik-worker.container index 20d62c5..d29cc61 100644 --- a/roles/authentik/templates/authentik-worker.container +++ b/roles/authentik/templates/authentik-worker.container @@ -11,11 +11,6 @@ EnvironmentFile={{ authentik_home }}/.env User={{ authentik_uid }}:{{ authentik_gid }} PodmanArgs=--group-add {{ postgresql_client_group_gid }} --group-add {{ valkey_client_group_gid }} -# Logging configuration -LogDriver=k8s-file -LogOpt=path={{ authentik_home }}/logs/worker.log -Volume={{ authentik_home }}/logs:{{ authentik_home }}/logs - # Volume mounts for data and sockets Volume={{ authentik_media_dir }}:/media Volume={{ authentik_data_dir }}:/data diff --git a/roles/gitea/defaults/main.yml b/roles/gitea/defaults/main.yml index 3e701ad..4517af6 100644 --- a/roles/gitea/defaults/main.yml +++ b/roles/gitea/defaults/main.yml @@ -39,9 +39,10 @@ caddy_sites_enabled_dir: "/etc/caddy/sites-enabled" # ================================================================= # Gitea manages its own database (Unix socket connection) +# Uses infrastructure variables for consistent socket path reference gitea_db_type: "postgres" -gitea_db_host: "/run/postgresql" # Unix socket directory -gitea_db_port: 5432 +gitea_db_host: "{{ postgresql_unix_socket_directories }}" # Unix socket from infrastructure +gitea_db_port: "{{ postgresql_port }}" gitea_db_name: "gitea" gitea_db_user: "gitea" gitea_db_password: "{{ vault_gitea_db_password }}" @@ -65,6 +66,16 @@ gitea_require_signin: false # SSH settings gitea_start_ssh_server: true +# ================================================================= +# Infrastructure Dependencies (Read-only) +# ================================================================= +# These variables reference infrastructure services defined by their roles +# Applications MUST NOT modify these values - they are provided by infrastructure + +postgresql_unix_socket_directories: "/var/run/postgresql" +postgresql_client_group: "postgres-clients" +postgresql_port: 5432 + # ================================================================= # Rick-Infra Integration Notes # ================================================================= diff --git a/roles/gitea/tasks/main.yml b/roles/gitea/tasks/main.yml index 0148580..0422198 100644 --- a/roles/gitea/tasks/main.yml +++ b/roles/gitea/tasks/main.yml @@ -2,6 +2,15 @@ # Gitea Service Role - Self-Contained Implementation # Manages Gitea Git service with own database +- name: Validate infrastructure variables are defined + assert: + that: + - postgresql_unix_socket_directories is defined + - postgresql_client_group is defined + - postgresql_port is defined + fail_msg: "Missing required infrastructure variables. Ensure PostgreSQL role has run first." + success_msg: "Infrastructure variables validated successfully" + - name: Install Gitea from Arch repository pacman: name: gitea @@ -43,23 +52,16 @@ group: "{{ gitea_group }}" mode: '0700' -# Socket access setup (following Authentik pattern) -- name: Add git user to postgres group for socket access +# Socket access setup (using infrastructure variables) +- name: Add git user to PostgreSQL client group for socket access user: name: "{{ gitea_user }}" - groups: postgres + groups: "{{ postgresql_client_group }}" append: true -- name: Ensure git can access PostgreSQL socket directory - file: - path: "/var/run/postgresql" - mode: '0770' - group: postgres - become: true - - name: Test PostgreSQL socket connectivity postgresql_ping: - login_unix_socket: "/var/run/postgresql" + login_unix_socket: "{{ postgresql_unix_socket_directories }}" login_user: "{{ gitea_user }}" become: true become_user: "{{ gitea_user }}" @@ -70,7 +72,7 @@ name: "{{ gitea_db_user }}" password: "{{ gitea_db_password }}" encrypted: yes - login_unix_socket: "/var/run/postgresql" + login_unix_socket: "{{ postgresql_unix_socket_directories }}" login_user: postgres become: true become_user: postgres @@ -81,7 +83,7 @@ owner: "{{ gitea_db_user }}" encoding: UTF8 template: template0 - login_unix_socket: "/var/run/postgresql" + login_unix_socket: "{{ postgresql_unix_socket_directories }}" login_user: postgres become: true become_user: postgres @@ -92,7 +94,7 @@ privs: ALL type: database role: "{{ gitea_db_user }}" - login_unix_socket: "/var/run/postgresql" + login_unix_socket: "{{ postgresql_unix_socket_directories }}" login_user: postgres become: true become_user: postgres