Infra: Introduce AWS Secrets Manager

.env.example

@@ -96,3 +96,8 @@ SMTP_PASSWORD=
 EMAILS_FROM_EMAIL=
 EMAILS_FROM_NAME=Kaapi
 FRONTEND_HOST=
+
+# AWS Secrets Manager
+USE_AWS_SECRETS=false
+AWS_SECRETS_REGION=
+AWS_POSTGRES_SECRET_NAME=

backend/app/core/config.py

@@ -1,3 +1,6 @@
+import boto3
+import json
+import logging
 import secrets
 import warnings
 import os
@@ -12,9 +15,14 @@
     model_validator,
 )
 from pydantic_core import MultiHostUrl
-from pydantic_settings import BaseSettings, SettingsConfigDict
+from pydantic_settings import (
+    BaseSettings,
+    SettingsConfigDict,
+)
 from typing_extensions import Self
 
+logger = logging.getLogger(__name__)
+
 
 def parse_cors(origins: Any) -> list[str] | str:
     # If it's a plain comma-separated string, split it into a list
@@ -46,9 +54,9 @@ class Settings(BaseSettings):
     PROJECT_NAME: str
     API_VERSION: str = "0.5.0"
     SENTRY_DSN: HttpUrl | None = None
-    POSTGRES_SERVER: str
+    POSTGRES_SERVER: str = ""
     POSTGRES_PORT: int = 5432
-    POSTGRES_USER: str
+    POSTGRES_USER: str = ""
     POSTGRES_PASSWORD: str = ""
     POSTGRES_DB: str = ""
     KAAPI_GUARDRAILS_AUTH: str = ""
@@ -104,6 +112,11 @@ def SQLALCHEMY_DATABASE_URI(self) -> PostgresDsn:
     AWS_DEFAULT_REGION: str = ""
     AWS_S3_BUCKET_PREFIX: str = ""
 
+    # AWS Secrets Manager
+    USE_AWS_SECRETS: bool = False
+    AWS_SECRETS_REGION: str = ""
+    AWS_POSTGRES_SECRET_NAME: str = ""
+
     # RabbitMQ configuration for Celery broker
     RABBITMQ_HOST: str = "localhost"
     RABBITMQ_PORT: int = 5672
@@ -182,6 +195,64 @@ def _check_default_secret(self, var_name: str, value: str | None) -> None:
             else:
                 raise ValueError(message)
 
+    @model_validator(mode="after")
+    def _apply_aws_secrets(self) -> Self:
+        """Overlay service credentials from AWS Secrets Manager when enabled.
+
+        Each AWS_*_SECRET_NAME points to a secret whose SecretString is a
+        JSON object (e.g. the format produced by RDS/ElastiCache rotation).
+        Each secret is processed independently with its own field map so
+        services that share key names (username/password/host/port) do not
+        overwrite each other. Keys absent from a secret leave the existing
+        .env-derived value untouched.
+        """
+        if not self.USE_AWS_SECRETS:
+            return self
+
+        secret_configs: list[tuple[str, dict[str, str]]] = [
+            (
+                self.AWS_POSTGRES_SECRET_NAME,
+                {
+                    "username": "POSTGRES_USER",
+                    "password": "POSTGRES_PASSWORD",
+                    "host": "POSTGRES_SERVER",
+                    "port": "POSTGRES_PORT",
+                },
+            ),
+        ]
+
+        active = [(name, field_map) for name, field_map in secret_configs if name]
+        if not active:
+            return self
+
+        region = self.AWS_SECRETS_REGION or self.AWS_DEFAULT_REGION
+        client_kwargs: dict[str, Any] = {"region_name": region} if region else {}
+        client = boto3.client("secretsmanager", **client_kwargs)
+
+        for secret_name, field_map in active:
+            self._load_secret(client, secret_name, field_map)
+
+        return self
+
+    def _load_secret(
+        self, client: Any, secret_name: str, field_map: dict[str, str]
+    ) -> None:
+        data = json.loads(client.get_secret_value(SecretId=secret_name)["SecretString"])
+        if not isinstance(data, dict):
+            raise ValueError(
+                f"Secret '{secret_name}' must be a JSON object "
+                f"(got {type(data).__name__})"
+            )
+        applied: list[str] = []
+        for key, field in field_map.items():
+            if key in data:
+                setattr(self, field, data[key])
+                applied.append(field)
+        logger.info(
+            f"[_load_secret] Loaded AWS secret | "
+            f"{{'name': '{secret_name}', 'fields': {sorted(applied)}}}"
+        )
+
     @model_validator(mode="after")
     def _enforce_non_default_secrets(self) -> Self:
         self._check_default_secret("SECRET_KEY", self.SECRET_KEY)
@@ -201,7 +272,6 @@ def get_settings() -> Settings:
     env_files = {"testing": "../.env.test", "development": "../.env"}
     env_file = env_files.get(environment, "../.env")
 
-    # Create Settings instance with the appropriate env file
     return Settings(_env_file=env_file)
 
 

backend/app/tests/core/test_config.py

@@ -0,0 +1,132 @@
+"""Tests for the _apply_aws_secrets validator on Settings."""
+
+import json
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from app.core.config import Settings
+
+_REQUIRED_FIELDS = {
+    "PROJECT_NAME": "test",
+    "POSTGRES_SERVER": "localhost",
+    "POSTGRES_USER": "env-user",
+    "POSTGRES_PASSWORD": "env-pw",
+    "EMAIL_TEST_USER": "t@example.com",
+    "FIRST_SUPERUSER": "s@example.com",
+    "FIRST_SUPERUSER_PASSWORD": "superpw",
+}
+
+
+@pytest.fixture(autouse=True)
+def _clean_aws_env(monkeypatch):
+    """Strip any AWS-related env vars that may leak from .env.test."""
+    for var in (
+        "USE_AWS_SECRETS",
+        "AWS_POSTGRES_SECRET_NAME",
+        "AWS_SECRETS_REGION",
+        "AWS_DEFAULT_REGION",
+    ):
+        monkeypatch.delenv(var, raising=False)
+
+
+def _make_settings(**overrides) -> Settings:
+    """Build a Settings instance bypassing the .env file."""
+    return Settings(_env_file=None, **{**_REQUIRED_FIELDS, **overrides})
+
+
+class TestApplyAwsSecrets:
+    def test_toggle_off_keeps_env_values(self):
+        s = _make_settings(USE_AWS_SECRETS=False)
+        assert s.POSTGRES_PASSWORD == "env-pw"
+        assert s.POSTGRES_USER == "env-user"
+
+    def test_toggle_on_but_no_secret_names_keeps_env_values(self):
+        s = _make_settings(USE_AWS_SECRETS=True)
+        assert s.POSTGRES_PASSWORD == "env-pw"
+        assert s.POSTGRES_USER == "env-user"
+
+    @patch("boto3.client")
+    def test_postgres_secret_overrides_postgres_fields(self, mock_boto):
+        mock_client = MagicMock()
+        mock_client.get_secret_value.return_value = {
+            "SecretString": json.dumps(
+                {"username": "u", "password": "p", "host": "h", "port": 5433}
+            )
+        }
+        mock_boto.return_value = mock_client
+
+        setting = _make_settings(
+            USE_AWS_SECRETS=True,
+            AWS_POSTGRES_SECRET_NAME="kaapi-db",
+            AWS_SECRETS_REGION="ap-south-1",
+        )
+
+        assert setting.POSTGRES_USER == "u"
+        assert setting.POSTGRES_PASSWORD == "p"
+        assert setting.POSTGRES_SERVER == "h"
+        assert setting.POSTGRES_PORT == 5433
+        mock_boto.assert_called_once_with("secretsmanager", region_name="ap-south-1")
+        mock_client.get_secret_value.assert_called_once_with(SecretId="kaapi-db")
+
+    @patch("boto3.client")
+    def test_missing_keys_fall_back_to_env(self, mock_boto):
+        mock_client = MagicMock()
+        mock_client.get_secret_value.return_value = {
+            "SecretString": json.dumps({"password": "aws-pw"})
+        }
+        mock_boto.return_value = mock_client
+
+        setting = _make_settings(
+            USE_AWS_SECRETS=True,
+            AWS_POSTGRES_SECRET_NAME="kaapi-db",
+        )
+
+        assert setting.POSTGRES_PASSWORD == "aws-pw"
+        assert setting.POSTGRES_USER == "env-user"
+        assert setting.POSTGRES_SERVER == "localhost"
+
+    @patch("boto3.client")
+    def test_non_dict_payload_raises(self, mock_boto):
+        mock_client = MagicMock()
+        mock_client.get_secret_value.return_value = {
+            "SecretString": json.dumps(["not", "a", "dict"])
+        }
+        mock_boto.return_value = mock_client
+
+        with pytest.raises(ValueError, match="must be a JSON object"):
+            _make_settings(
+                USE_AWS_SECRETS=True,
+                AWS_POSTGRES_SECRET_NAME="kaapi-db",
+            )
+
+    @patch("boto3.client")
+    def test_region_falls_back_to_aws_default_region(self, mock_boto):
+        mock_client = MagicMock()
+        mock_client.get_secret_value.return_value = {
+            "SecretString": json.dumps({"password": "p"})
+        }
+        mock_boto.return_value = mock_client
+
+        _make_settings(
+            USE_AWS_SECRETS=True,
+            AWS_POSTGRES_SECRET_NAME="kaapi-db",
+            AWS_DEFAULT_REGION="us-east-1",
+        )
+
+        mock_boto.assert_called_once_with("secretsmanager", region_name="us-east-1")
+
+    @patch("boto3.client")
+    def test_no_region_builds_client_without_kwarg(self, mock_boto):
+        mock_client = MagicMock()
+        mock_client.get_secret_value.return_value = {
+            "SecretString": json.dumps({"password": "p"})
+        }
+        mock_boto.return_value = mock_client
+
+        _make_settings(
+            USE_AWS_SECRETS=True,
+            AWS_POSTGRES_SECRET_NAME="kaapi-db",
+        )
+
+        mock_boto.assert_called_once_with("secretsmanager")

docker-compose.yml

@@ -8,14 +8,14 @@ services:
     env_file:
       - .env
     environment:
-      POSTGRES_USER: ${POSTGRES_USER:?POSTGRES_USER not set}
-      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD not set}
-      POSTGRES_DB: ${POSTGRES_DB:?POSTGRES_DB not set}
+      POSTGRES_USER: ${POSTGRES_USER:-postgres}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres}
+      POSTGRES_DB: ${POSTGRES_DB:-kaapi}
       PGDATA: /var/lib/postgresql/data/pgdata
     volumes:
       - kaapi-postgres:/var/lib/postgresql/data
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-postgres} -d ${POSTGRES_DB:-kaapi}"]
       interval: 10s
       timeout: 10s
       retries: 5
@@ -30,8 +30,8 @@ services:
     env_file:
       - .env
     command: >
-      sh -c "if [ -n \"${REDIS_PASSWORD}\" ]; then
-               redis-server --requirepass \"${REDIS_PASSWORD}\";
+      sh -c "if [ -n \"${REDIS_PASSWORD:-}\" ]; then
+               redis-server --requirepass \"${REDIS_PASSWORD:-}\";
              else
                redis-server;
              fi"
@@ -40,7 +40,7 @@ services:
     ports:
       - "6379:6379"
     healthcheck:
-      test: ["CMD-SHELL", "if [ -n \"${REDIS_PASSWORD}\" ]; then redis-cli -a \"${REDIS_PASSWORD}\" ping; else redis-cli ping; fi"]
+      test: ["CMD-SHELL", "if [ -n \"${REDIS_PASSWORD:-}\" ]; then redis-cli -a \"${REDIS_PASSWORD:-}\" ping; else redis-cli ping; fi"]
       interval: 10s
       timeout: 10s
       retries: 5
@@ -53,9 +53,9 @@ services:
     env_file:
       - .env
     environment:
-      RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:?RABBITMQ_USER not set}
-      RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:?RABBITMQ_PASSWORD not set}
-      RABBITMQ_DEFAULT_VHOST: ${RABBITMQ_VHOST:?RABBITMQ_VHOST not set}
+      RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-guest}
+      RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:-guest}
+      RABBITMQ_DEFAULT_VHOST: ${RABBITMQ_VHOST:-/}
     volumes:
       - kaapi-rabbitmq:/var/lib/rabbitmq
     ports: