diff --git a/.env.example b/.env.example index c05e3e17..c8ffae77 100644 --- a/.env.example +++ b/.env.example @@ -96,3 +96,8 @@ SMTP_PASSWORD= EMAILS_FROM_EMAIL= EMAILS_FROM_NAME=Kaapi FRONTEND_HOST= + +# AWS Secrets Manager +USE_AWS_SECRETS=false +AWS_SECRETS_REGION= +AWS_POSTGRES_SECRET_NAME= diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 6e46112d..0e51ab7a 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -1,3 +1,6 @@ +import boto3 +import json +import logging import secrets import warnings import os @@ -12,9 +15,14 @@ model_validator, ) from pydantic_core import MultiHostUrl -from pydantic_settings import BaseSettings, SettingsConfigDict +from pydantic_settings import ( + BaseSettings, + SettingsConfigDict, +) from typing_extensions import Self +logger = logging.getLogger(__name__) + def parse_cors(origins: Any) -> list[str] | str: # If it's a plain comma-separated string, split it into a list @@ -46,9 +54,9 @@ class Settings(BaseSettings): PROJECT_NAME: str API_VERSION: str = "0.5.0" SENTRY_DSN: HttpUrl | None = None - POSTGRES_SERVER: str + POSTGRES_SERVER: str = "" POSTGRES_PORT: int = 5432 - POSTGRES_USER: str + POSTGRES_USER: str = "" POSTGRES_PASSWORD: str = "" POSTGRES_DB: str = "" KAAPI_GUARDRAILS_AUTH: str = "" @@ -104,6 +112,11 @@ def SQLALCHEMY_DATABASE_URI(self) -> PostgresDsn: AWS_DEFAULT_REGION: str = "" AWS_S3_BUCKET_PREFIX: str = "" + # AWS Secrets Manager + USE_AWS_SECRETS: bool = False + AWS_SECRETS_REGION: str = "" + AWS_POSTGRES_SECRET_NAME: str = "" + # RabbitMQ configuration for Celery broker RABBITMQ_HOST: str = "localhost" RABBITMQ_PORT: int = 5672 @@ -182,6 +195,64 @@ def _check_default_secret(self, var_name: str, value: str | None) -> None: else: raise ValueError(message) + @model_validator(mode="after") + def _apply_aws_secrets(self) -> Self: + """Overlay service credentials from AWS Secrets Manager when enabled. + + Each AWS_*_SECRET_NAME points to a secret whose SecretString is a + JSON object (e.g. the format produced by RDS/ElastiCache rotation). + Each secret is processed independently with its own field map so + services that share key names (username/password/host/port) do not + overwrite each other. Keys absent from a secret leave the existing + .env-derived value untouched. + """ + if not self.USE_AWS_SECRETS: + return self + + secret_configs: list[tuple[str, dict[str, str]]] = [ + ( + self.AWS_POSTGRES_SECRET_NAME, + { + "username": "POSTGRES_USER", + "password": "POSTGRES_PASSWORD", + "host": "POSTGRES_SERVER", + "port": "POSTGRES_PORT", + }, + ), + ] + + active = [(name, field_map) for name, field_map in secret_configs if name] + if not active: + return self + + region = self.AWS_SECRETS_REGION or self.AWS_DEFAULT_REGION + client_kwargs: dict[str, Any] = {"region_name": region} if region else {} + client = boto3.client("secretsmanager", **client_kwargs) + + for secret_name, field_map in active: + self._load_secret(client, secret_name, field_map) + + return self + + def _load_secret( + self, client: Any, secret_name: str, field_map: dict[str, str] + ) -> None: + data = json.loads(client.get_secret_value(SecretId=secret_name)["SecretString"]) + if not isinstance(data, dict): + raise ValueError( + f"Secret '{secret_name}' must be a JSON object " + f"(got {type(data).__name__})" + ) + applied: list[str] = [] + for key, field in field_map.items(): + if key in data: + setattr(self, field, data[key]) + applied.append(field) + logger.info( + f"[_load_secret] Loaded AWS secret | " + f"{{'name': '{secret_name}', 'fields': {sorted(applied)}}}" + ) + @model_validator(mode="after") def _enforce_non_default_secrets(self) -> Self: self._check_default_secret("SECRET_KEY", self.SECRET_KEY) @@ -201,7 +272,6 @@ def get_settings() -> Settings: env_files = {"testing": "../.env.test", "development": "../.env"} env_file = env_files.get(environment, "../.env") - # Create Settings instance with the appropriate env file return Settings(_env_file=env_file) diff --git a/backend/app/tests/core/test_config.py b/backend/app/tests/core/test_config.py new file mode 100644 index 00000000..aa16912a --- /dev/null +++ b/backend/app/tests/core/test_config.py @@ -0,0 +1,132 @@ +"""Tests for the _apply_aws_secrets validator on Settings.""" + +import json +from unittest.mock import MagicMock, patch + +import pytest + +from app.core.config import Settings + +_REQUIRED_FIELDS = { + "PROJECT_NAME": "test", + "POSTGRES_SERVER": "localhost", + "POSTGRES_USER": "env-user", + "POSTGRES_PASSWORD": "env-pw", + "EMAIL_TEST_USER": "t@example.com", + "FIRST_SUPERUSER": "s@example.com", + "FIRST_SUPERUSER_PASSWORD": "superpw", +} + + +@pytest.fixture(autouse=True) +def _clean_aws_env(monkeypatch): + """Strip any AWS-related env vars that may leak from .env.test.""" + for var in ( + "USE_AWS_SECRETS", + "AWS_POSTGRES_SECRET_NAME", + "AWS_SECRETS_REGION", + "AWS_DEFAULT_REGION", + ): + monkeypatch.delenv(var, raising=False) + + +def _make_settings(**overrides) -> Settings: + """Build a Settings instance bypassing the .env file.""" + return Settings(_env_file=None, **{**_REQUIRED_FIELDS, **overrides}) + + +class TestApplyAwsSecrets: + def test_toggle_off_keeps_env_values(self): + s = _make_settings(USE_AWS_SECRETS=False) + assert s.POSTGRES_PASSWORD == "env-pw" + assert s.POSTGRES_USER == "env-user" + + def test_toggle_on_but_no_secret_names_keeps_env_values(self): + s = _make_settings(USE_AWS_SECRETS=True) + assert s.POSTGRES_PASSWORD == "env-pw" + assert s.POSTGRES_USER == "env-user" + + @patch("boto3.client") + def test_postgres_secret_overrides_postgres_fields(self, mock_boto): + mock_client = MagicMock() + mock_client.get_secret_value.return_value = { + "SecretString": json.dumps( + {"username": "u", "password": "p", "host": "h", "port": 5433} + ) + } + mock_boto.return_value = mock_client + + setting = _make_settings( + USE_AWS_SECRETS=True, + AWS_POSTGRES_SECRET_NAME="kaapi-db", + AWS_SECRETS_REGION="ap-south-1", + ) + + assert setting.POSTGRES_USER == "u" + assert setting.POSTGRES_PASSWORD == "p" + assert setting.POSTGRES_SERVER == "h" + assert setting.POSTGRES_PORT == 5433 + mock_boto.assert_called_once_with("secretsmanager", region_name="ap-south-1") + mock_client.get_secret_value.assert_called_once_with(SecretId="kaapi-db") + + @patch("boto3.client") + def test_missing_keys_fall_back_to_env(self, mock_boto): + mock_client = MagicMock() + mock_client.get_secret_value.return_value = { + "SecretString": json.dumps({"password": "aws-pw"}) + } + mock_boto.return_value = mock_client + + setting = _make_settings( + USE_AWS_SECRETS=True, + AWS_POSTGRES_SECRET_NAME="kaapi-db", + ) + + assert setting.POSTGRES_PASSWORD == "aws-pw" + assert setting.POSTGRES_USER == "env-user" + assert setting.POSTGRES_SERVER == "localhost" + + @patch("boto3.client") + def test_non_dict_payload_raises(self, mock_boto): + mock_client = MagicMock() + mock_client.get_secret_value.return_value = { + "SecretString": json.dumps(["not", "a", "dict"]) + } + mock_boto.return_value = mock_client + + with pytest.raises(ValueError, match="must be a JSON object"): + _make_settings( + USE_AWS_SECRETS=True, + AWS_POSTGRES_SECRET_NAME="kaapi-db", + ) + + @patch("boto3.client") + def test_region_falls_back_to_aws_default_region(self, mock_boto): + mock_client = MagicMock() + mock_client.get_secret_value.return_value = { + "SecretString": json.dumps({"password": "p"}) + } + mock_boto.return_value = mock_client + + _make_settings( + USE_AWS_SECRETS=True, + AWS_POSTGRES_SECRET_NAME="kaapi-db", + AWS_DEFAULT_REGION="us-east-1", + ) + + mock_boto.assert_called_once_with("secretsmanager", region_name="us-east-1") + + @patch("boto3.client") + def test_no_region_builds_client_without_kwarg(self, mock_boto): + mock_client = MagicMock() + mock_client.get_secret_value.return_value = { + "SecretString": json.dumps({"password": "p"}) + } + mock_boto.return_value = mock_client + + _make_settings( + USE_AWS_SECRETS=True, + AWS_POSTGRES_SECRET_NAME="kaapi-db", + ) + + mock_boto.assert_called_once_with("secretsmanager") diff --git a/docker-compose.yml b/docker-compose.yml index 98ef86fb..e295a3a3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,14 +8,14 @@ services: env_file: - .env environment: - POSTGRES_USER: ${POSTGRES_USER:?POSTGRES_USER not set} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD not set} - POSTGRES_DB: ${POSTGRES_DB:?POSTGRES_DB not set} + POSTGRES_USER: ${POSTGRES_USER:-postgres} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} + POSTGRES_DB: ${POSTGRES_DB:-kaapi} PGDATA: /var/lib/postgresql/data/pgdata volumes: - kaapi-postgres:/var/lib/postgresql/data healthcheck: - test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-postgres} -d ${POSTGRES_DB:-kaapi}"] interval: 10s timeout: 10s retries: 5 @@ -30,8 +30,8 @@ services: env_file: - .env command: > - sh -c "if [ -n \"${REDIS_PASSWORD}\" ]; then - redis-server --requirepass \"${REDIS_PASSWORD}\"; + sh -c "if [ -n \"${REDIS_PASSWORD:-}\" ]; then + redis-server --requirepass \"${REDIS_PASSWORD:-}\"; else redis-server; fi" @@ -40,7 +40,7 @@ services: ports: - "6379:6379" healthcheck: - test: ["CMD-SHELL", "if [ -n \"${REDIS_PASSWORD}\" ]; then redis-cli -a \"${REDIS_PASSWORD}\" ping; else redis-cli ping; fi"] + test: ["CMD-SHELL", "if [ -n \"${REDIS_PASSWORD:-}\" ]; then redis-cli -a \"${REDIS_PASSWORD:-}\" ping; else redis-cli ping; fi"] interval: 10s timeout: 10s retries: 5 @@ -53,9 +53,9 @@ services: env_file: - .env environment: - RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:?RABBITMQ_USER not set} - RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:?RABBITMQ_PASSWORD not set} - RABBITMQ_DEFAULT_VHOST: ${RABBITMQ_VHOST:?RABBITMQ_VHOST not set} + RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-guest} + RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:-guest} + RABBITMQ_DEFAULT_VHOST: ${RABBITMQ_VHOST:-/} volumes: - kaapi-rabbitmq:/var/lib/rabbitmq ports: