From 47b4bde0444384cdd54ea90d922d62d1fc5e759d Mon Sep 17 00:00:00 2001 From: Prashant Vasudevan <71649489+vprashrex@users.noreply.github.com> Date: Tue, 31 Mar 2026 09:29:19 +0530 Subject: [PATCH 1/2] feat(assessment): Implement assessment evaluation orchestration service - Added `service.py` to handle assessment evaluation orchestration, including starting and retrying assessments. - Introduced utility functions for exporting assessment results in various formats (CSV, XLSX, JSON) in `export.py`. - Created parsing utilities for handling batch results in `parsing.py`. - Implemented validation for dataset file uploads in `validators.py`. - Updated batch job model to include assessment type. - Enhanced evaluation run model to reference parent assessments. - Added new fields for reasoning effort and summary preferences in LLM request model. --- .../050_add_assessment_manager_table.py | 207 ++++++ backend/app/api/main.py | 2 + backend/app/api/routes/cron.py | 38 +- backend/app/assessment/__init__.py | 1 + backend/app/assessment/batch.py | 571 +++++++++++++++ backend/app/assessment/cron.py | 189 +++++ backend/app/assessment/crud.py | 337 +++++++++ backend/app/assessment/dataset.py | 170 +++++ backend/app/assessment/events.py | 51 ++ backend/app/assessment/mappers.py | 218 ++++++ backend/app/assessment/models.py | 296 ++++++++ backend/app/assessment/processing.py | 483 +++++++++++++ backend/app/assessment/routes.py | 654 ++++++++++++++++++ backend/app/assessment/service.py | 271 ++++++++ backend/app/assessment/utils/__init__.py | 23 + backend/app/assessment/utils/export.py | 432 ++++++++++++ backend/app/assessment/utils/parsing.py | 32 + backend/app/assessment/validators.py | 67 ++ backend/app/models/batch_job.py | 1 + backend/app/models/evaluation.py | 13 + backend/app/models/llm/request.py | 19 + 21 files changed, 4071 insertions(+), 4 deletions(-) create mode 100644 backend/app/alembic/versions/050_add_assessment_manager_table.py create mode 100644 backend/app/assessment/__init__.py create mode 100644 backend/app/assessment/batch.py create mode 100644 backend/app/assessment/cron.py create mode 100644 backend/app/assessment/crud.py create mode 100644 backend/app/assessment/dataset.py create mode 100644 backend/app/assessment/events.py create mode 100644 backend/app/assessment/mappers.py create mode 100644 backend/app/assessment/models.py create mode 100644 backend/app/assessment/processing.py create mode 100644 backend/app/assessment/routes.py create mode 100644 backend/app/assessment/service.py create mode 100644 backend/app/assessment/utils/__init__.py create mode 100644 backend/app/assessment/utils/export.py create mode 100644 backend/app/assessment/utils/parsing.py create mode 100644 backend/app/assessment/validators.py diff --git a/backend/app/alembic/versions/050_add_assessment_manager_table.py b/backend/app/alembic/versions/050_add_assessment_manager_table.py new file mode 100644 index 000000000..ffecd0767 --- /dev/null +++ b/backend/app/alembic/versions/050_add_assessment_manager_table.py @@ -0,0 +1,207 @@ +"""add assessment manager table + +Revision ID: 050 +Revises: 049 +Create Date: 2026-03-26 23:30:00.000000 + +""" + +import sqlalchemy as sa +import sqlmodel.sql.sqltypes +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "050" +down_revision = "049" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "assessment", + sa.Column( + "id", + sa.Integer(), + nullable=False, + comment="Unique identifier for the assessment", + ), + sa.Column( + "experiment_name", + sqlmodel.sql.sqltypes.AutoString(), + nullable=False, + comment="Experiment name shared by child config runs", + ), + sa.Column( + "dataset_id", + sa.Integer(), + nullable=False, + comment="Reference to the evaluation dataset", + ), + sa.Column( + "dataset_name", + sqlmodel.sql.sqltypes.AutoString(), + nullable=False, + comment="Name of the dataset used by this assessment", + ), + sa.Column( + "status", + sqlmodel.sql.sqltypes.AutoString(), + nullable=False, + server_default="pending", + comment="Overall assessment status across all child evaluation runs", + ), + sa.Column( + "total_runs", + sa.Integer(), + nullable=False, + server_default="0", + comment="Total number of child evaluation runs", + ), + sa.Column( + "pending_runs", + sa.Integer(), + nullable=False, + server_default="0", + comment="Number of child runs in pending state", + ), + sa.Column( + "processing_runs", + sa.Integer(), + nullable=False, + server_default="0", + comment="Number of child runs in processing state", + ), + sa.Column( + "completed_runs", + sa.Integer(), + nullable=False, + server_default="0", + comment="Number of child runs in completed state", + ), + sa.Column( + "failed_runs", + sa.Integer(), + nullable=False, + server_default="0", + comment="Number of child runs in failed state", + ), + sa.Column( + "run_stats", + postgresql.JSONB(astext_type=sa.Text()), + nullable=False, + server_default=sa.text("'[]'::jsonb"), + comment="Cached status snapshot for child evaluation runs", + ), + sa.Column( + "error_message", + sa.Text(), + nullable=True, + comment="Aggregated error message for child run failures", + ), + sa.Column( + "callback_url", + sqlmodel.sql.sqltypes.AutoString(), + nullable=True, + comment="Optional frontend callback URL for status updates", + ), + sa.Column( + "organization_id", + sa.Integer(), + nullable=False, + comment="Reference to the organization", + ), + sa.Column( + "project_id", + sa.Integer(), + nullable=False, + comment="Reference to the project", + ), + sa.Column( + "inserted_at", + sa.DateTime(), + nullable=False, + comment="Timestamp when the assessment was created", + ), + sa.Column( + "updated_at", + sa.DateTime(), + nullable=False, + comment="Timestamp when the assessment was last updated", + ), + sa.ForeignKeyConstraint( + ["dataset_id"], + ["evaluation_dataset.id"], + name="fk_assessment_dataset_id", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["organization_id"], + ["organization.id"], + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["project_id"], + ["project.id"], + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + op.f("ix_assessment_experiment_name"), + "assessment", + ["experiment_name"], + unique=False, + ) + op.create_index( + "idx_assessment_status_org", + "assessment", + ["status", "organization_id"], + unique=False, + ) + op.create_index( + "idx_assessment_status_project", + "assessment", + ["status", "project_id"], + unique=False, + ) + + op.add_column( + "evaluation_run", + sa.Column( + "assessment_id", + sa.Integer(), + nullable=True, + comment="Reference to parent assessment manager row, if applicable", + ), + ) + op.create_foreign_key( + "fk_evaluation_run_assessment_id", + "evaluation_run", + "assessment", + ["assessment_id"], + ["id"], + ondelete="SET NULL", + ) + op.create_index( + "idx_eval_run_assessment_id", + "evaluation_run", + ["assessment_id"], + unique=False, + ) + + +def downgrade(): + op.drop_index("idx_eval_run_assessment_id", table_name="evaluation_run") + op.drop_constraint( + "fk_evaluation_run_assessment_id", + "evaluation_run", + type_="foreignkey", + ) + op.drop_column("evaluation_run", "assessment_id") + + op.drop_index("idx_assessment_status_project", table_name="assessment") + op.drop_index("idx_assessment_status_org", table_name="assessment") + op.drop_index(op.f("ix_assessment_experiment_name"), table_name="assessment") + op.drop_table("assessment") diff --git a/backend/app/api/main.py b/backend/app/api/main.py index 5ab1cbd9e..8abb8a0bf 100644 --- a/backend/app/api/main.py +++ b/backend/app/api/main.py @@ -27,6 +27,7 @@ collection_job, ) from app.api.routes import evaluations +from app.assessment import routes as assessment_routes from app.core.config import settings api_router = APIRouter() @@ -54,6 +55,7 @@ api_router.include_router(utils.router) api_router.include_router(fine_tuning.router) api_router.include_router(model_evaluation.router) +api_router.include_router(assessment_routes.router) if settings.ENVIRONMENT in ["development", "testing"]: diff --git a/backend/app/api/routes/cron.py b/backend/app/api/routes/cron.py index ab072970b..2c7b3c81e 100644 --- a/backend/app/api/routes/cron.py +++ b/backend/app/api/routes/cron.py @@ -4,7 +4,7 @@ from fastapi import APIRouter, Depends from app.api.deps import SessionDep -from app.crud.evaluations import process_all_pending_evaluations_sync +from app.crud.evaluations import process_all_pending_evaluations logger = logging.getLogger(__name__) @@ -16,7 +16,7 @@ include_in_schema=False, dependencies=[Depends(require_permission(Permission.SUPERUSER))], ) -def evaluation_cron_job( +async def evaluation_cron_job( session: SessionDep, ) -> dict: """ @@ -26,7 +26,8 @@ def evaluation_cron_job( 1. Fetches all evaluation runs with status='processing' 2. Groups them by project_id 3. Processes each project with its OpenAI/Langfuse clients - 4. Returns aggregated results + 4. Also polls pending assessment evaluations + 5. Returns aggregated results Hidden from Swagger documentation. Requires authentication via FIRST_SUPERUSER credentials. @@ -35,7 +36,36 @@ def evaluation_cron_job( try: # Process all pending evaluations across all organizations - result = process_all_pending_evaluations_sync(session=session) + result = await process_all_pending_evaluations(session=session) + + # Also poll assessment evaluations (must await in the same event loop + # so that SSE publish reaches subscribers via the shared broker). + try: + from app.assessment.cron import ( + poll_all_pending_assessment_evaluations, + ) + + assessment_result = await poll_all_pending_assessment_evaluations( + session=session + ) + + # Merge assessment results into the main result + result["assessment"] = assessment_result + result["total_processed"] = result.get( + "total_processed", 0 + ) + assessment_result.get("processed", 0) + result["total_failed"] = result.get( + "total_failed", 0 + ) + assessment_result.get("failed", 0) + result["total_still_processing"] = result.get( + "total_still_processing", 0 + ) + assessment_result.get("still_processing", 0) + except Exception as ae: + logger.error( + f"[evaluation_cron_job] Assessment polling failed: {ae}", + exc_info=True, + ) + result["assessment_error"] = str(ae) logger.info( f"[evaluation_cron_job] Completed: " diff --git a/backend/app/assessment/__init__.py b/backend/app/assessment/__init__.py new file mode 100644 index 000000000..a00a9f1dd --- /dev/null +++ b/backend/app/assessment/__init__.py @@ -0,0 +1 @@ +"""Assessment module — multi-config, multi-provider batch evaluation.""" diff --git a/backend/app/assessment/batch.py b/backend/app/assessment/batch.py new file mode 100644 index 000000000..ee67da1f4 --- /dev/null +++ b/backend/app/assessment/batch.py @@ -0,0 +1,571 @@ +"""Assessment batch JSONL construction and submission. + +Builds provider-specific JSONL files from dataset rows + config, +then submits them via the core batch infrastructure. +""" + +import csv +import io +import logging +import re +from typing import Any +from uuid import UUID + +from sqlmodel import Session + +from app.assessment.mappers import ( + map_kaapi_to_google_params, + map_kaapi_to_openai_params, + normalize_llm_text, +) +from app.assessment.models import AssessmentAttachment, AssessmentTextLLMParams +from app.core.batch import BATCH_KEY, start_batch_job +from app.core.batch.openai import OpenAIBatchProvider +from app.core.cloud import get_cloud_storage +from app.core.storage_utils import load_json_from_object_store +from app.crud.config.version import ConfigVersionCrud +from app.models.batch_job import BatchJob, BatchJobType +from app.models.evaluation import EvaluationDataset, EvaluationRun +from app.models.llm.request import ConfigBlob, KaapiCompletionConfig, LLMCallConfig +from app.services.llm.jobs import resolve_config_blob + +logger = logging.getLogger(__name__) + +# Provider name → native provider suffix mapping +_NATIVE_PROVIDERS = { + "openai": "openai", + "openai-native": "openai", + "google": "google", + "google-native": "google", +} + + +def _resolve_config( + session: Session, + config_id: UUID, + config_version: int, + project_id: int, +) -> tuple[ConfigBlob | None, str | None]: + """Resolve a stored config into a ConfigBlob.""" + config_crud = ConfigVersionCrud( + session=session, + config_id=config_id, + project_id=project_id, + ) + return resolve_config_blob( + config_crud=config_crud, + config=LLMCallConfig(id=config_id, version=config_version), + ) + + +def _load_dataset_rows( + session: Session, + dataset: EvaluationDataset, +) -> list[dict[str, str]]: + """Load dataset rows from object store. + + Returns a list of dicts (one per row) with column-name keys. + """ + if not dataset.object_store_url: + raise ValueError(f"Dataset {dataset.id} has no object_store_url") + + storage = get_cloud_storage(session=session, project_id=dataset.project_id) + + # Download the file content via stream() + body = storage.stream(dataset.object_store_url) + file_content = body.read() + if not file_content: + raise ValueError(f"Failed to download dataset from {dataset.object_store_url}") + + metadata = dataset.dataset_metadata or {} + file_ext = metadata.get("file_extension", ".csv") + + if file_ext in (".xlsx", ".xls"): + return _parse_excel_rows(file_content) + return _parse_csv_rows(file_content) + + +def _parse_csv_rows(content: bytes) -> list[dict[str, str]]: + """Parse CSV content into list of row dicts.""" + for encoding in ("utf-8-sig", "utf-8", "latin-1"): + try: + text = content.decode(encoding) + break + except (UnicodeDecodeError, ValueError): + continue + else: + text = content.decode("utf-8", errors="replace") + + reader = csv.DictReader(io.StringIO(text)) + return [row for row in reader if any(v and v.strip() for v in row.values())] + + +def _parse_excel_rows(content: bytes) -> list[dict[str, str]]: + """Parse Excel content into list of row dicts.""" + import openpyxl + + wb = openpyxl.load_workbook(io.BytesIO(content), read_only=True, data_only=True) + ws = wb.active + if ws is None: + wb.close() + return [] + + rows_iter = ws.iter_rows(values_only=True) + header = next(rows_iter, None) + if header is None: + wb.close() + return [] + + columns = [str(h) if h is not None else f"col_{i}" for i, h in enumerate(header)] + result = [] + for row in rows_iter: + if row and any(cell is not None for cell in row): + row_dict = { + columns[i]: str(cell) if cell is not None else "" + for i, cell in enumerate(row) + if i < len(columns) + } + result.append(row_dict) + + wb.close() + return result + + +def _build_text_prompt( + row: dict[str, str], + text_columns: list[str], + prompt_template: str | None, +) -> str: + """Build the text prompt for a single row. + + If prompt_template is provided, placeholders like {column_name} are replaced. + Otherwise, all text column values are concatenated with newlines. + """ + if prompt_template: + prompt = normalize_llm_text(prompt_template) + for col in text_columns: + placeholder = "{" + col + "}" + prompt = prompt.replace(placeholder, normalize_llm_text(row.get(col, ""))) + return prompt + + # No template: concatenate text columns + parts = [normalize_llm_text(row.get(col, "")) for col in text_columns if row.get(col, "").strip()] + return "\n".join(parts) + + +def _split_attachment_urls(value: str) -> list[str]: + """Split comma/newline separated attachment URLs from a single dataset cell.""" + return [part.strip() for part in re.split(r"[\n,]+", value) if part.strip()] + + +def _to_direct_attachment_url(url: str, attachment_type: str) -> str: + """Normalize share-page attachment URLs into provider-fetchable direct URLs. + + This currently handles common Google Drive share URL shapes. The file must + still be publicly accessible to the model provider. + """ + url = url.strip() + file_id = None + + match = re.match(r"https://drive\.google\.com/file/d/([^/]+)", url) + if match: + file_id = match.group(1) + + if not file_id: + match = re.search(r"[?&]id=([a-zA-Z0-9_-]+)", url) + if match and ( + "drive.google.com" in url or "drive.usercontent.google.com" in url + ): + file_id = match.group(1) + + if not file_id: + return url + + if attachment_type == "image": + return f"https://lh3.googleusercontent.com/d/{file_id}" + + return f"https://drive.google.com/uc?export=download&id={file_id}" + + +def _resolve_attachment_values( + value: str, + att: AssessmentAttachment, +) -> list[dict[str, Any]]: + """Convert one dataset cell into one or more OpenAI-style input objects.""" + value = value.strip() + if not value: + return [] + + if att.format == "url": + values = _split_attachment_urls(value) + else: + values = [value] + + resolved: list[dict[str, Any]] = [] + for item_value in values: + normalized_value = ( + _to_direct_attachment_url(item_value, att.type) + if att.format == "url" + else item_value + ) + + if att.type == "image": + if att.format == "url": + resolved.append({"type": "input_image", "image_url": normalized_value}) + else: + resolved.append( + { + "type": "input_image", + "image_url": f"data:image/png;base64,{normalized_value}", + } + ) + elif att.type == "pdf": + if att.format == "url": + resolved.append( + { + "type": "input_file", + "file_data": normalized_value, + "filename": "document.pdf", + } + ) + else: + resolved.append( + { + "type": "input_file", + "file_data": f"data:application/pdf;base64,{normalized_value}", + "filename": "document.pdf", + } + ) + + return resolved + + +def build_openai_jsonl( + rows: list[dict[str, str]], + text_columns: list[str], + attachments: list[AssessmentAttachment], + prompt_template: str | None, + openai_params: dict, +) -> list[dict[str, Any]]: + """Build OpenAI batch JSONL data from dataset rows. + + Each line follows the OpenAI batch format: + { + "custom_id": "row_0", + "method": "POST", + "url": "/v1/responses", + "body": { model, instructions, temperature, input: [{role, content: [...]}] } + } + """ + jsonl_data = [] + + for idx, row in enumerate(rows): + # Build input array + input_parts: list[dict[str, Any]] = [] + + # Text prompt + text_prompt = _build_text_prompt(row, text_columns, prompt_template) + if text_prompt.strip(): + input_parts.append({"type": "input_text", "text": text_prompt}) + + # Attachments + for att in attachments: + cell_value = row.get(att.column, "") + input_parts.extend(_resolve_attachment_values(cell_value, att)) + + if not input_parts: + logger.warning(f"[build_openai_jsonl] Skipping empty row | idx={idx}") + continue + + # Build body from mapped params + body = dict(openai_params) + body["input"] = [ + { + "role": "user", + "content": input_parts, + } + ] + + jsonl_data.append( + { + BATCH_KEY: f"row_{idx}", + "method": "POST", + "url": "/v1/responses", + "body": body, + } + ) + + return jsonl_data + + +def build_google_jsonl( + rows: list[dict[str, str]], + text_columns: list[str], + attachments: list[AssessmentAttachment], + prompt_template: str | None, + google_params: dict, +) -> list[dict[str, Any]]: + """Build Google (Gemini) batch JSONL data from dataset rows. + + Each line follows the Gemini batch format: + { + "key": "row_0", + "request": { "contents": [{ "parts": [...], "role": "user" }] } + } + """ + jsonl_data = [] + + for idx, row in enumerate(rows): + parts: list[dict[str, Any]] = [] + + # Text prompt + text_prompt = _build_text_prompt(row, text_columns, prompt_template) + if text_prompt.strip(): + parts.append({"text": text_prompt}) + + # Attachments (Gemini uses file_data for inline content) + for att in attachments: + cell_value = row.get(att.column, "").strip() + if not cell_value: + continue + + cell_values = ( + _split_attachment_urls(cell_value) + if att.format == "url" + else [cell_value] + ) + + for item_value in cell_values: + normalized_value = ( + _to_direct_attachment_url(item_value, att.type) + if att.format == "url" + else item_value + ) + if att.type == "image": + if att.format == "url": + parts.append( + { + "fileData": { + "mimeType": "image/png", + "fileUri": normalized_value, + } + } + ) + else: + parts.append( + { + "inlineData": { + "mimeType": "image/png", + "data": normalized_value, + } + } + ) + elif att.type == "pdf": + if att.format == "url": + parts.append( + { + "fileData": { + "mimeType": "application/pdf", + "fileUri": normalized_value, + } + } + ) + else: + parts.append( + { + "inlineData": { + "mimeType": "application/pdf", + "data": normalized_value, + } + } + ) + + if not parts: + logger.warning(f"[build_google_jsonl] Skipping empty row | idx={idx}") + continue + + system_instruction = google_params.get("instructions") + request: dict[str, Any] = { + "contents": [{"parts": parts, "role": "user"}], + } + if system_instruction: + request["systemInstruction"] = { + "parts": [{"text": system_instruction}] + } + + generation_config: dict[str, Any] = {} + temperature = google_params.get("temperature") + if temperature is not None: + generation_config["temperature"] = temperature + top_p = google_params.get("top_p") + if top_p is not None: + generation_config["topP"] = top_p + max_output_tokens = google_params.get("max_output_tokens") + if max_output_tokens is not None: + generation_config["maxOutputTokens"] = max_output_tokens + thinking_config = google_params.get("thinking_config") + if thinking_config: + generation_config["thinkingConfig"] = thinking_config + output_schema = google_params.get("output_schema") + if output_schema: + generation_config["responseMimeType"] = "application/json" + generation_config["responseSchema"] = output_schema + if generation_config: + request["generationConfig"] = generation_config + + jsonl_data.append( + { + "metadata": {"key": f"row_{idx}"}, + "request": request, + } + ) + + return jsonl_data + + +def submit_assessment_batch( + session: Session, + eval_run: EvaluationRun, + dataset: EvaluationDataset, + config_blob: ConfigBlob, + assessment_config: dict[str, Any], + organization_id: int, + project_id: int, +) -> BatchJob: + """Build JSONL and submit a batch for one assessment run. + + Args: + session: Database session + eval_run: The EvaluationRun to process + dataset: The dataset to read rows from + config_blob: Resolved configuration blob + assessment_config: Assessment-specific config (prompt_template, text_columns, etc.) + organization_id: Organization ID + project_id: Project ID + + Returns: + Created BatchJob record + """ + text_columns = assessment_config.get("text_columns", []) + prompt_template = assessment_config.get("prompt_template") + attachments_raw = assessment_config.get("attachments", []) + output_schema = assessment_config.get("output_schema") + attachments = [AssessmentAttachment(**a) for a in attachments_raw] + + # Load dataset rows + rows = _load_dataset_rows(session, dataset) + if not rows: + raise ValueError(f"Dataset {dataset.id} has no rows") + + logger.info( + f"[submit_assessment_batch] Building JSONL | " + f"run_id={eval_run.id} | rows={len(rows)} | " + f"provider={config_blob.completion.provider}" + ) + + # Determine provider and build params + completion = config_blob.completion + provider_name = completion.provider or "openai" + + params = dict(completion.params) + if output_schema: + params["output_schema"] = output_schema + + # Determine the base provider (openai or google) + base_provider = _NATIVE_PROVIDERS.get(provider_name, "openai") + + if base_provider == "openai": + mapped_params, warnings = map_kaapi_to_openai_params(params) + if warnings: + logger.info(f"[submit_assessment_batch] Mapper warnings: {warnings}") + + jsonl_data = build_openai_jsonl( + rows=rows, + text_columns=text_columns, + attachments=attachments, + prompt_template=prompt_template, + openai_params=mapped_params, + ) + + # Get OpenAI client and submit + from app.utils import get_openai_client + + openai_client = get_openai_client( + session=session, + org_id=organization_id, + project_id=project_id, + ) + provider = OpenAIBatchProvider(client=openai_client) + + batch_config = { + "endpoint": "/v1/responses", + "description": f"Assessment: {eval_run.run_name}", + "completion_window": "24h", + } + + batch_job = start_batch_job( + session=session, + provider=provider, + provider_name="openai", + job_type=BatchJobType.ASSESSMENT, + organization_id=organization_id, + project_id=project_id, + jsonl_data=jsonl_data, + config=batch_config, + ) + + elif base_provider == "google": + mapped_params, warnings = map_kaapi_to_google_params(params) + if warnings: + logger.info(f"[submit_assessment_batch] Mapper warnings: {warnings}") + + jsonl_data = build_google_jsonl( + rows=rows, + text_columns=text_columns, + attachments=attachments, + prompt_template=prompt_template, + google_params=mapped_params, + ) + + # Get Gemini client and submit + from app.core.batch import GeminiBatchProvider + from app.core.batch.client import GeminiClient + + gemini_client = GeminiClient.from_credentials( + session=session, + org_id=organization_id, + project_id=project_id, + ) + provider = GeminiBatchProvider( + client=gemini_client.client, + model=f"models/{mapped_params.get('model', 'gemini-2.5-pro')}", + ) + + batch_config = { + "display_name": f"assessment-{eval_run.run_name}", + "model": f"models/{mapped_params.get('model', 'gemini-2.5-pro')}", + } + + batch_job = start_batch_job( + session=session, + provider=provider, + provider_name="google", + job_type=BatchJobType.ASSESSMENT, + organization_id=organization_id, + project_id=project_id, + jsonl_data=jsonl_data, + config=batch_config, + ) + + else: + raise ValueError( + f"Unsupported provider for assessment batches: {provider_name}" + ) + + logger.info( + f"[submit_assessment_batch] Submitted batch | " + f"run_id={eval_run.id} | batch_job_id={batch_job.id} | " + f"provider={base_provider} | items={len(jsonl_data)}" + ) + + return batch_job diff --git a/backend/app/assessment/cron.py b/backend/app/assessment/cron.py new file mode 100644 index 000000000..7f1d0807b --- /dev/null +++ b/backend/app/assessment/cron.py @@ -0,0 +1,189 @@ +"""Cron processing functions for assessment evaluations.""" + +import logging +from typing import Any + +from sqlmodel import Session, select + +from app.assessment.crud import ( + get_assessment_runs_for_manager, + recompute_assessment_status, + update_assessment_run_status, +) +from app.assessment.events import assessment_event_broker +from app.assessment.processing import check_and_process_assessment +from app.assessment.models import Assessment +from app.models.evaluation import EvaluationRun +from app.utils import APIResponse + +logger = logging.getLogger(__name__) + + +def _log_config_progress(result: dict[str, Any], eval_run: EvaluationRun) -> None: + """Emit explicit config-level logs for grouped assessment experiments.""" + action = result.get("action") + if action not in {"processed", "failed"}: + return + + logger.info( + "[poll_all_pending_assessment_evaluations] " + "Experiment config update | " + f"experiment={eval_run.run_name} | " + f"assessment_id={eval_run.assessment_id} | " + f"run_id={eval_run.id} | " + f"config_id={eval_run.config_id} | " + f"config_version={eval_run.config_version} | " + f"action={action} | " + f"status={result.get('current_status')} | " + f"provider_status={result.get('provider_status')}" + ) + + +def _build_callback_payload( + assessment: Assessment, + eval_run: EvaluationRun, + result: dict[str, Any], +) -> dict[str, Any]: + """Build minimal SSE payload for assessment invalidation.""" + return APIResponse.success_response( + data={ + "type": "assessment.child_status_changed", + "assessment_id": assessment.id, + "assessment_status": assessment.status, + "run": { + "id": eval_run.id, + "config_id": str(eval_run.config_id) if eval_run.config_id else None, + "config_version": eval_run.config_version, + "status": result.get("current_status"), + "error": result.get("error"), + "updated_at": eval_run.updated_at.isoformat() + if eval_run.updated_at + else None, + }, + } + ).model_dump() + + +async def poll_all_pending_assessment_evaluations( + session: Session, +) -> dict[str, Any]: + """Poll all non-terminal parent assessments and their active child runs.""" + statement = select(Assessment).where( + Assessment.status.in_(("pending", "processing")), + ) + pending_assessments = list(session.exec(statement).all()) + + if not pending_assessments: + return { + "total": 0, + "processed": 0, + "failed": 0, + "still_processing": 0, + "details": [], + } + + logger.info( + "[poll_all_pending_assessment_evaluations] " + f"Found {len(pending_assessments)} active assessments" + ) + + all_results: list[dict[str, Any]] = [] + processed = 0 + failed = 0 + still_processing = 0 + + for assessment in pending_assessments: + runs = get_assessment_runs_for_manager(session=session, assessment=assessment) + active_runs = [ + run for run in runs if run.status in {"processing", "in_progress"} + ] + + if not active_runs: + refreshed = recompute_assessment_status( + session=session, assessment_id=assessment.id + ) + if refreshed.status in {"pending", "processing"}: + still_processing += 1 + continue + + for eval_run in active_runs: + try: + result = await check_and_process_assessment( + eval_run=eval_run, + session=session, + ) + all_results.append(result) + _log_config_progress(result, eval_run) + + if result["action"] in {"processed", "failed"}: + refreshed_assessment = session.get(Assessment, assessment.id) + if refreshed_assessment: + assessment_event_broker.publish( + _build_callback_payload( + refreshed_assessment, + eval_run, + result, + ) + ) + + if result["action"] == "processed": + processed += 1 + elif result["action"] == "failed": + failed += 1 + else: + still_processing += 1 + + except Exception as e: + logger.error( + "[poll_all_pending_assessment_evaluations] " + f"Failed run {eval_run.id} | " + f"experiment={eval_run.run_name} | " + f"assessment_id={eval_run.assessment_id} | " + f"config_id={eval_run.config_id} | " + f"config_version={eval_run.config_version} | " + f"error={e}", + exc_info=True, + ) + update_assessment_run_status( + session=session, + eval_run=eval_run, + status="failed", + error_message=f"Poll failed: {str(e)}", + ) + refreshed_assessment = recompute_assessment_status( + session=session, assessment_id=assessment.id + ) + failure_result = { + "assessment_id": eval_run.assessment_id, + "run_id": eval_run.id, + "run_name": eval_run.run_name, + "config_id": str(eval_run.config_id) + if eval_run.config_id + else None, + "config_version": eval_run.config_version, + "action": "failed", + "error": str(e), + "current_status": "failed", + } + all_results.append(failure_result) + assessment_event_broker.publish( + _build_callback_payload( + refreshed_assessment, + eval_run, + failure_result, + ) + ) + failed += 1 + + logger.info( + "[poll_all_pending_assessment_evaluations] Summary | " + f"processed={processed} | failed={failed} | still_processing={still_processing}" + ) + + return { + "total": len(pending_assessments), + "processed": processed, + "failed": failed, + "still_processing": still_processing, + "details": all_results, + } diff --git a/backend/app/assessment/crud.py b/backend/app/assessment/crud.py new file mode 100644 index 000000000..b340b4596 --- /dev/null +++ b/backend/app/assessment/crud.py @@ -0,0 +1,337 @@ +"""Assessment CRUD — thin wrappers around EvaluationRun for type='assessment'.""" + +import logging +from typing import Any +from uuid import UUID + +from sqlmodel import Session, select + +from app.core.util import now +from app.assessment.models import Assessment +from app.models.evaluation import EvaluationRun + +logger = logging.getLogger(__name__) + +ASSESSMENT_TYPE = "assessment" + + +def create_assessment( + session: Session, + experiment_name: str, + dataset_id: int, + dataset_name: str, + organization_id: int, + project_id: int, + total_runs: int, +) -> Assessment: + """Create a parent assessment manager row.""" + assessment = Assessment( + experiment_name=experiment_name, + dataset_id=dataset_id, + dataset_name=dataset_name, + status="pending", + total_runs=total_runs, + pending_runs=total_runs, + processing_runs=0, + completed_runs=0, + failed_runs=0, + run_stats=[], + organization_id=organization_id, + project_id=project_id, + inserted_at=now(), + updated_at=now(), + ) + + session.add(assessment) + try: + session.commit() + session.refresh(assessment) + except Exception as e: + session.rollback() + logger.error(f"[create_assessment] Failed: {e}", exc_info=True) + raise + + logger.info( + f"[create_assessment] Created assessment id={assessment.id} | " + f"experiment={experiment_name} | total_runs={total_runs}" + ) + return assessment + + +def get_assessment_runs_for_manager( + session: Session, + assessment: Assessment, +) -> list[EvaluationRun]: + """List child evaluation runs for a parent assessment row.""" + statement = ( + select(EvaluationRun) + .where(EvaluationRun.assessment_id == assessment.id) + .where(EvaluationRun.type == ASSESSMENT_TYPE) + .order_by(EvaluationRun.inserted_at.desc()) + ) + return list(session.exec(statement).all()) + + +def create_assessment_run( + session: Session, + run_name: str, + dataset_name: str, + dataset_id: int, + assessment_id: int | None, + config_id: UUID, + config_version: int, + organization_id: int, + project_id: int, + assessment_config: dict[str, Any] | None = None, +) -> EvaluationRun: + """Create an assessment evaluation run record. + + Re-uses EvaluationRun with type='assessment' and stores assessment-specific + config in the score JSONB field under 'assessment_config'. + """ + eval_run = EvaluationRun( + run_name=run_name, + dataset_name=dataset_name, + dataset_id=dataset_id, + assessment_id=assessment_id, + type=ASSESSMENT_TYPE, + config_id=config_id, + config_version=config_version, + status="pending", + score={"assessment_config": assessment_config} if assessment_config else None, + organization_id=organization_id, + project_id=project_id, + inserted_at=now(), + updated_at=now(), + ) + + session.add(eval_run) + try: + session.commit() + session.refresh(eval_run) + except Exception as e: + session.rollback() + logger.error(f"[create_assessment_run] Failed: {e}", exc_info=True) + raise + + logger.info( + f"[create_assessment_run] Created run id={eval_run.id} | " + f"name={run_name} | config_id={config_id} v{config_version}" + ) + return eval_run + + +def list_assessments( + session: Session, + organization_id: int, + project_id: int, + limit: int = 50, + offset: int = 0, +) -> list[Assessment]: + """List parent assessment manager rows.""" + statement = ( + select(Assessment) + .where(Assessment.organization_id == organization_id) + .where(Assessment.project_id == project_id) + .order_by(Assessment.inserted_at.desc()) + .limit(limit) + .offset(offset) + ) + return list(session.exec(statement).all()) + + +def get_assessment_by_id( + session: Session, + assessment_id: int, + organization_id: int, + project_id: int, +) -> Assessment | None: + """Get a specific parent assessment manager row.""" + statement = ( + select(Assessment) + .where(Assessment.id == assessment_id) + .where(Assessment.organization_id == organization_id) + .where(Assessment.project_id == project_id) + ) + return session.exec(statement).first() + + +def list_assessment_runs( + session: Session, + organization_id: int, + project_id: int, + assessment_id: int | None = None, + limit: int = 50, + offset: int = 0, +) -> list[EvaluationRun]: + """List child assessment evaluation runs by traversing from assessments.""" + if assessment_id is not None: + assessment = get_assessment_by_id( + session=session, + assessment_id=assessment_id, + organization_id=organization_id, + project_id=project_id, + ) + if not assessment: + return [] + runs = get_assessment_runs_for_manager(session=session, assessment=assessment) + return runs[offset : offset + limit] + + assessments = list_assessments( + session=session, + organization_id=organization_id, + project_id=project_id, + limit=limit, + offset=offset, + ) + runs: list[EvaluationRun] = [] + for assessment in assessments: + runs.extend( + get_assessment_runs_for_manager(session=session, assessment=assessment) + ) + return runs + + +def get_assessment_run_by_id( + session: Session, + run_id: int, + organization_id: int, + project_id: int, +) -> EvaluationRun | None: + """Get a specific assessment evaluation run by ID.""" + statement = ( + select(EvaluationRun) + .where(EvaluationRun.id == run_id) + .where(EvaluationRun.organization_id == organization_id) + .where(EvaluationRun.project_id == project_id) + .where(EvaluationRun.type == ASSESSMENT_TYPE) + ) + return session.exec(statement).first() + + +def _determine_assessment_status( + total_runs: int, + pending_runs: int, + processing_runs: int, + completed_runs: int, + failed_runs: int, +) -> str: + """Compute parent assessment status from child evaluation runs.""" + if total_runs == 0: + return "pending" + if completed_runs == total_runs: + return "completed" + if failed_runs == total_runs: + return "failed" + if ( + completed_runs > 0 + and failed_runs > 0 + and pending_runs == 0 + and processing_runs == 0 + ): + return "completed_with_errors" + if pending_runs > 0 and pending_runs == total_runs: + return "pending" + return "processing" + + +def recompute_assessment_status( + session: Session, + assessment_id: int, +) -> Assessment: + """Recompute cached parent assessment counters from child runs.""" + assessment = session.get(Assessment, assessment_id) + if not assessment: + raise ValueError(f"Assessment {assessment_id} not found") + + statement = ( + select(EvaluationRun) + .where(EvaluationRun.assessment_id == assessment_id) + .where(EvaluationRun.type == ASSESSMENT_TYPE) + .order_by(EvaluationRun.id.asc()) + ) + runs = list(session.exec(statement).all()) + + pending_runs = sum(1 for run in runs if run.status == "pending") + processing_runs = sum( + 1 for run in runs if run.status in {"processing", "in_progress"} + ) + completed_runs = sum(1 for run in runs if run.status == "completed") + failed_runs = sum(1 for run in runs if run.status == "failed") + total_runs = len(runs) + + assessment.total_runs = total_runs + assessment.pending_runs = pending_runs + assessment.processing_runs = processing_runs + assessment.completed_runs = completed_runs + assessment.failed_runs = failed_runs + assessment.status = _determine_assessment_status( + total_runs=total_runs, + pending_runs=pending_runs, + processing_runs=processing_runs, + completed_runs=completed_runs, + failed_runs=failed_runs, + ) + assessment.error_message = ( + f"{failed_runs} of {total_runs} evaluation run(s) failed" + if failed_runs > 0 + else None + ) + assessment.run_stats = [ + { + "run_id": run.id, + "config_id": str(run.config_id) if run.config_id else None, + "config_version": run.config_version, + "status": run.status, + "total_items": run.total_items, + "error_message": run.error_message, + "updated_at": run.updated_at.isoformat() if run.updated_at else None, + } + for run in runs + ] + assessment.updated_at = now() + + session.add(assessment) + try: + session.commit() + session.refresh(assessment) + except Exception as e: + session.rollback() + logger.error(f"[recompute_assessment_status] Failed: {e}", exc_info=True) + raise + + return assessment + + +def update_assessment_run_status( + session: Session, + eval_run: EvaluationRun, + status: str, + error_message: str | None = None, + batch_job_id: int | None = None, + total_items: int | None = None, + object_store_url: str | None = None, +) -> EvaluationRun: + """Update an assessment run's status and optional fields.""" + eval_run.status = status + eval_run.updated_at = now() + + if error_message is not None: + eval_run.error_message = error_message + if batch_job_id is not None: + eval_run.batch_job_id = batch_job_id + if total_items is not None: + eval_run.total_items = total_items + if object_store_url is not None: + eval_run.object_store_url = object_store_url + + session.add(eval_run) + try: + session.commit() + session.refresh(eval_run) + except Exception as e: + session.rollback() + logger.error(f"[update_assessment_run_status] Failed: {e}", exc_info=True) + raise + + return eval_run diff --git a/backend/app/assessment/dataset.py b/backend/app/assessment/dataset.py new file mode 100644 index 000000000..c3c1f56ec --- /dev/null +++ b/backend/app/assessment/dataset.py @@ -0,0 +1,170 @@ +"""Dataset management service for assessment evaluations (CSV + Excel). + +Upload stores files directly to object store as-is (no column validation, +no format conversion). Row count is computed for metadata. +""" + +import csv +import io +import logging + +from fastapi import HTTPException +from sqlmodel import Session + +from app.core.cloud import get_cloud_storage +from app.core.storage_utils import generate_timestamped_filename, upload_to_object_store +from app.crud.evaluations import create_evaluation_dataset +from app.models.evaluation import EvaluationDataset +from app.services.evaluations.validators import sanitize_dataset_name + +logger = logging.getLogger(__name__) + +_MIME_TYPES = { + ".csv": "text/csv", + ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ".xls": "application/vnd.ms-excel", +} + + +def _upload_file_to_object_store( + session: Session, + project_id: int, + file_content: bytes, + file_ext: str, + dataset_name: str, +) -> str | None: + """Upload the raw file to object store, preserving original format.""" + extension = file_ext.lstrip(".") + filename = generate_timestamped_filename(dataset_name, extension=extension) + content_type = _MIME_TYPES.get(file_ext, "application/octet-stream") + + try: + storage = get_cloud_storage(session=session, project_id=project_id) + return upload_to_object_store( + storage=storage, + content=file_content, + filename=filename, + subdirectory="datasets", + content_type=content_type, + ) + except Exception as e: + logger.warning( + f"[_upload_file_to_object_store] Failed to upload | {e}", + exc_info=True, + ) + return None + + +def _count_csv_rows(content: bytes) -> int: + """Count data rows in a CSV file (excluding header).""" + try: + for encoding in ("utf-8-sig", "utf-8", "latin-1"): + try: + text = content.decode(encoding) + break + except (UnicodeDecodeError, ValueError): + continue + else: + text = content.decode("utf-8", errors="replace") + + reader = csv.reader(io.StringIO(text)) + next(reader, None) + return sum(1 for row in reader if any(cell.strip() for cell in row)) + except Exception as e: + logger.warning(f"[_count_csv_rows] Failed to count rows | {e}") + return 0 + + +def _count_excel_rows(content: bytes) -> int: + """Count data rows in an Excel file (excluding header).""" + try: + import openpyxl + + wb = openpyxl.load_workbook(io.BytesIO(content), read_only=True, data_only=True) + ws = wb.active + if ws is None: + return 0 + + rows_iter = ws.iter_rows(values_only=True) + header = next(rows_iter, None) + if header is None: + wb.close() + return 0 + + count = sum( + 1 for row in rows_iter if row and any(cell is not None for cell in row) + ) + wb.close() + return count + except Exception as e: + logger.warning(f"[_count_excel_rows] Failed to count rows | {e}") + return 0 + + +def _count_rows(content: bytes, file_ext: str) -> int: + """Count data rows in a file (CSV or Excel), excluding the header.""" + if file_ext in (".xlsx", ".xls"): + return _count_excel_rows(content) + return _count_csv_rows(content) + + +def upload_dataset( + session: Session, + file_content: bytes, + file_ext: str, + dataset_name: str, + description: str | None, + organization_id: int, + project_id: int, +) -> EvaluationDataset: + """Upload a dataset file directly to object store and record metadata.""" + original_name = dataset_name + try: + dataset_name = sanitize_dataset_name(dataset_name) + except ValueError as e: + raise HTTPException(status_code=422, detail=f"Invalid dataset name: {str(e)}") + + if original_name != dataset_name: + logger.info( + f"[upload_dataset] Dataset name sanitized | '{original_name}' -> '{dataset_name}'" + ) + + row_count = _count_rows(file_content, file_ext) + + logger.info( + f"[upload_dataset] Uploading dataset | dataset={dataset_name} | " + f"file_type={file_ext} | rows={row_count} | " + f"org_id={organization_id} | project_id={project_id}" + ) + + object_store_url = _upload_file_to_object_store( + session=session, + project_id=project_id, + file_content=file_content, + file_ext=file_ext, + dataset_name=dataset_name, + ) + + metadata = { + "file_extension": file_ext, + "file_size_bytes": len(file_content), + "total_items_count": row_count, + } + + dataset = create_evaluation_dataset( + session=session, + name=dataset_name, + description=description, + dataset_metadata=metadata, + object_store_url=object_store_url, + langfuse_dataset_id=None, + organization_id=organization_id, + project_id=project_id, + ) + + logger.info( + f"[upload_dataset] Created dataset record | " + f"id={dataset.id} | name={dataset_name} | rows={row_count}" + ) + + return dataset diff --git a/backend/app/assessment/events.py b/backend/app/assessment/events.py new file mode 100644 index 000000000..832f5c7d1 --- /dev/null +++ b/backend/app/assessment/events.py @@ -0,0 +1,51 @@ +"""Assessment SSE event broadcaster.""" + +import asyncio +import json +import logging +from collections.abc import AsyncIterator + +logger = logging.getLogger(__name__) + + +class AssessmentEventBroker: + def __init__(self) -> None: + self._subscribers: set[asyncio.Queue[dict]] = set() + + async def subscribe(self) -> AsyncIterator[str]: + queue: asyncio.Queue[dict] = asyncio.Queue() + self._subscribers.add(queue) + logger.info( + "[subscribe] New SSE subscriber | total=%d", len(self._subscribers) + ) + try: + yield "event: ready\ndata: {}\n\n" + while True: + try: + payload = await asyncio.wait_for(queue.get(), timeout=15) + except asyncio.TimeoutError: + yield ": keep-alive\n\n" + continue + event_type = payload.get("type", "message") + yield f"event: {event_type}\ndata: {json.dumps(payload)}\n\n" + finally: + self._subscribers.discard(queue) + logger.info( + "[subscribe] SSE subscriber disconnected | remaining=%d", + len(self._subscribers), + ) + + def publish(self, payload: dict) -> None: + if not self._subscribers: + logger.debug("[publish] No subscribers, event dropped | type=%s", payload.get("type")) + return + logger.info( + "[publish] Broadcasting event | type=%s | subscribers=%d", + payload.get("type"), + len(self._subscribers), + ) + for queue in list(self._subscribers): + queue.put_nowait(payload) + + +assessment_event_broker = AssessmentEventBroker() diff --git a/backend/app/assessment/mappers.py b/backend/app/assessment/mappers.py new file mode 100644 index 000000000..79cb65aa9 --- /dev/null +++ b/backend/app/assessment/mappers.py @@ -0,0 +1,218 @@ +import logging +import unicodedata + +import litellm +from google.genai import _transformers as genai_transformers + +logger = logging.getLogger(__name__) + + +def normalize_llm_text(text: str) -> str: + if not isinstance(text, str) or not text: + return text + + text = text.replace("\\n", "\n") + text = text.replace("\\t", "\t") + text = text.replace("\\r", "\r") + text = text.replace('\\"', '"') + text = text.replace("\\\\", "\\") + + text = unicodedata.normalize("NFC", text) + + return text + + +def _ensure_openai_strict_schema(schema: dict) -> dict: + """Recursively add additionalProperties: false for OpenAI strict JSON schema validation.""" + normalized = dict(schema) + + if normalized.get("type") == "object": + normalized["additionalProperties"] = False + + if "properties" in normalized: + normalized["properties"] = { + key: _ensure_openai_strict_schema(value) if isinstance(value, dict) else value + for key, value in normalized["properties"].items() + } + + items = normalized.get("items") + if isinstance(items, dict): + normalized["items"] = _ensure_openai_strict_schema(items) + + return normalized + + +def _strip_additional_properties(schema: dict) -> dict: + """Recursively strip additionalProperties — unsupported by Google GenAI.""" + schema = dict(schema) + schema.pop("additionalProperties", None) + + if "properties" in schema: + schema["properties"] = { + k: _strip_additional_properties(v) if isinstance(v, dict) else v + for k, v in schema["properties"].items() + } + + if "items" in schema and isinstance(schema["items"], dict): + schema["items"] = _strip_additional_properties(schema["items"]) + + return schema + + +def _convert_json_schema_to_google(schema: dict) -> dict: + """Convert a JSON Schema dict to Google GenAI's OpenAPI-style schema. + + Strips unsupported fields, then normalizes the schema through the Gemini SDK + so enum/type values match Gemini's expected OpenAPI-flavored shape. + """ + normalized_schema = _strip_additional_properties(schema) + converted = genai_transformers.t_schema(None, normalized_schema) + google_schema = ( + converted.model_dump(mode="json", exclude_none=True) + if converted is not None + else normalized_schema + ) + + if "properties" in google_schema and "propertyOrdering" not in google_schema: + google_schema["propertyOrdering"] = list( + normalized_schema.get("required", []) + ) or list(google_schema["properties"].keys()) + + return google_schema + + +def map_kaapi_to_openai_params(kaapi_params: dict) -> tuple[dict, list[str]]: + """Map Kaapi-abstracted parameters to OpenAI API parameters. + + Returns: + Tuple of (OpenAI API params dict, list of warning strings) + """ + openai_params: dict = {} + warnings: list[str] = [] + + model = kaapi_params.get("model") + reasoning = kaapi_params.get("reasoning") + effort = kaapi_params.get("effort") or reasoning + summary = kaapi_params.get("summary") + temperature = kaapi_params.get("temperature") + top_p = kaapi_params.get("top_p") + + instructions = normalize_llm_text(kaapi_params.get("instructions")) + knowledge_base_ids = kaapi_params.get("knowledge_base_ids") + max_num_results = kaapi_params.get("max_num_results") + response_format = kaapi_params.get("response_format") + output_schema = kaapi_params.get("output_schema") + + support_reasoning = litellm.supports_reasoning(model=f"openai/{model}") + + # max_output_tokens is intentionally omitted for batch assessment — + # Indic feedback responses can be long and a stored token limit would truncate them. + + if support_reasoning: + reasoning_payload: dict[str, object] = {} + if effort is not None: + reasoning_payload["effort"] = effort + if summary is not None: + reasoning_payload["summary"] = None if summary == "null" else summary + if reasoning_payload: + openai_params["reasoning"] = reasoning_payload + if temperature is not None: + warnings.append( + "Parameter 'temperature' was suppressed because the selected model " + "supports reasoning, and temperature is ignored when reasoning is enabled." + ) + if top_p is not None: + warnings.append( + "Parameter 'top_p' was suppressed because the selected model " + "supports reasoning, and top_p is ignored when reasoning is enabled." + ) + else: + if effort is not None or summary is not None: + warnings.append( + "Parameters 'effort'/'summary' were suppressed because the selected model " + "does not support reasoning." + ) + if temperature is not None: + openai_params["temperature"] = temperature + if top_p is not None: + openai_params["top_p"] = top_p + + if model: + openai_params["model"] = model + + if instructions: + openai_params["instructions"] = instructions + + if output_schema is not None: + openai_params["text"] = { + "format": { + "type": "json_schema", + "name": "output", + "strict": True, + "schema": _ensure_openai_strict_schema(output_schema), + } + } + elif response_format and response_format != "text": + openai_params["text"] = {"format": {"type": response_format}} + + if knowledge_base_ids: + openai_params["tools"] = [ + { + "type": "file_search", + "vector_store_ids": knowledge_base_ids, + "max_num_results": max_num_results or 20, + } + ] + + return openai_params, warnings + + +def map_kaapi_to_google_params(kaapi_params: dict) -> tuple[dict, list[str]]: + """Map Kaapi-abstracted parameters to Google AI (Gemini) API parameters. + + Returns: + Tuple of (Google AI params dict, list of warning strings) + """ + google_params: dict = {} + warnings: list[str] = [] + + model = kaapi_params.get("model") + if not model: + return {}, ["Missing required 'model' parameter"] + + google_params["model"] = model + + instructions = normalize_llm_text(kaapi_params.get("instructions")) + if instructions: + google_params["instructions"] = instructions + + temperature = kaapi_params.get("temperature") + if temperature is not None: + google_params["temperature"] = temperature + + top_p = kaapi_params.get("top_p") + if top_p is not None: + google_params["top_p"] = top_p + + max_output_tokens = kaapi_params.get("max_output_tokens") + if max_output_tokens is not None: + google_params["max_output_tokens"] = max_output_tokens + + thinking_level = kaapi_params.get("thinking_level") + if thinking_level: + google_params["thinking_config"] = {"thinking_level": thinking_level} + + reasoning = kaapi_params.get("reasoning") + if reasoning: + google_params["reasoning"] = reasoning + + output_schema = kaapi_params.get("output_schema") + if output_schema is not None: + google_params["output_schema"] = _convert_json_schema_to_google(output_schema) + + if kaapi_params.get("knowledge_base_ids"): + warnings.append( + "Parameter 'knowledge_base_ids' is not supported by Google AI and was ignored." + ) + + return google_params, warnings diff --git a/backend/app/assessment/models.py b/backend/app/assessment/models.py new file mode 100644 index 000000000..6fad58e10 --- /dev/null +++ b/backend/app/assessment/models.py @@ -0,0 +1,296 @@ +"""Assessment models — DB table, Pydantic schemas, and LLM param wrappers.""" + +from datetime import datetime +from typing import Any, Literal +from uuid import UUID + +from pydantic import BaseModel, Field +from sqlalchemy import Column, Index, Text +from sqlalchemy.dialects.postgresql import JSONB +from sqlmodel import Field as SQLField +from sqlmodel import SQLModel + +from app.core.util import now +from app.models.llm.request import TextLLMParams + + +# ── Database model ────────────────────────────────────────────── + + +class Assessment(SQLModel, table=True): + """Manager table for multi-config assessment evaluations.""" + + __tablename__ = "assessment" + __table_args__ = ( + Index("idx_assessment_status_org", "status", "organization_id"), + Index("idx_assessment_status_project", "status", "project_id"), + ) + + id: int = SQLField( + default=None, + primary_key=True, + sa_column_kwargs={"comment": "Unique identifier for the assessment"}, + ) + experiment_name: str = SQLField( + index=True, + description="Experiment name shared by child config runs", + sa_column_kwargs={"comment": "Experiment name shared by child config runs"}, + ) + dataset_id: int = SQLField( + foreign_key="evaluation_dataset.id", + nullable=False, + ondelete="CASCADE", + sa_column_kwargs={"comment": "Reference to the evaluation dataset"}, + ) + dataset_name: str = SQLField( + nullable=False, + description="Name of the dataset used by this assessment", + sa_column_kwargs={"comment": "Name of the dataset used by this assessment"}, + ) + status: str = SQLField( + default="pending", + description="Overall assessment status across all child evaluation runs", + sa_column_kwargs={ + "comment": "Overall assessment status across all child evaluation runs" + }, + ) + total_runs: int = SQLField( + default=0, + nullable=False, + sa_column_kwargs={"comment": "Total number of child evaluation runs"}, + ) + pending_runs: int = SQLField( + default=0, + nullable=False, + sa_column_kwargs={"comment": "Number of child runs in pending state"}, + ) + processing_runs: int = SQLField( + default=0, + nullable=False, + sa_column_kwargs={"comment": "Number of child runs in processing state"}, + ) + completed_runs: int = SQLField( + default=0, + nullable=False, + sa_column_kwargs={"comment": "Number of child runs in completed state"}, + ) + failed_runs: int = SQLField( + default=0, + nullable=False, + sa_column_kwargs={"comment": "Number of child runs in failed state"}, + ) + run_stats: list[dict[str, Any]] = SQLField( + default_factory=list, + sa_column=Column( + JSONB, + nullable=False, + comment="Cached status snapshot for child evaluation runs", + ), + description="Cached status snapshot for child evaluation runs", + ) + error_message: str | None = SQLField( + default=None, + sa_column=Column( + Text, + nullable=True, + comment="Aggregated error message for child run failures", + ), + description="Aggregated error message for child run failures", + ) + callback_url: str | None = SQLField( + default=None, + nullable=True, + sa_column_kwargs={ + "comment": "Optional frontend callback URL for status updates" + }, + ) + organization_id: int = SQLField( + foreign_key="organization.id", + nullable=False, + ondelete="CASCADE", + sa_column_kwargs={"comment": "Reference to the organization"}, + ) + project_id: int = SQLField( + foreign_key="project.id", + nullable=False, + ondelete="CASCADE", + sa_column_kwargs={"comment": "Reference to the project"}, + ) + inserted_at: datetime = SQLField( + default_factory=now, + nullable=False, + sa_column_kwargs={"comment": "Timestamp when the assessment was created"}, + ) + updated_at: datetime = SQLField( + default_factory=now, + nullable=False, + sa_column_kwargs={"comment": "Timestamp when the assessment was last updated"}, + ) + + +class AssessmentPublic(BaseModel): + """Public model for assessment manager rows.""" + + id: int + experiment_name: str + dataset_id: int + dataset_name: str + status: str + total_runs: int + pending_runs: int + processing_runs: int + completed_runs: int + failed_runs: int + run_stats: list[dict[str, Any]] + error_message: str | None + organization_id: int + project_id: int + inserted_at: datetime + updated_at: datetime + + +# ── Extended LLM params ────────────────────────────────────────── + + +class AssessmentTextLLMParams(TextLLMParams): + """TextLLMParams extended with response_format and output_schema for assessments.""" + + response_format: Literal["text", "json_object"] = Field( + default="text", + description="Response format: 'text' or 'json_object'", + ) + output_schema: dict[str, Any] | None = Field( + default=None, + description="JSON Schema for structured output", + ) + + +# ── Attachment / config references ─────────────────────────────── + + +class AssessmentAttachment(BaseModel): + """Attachment column configuration.""" + + column: str = Field(..., description="Column name containing the attachment data") + type: str = Field(..., description="Attachment type: 'image' or 'pdf'") + format: str = Field(..., description="Data format: 'url' or 'base64'") + + +class AssessmentConfigRef(BaseModel): + """Reference to a stored config version.""" + + config_id: UUID = Field(..., description="Stored config UUID") + config_version: int = Field(..., ge=1, description="Config version number") + + +# ── Request model ──────────────────────────────────────────────── + + +class AssessmentCreate(BaseModel): + """Request body for creating an assessment evaluation.""" + + experiment_name: str = Field( + ..., min_length=1, description="Name for this evaluation experiment" + ) + dataset_id: int = Field(..., description="ID of the uploaded dataset") + prompt_template: str | None = Field( + None, + description=( + "Prompt template with {column} placeholders. " + "If null, all text columns are concatenated." + ), + ) + text_columns: list[str] = Field( + default_factory=list, description="Column names mapped as text input" + ) + attachments: list[AssessmentAttachment] = Field( + default_factory=list, description="Attachment column configurations" + ) + output_schema: dict[str, Any] | None = Field( + None, description="JSON Schema for structured output" + ) + configs: list[AssessmentConfigRef] = Field( + ..., min_length=1, max_length=4, description="Config versions to evaluate" + ) + + +# ── Response models ────────────────────────────────────────────── + + +class AssessmentRunSummary(BaseModel): + """Summary of a single evaluation run created for one config.""" + + run_id: int + assessment_id: int | None = None + config_id: str + config_version: int + status: str + + +class AssessmentResponse(BaseModel): + """Response after submitting an assessment evaluation.""" + + assessment_id: int + experiment_name: str + dataset_id: int + dataset_name: str + num_configs: int + runs: list[AssessmentRunSummary] + + +class AssessmentRunPublic(BaseModel): + """Public view of an assessment evaluation run.""" + + id: int + assessment_id: int | None + run_name: str + dataset_name: str + dataset_id: int + config_id: UUID | None + config_version: int | None + status: str + total_items: int + error_message: str | None + organization_id: int + project_id: int + assessment_config: dict[str, Any] | None = Field( + None, description="Assessment-specific configuration (prompt, columns, schema)" + ) + inserted_at: datetime + updated_at: datetime + + +class AssessmentExportRow(BaseModel): + """Flattened assessment result row for CSV/XLSX export.""" + + assessment_id: int + experiment_name: str + dataset_id: int | None + dataset_name: str | None + run_id: int + run_name: str + run_status: str + config_id: UUID | None + config_version: int | None + row_id: str + result_status: str + input_data: dict[str, str] | None = None + output: str | None = None + error: str | None = None + response_id: str | None = None + input_tokens: int | None = None + output_tokens: int | None = None + total_tokens: int | None = None + updated_at: datetime + + +class AssessmentDatasetResponse(BaseModel): + """Response model for assessment dataset.""" + + dataset_id: int + dataset_name: str + description: str | None = None + total_items: int = 0 + file_extension: str | None = None + object_store_url: str | None = None + signed_url: str | None = None diff --git a/backend/app/assessment/processing.py b/backend/app/assessment/processing.py new file mode 100644 index 000000000..66e47c929 --- /dev/null +++ b/backend/app/assessment/processing.py @@ -0,0 +1,483 @@ +"""Assessment batch result processing and polling. + +Handles downloading completed batch results, parsing them, and updating +the assessment run status. Follows the same pattern as text evaluation +processing but adapted for multi-provider (OpenAI + Google) support. +""" + +import json +import logging +from typing import Any + +from fastapi import HTTPException +from sqlmodel import Session + +from app.assessment.crud import ( + recompute_assessment_status, + update_assessment_run_status, +) +from app.assessment.events import assessment_event_broker +from app.core.batch import ( + BATCH_KEY, + OpenAIBatchProvider, + GeminiBatchProvider, + download_batch_results, + poll_batch_status, + upload_batch_results_to_object_store, +) +from app.core.batch.base import BatchProvider +from app.core.batch.client import GeminiClient +from app.core.batch.gemini import BatchJobState, extract_text_from_response_dict +from app.crud.job import get_batch_job +from app.models.evaluation import EvaluationRun +from app.utils import get_openai_client + +logger = logging.getLogger(__name__) + + +def _sanitize_json_output(raw: str) -> str: + """Escape control characters inside JSON string values that the model emitted literally. + + Strict structured-output mode should prevent this, but long Indic-language + responses sometimes contain literal newlines / tabs inside string values, + making the JSON unparseable. This function walks the raw text once and + replaces any bare control characters found while inside a JSON string with + their JSON escape equivalents, producing valid JSON without touching the + surrounding structure. + """ + result: list[str] = [] + in_string = False + escape_next = False + + for ch in raw: + if escape_next: + result.append(ch) + escape_next = False + elif ch == "\\": + result.append(ch) + escape_next = True + elif ch == '"': + in_string = not in_string + result.append(ch) + elif in_string and ch == "\n": + result.append("\\n") + elif in_string and ch == "\r": + result.append("\\r") + elif in_string and ch == "\t": + result.append("\\t") + else: + result.append(ch) + + return "".join(result) + + +def _get_batch_provider( + session: Session, + provider_name: str, + organization_id: int, + project_id: int, +) -> BatchProvider: + """Get the appropriate batch provider instance.""" + if provider_name in ("openai", "openai-native"): + openai_client = get_openai_client( + session=session, + org_id=organization_id, + project_id=project_id, + ) + return OpenAIBatchProvider(client=openai_client) + + if provider_name in ("google", "google-native"): + gemini_client = GeminiClient.from_credentials( + session=session, + org_id=organization_id, + project_id=project_id, + ) + return GeminiBatchProvider(client=gemini_client.client) + + raise ValueError(f"Unsupported provider for assessment polling: {provider_name}") + + +def parse_assessment_output( + raw_results: list[dict[str, Any]], + provider_name: str, +) -> list[dict[str, Any]]: + """Parse batch results into assessment output format. + + Args: + raw_results: Raw results from batch provider + provider_name: Provider name ('openai' or 'google') + + Returns: + List of parsed results with row_id, output text, usage, etc. + """ + results = [] + + for result in raw_results: + row_id = result.get(BATCH_KEY) or result.get("key", "unknown") + + if provider_name in ("openai", "openai-native"): + response = result.get("response", {}) + response_status = response.get("status_code") + response_body = result.get("response", {}).get("body", {}) + error = result.get("error") + + if error: + results.append( + { + "row_id": row_id, + "output": None, + "error": error.get("message", str(error)), + "usage": None, + } + ) + continue + + if response_status and response_status >= 400: + response_error = response_body.get("error", {}) + results.append( + { + "row_id": row_id, + "output": None, + "error": response_error.get( + "message", f"Request failed with status {response_status}" + ), + "usage": None, + "response_id": response_body.get("id"), + } + ) + continue + + # Prefer the convenience field when present; otherwise concatenate all + # output_text fragments so structured JSON isn't truncated mid-object. + generated_text = response_body.get("output_text") or "" + + if not isinstance(generated_text, str) or not generated_text: + output = response_body.get("output", "") + text_chunks: list[str] = [] + + if isinstance(output, list): + for item in output: + if isinstance(item, dict) and item.get("type") == "message": + for content in item.get("content", []): + if ( + isinstance(content, dict) + and content.get("type") == "output_text" + ): + text = content.get("text") + if isinstance(text, str) and text: + text_chunks.append(text) + generated_text = "".join(text_chunks) + elif isinstance(output, str): + generated_text = output + + if generated_text: + try: + generated_text = json.dumps( + json.loads(generated_text), ensure_ascii=False + ) + except (json.JSONDecodeError, TypeError): + # Model emitted literal control characters inside string values. + # Sanitize and retry once. + try: + sanitized = _sanitize_json_output(generated_text) + generated_text = json.dumps( + json.loads(sanitized), ensure_ascii=False + ) + except (json.JSONDecodeError, TypeError): + pass + + results.append( + { + "row_id": row_id, + "output": generated_text, + "error": None if generated_text else "Empty response output", + "usage": response_body.get("usage"), + "response_id": response_body.get("id"), + } + ) + + elif provider_name in ("google", "google-native"): + response = result.get("response") + error = result.get("error") + + if error: + results.append( + { + "row_id": row_id, + "output": None, + "error": str(error), + "usage": None, + } + ) + continue + + if response: + text = extract_text_from_response_dict(response) + results.append( + { + "row_id": row_id, + "output": text, + "error": None, + "usage": None, + } + ) + else: + results.append( + { + "row_id": row_id, + "output": None, + "error": "Empty response", + "usage": None, + } + ) + + logger.info( + f"[parse_assessment_output] Parsed {len(results)} results | " + f"provider={provider_name}" + ) + return results + + +async def check_and_process_assessment( + eval_run: EvaluationRun, + session: Session, +) -> dict[str, Any]: + """Check assessment batch status and process if completed. + + Args: + eval_run: EvaluationRun with type='assessment' + session: Database session + + Returns: + Dict with status information + """ + log_prefix = f"[assessment={eval_run.id}]" + previous_status = eval_run.status + + try: + if not eval_run.batch_job_id: + raise ValueError(f"Assessment run {eval_run.id} has no batch_job_id") + + batch_job = get_batch_job(session=session, batch_job_id=eval_run.batch_job_id) + if not batch_job: + raise ValueError(f"BatchJob {eval_run.batch_job_id} not found") + + # Get provider and poll status + provider = _get_batch_provider( + session=session, + provider_name=batch_job.provider, + organization_id=eval_run.organization_id, + project_id=eval_run.project_id, + ) + status_result = poll_batch_status( + session=session, + provider=provider, + batch_job=batch_job, + ) + session.refresh(batch_job) + + provider_status = batch_job.provider_status + + if ( + provider_status == "completed" + or provider_status == BatchJobState.SUCCEEDED.value + ): + if not batch_job.provider_output_file_id: + request_counts = status_result.get("request_counts") or {} + error_file_id = status_result.get("error_file_id") + failed_count = request_counts.get("failed", 0) + completed_count = request_counts.get("completed", 0) + total_count = request_counts.get("total", 0) + + if error_file_id and failed_count > 0 and completed_count == 0: + error_msg = ( + f"Batch completed with {failed_count} failed request(s)" + f" and no successful outputs" + ) + if total_count: + error_msg += f" out of {total_count}" + error_msg += f" (error_file_id: {error_file_id})" + + update_assessment_run_status( + session=session, + eval_run=eval_run, + status="failed", + error_message=error_msg, + ) + if eval_run.assessment_id is not None: + recompute_assessment_status( + session=session, assessment_id=eval_run.assessment_id + ) + + return { + "run_id": eval_run.id, + "assessment_id": eval_run.assessment_id, + "run_name": eval_run.run_name, + "previous_status": previous_status, + "current_status": "failed", + "provider_status": provider_status, + "action": "failed", + "error": error_msg, + } + + logger.info( + f"{log_prefix} Batch completed but output file is not ready yet | " + f"batch_job_id={batch_job.id} | provider_status={provider_status}" + ) + return { + "run_id": eval_run.id, + "assessment_id": eval_run.assessment_id, + "run_name": eval_run.run_name, + "previous_status": previous_status, + "current_status": eval_run.status, + "provider_status": provider_status, + "action": "no_change", + } + + # Emit SSE: results are being prepared + assessment_event_broker.publish({ + "type": "assessment.results_preparing", + "assessment_id": eval_run.assessment_id, + "run_id": eval_run.id, + "message": "Results are being prepared", + }) + + # Download and process results + raw_results = download_batch_results(provider=provider, batch_job=batch_job) + + # Upload raw results to object store + object_store_url = None + try: + object_store_url = upload_batch_results_to_object_store( + session=session, batch_job=batch_job, results=raw_results + ) + except Exception as e: + logger.warning(f"{log_prefix} Object store upload failed: {e}") + + # Parse results + parsed = parse_assessment_output(raw_results, batch_job.provider) + error_count = sum(1 for r in parsed if r.get("error")) + success_count = sum(1 for r in parsed if not r.get("error")) + + # Update run status + error_msg = f"{error_count} item(s) failed" if error_count > 0 else None + run_status = ( + "failed" + if parsed and success_count == 0 and error_count > 0 + else "completed" + ) + + if not parsed: + run_status = "failed" + error_msg = "Batch completed but no valid results were produced" + + update_assessment_run_status( + session=session, + eval_run=eval_run, + status=run_status, + error_message=error_msg, + object_store_url=object_store_url, + ) + if eval_run.assessment_id is not None: + recompute_assessment_status( + session=session, assessment_id=eval_run.assessment_id + ) + + # Emit SSE: results are ready + assessment_event_broker.publish({ + "type": "assessment.results_ready", + "assessment_id": eval_run.assessment_id, + "run_id": eval_run.id, + "status": run_status, + "total_results": len(parsed), + "errors": error_count, + "message": "Results are ready", + }) + + return { + "run_id": eval_run.id, + "assessment_id": eval_run.assessment_id, + "run_name": eval_run.run_name, + "previous_status": previous_status, + "current_status": run_status, + "provider_status": provider_status, + "action": "processed" if run_status == "completed" else "failed", + "total_results": len(parsed), + "errors": error_count, + } + + elif provider_status in ( + "failed", + "expired", + "cancelled", + BatchJobState.FAILED.value, + BatchJobState.CANCELLED.value, + BatchJobState.EXPIRED.value, + ): + error_msg = batch_job.error_message or f"Batch {provider_status}" + update_assessment_run_status( + session=session, + eval_run=eval_run, + status="failed", + error_message=error_msg, + ) + if eval_run.assessment_id is not None: + recompute_assessment_status( + session=session, assessment_id=eval_run.assessment_id + ) + + return { + "run_id": eval_run.id, + "assessment_id": eval_run.assessment_id, + "run_name": eval_run.run_name, + "previous_status": previous_status, + "current_status": "failed", + "provider_status": provider_status, + "action": "failed", + "error": error_msg, + } + + else: + # Still processing + return { + "run_id": eval_run.id, + "assessment_id": eval_run.assessment_id, + "run_name": eval_run.run_name, + "previous_status": previous_status, + "current_status": eval_run.status, + "provider_status": provider_status, + "action": "no_change", + } + + except Exception as e: + logger.error( + f"{log_prefix} Error checking assessment: {e}", + exc_info=True, + ) + update_assessment_run_status( + session=session, + eval_run=eval_run, + status="failed", + error_message=f"Check failed: {str(e)}", + ) + if eval_run.assessment_id is not None: + recompute_assessment_status( + session=session, assessment_id=eval_run.assessment_id + ) + return { + "run_id": eval_run.id, + "assessment_id": eval_run.assessment_id, + "run_name": eval_run.run_name, + "previous_status": previous_status, + "current_status": "failed", + "provider_status": "unknown", + "action": "failed", + "error": str(e), + } + + +async def poll_all_pending_assessments(session: Session) -> dict[str, Any]: + """Backward-compatible wrapper for parent-first assessment polling.""" + from app.assessment.cron import poll_all_pending_assessment_evaluations + + return await poll_all_pending_assessment_evaluations(session=session) diff --git a/backend/app/assessment/routes.py b/backend/app/assessment/routes.py new file mode 100644 index 000000000..dc794cc10 --- /dev/null +++ b/backend/app/assessment/routes.py @@ -0,0 +1,654 @@ +"""Assessment API routes.""" + +import asyncio +import io +import logging +import zipfile +from typing import Any, Literal + +from fastapi import ( + APIRouter, + Depends, + File, + Form, + HTTPException, + Query, + UploadFile, +) +from fastapi.responses import StreamingResponse + +from app.api.deps import AuthContextDep, SessionDep +from app.api.permissions import Permission, require_permission +from app.assessment.crud import ( + get_assessment_by_id, + get_assessment_run_by_id, + list_assessment_runs, + list_assessments, +) +from app.assessment.dataset import upload_dataset as upload_assessment_dataset +from app.assessment.events import assessment_event_broker +from app.assessment.models import ( + AssessmentCreate, + AssessmentDatasetResponse, + AssessmentExportRow, + AssessmentPublic, + AssessmentResponse, + AssessmentRunPublic, +) +from app.assessment.service import ( + retry_assessment, + retry_assessment_run, + start_assessment, +) +from app.assessment.utils import ( + build_export_response, + build_json_export_rows, + load_export_rows_for_run, + serialize_export_rows, + sort_export_rows, +) +from app.assessment.utils.export import _safe_filename_part +from app.assessment.validators import validate_dataset_file +from app.core.cloud import get_cloud_storage +from app.core.storage_utils import generate_timestamped_filename +from app.crud.evaluations import get_dataset_by_id +from app.crud.evaluations import list_datasets as list_evaluation_datasets +from app.crud.evaluations.dataset import delete_dataset as delete_dataset_crud +from app.models.evaluation import EvaluationDataset, EvaluationRun +from app.utils import APIResponse, load_description + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/assessment", tags=["Assessment"]) + + +# ── Dataset routes ─────────────────────────────────────────────── + + +def _dataset_to_response( + dataset: EvaluationDataset, + signed_url: str | None = None, +) -> AssessmentDatasetResponse: + """Convert a dataset model to an AssessmentDatasetResponse.""" + metadata = dataset.dataset_metadata or {} + return AssessmentDatasetResponse( + dataset_id=dataset.id, + dataset_name=dataset.name, + description=dataset.description, + total_items=metadata.get("total_items_count", 0), + file_extension=metadata.get("file_extension"), + object_store_url=dataset.object_store_url, + signed_url=signed_url, + ) + + +@router.post( + "/datasets", + description=load_description("evaluation/upload_dataset.md"), + response_model=APIResponse[AssessmentDatasetResponse], + dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))], +) +async def upload_dataset( + session: SessionDep, + auth_context: AuthContextDep, + file: UploadFile = File( + ..., description="CSV or Excel file to upload as a dataset" + ), + dataset_name: str = Form(..., description="Name for the dataset"), + description: str | None = Form(None, description="Optional dataset description"), +) -> APIResponse[AssessmentDatasetResponse]: + """Upload an assessment dataset (any CSV/Excel file, no column requirements).""" + file_content, file_ext = await validate_dataset_file(file) + + dataset = upload_assessment_dataset( + session=session, + file_content=file_content, + file_ext=file_ext, + dataset_name=dataset_name, + description=description, + organization_id=auth_context.organization_.id, + project_id=auth_context.project_.id, + ) + + return APIResponse.success_response(data=_dataset_to_response(dataset)) + + +@router.get( + "/datasets", + description=load_description("evaluation/list_datasets.md"), + response_model=APIResponse[list[AssessmentDatasetResponse]], + dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))], +) +def list_datasets( + session: SessionDep, + auth_context: AuthContextDep, + limit: int = Query( + default=50, ge=1, le=100, description="Maximum number of datasets to return" + ), + offset: int = Query(default=0, ge=0, description="Number of datasets to skip"), +) -> APIResponse[list[AssessmentDatasetResponse]]: + """List assessment datasets.""" + datasets = list_evaluation_datasets( + session=session, + organization_id=auth_context.organization_.id, + project_id=auth_context.project_.id, + limit=limit, + offset=offset, + ) + + return APIResponse.success_response( + data=[_dataset_to_response(dataset) for dataset in datasets] + ) + + +@router.get( + "/datasets/{dataset_id}", + description=load_description("evaluation/get_dataset.md"), + response_model=APIResponse[AssessmentDatasetResponse], + dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))], +) +def get_dataset( + dataset_id: int, + session: SessionDep, + auth_context: AuthContextDep, + include_signed_url: bool = Query( + False, description="Include a signed URL for downloading the raw file from S3" + ), +) -> APIResponse[AssessmentDatasetResponse]: + """Get a specific assessment dataset.""" + dataset = get_dataset_by_id( + session=session, + dataset_id=dataset_id, + organization_id=auth_context.organization_.id, + project_id=auth_context.project_.id, + ) + + if not dataset: + raise HTTPException( + status_code=404, detail=f"Dataset {dataset_id} not found or not accessible" + ) + + signed_url = None + if include_signed_url and dataset.object_store_url: + storage = get_cloud_storage( + session=session, project_id=auth_context.project_.id + ) + signed_url = storage.get_signed_url(dataset.object_store_url) + + return APIResponse.success_response( + data=_dataset_to_response(dataset, signed_url=signed_url) + ) + + +@router.delete( + "/datasets/{dataset_id}", + description=load_description("evaluation/delete_dataset.md"), + response_model=APIResponse[dict], + dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))], +) +def delete_dataset( + dataset_id: int, + session: SessionDep, + auth_context: AuthContextDep, +) -> APIResponse[dict]: + """Delete an assessment dataset.""" + dataset = get_dataset_by_id( + session=session, + dataset_id=dataset_id, + organization_id=auth_context.organization_.id, + project_id=auth_context.project_.id, + ) + + if not dataset: + raise HTTPException( + status_code=404, detail=f"Dataset {dataset_id} not found or not accessible" + ) + + dataset_name = dataset.name + error = delete_dataset_crud(session=session, dataset=dataset) + if error: + raise HTTPException(status_code=400, detail=error) + + return APIResponse.success_response( + data={ + "message": f"Successfully deleted dataset '{dataset_name}' (id={dataset_id})", + "dataset_id": dataset_id, + } + ) + + +# ── Evaluation routes ──────────────────────────────────────────── + + +@router.post( + "/evaluations", + response_model=APIResponse[AssessmentResponse], + dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))], +) +def create_evaluation( + request: AssessmentCreate, + session: SessionDep, + auth_context: AuthContextDep, +) -> APIResponse[AssessmentResponse]: + """Submit an assessment evaluation run.""" + logger.info( + f"[create_evaluation] Assessment evaluation | " + f"experiment={request.experiment_name} | " + f"dataset_id={request.dataset_id} | " + f"configs={len(request.configs)}" + ) + + result = start_assessment( + session=session, + request=request, + organization_id=auth_context.organization_.id, + project_id=auth_context.project_.id, + ) + + return APIResponse.success_response(data=result) + + +@router.post( + "/assessments/{assessment_id}/retry", + response_model=APIResponse[AssessmentResponse], + dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))], +) +def retry_assessment_manager( + assessment_id: int, + session: SessionDep, + auth_context: AuthContextDep, +) -> APIResponse[AssessmentResponse]: + """Retry a parent assessment using the same dataset/config inputs.""" + assessment = get_assessment_by_id( + session=session, + assessment_id=assessment_id, + organization_id=auth_context.organization_.id, + project_id=auth_context.project_.id, + ) + if not assessment: + raise HTTPException( + status_code=404, + detail=f"Assessment {assessment_id} not found or not accessible", + ) + + result = retry_assessment( + session=session, + assessment=assessment, + organization_id=auth_context.organization_.id, + project_id=auth_context.project_.id, + ) + return APIResponse.success_response(data=result) + + +@router.post( + "/evaluations/{evaluation_id}/retry", + response_model=APIResponse[AssessmentResponse], + dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))], +) +def retry_assessment_evaluation( + evaluation_id: int, + session: SessionDep, + auth_context: AuthContextDep, +) -> APIResponse[AssessmentResponse]: + """Retry a single child assessment run using the same inputs.""" + run = get_assessment_run_by_id( + session=session, + run_id=evaluation_id, + organization_id=auth_context.organization_.id, + project_id=auth_context.project_.id, + ) + if not run: + raise HTTPException( + status_code=404, + detail=f"Assessment evaluation {evaluation_id} not found or not accessible", + ) + + result = retry_assessment_run( + session=session, + run=run, + organization_id=auth_context.organization_.id, + project_id=auth_context.project_.id, + ) + return APIResponse.success_response(data=result) + + +@router.get( + "/events", + dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))], + include_in_schema=False, +) +async def stream_assessment_events( + _session: SessionDep, + _auth_context: AuthContextDep, +) -> StreamingResponse: + """SSE stream for assessment invalidation events.""" + + async def event_stream(): + async for chunk in assessment_event_broker.subscribe(): + yield chunk + await asyncio.sleep(0) + + return StreamingResponse( + event_stream(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache, no-transform", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", + }, + ) + + +@router.get( + "/assessments", + response_model=APIResponse[list[AssessmentPublic]], + dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))], +) +def list_assessment_managers( + session: SessionDep, + auth_context: AuthContextDep, + limit: int = Query(default=50, ge=1, le=100), + offset: int = Query(default=0, ge=0), +) -> APIResponse[list[AssessmentPublic]]: + """List parent assessment manager rows.""" + assessments = list_assessments( + session=session, + organization_id=auth_context.organization_.id, + project_id=auth_context.project_.id, + limit=limit, + offset=offset, + ) + + return APIResponse.success_response( + data=[ + AssessmentPublic( + id=a.id, + experiment_name=a.experiment_name, + dataset_id=a.dataset_id, + dataset_name=a.dataset_name, + status=a.status, + total_runs=a.total_runs, + pending_runs=a.pending_runs, + processing_runs=a.processing_runs, + completed_runs=a.completed_runs, + failed_runs=a.failed_runs, + run_stats=a.run_stats, + error_message=a.error_message, + organization_id=a.organization_id, + project_id=a.project_id, + inserted_at=a.inserted_at, + updated_at=a.updated_at, + ) + for a in assessments + ] + ) + + +@router.get( + "/assessments/{assessment_id}", + response_model=APIResponse[AssessmentPublic], + dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))], +) +def get_assessment_manager( + assessment_id: int, + session: SessionDep, + auth_context: AuthContextDep, +) -> APIResponse[AssessmentPublic]: + """Get a specific parent assessment manager row.""" + assessment = get_assessment_by_id( + session=session, + assessment_id=assessment_id, + organization_id=auth_context.organization_.id, + project_id=auth_context.project_.id, + ) + + if not assessment: + raise HTTPException( + status_code=404, + detail=f"Assessment {assessment_id} not found or not accessible", + ) + + return APIResponse.success_response( + data=AssessmentPublic( + id=assessment.id, + experiment_name=assessment.experiment_name, + dataset_id=assessment.dataset_id, + dataset_name=assessment.dataset_name, + status=assessment.status, + total_runs=assessment.total_runs, + pending_runs=assessment.pending_runs, + processing_runs=assessment.processing_runs, + completed_runs=assessment.completed_runs, + failed_runs=assessment.failed_runs, + run_stats=assessment.run_stats, + error_message=assessment.error_message, + organization_id=assessment.organization_id, + project_id=assessment.project_id, + inserted_at=assessment.inserted_at, + updated_at=assessment.updated_at, + ) + ) + + +@router.get( + "/assessments/{assessment_id}/results", + response_model=APIResponse[list[dict[str, Any]]], + dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))], +) +def export_assessment_results( + assessment_id: int, + session: SessionDep, + auth_context: AuthContextDep, + export_format: Literal["json", "csv", "xlsx"] = Query(default="json"), +) -> APIResponse[list[dict[str, Any]]] | StreamingResponse: + """Return child-run results. For CSV/XLSX with multiple runs, returns a ZIP.""" + assessment = get_assessment_by_id( + session=session, + assessment_id=assessment_id, + organization_id=auth_context.organization_.id, + project_id=auth_context.project_.id, + ) + if not assessment: + raise HTTPException( + status_code=404, + detail=f"Assessment {assessment_id} not found or not accessible", + ) + + runs = list_assessment_runs( + session=session, + organization_id=auth_context.organization_.id, + project_id=auth_context.project_.id, + assessment_id=assessment_id, + limit=max(assessment.total_runs, 1), + offset=0, + ) + + # Build per-run export data + runs_with_rows: list[tuple[EvaluationRun, list[AssessmentExportRow]]] = [] + all_rows: list[AssessmentExportRow] = [] + for run in runs: + rows = load_export_rows_for_run(session=session, run=run, assessment=assessment) + if rows: + runs_with_rows.append((run, sort_export_rows(rows))) + all_rows.extend(rows) + + all_rows = sort_export_rows(all_rows) + + # JSON: return flat list + if export_format == "json": + return APIResponse.success_response(data=build_json_export_rows(all_rows)) + + # Single run: return a single file directly + if len(runs_with_rows) <= 1: + return build_export_response( + export_rows=all_rows, + export_format=export_format, + base_name=f"{assessment.experiment_name}_assessment_{assessment.id}_results", + ) + + # Multiple runs: ZIP with one file per run + zip_buffer = io.BytesIO() + with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf: + for run, rows in runs_with_rows: + config_label = ( + f"config_v{run.config_version}" + if run.config_version + else f"run_{run.id}" + ) + config_id_short = (run.config_id or "")[:8] + file_base = _safe_filename_part(f"{config_label}_{config_id_short}") + file_bytes, _ = serialize_export_rows(rows, export_format) + zf.writestr(f"{file_base}.{export_format}", file_bytes) + + zip_buffer.seek(0) + zip_filename = generate_timestamped_filename( + _safe_filename_part(f"{assessment.experiment_name}_assessment_{assessment.id}"), + extension="zip", + ) + return StreamingResponse( + zip_buffer, + media_type="application/zip", + headers={"Content-Disposition": f'attachment; filename="{zip_filename}"'}, + ) + + +@router.get( + "/evaluations/{evaluation_id}/results", + response_model=APIResponse[list[dict[str, Any]]], + dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))], +) +def export_assessment_run_results( + evaluation_id: int, + session: SessionDep, + auth_context: AuthContextDep, + export_format: Literal["json", "csv", "xlsx"] = Query(default="json"), +) -> APIResponse[list[dict[str, Any]]] | StreamingResponse: + """Return flattened results for a single child assessment run.""" + run = get_assessment_run_by_id( + session=session, + run_id=evaluation_id, + organization_id=auth_context.organization_.id, + project_id=auth_context.project_.id, + ) + if not run: + raise HTTPException( + status_code=404, + detail=f"Assessment evaluation {evaluation_id} not found or not accessible", + ) + + assessment = None + if run.assessment_id is not None: + assessment = get_assessment_by_id( + session=session, + assessment_id=run.assessment_id, + organization_id=auth_context.organization_.id, + project_id=auth_context.project_.id, + ) + + export_rows = sort_export_rows( + load_export_rows_for_run( + session=session, + run=run, + assessment=assessment, + ) + ) + + if export_format != "json": + return build_export_response( + export_rows=export_rows, + export_format=export_format, + base_name=f"{run.run_name}_evaluation_{run.id}_results", + ) + + return APIResponse.success_response(data=build_json_export_rows(export_rows)) + + +@router.get( + "/evaluations", + response_model=APIResponse[list[AssessmentRunPublic]], + dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))], +) +def list_evaluations( + session: SessionDep, + auth_context: AuthContextDep, + assessment_id: int | None = Query(default=None, ge=1), + limit: int = Query(default=50, ge=1, le=100), + offset: int = Query(default=0, ge=0), +) -> APIResponse[list[AssessmentRunPublic]]: + """List assessment evaluation runs.""" + runs = list_assessment_runs( + session=session, + organization_id=auth_context.organization_.id, + project_id=auth_context.project_.id, + assessment_id=assessment_id, + limit=limit, + offset=offset, + ) + + return APIResponse.success_response( + data=[ + AssessmentRunPublic( + id=r.id, + assessment_id=r.assessment_id, + run_name=r.run_name, + dataset_name=r.dataset_name, + dataset_id=r.dataset_id, + config_id=r.config_id, + config_version=r.config_version, + status=r.status, + total_items=r.total_items, + error_message=r.error_message, + organization_id=r.organization_id, + project_id=r.project_id, + assessment_config=(r.score or {}).get("assessment_config"), + inserted_at=r.inserted_at, + updated_at=r.updated_at, + ) + for r in runs + ] + ) + + +@router.get( + "/evaluations/{evaluation_id}", + response_model=APIResponse[AssessmentRunPublic], + dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))], +) +def get_evaluation( + evaluation_id: int, + session: SessionDep, + auth_context: AuthContextDep, +) -> APIResponse[AssessmentRunPublic]: + """Get a specific assessment evaluation run.""" + run = get_assessment_run_by_id( + session=session, + run_id=evaluation_id, + organization_id=auth_context.organization_.id, + project_id=auth_context.project_.id, + ) + + if not run: + raise HTTPException( + status_code=404, + detail=f"Assessment evaluation {evaluation_id} not found or not accessible", + ) + + return APIResponse.success_response( + data=AssessmentRunPublic( + id=run.id, + assessment_id=run.assessment_id, + run_name=run.run_name, + dataset_name=run.dataset_name, + dataset_id=run.dataset_id, + config_id=run.config_id, + config_version=run.config_version, + status=run.status, + total_items=run.total_items, + error_message=run.error_message, + organization_id=run.organization_id, + project_id=run.project_id, + assessment_config=(run.score or {}).get("assessment_config"), + inserted_at=run.inserted_at, + updated_at=run.updated_at, + ) + ) diff --git a/backend/app/assessment/service.py b/backend/app/assessment/service.py new file mode 100644 index 000000000..ebb84d132 --- /dev/null +++ b/backend/app/assessment/service.py @@ -0,0 +1,271 @@ +"""Assessment evaluation orchestration service.""" + +import logging +from typing import Any +from uuid import UUID + +from fastapi import HTTPException +from sqlmodel import Session + +from app.assessment.batch import _resolve_config, submit_assessment_batch +from app.assessment.crud import ( + create_assessment, + create_assessment_run, + get_assessment_runs_for_manager, + recompute_assessment_status, + update_assessment_run_status, +) +from app.assessment.models import ( + AssessmentAttachment, + AssessmentConfigRef, + AssessmentCreate, + AssessmentResponse, + AssessmentRunSummary, +) +from app.crud.evaluations import get_dataset_by_id +from app.assessment.models import Assessment +from app.models.evaluation import EvaluationRun + +logger = logging.getLogger(__name__) + + +def _build_retry_request( + *, + experiment_name: str, + dataset_id: int, + runs: list[EvaluationRun], +) -> AssessmentCreate: + if not runs: + raise HTTPException(status_code=400, detail="No assessment runs found to retry") + + first_run = runs[0] + assessment_config = (first_run.score or {}).get("assessment_config") + if not isinstance(assessment_config, dict): + raise HTTPException( + status_code=400, + detail="Assessment configuration is missing for retry", + ) + + attachments = assessment_config.get("attachments") or [] + configs: list[AssessmentConfigRef] = [] + for run in runs: + if not run.config_id or run.config_version is None: + raise HTTPException( + status_code=400, + detail=f"Config reference is missing for run {run.id}", + ) + configs.append( + AssessmentConfigRef( + config_id=UUID(str(run.config_id)), + config_version=run.config_version, + ) + ) + + return AssessmentCreate( + experiment_name=experiment_name, + dataset_id=dataset_id, + prompt_template=assessment_config.get("prompt_template"), + text_columns=list(assessment_config.get("text_columns") or []), + attachments=[AssessmentAttachment.model_validate(item) for item in attachments], + output_schema=assessment_config.get("output_schema"), + configs=configs, + ) + + +def start_assessment( + session: Session, + request: AssessmentCreate, + organization_id: int, + project_id: int, +) -> AssessmentResponse: + """Start an assessment evaluation. + + Validates the dataset, resolves each config, creates one EvaluationRun per config, + and kicks off batch processing for each. + + Args: + session: Database session + request: Validated request body + organization_id: Organization ID + project_id: Project ID + + Returns: + AssessmentResponse with created run summaries + + Raises: + HTTPException: If dataset not found or configs invalid + """ + logger.info( + f"[start_assessment] Starting | " + f"experiment={request.experiment_name} | " + f"dataset_id={request.dataset_id} | " + f"configs={len(request.configs)} | " + f"org_id={organization_id}" + ) + + # 1. Validate dataset + dataset = get_dataset_by_id( + session=session, + dataset_id=request.dataset_id, + organization_id=organization_id, + project_id=project_id, + ) + if not dataset: + raise HTTPException( + status_code=404, + detail=f"Dataset {request.dataset_id} not found or not accessible", + ) + + # 2. Build assessment-specific config to store with each run + assessment_config: dict[str, Any] = { + "prompt_template": request.prompt_template, + "text_columns": request.text_columns, + "attachments": [a.model_dump() for a in request.attachments], + } + if request.output_schema: + assessment_config["output_schema"] = request.output_schema + + # 3. Validate all configs first before creating any runs + resolved_configs = [] + for cfg in request.configs: + config_blob, error = _resolve_config( + session=session, + config_id=cfg.config_id, + config_version=cfg.config_version, + project_id=project_id, + ) + if error or config_blob is None: + raise HTTPException( + status_code=400, + detail=f"Failed to resolve config {cfg.config_id} v{cfg.config_version}: {error}", + ) + resolved_configs.append((cfg, config_blob)) + + # 4. Create parent assessment manager row + assessment = create_assessment( + session=session, + experiment_name=request.experiment_name, + dataset_id=request.dataset_id, + dataset_name=dataset.name, + organization_id=organization_id, + project_id=project_id, + total_runs=len(resolved_configs), + ) + + # 5. Create one EvaluationRun per config and submit batches + runs: list[EvaluationRun] = [] + for cfg, config_blob in resolved_configs: + run = create_assessment_run( + session=session, + run_name=request.experiment_name, + dataset_name=dataset.name, + dataset_id=request.dataset_id, + assessment_id=assessment.id, + config_id=cfg.config_id, + config_version=cfg.config_version, + organization_id=organization_id, + project_id=project_id, + assessment_config=assessment_config, + ) + + # Submit batch for this run + try: + batch_job = submit_assessment_batch( + session=session, + eval_run=run, + dataset=dataset, + config_blob=config_blob, + assessment_config=assessment_config, + organization_id=organization_id, + project_id=project_id, + ) + + run = update_assessment_run_status( + session=session, + eval_run=run, + status="processing", + batch_job_id=batch_job.id, + total_items=batch_job.total_items, + ) + recompute_assessment_status(session=session, assessment_id=assessment.id) + + except Exception as e: + logger.error( + f"[start_assessment] Failed to submit batch for run {run.id}: {e}", + exc_info=True, + ) + run = update_assessment_run_status( + session=session, + eval_run=run, + status="failed", + error_message=f"Batch submission failed: {str(e)}", + ) + recompute_assessment_status(session=session, assessment_id=assessment.id) + + runs.append(run) + + recompute_assessment_status(session=session, assessment_id=assessment.id) + + logger.info( + f"[start_assessment] Created assessment {assessment.id} with {len(runs)} runs | " + f"run_ids={[r.id for r in runs]}" + ) + + return AssessmentResponse( + assessment_id=assessment.id, + experiment_name=request.experiment_name, + dataset_id=request.dataset_id, + dataset_name=dataset.name, + num_configs=len(runs), + runs=[ + AssessmentRunSummary( + run_id=r.id, + assessment_id=r.assessment_id, + config_id=str(r.config_id), + config_version=r.config_version, + status=r.status, + ) + for r in runs + ], + ) + + +def retry_assessment( + session: Session, + assessment: Assessment, + organization_id: int, + project_id: int, +) -> AssessmentResponse: + """Create a new assessment run using the same parent assessment inputs.""" + runs = get_assessment_runs_for_manager(session=session, assessment=assessment) + request = _build_retry_request( + experiment_name=assessment.experiment_name, + dataset_id=assessment.dataset_id, + runs=runs, + ) + return start_assessment( + session=session, + request=request, + organization_id=organization_id, + project_id=project_id, + ) + + +def retry_assessment_run( + session: Session, + run: EvaluationRun, + organization_id: int, + project_id: int, +) -> AssessmentResponse: + """Create a new assessment using the same inputs as a single child run.""" + request = _build_retry_request( + experiment_name=run.run_name, + dataset_id=run.dataset_id, + runs=[run], + ) + return start_assessment( + session=session, + request=request, + organization_id=organization_id, + project_id=project_id, + ) diff --git a/backend/app/assessment/utils/__init__.py b/backend/app/assessment/utils/__init__.py new file mode 100644 index 000000000..78e127cf3 --- /dev/null +++ b/backend/app/assessment/utils/__init__.py @@ -0,0 +1,23 @@ +"""Assessment utility functions.""" + +from app.assessment.utils.export import ( + build_export_response, + build_json_export_rows, + load_export_rows_for_run, + serialize_export_rows, + sort_export_rows, +) +from app.assessment.utils.parsing import ( + parse_stored_results, + usage_totals, +) + +__all__ = [ + "build_export_response", + "build_json_export_rows", + "load_export_rows_for_run", + "parse_stored_results", + "serialize_export_rows", + "sort_export_rows", + "usage_totals", +] diff --git a/backend/app/assessment/utils/export.py b/backend/app/assessment/utils/export.py new file mode 100644 index 000000000..8f7800384 --- /dev/null +++ b/backend/app/assessment/utils/export.py @@ -0,0 +1,432 @@ +"""Export utilities for assessment results (CSV, XLSX, JSON).""" + +import csv +import io +import json +import logging +import re +from typing import Any, Literal + +from fastapi import HTTPException +from fastapi.responses import StreamingResponse + +from app.assessment.batch import _load_dataset_rows +from app.assessment.models import AssessmentExportRow +from app.assessment.processing import parse_assessment_output +from app.assessment.utils.parsing import parse_stored_results, usage_totals +from app.core.cloud import get_cloud_storage +from app.core.storage_utils import generate_timestamped_filename +from app.crud.job import get_batch_job +from app.assessment.models import Assessment +from app.models.batch_job import BatchJob +from app.models.evaluation import EvaluationDataset, EvaluationRun + +logger = logging.getLogger(__name__) + + +def _safe_filename_part(value: str) -> str: + """Build a filesystem-safe filename component.""" + sanitized = re.sub(r"[^A-Za-z0-9._-]+", "_", value).strip("._") + return sanitized or "assessment_results" + + +def _expand_input_columns( + row_payload: list[dict[str, Any]], +) -> tuple[list[dict[str, Any]], list[str]]: + """Expand ``input_data`` dict into separate input columns. + + Uses the original column names from the dataset (no prefix). + + Returns: + (expanded_rows with input_data replaced by individual columns, + ordered list of input column names) + """ + input_keys: list[str] = [] + seen_keys: dict[str, None] = {} + + for row in row_payload: + input_data = row.get("input_data") + if isinstance(input_data, dict): + for k in input_data: + if k not in seen_keys: + seen_keys[k] = None + input_keys.append(k) + + if not input_keys: + for row in row_payload: + row.pop("input_data", None) + return row_payload, [] + + expanded: list[dict[str, Any]] = [] + for row in row_payload: + input_data = row.pop("input_data", None) or {} + new_row = {} + for k in input_keys: + new_row[k] = input_data.get(k) + new_row.update(row) + expanded.append(new_row) + + return expanded, list(input_keys) + + +def _drop_empty_columns( + rows: list[dict[str, Any]], + fieldnames: list[str], +) -> tuple[list[dict[str, Any]], list[str]]: + """Remove columns where every row has a null or empty-string value.""" + non_empty_fields: list[str] = [] + for field in fieldnames: + if any( + row.get(field) is not None and str(row.get(field, "")).strip() != "" + for row in rows + ): + non_empty_fields.append(field) + + if len(non_empty_fields) == len(fieldnames): + return rows, fieldnames + + pruned = [{k: row.get(k) for k in non_empty_fields} for row in rows] + return pruned, non_empty_fields + + +def _expand_output_columns( + row_payload: list[dict[str, Any]], +) -> tuple[list[dict[str, Any]], list[str]]: + """Expand the ``output`` field into separate columns when it contains valid JSON. + + Returns: + (expanded_rows, ordered_fieldnames) + """ + # First expand input columns + row_payload, input_col_names = _expand_input_columns(row_payload) + + base_fields = [ + f for f in AssessmentExportRow.model_fields.keys() + if f not in ("output", "input_data") + ] + + parsed_outputs: list[dict[str, Any] | None] = [] + output_keys: list[str] = [] + seen_keys: dict[str, None] = {} # ordered set + has_unparsed_output = False + + for row in row_payload: + raw = row.get("output") + if raw is None: + parsed_outputs.append(None) + continue + + if isinstance(raw, str): + try: + parsed = json.loads(raw) + except (json.JSONDecodeError, TypeError): + parsed = None + elif isinstance(raw, dict): + parsed = raw + else: + parsed = None + + if not isinstance(parsed, dict): + has_unparsed_output = True + parsed_outputs.append(None) + continue + + parsed_outputs.append(parsed) + for k in parsed: + if k not in seen_keys: + seen_keys[k] = None + output_keys.append(k) + + if not output_keys: + # Keep original layout with output as a single column + fieldnames = input_col_names + list(AssessmentExportRow.model_fields.keys()) + fieldnames = [f for f in fieldnames if f != "input_data"] + return row_payload, fieldnames + + # Build expanded rows + expanded: list[dict[str, Any]] = [] + for row, parsed in zip(row_payload, parsed_outputs, strict=True): + new_row = {k: v for k, v in row.items() if k != "output"} + if parsed: + for k in output_keys: + new_row[k] = parsed.get(k) + else: + for k in output_keys: + new_row[k] = None + if row.get("output") is not None: + new_row["output_raw"] = row.get("output") + expanded.append(new_row) + + # Build fieldnames: input columns + base fields + output columns + output_idx = base_fields.index("result_status") + 1 # after result_status + fieldnames = ( + input_col_names + + base_fields[:output_idx] + + output_keys + + base_fields[output_idx:] + ) + if has_unparsed_output: + fieldnames.insert( + len(input_col_names) + output_idx + len(output_keys), "output_raw" + ) + + return expanded, fieldnames + + +def serialize_export_rows( + export_rows: list[AssessmentExportRow], + export_format: Literal["json", "csv", "xlsx"], +) -> tuple[bytes, str]: + """Serialize export rows into the requested file format.""" + row_payload = [row.model_dump(mode="json") for row in export_rows] + + if export_format == "json": + expanded, _ = _expand_output_columns(row_payload) + return ( + json.dumps(expanded, ensure_ascii=False, indent=2).encode("utf-8"), + "application/json", + ) + + # For CSV/XLSX, expand output keys into separate columns + expanded, fieldnames = _expand_output_columns(row_payload) + + if export_format == "csv": + output = io.StringIO() + writer = csv.DictWriter(output, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(expanded) + return output.getvalue().encode("utf-8"), "text/csv" + + try: + import pandas as pd + except ImportError as exc: + raise HTTPException( + status_code=500, + detail="XLSX export requires pandas/openpyxl support in the backend runtime", + ) from exc + + # XLSX shows input columns + output columns only (no metadata fields). + metadata_fields = { + f for f in AssessmentExportRow.model_fields.keys() + if f not in ("output", "input_data") + } + excel_fields = [f for f in fieldnames if f not in metadata_fields] + if not excel_fields: + excel_fields = ["output"] + + # Drop columns where every row is null/empty + expanded, excel_fields = _drop_empty_columns(expanded, excel_fields) + + buf = io.BytesIO() + data_frame = pd.DataFrame(expanded, columns=excel_fields) + with pd.ExcelWriter(buf) as writer: + data_frame.to_excel(writer, index=False, sheet_name="results") + return ( + buf.getvalue(), + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ) + + +def build_json_export_rows( + export_rows: list[AssessmentExportRow], +) -> list[dict[str, Any]]: + """Return JSON rows with structured output expanded into top-level keys.""" + row_payload = [row.model_dump(mode="json") for row in export_rows] + expanded, _ = _expand_output_columns(row_payload) + return expanded + + +def build_export_response( + export_rows: list[AssessmentExportRow], + export_format: Literal["json", "csv", "xlsx"], + base_name: str, +) -> StreamingResponse: + """Return a file download response for assessment exports.""" + payload, media_type = serialize_export_rows(export_rows, export_format) + filename = generate_timestamped_filename( + _safe_filename_part(base_name), + extension=export_format, + ) + return StreamingResponse( + io.BytesIO(payload), + media_type=media_type, + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + +def _load_parsed_results_for_run( + session: Any, + run: EvaluationRun, + batch_job: BatchJob, +) -> list[dict[str, Any]] | None: + """Fetch and parse the stored batch results for a run. + + Tries object store first; falls back to downloading directly from the + batch provider (e.g. OpenAI file API) when the S3 copy is unavailable. + """ + # 1. Try object store (S3) + if run.object_store_url: + try: + storage = get_cloud_storage(session, project_id=run.project_id) + body = storage.stream(run.object_store_url) + raw_results = parse_stored_results(body.read().decode("utf-8")) + if raw_results: + return parse_assessment_output(raw_results, batch_job.provider) + logger.warning( + "[_load_parsed_results_for_run] S3 file was empty for run id=%s", + run.id, + ) + except Exception as exc: + logger.warning( + "[_load_parsed_results_for_run] S3 download failed for run id=%s: %s", + run.id, + exc, + ) + + # 2. Fallback: download directly from batch provider + if batch_job.provider_output_file_id: + try: + from app.assessment.processing import _get_batch_provider + from app.core.batch import download_batch_results + + provider = _get_batch_provider( + session=session, + provider_name=batch_job.provider, + organization_id=run.organization_id, + project_id=run.project_id, + ) + raw_results = download_batch_results(provider=provider, batch_job=batch_job) + return parse_assessment_output(raw_results, batch_job.provider) + except Exception as exc: + logger.error( + "[_load_parsed_results_for_run] Provider download also failed for run id=%s: %s", + run.id, + exc, + exc_info=True, + ) + + logger.warning( + "[_load_parsed_results_for_run] No results available for run id=%s " + "(object_store_url=%s, provider_output_file_id=%s)", + run.id, + run.object_store_url, + batch_job.provider_output_file_id, + ) + return None + + +def _load_dataset_rows_for_run( + session: Any, + run: EvaluationRun, +) -> list[dict[str, str]]: + """Load original dataset rows for input-output correlation. + + Returns an empty list if the dataset is not available. + """ + try: + dataset = session.get(EvaluationDataset, run.dataset_id) + if not dataset or not dataset.object_store_url: + logger.warning( + "[_load_dataset_rows_for_run] Dataset not available for run id=%s", + run.id, + ) + return [] + return _load_dataset_rows(session, dataset) + except Exception as exc: + logger.warning( + "[_load_dataset_rows_for_run] Failed to load dataset for run id=%s: %s", + run.id, + exc, + ) + return [] + + +def load_export_rows_for_run( + session: Any, + run: EvaluationRun, + assessment: Assessment | None = None, +) -> list[AssessmentExportRow]: + """Load flattened export rows for a single child assessment run.""" + if not run.batch_job_id: + logger.warning( + "[load_export_rows_for_run] No batch_job_id for run id=%s", run.id + ) + return [] + + batch_job = get_batch_job(session=session, batch_job_id=run.batch_job_id) + if not batch_job: + logger.warning( + "[load_export_rows_for_run] Missing batch job for run id=%s", + run.id, + ) + return [] + + parsed_results = _load_parsed_results_for_run( + session=session, + run=run, + batch_job=batch_job, + ) + if parsed_results is None: + return [] + + # Load original dataset rows for input correlation + dataset_rows = _load_dataset_rows_for_run(session, run) + + experiment_name = assessment.experiment_name if assessment else run.run_name + dataset_id = assessment.dataset_id if assessment else run.dataset_id + dataset_name = assessment.dataset_name if assessment else run.dataset_name + + export_rows: list[AssessmentExportRow] = [] + for item in parsed_results: + input_tokens, output_tokens, total_tokens = usage_totals(item.get("usage")) + + # Correlate with original input row via row_id (format: "row_{idx}") + input_data: dict[str, str] | None = None + row_id_str = str(item.get("row_id", "")) + if dataset_rows and row_id_str.startswith("row_"): + try: + row_idx = int(row_id_str.split("_", 1)[1]) + if 0 <= row_idx < len(dataset_rows): + input_data = dataset_rows[row_idx] + except (ValueError, IndexError): + pass + + export_rows.append( + AssessmentExportRow( + assessment_id=run.assessment_id or 0, + experiment_name=experiment_name, + dataset_id=dataset_id, + dataset_name=dataset_name, + run_id=run.id, + run_name=run.run_name, + run_status=run.status, + config_id=run.config_id, + config_version=run.config_version, + row_id=row_id_str, + result_status="failed" if item.get("error") else "passed", + input_data=input_data, + output=item.get("output"), + error=item.get("error"), + response_id=item.get("response_id"), + input_tokens=input_tokens, + output_tokens=output_tokens, + total_tokens=total_tokens, + updated_at=run.updated_at, + ) + ) + + return export_rows + + +def sort_export_rows( + export_rows: list[AssessmentExportRow], +) -> list[AssessmentExportRow]: + """Sort exported rows for stable downloads across runs/configs.""" + export_rows.sort( + key=lambda row: ( + row.config_version or 0, + row.row_id, + row.run_id, + ) + ) + return export_rows diff --git a/backend/app/assessment/utils/parsing.py b/backend/app/assessment/utils/parsing.py new file mode 100644 index 000000000..fbb064484 --- /dev/null +++ b/backend/app/assessment/utils/parsing.py @@ -0,0 +1,32 @@ +"""Parsing utilities for assessment batch results.""" + +import json +from typing import Any + + +def parse_stored_results(raw_content: str) -> list[dict[str, Any]]: + """Parse stored batch results from JSONL or JSON array.""" + content = raw_content.strip() + if not content: + return [] + + if content.startswith("["): + parsed = json.loads(content) + return parsed if isinstance(parsed, list) else [] + + return [json.loads(line) for line in content.splitlines() if line.strip()] + + +def usage_totals(usage: Any) -> tuple[int | None, int | None, int | None]: + """Extract common token totals from provider usage payloads.""" + if not isinstance(usage, dict): + return None, None, None + + input_tokens = usage.get("input_tokens") or usage.get("prompt_tokens") + output_tokens = usage.get("output_tokens") or usage.get("completion_tokens") + total_tokens = usage.get("total_tokens") + + if total_tokens is None and input_tokens is not None and output_tokens is not None: + total_tokens = input_tokens + output_tokens + + return input_tokens, output_tokens, total_tokens diff --git a/backend/app/assessment/validators.py b/backend/app/assessment/validators.py new file mode 100644 index 000000000..96d8718a6 --- /dev/null +++ b/backend/app/assessment/validators.py @@ -0,0 +1,67 @@ +"""Validation utilities for assessment dataset file uploads (CSV + Excel). + +Only validates file type and size — no column requirements. +""" + +import logging +from pathlib import Path + +from fastapi import HTTPException, UploadFile + +logger = logging.getLogger(__name__) + +MAX_FILE_SIZE = 10 * 1024 * 1024 # 10 MB + +ALLOWED_EXTENSIONS = {".csv", ".xlsx", ".xls"} +ALLOWED_MIME_TYPES = { + "text/csv", + "application/csv", + "text/plain", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.ms-excel", +} + + +async def validate_dataset_file(file: UploadFile) -> tuple[bytes, str]: + """Validate an uploaded dataset file (CSV or Excel). + + Only checks file type and size — does NOT inspect columns. + + Returns: + Tuple of (file content as bytes, file extension) + + Raises: + HTTPException: If validation fails + """ + if not file.filename: + raise HTTPException(status_code=422, detail="File must have a filename") + + file_ext = Path(file.filename).suffix.lower() + if file_ext not in ALLOWED_EXTENSIONS: + raise HTTPException( + status_code=422, + detail=f"Invalid file type. Allowed: CSV, XLSX, XLS. Got: {file_ext}", + ) + + content_type = file.content_type + if content_type not in ALLOWED_MIME_TYPES: + logger.warning( + f"[validate_dataset_file] Unexpected content type '{content_type}' " + f"for extension '{file_ext}', proceeding based on extension" + ) + + file.file.seek(0, 2) + file_size = file.file.tell() + file.file.seek(0) + + if file_size > MAX_FILE_SIZE: + raise HTTPException( + status_code=413, + detail=f"File too large. Maximum size: {MAX_FILE_SIZE / (1024 * 1024):.0f}MB", + ) + + if file_size == 0: + raise HTTPException(status_code=422, detail="Empty file uploaded") + + content = await file.read() + return content, file_ext diff --git a/backend/app/models/batch_job.py b/backend/app/models/batch_job.py index a01667831..426d44f59 100644 --- a/backend/app/models/batch_job.py +++ b/backend/app/models/batch_job.py @@ -16,6 +16,7 @@ class BatchJobType(str, Enum): STT_EVALUATION = "stt_evaluation" TTS_EVALUATION = "tts_evaluation" EMBEDDING = "embedding" + ASSESSMENT = "assessment" if TYPE_CHECKING: diff --git a/backend/app/models/evaluation.py b/backend/app/models/evaluation.py index d2d2beecc..ab667878b 100644 --- a/backend/app/models/evaluation.py +++ b/backend/app/models/evaluation.py @@ -255,6 +255,18 @@ class EvaluationRun(SQLModel, table=True): sa_column_kwargs={"comment": "Reference to the evaluation dataset"}, ) + # Assessment reference (set when run belongs to a parent Assessment) + assessment_id: int | None = SQLField( + default=None, + foreign_key="assessment.id", + nullable=True, + ondelete="SET NULL", + description="Reference to parent assessment manager row, if applicable", + sa_column_kwargs={ + "comment": "Reference to parent assessment manager row, if applicable" + }, + ) + # Batch job references batch_job_id: int | None = SQLField( default=None, @@ -391,6 +403,7 @@ class EvaluationRunPublic(SQLModel): config_id: UUID | None config_version: int | None dataset_id: int + assessment_id: int | None batch_job_id: int | None embedding_batch_job_id: int | None status: str diff --git a/backend/app/models/llm/request.py b/backend/app/models/llm/request.py index 1317c9ef3..2ea2a6d00 100644 --- a/backend/app/models/llm/request.py +++ b/backend/app/models/llm/request.py @@ -30,11 +30,30 @@ class TextLLMParams(SQLModel): default=None, description="Reasoning configuration or instructions", ) + effort: Literal["none", "minimal", "low", "medium", "high", "xhigh"] | None = Field( + default=None, + description="Model-specific reasoning effort setting for reasoning-capable models", + ) + summary: Literal["auto", "detailed", "concise", "null"] | None = Field( + default=None, + description="Model-specific reasoning summary preference", + ) temperature: float | None = Field( default=0.1, ge=0.0, le=2.0, ) + top_p: float | None = Field( + default=None, + ge=0.0, + le=1.0, + description="Nucleus sampling parameter", + ) + max_output_tokens: int | None = Field( + default=None, + ge=1, + description="Maximum tokens to generate in the response", + ) max_num_results: int | None = Field( default=None, ge=1, From a0c6315fd09c0591947dc34301bb16138925946e Mon Sep 17 00:00:00 2001 From: Prashant Vasudevan <71649489+vprashrex@users.noreply.github.com> Date: Sun, 5 Apr 2026 01:09:50 +0530 Subject: [PATCH 2/2] Refactor code structure for improved readability and maintainability --- .env.example | 4 + .gitignore | 3 + .../050_add_assessment_manager_table.py | 194 +++++++++++-- backend/app/api/main.py | 3 +- backend/app/api/permissions.py | 29 ++ backend/app/api/routes/cron.py | 2 +- backend/app/api/routes/features.py | 24 ++ backend/app/assessment/batch.py | 42 +-- backend/app/assessment/cron.py | 76 ++--- backend/app/assessment/crud.py | 270 ++++++++---------- backend/app/assessment/models.py | 143 +++++++++- backend/app/assessment/processing.py | 125 ++++---- backend/app/assessment/routes.py | 20 +- backend/app/assessment/service.py | 58 ++-- backend/app/assessment/utils/export.py | 17 +- backend/app/core/config.py | 5 + backend/app/core/feature_flags/__init__.py | 54 ++++ backend/app/core/feature_flags/client.py | 96 +++++++ backend/app/core/feature_flags/context.py | 21 ++ backend/app/main.py | 18 ++ backend/app/models/__init__.py | 2 + backend/app/models/evaluation.py | 13 - backend/pyproject.toml | 2 + backend/uv.lock | 236 +++++++++++++++ 24 files changed, 1107 insertions(+), 350 deletions(-) create mode 100644 backend/app/api/routes/features.py create mode 100644 backend/app/core/feature_flags/__init__.py create mode 100644 backend/app/core/feature_flags/client.py create mode 100644 backend/app/core/feature_flags/context.py diff --git a/.env.example b/.env.example index 98ac7d108..771fb3d8c 100644 --- a/.env.example +++ b/.env.example @@ -84,3 +84,7 @@ OPENAI_API_KEY="" KAAPI_GUARDRAILS_AUTH="" KAAPI_GUARDRAILS_URL="" + +# Unleash feature flags +UNLEASH_URL= +UNLEASH_API_KEY= diff --git a/.gitignore b/.gitignore index ad2127f45..0c8e46e86 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ ENV/ # /backend/app/logs + +# OpenObserve local data +/backend/data/ diff --git a/backend/app/alembic/versions/050_add_assessment_manager_table.py b/backend/app/alembic/versions/050_add_assessment_manager_table.py index ffecd0767..a45288acd 100644 --- a/backend/app/alembic/versions/050_add_assessment_manager_table.py +++ b/backend/app/alembic/versions/050_add_assessment_manager_table.py @@ -1,4 +1,4 @@ -"""add assessment manager table +"""add assessment and assessment_run tables Revision ID: 050 Revises: 049 @@ -166,41 +166,191 @@ def upgrade(): ["status", "project_id"], unique=False, ) - - op.add_column( - "evaluation_run", + op.create_table( + "assessment_run", + sa.Column( + "id", + sa.Integer(), + nullable=False, + comment="Unique identifier for the assessment run", + ), + sa.Column( + "run_name", + sqlmodel.sql.sqltypes.AutoString(), + nullable=False, + comment="Name of the assessment run", + ), sa.Column( "assessment_id", sa.Integer(), nullable=True, - comment="Reference to parent assessment manager row, if applicable", + comment="Reference to parent assessment manager row", + ), + sa.Column( + "dataset_id", + sa.Integer(), + nullable=False, + comment="Reference to the evaluation dataset", + ), + sa.Column( + "dataset_name", + sqlmodel.sql.sqltypes.AutoString(), + nullable=False, + comment="Name of the dataset used", + ), + sa.Column( + "config_id", + sa.Uuid(), + nullable=True, + comment="Reference to the stored config used", + ), + sa.Column( + "config_version", + sa.Integer(), + nullable=True, + comment="Version of the config used", + ), + sa.Column( + "status", + sqlmodel.sql.sqltypes.AutoString(), + nullable=False, + server_default="pending", + comment="Run status: pending, processing, completed, failed", + ), + sa.Column( + "batch_job_id", + sa.Integer(), + nullable=True, + comment="Reference to the batch job processing this run", ), + sa.Column( + "total_items", + sa.Integer(), + nullable=False, + server_default="0", + comment="Total number of dataset items in this run", + ), + sa.Column( + "input", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + comment="Assessment input config: prompt_template, text_columns, attachments, output_schema", + ), + sa.Column( + "object_store_url", + sqlmodel.sql.sqltypes.AutoString(), + nullable=True, + comment="S3 URL of processed batch results", + ), + sa.Column( + "error_message", + sa.Text(), + nullable=True, + comment="Error message if the run failed", + ), + sa.Column( + "eval_score", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + comment="Evaluation scores (reserved for future use)", + ), + sa.Column( + "eval_score_trace_url", + sqlmodel.sql.sqltypes.AutoString(), + nullable=True, + comment="S3 URL for evaluation score traces (reserved)", + ), + sa.Column( + "organization_id", + sa.Integer(), + nullable=False, + comment="Reference to the organization", + ), + sa.Column( + "project_id", + sa.Integer(), + nullable=False, + comment="Reference to the project", + ), + sa.Column( + "inserted_at", + sa.DateTime(), + nullable=False, + comment="Timestamp when the run was created", + ), + sa.Column( + "updated_at", + sa.DateTime(), + nullable=False, + comment="Timestamp when the run was last updated", + ), + sa.ForeignKeyConstraint( + ["assessment_id"], + ["assessment.id"], + name="fk_assessment_run_assessment_id", + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["dataset_id"], + ["evaluation_dataset.id"], + name="fk_assessment_run_dataset_id", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["config_id"], + ["config.id"], + name="fk_assessment_run_config_id", + ), + sa.ForeignKeyConstraint( + ["batch_job_id"], + ["batch_job.id"], + name="fk_assessment_run_batch_job_id", + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["organization_id"], + ["organization.id"], + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["project_id"], + ["project.id"], + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id"), ) - op.create_foreign_key( - "fk_evaluation_run_assessment_id", - "evaluation_run", - "assessment", - ["assessment_id"], - ["id"], - ondelete="SET NULL", + op.create_index( + op.f("ix_assessment_run_run_name"), + "assessment_run", + ["run_name"], + unique=False, ) op.create_index( - "idx_eval_run_assessment_id", - "evaluation_run", + "idx_assessment_run_status_org", + "assessment_run", + ["status", "organization_id"], + unique=False, + ) + op.create_index( + "idx_assessment_run_status_project", + "assessment_run", + ["status", "project_id"], + unique=False, + ) + op.create_index( + "idx_assessment_run_assessment_id", + "assessment_run", ["assessment_id"], unique=False, ) def downgrade(): - op.drop_index("idx_eval_run_assessment_id", table_name="evaluation_run") - op.drop_constraint( - "fk_evaluation_run_assessment_id", - "evaluation_run", - type_="foreignkey", - ) - op.drop_column("evaluation_run", "assessment_id") - + op.drop_index("idx_assessment_run_assessment_id", table_name="assessment_run") + op.drop_index("idx_assessment_run_status_project", table_name="assessment_run") + op.drop_index("idx_assessment_run_status_org", table_name="assessment_run") + op.drop_index(op.f("ix_assessment_run_run_name"), table_name="assessment_run") + op.drop_table("assessment_run") op.drop_index("idx_assessment_status_project", table_name="assessment") op.drop_index("idx_assessment_status_org", table_name="assessment") op.drop_index(op.f("ix_assessment_experiment_name"), table_name="assessment") diff --git a/backend/app/api/main.py b/backend/app/api/main.py index 8abb8a0bf..b60973baf 100644 --- a/backend/app/api/main.py +++ b/backend/app/api/main.py @@ -26,7 +26,7 @@ model_evaluation, collection_job, ) -from app.api.routes import evaluations +from app.api.routes import evaluations, features from app.assessment import routes as assessment_routes from app.core.config import settings @@ -53,6 +53,7 @@ api_router.include_router(threads.router) api_router.include_router(users.router) api_router.include_router(utils.router) +api_router.include_router(features.router) api_router.include_router(fine_tuning.router) api_router.include_router(model_evaluation.router) api_router.include_router(assessment_routes.router) diff --git a/backend/app/api/permissions.py b/backend/app/api/permissions.py index 4142b7a4c..89a60229b 100644 --- a/backend/app/api/permissions.py +++ b/backend/app/api/permissions.py @@ -5,6 +5,7 @@ from app.models import AuthContext from app.api.deps import AuthContextDep, SessionDep +from app.core.feature_flags import FeatureFlag class Permission(str, Enum): @@ -68,3 +69,31 @@ def permission_checker( ) return permission_checker + + +def require_feature(flag: FeatureFlag): + """Dependency factory that gates a route behind an Unleash feature flag. + + Returns 404 when the flag is disabled for the caller's org/project, + so the feature appears non-existent rather than forbidden. + + Usage:: + + router = APIRouter( + dependencies=[Depends(require_feature(FeatureFlag.ASSESSMENT))] + ) + """ + from app.core.feature_flags import is_enabled + + def _check(auth_context: AuthContextDep): + org_id = auth_context.organization.id if auth_context.organization else None + project_id = auth_context.project.id if auth_context.project else None + + if org_id is None or not is_enabled( + flag, + organization_id=org_id, + project_id=project_id, + ): + raise HTTPException(status_code=404) + + return _check diff --git a/backend/app/api/routes/cron.py b/backend/app/api/routes/cron.py index 2c7b3c81e..20bb209c3 100644 --- a/backend/app/api/routes/cron.py +++ b/backend/app/api/routes/cron.py @@ -65,7 +65,7 @@ async def evaluation_cron_job( f"[evaluation_cron_job] Assessment polling failed: {ae}", exc_info=True, ) - result["assessment_error"] = str(ae) + result["assessment_error"] = "Assessment polling failed" logger.info( f"[evaluation_cron_job] Completed: " diff --git a/backend/app/api/routes/features.py b/backend/app/api/routes/features.py new file mode 100644 index 000000000..33329121a --- /dev/null +++ b/backend/app/api/routes/features.py @@ -0,0 +1,24 @@ +"""Feature flags endpoint — returns resolved flags for the caller's scope.""" + +from fastapi import APIRouter, Depends + +from app.api.deps import AuthContextDep +from app.api.permissions import Permission, require_permission +from app.core.feature_flags import resolve_all_flags + +router = APIRouter(tags=["Features"]) + + +@router.get( + "/features", + response_model=dict[str, bool], + dependencies=[Depends(require_permission(Permission.REQUIRE_ORGANIZATION))], +) +def get_features(auth_context: AuthContextDep) -> dict[str, bool]: + """Return all feature flags resolved for the caller's org + project.""" + org_id = auth_context.organization_.id + project_id = auth_context.project.id if auth_context.project else None + return resolve_all_flags( + organization_id=org_id, + project_id=project_id, + ) diff --git a/backend/app/assessment/batch.py b/backend/app/assessment/batch.py index ee67da1f4..181db668e 100644 --- a/backend/app/assessment/batch.py +++ b/backend/app/assessment/batch.py @@ -18,14 +18,18 @@ map_kaapi_to_openai_params, normalize_llm_text, ) -from app.assessment.models import AssessmentAttachment, AssessmentTextLLMParams +from app.assessment.models import ( + AssessmentAttachment, + AssessmentRun, + AssessmentTextLLMParams, +) from app.core.batch import BATCH_KEY, start_batch_job from app.core.batch.openai import OpenAIBatchProvider from app.core.cloud import get_cloud_storage from app.core.storage_utils import load_json_from_object_store from app.crud.config.version import ConfigVersionCrud from app.models.batch_job import BatchJob, BatchJobType -from app.models.evaluation import EvaluationDataset, EvaluationRun +from app.models.evaluation import EvaluationDataset from app.models.llm.request import ConfigBlob, KaapiCompletionConfig, LLMCallConfig from app.services.llm.jobs import resolve_config_blob @@ -149,7 +153,11 @@ def _build_text_prompt( return prompt # No template: concatenate text columns - parts = [normalize_llm_text(row.get(col, "")) for col in text_columns if row.get(col, "").strip()] + parts = [ + normalize_llm_text(row.get(col, "")) + for col in text_columns + if row.get(col, "").strip() + ] return "\n".join(parts) @@ -389,9 +397,7 @@ def build_google_jsonl( "contents": [{"parts": parts, "role": "user"}], } if system_instruction: - request["systemInstruction"] = { - "parts": [{"text": system_instruction}] - } + request["systemInstruction"] = {"parts": [{"text": system_instruction}]} generation_config: dict[str, Any] = {} temperature = google_params.get("temperature") @@ -425,10 +431,10 @@ def build_google_jsonl( def submit_assessment_batch( session: Session, - eval_run: EvaluationRun, + run: AssessmentRun, dataset: EvaluationDataset, config_blob: ConfigBlob, - assessment_config: dict[str, Any], + assessment_input: dict[str, Any], organization_id: int, project_id: int, ) -> BatchJob: @@ -436,20 +442,20 @@ def submit_assessment_batch( Args: session: Database session - eval_run: The EvaluationRun to process + run: The AssessmentRun to process dataset: The dataset to read rows from config_blob: Resolved configuration blob - assessment_config: Assessment-specific config (prompt_template, text_columns, etc.) + assessment_input: Assessment input config (prompt_template, text_columns, etc.) organization_id: Organization ID project_id: Project ID Returns: Created BatchJob record """ - text_columns = assessment_config.get("text_columns", []) - prompt_template = assessment_config.get("prompt_template") - attachments_raw = assessment_config.get("attachments", []) - output_schema = assessment_config.get("output_schema") + text_columns = assessment_input.get("text_columns", []) + prompt_template = assessment_input.get("prompt_template") + attachments_raw = assessment_input.get("attachments", []) + output_schema = assessment_input.get("output_schema") attachments = [AssessmentAttachment(**a) for a in attachments_raw] # Load dataset rows @@ -459,7 +465,7 @@ def submit_assessment_batch( logger.info( f"[submit_assessment_batch] Building JSONL | " - f"run_id={eval_run.id} | rows={len(rows)} | " + f"run_id={run.id} | rows={len(rows)} | " f"provider={config_blob.completion.provider}" ) @@ -499,7 +505,7 @@ def submit_assessment_batch( batch_config = { "endpoint": "/v1/responses", - "description": f"Assessment: {eval_run.run_name}", + "description": f"Assessment: {run.run_name}", "completion_window": "24h", } @@ -542,7 +548,7 @@ def submit_assessment_batch( ) batch_config = { - "display_name": f"assessment-{eval_run.run_name}", + "display_name": f"assessment-{run.run_name}", "model": f"models/{mapped_params.get('model', 'gemini-2.5-pro')}", } @@ -564,7 +570,7 @@ def submit_assessment_batch( logger.info( f"[submit_assessment_batch] Submitted batch | " - f"run_id={eval_run.id} | batch_job_id={batch_job.id} | " + f"run_id={run.id} | batch_job_id={batch_job.id} | " f"provider={base_provider} | items={len(jsonl_data)}" ) diff --git a/backend/app/assessment/cron.py b/backend/app/assessment/cron.py index 7f1d0807b..25bdd63d0 100644 --- a/backend/app/assessment/cron.py +++ b/backend/app/assessment/cron.py @@ -11,15 +11,14 @@ update_assessment_run_status, ) from app.assessment.events import assessment_event_broker +from app.assessment.models import Assessment, AssessmentRun from app.assessment.processing import check_and_process_assessment -from app.assessment.models import Assessment -from app.models.evaluation import EvaluationRun from app.utils import APIResponse logger = logging.getLogger(__name__) -def _log_config_progress(result: dict[str, Any], eval_run: EvaluationRun) -> None: +def _log_config_progress(result: dict[str, Any], run: AssessmentRun) -> None: """Emit explicit config-level logs for grouped assessment experiments.""" action = result.get("action") if action not in {"processed", "failed"}: @@ -28,11 +27,11 @@ def _log_config_progress(result: dict[str, Any], eval_run: EvaluationRun) -> Non logger.info( "[poll_all_pending_assessment_evaluations] " "Experiment config update | " - f"experiment={eval_run.run_name} | " - f"assessment_id={eval_run.assessment_id} | " - f"run_id={eval_run.id} | " - f"config_id={eval_run.config_id} | " - f"config_version={eval_run.config_version} | " + f"experiment={run.run_name} | " + f"assessment_id={run.assessment_id} | " + f"run_id={run.id} | " + f"config_id={run.config_id} | " + f"config_version={run.config_version} | " f"action={action} | " f"status={result.get('current_status')} | " f"provider_status={result.get('provider_status')}" @@ -41,7 +40,7 @@ def _log_config_progress(result: dict[str, Any], eval_run: EvaluationRun) -> Non def _build_callback_payload( assessment: Assessment, - eval_run: EvaluationRun, + run: AssessmentRun, result: dict[str, Any], ) -> dict[str, Any]: """Build minimal SSE payload for assessment invalidation.""" @@ -51,14 +50,12 @@ def _build_callback_payload( "assessment_id": assessment.id, "assessment_status": assessment.status, "run": { - "id": eval_run.id, - "config_id": str(eval_run.config_id) if eval_run.config_id else None, - "config_version": eval_run.config_version, + "id": run.id, + "config_id": str(run.config_id) if run.config_id else None, + "config_version": run.config_version, "status": result.get("current_status"), "error": result.get("error"), - "updated_at": eval_run.updated_at.isoformat() - if eval_run.updated_at - else None, + "updated_at": run.updated_at.isoformat() if run.updated_at else None, }, } ).model_dump() @@ -74,6 +71,9 @@ async def poll_all_pending_assessment_evaluations( pending_assessments = list(session.exec(statement).all()) if not pending_assessments: + logger.info( + "[poll_all_pending_assessment_evaluations] " "No active assessments found" + ) return { "total": 0, "processed": 0, @@ -102,18 +102,26 @@ async def poll_all_pending_assessment_evaluations( refreshed = recompute_assessment_status( session=session, assessment_id=assessment.id ) + logger.info( + "[poll_all_pending_assessment_evaluations] " + f"No active runs for assessment {assessment.id} | " + f"recomputed status={refreshed.status} | " + f"total_runs={refreshed.total_runs} | " + f"completed={refreshed.completed_runs} | " + f"failed={refreshed.failed_runs}" + ) if refreshed.status in {"pending", "processing"}: still_processing += 1 continue - for eval_run in active_runs: + for run in active_runs: try: result = await check_and_process_assessment( - eval_run=eval_run, + run=run, session=session, ) all_results.append(result) - _log_config_progress(result, eval_run) + _log_config_progress(result, run) if result["action"] in {"processed", "failed"}: refreshed_assessment = session.get(Assessment, assessment.id) @@ -121,7 +129,7 @@ async def poll_all_pending_assessment_evaluations( assessment_event_broker.publish( _build_callback_payload( refreshed_assessment, - eval_run, + run, result, ) ) @@ -136,40 +144,38 @@ async def poll_all_pending_assessment_evaluations( except Exception as e: logger.error( "[poll_all_pending_assessment_evaluations] " - f"Failed run {eval_run.id} | " - f"experiment={eval_run.run_name} | " - f"assessment_id={eval_run.assessment_id} | " - f"config_id={eval_run.config_id} | " - f"config_version={eval_run.config_version} | " + f"Failed run {run.id} | " + f"experiment={run.run_name} | " + f"assessment_id={run.assessment_id} | " + f"config_id={run.config_id} | " + f"config_version={run.config_version} | " f"error={e}", exc_info=True, ) update_assessment_run_status( session=session, - eval_run=eval_run, + run=run, status="failed", - error_message=f"Poll failed: {str(e)}", + error_message="Processing failed. Check server logs for details.", ) refreshed_assessment = recompute_assessment_status( session=session, assessment_id=assessment.id ) failure_result = { - "assessment_id": eval_run.assessment_id, - "run_id": eval_run.id, - "run_name": eval_run.run_name, - "config_id": str(eval_run.config_id) - if eval_run.config_id - else None, - "config_version": eval_run.config_version, + "assessment_id": run.assessment_id, + "run_id": run.id, + "run_name": run.run_name, + "config_id": str(run.config_id) if run.config_id else None, + "config_version": run.config_version, "action": "failed", - "error": str(e), + "error": "Processing failed", "current_status": "failed", } all_results.append(failure_result) assessment_event_broker.publish( _build_callback_payload( refreshed_assessment, - eval_run, + run, failure_result, ) ) diff --git a/backend/app/assessment/crud.py b/backend/app/assessment/crud.py index b340b4596..9a77ef119 100644 --- a/backend/app/assessment/crud.py +++ b/backend/app/assessment/crud.py @@ -1,4 +1,4 @@ -"""Assessment CRUD — thin wrappers around EvaluationRun for type='assessment'.""" +"""Assessment CRUD — operations for Assessment and AssessmentRun tables.""" import logging from typing import Any @@ -7,12 +7,12 @@ from sqlmodel import Session, select from app.core.util import now -from app.assessment.models import Assessment -from app.models.evaluation import EvaluationRun +from app.assessment.models import Assessment, AssessmentRun logger = logging.getLogger(__name__) -ASSESSMENT_TYPE = "assessment" + +# ── Assessment (parent) ───────────────────────────────────────── def create_assessment( @@ -58,20 +58,44 @@ def create_assessment( return assessment -def get_assessment_runs_for_manager( +def get_assessment_by_id( session: Session, - assessment: Assessment, -) -> list[EvaluationRun]: - """List child evaluation runs for a parent assessment row.""" + assessment_id: int, + organization_id: int, + project_id: int, +) -> Assessment | None: + """Get a specific parent assessment manager row.""" statement = ( - select(EvaluationRun) - .where(EvaluationRun.assessment_id == assessment.id) - .where(EvaluationRun.type == ASSESSMENT_TYPE) - .order_by(EvaluationRun.inserted_at.desc()) + select(Assessment) + .where(Assessment.id == assessment_id) + .where(Assessment.organization_id == organization_id) + .where(Assessment.project_id == project_id) + ) + return session.exec(statement).first() + + +def list_assessments( + session: Session, + organization_id: int, + project_id: int, + limit: int = 50, + offset: int = 0, +) -> list[Assessment]: + """List parent assessment manager rows.""" + statement = ( + select(Assessment) + .where(Assessment.organization_id == organization_id) + .where(Assessment.project_id == project_id) + .order_by(Assessment.inserted_at.desc()) + .limit(limit) + .offset(offset) ) return list(session.exec(statement).all()) +# ── AssessmentRun (child) ─────────────────────────────────────── + + def create_assessment_run( session: Session, run_name: str, @@ -82,78 +106,68 @@ def create_assessment_run( config_version: int, organization_id: int, project_id: int, - assessment_config: dict[str, Any] | None = None, -) -> EvaluationRun: - """Create an assessment evaluation run record. - - Re-uses EvaluationRun with type='assessment' and stores assessment-specific - config in the score JSONB field under 'assessment_config'. - """ - eval_run = EvaluationRun( + assessment_input: dict[str, Any] | None = None, +) -> AssessmentRun: + """Create an assessment run record in the assessment_run table.""" + run = AssessmentRun( run_name=run_name, dataset_name=dataset_name, dataset_id=dataset_id, assessment_id=assessment_id, - type=ASSESSMENT_TYPE, config_id=config_id, config_version=config_version, status="pending", - score={"assessment_config": assessment_config} if assessment_config else None, + total_items=0, + input=assessment_input, organization_id=organization_id, project_id=project_id, inserted_at=now(), updated_at=now(), ) - session.add(eval_run) + session.add(run) try: session.commit() - session.refresh(eval_run) + session.refresh(run) except Exception as e: session.rollback() logger.error(f"[create_assessment_run] Failed: {e}", exc_info=True) raise logger.info( - f"[create_assessment_run] Created run id={eval_run.id} | " + f"[create_assessment_run] Created run id={run.id} | " f"name={run_name} | config_id={config_id} v{config_version}" ) - return eval_run + return run -def list_assessments( +def get_assessment_run_by_id( session: Session, + run_id: int, organization_id: int, project_id: int, - limit: int = 50, - offset: int = 0, -) -> list[Assessment]: - """List parent assessment manager rows.""" +) -> AssessmentRun | None: + """Get a specific assessment run by ID.""" statement = ( - select(Assessment) - .where(Assessment.organization_id == organization_id) - .where(Assessment.project_id == project_id) - .order_by(Assessment.inserted_at.desc()) - .limit(limit) - .offset(offset) + select(AssessmentRun) + .where(AssessmentRun.id == run_id) + .where(AssessmentRun.organization_id == organization_id) + .where(AssessmentRun.project_id == project_id) ) - return list(session.exec(statement).all()) + return session.exec(statement).first() -def get_assessment_by_id( +def get_assessment_runs_for_manager( session: Session, - assessment_id: int, - organization_id: int, - project_id: int, -) -> Assessment | None: - """Get a specific parent assessment manager row.""" + assessment: Assessment, +) -> list[AssessmentRun]: + """List child runs for a parent assessment.""" statement = ( - select(Assessment) - .where(Assessment.id == assessment_id) - .where(Assessment.organization_id == organization_id) - .where(Assessment.project_id == project_id) + select(AssessmentRun) + .where(AssessmentRun.assessment_id == assessment.id) + .order_by(AssessmentRun.inserted_at.desc()) ) - return session.exec(statement).first() + return list(session.exec(statement).all()) def list_assessment_runs( @@ -163,50 +177,57 @@ def list_assessment_runs( assessment_id: int | None = None, limit: int = 50, offset: int = 0, -) -> list[EvaluationRun]: - """List child assessment evaluation runs by traversing from assessments.""" +) -> list[AssessmentRun]: + """List assessment runs, optionally filtered by assessment_id.""" + statement = ( + select(AssessmentRun) + .where(AssessmentRun.organization_id == organization_id) + .where(AssessmentRun.project_id == project_id) + ) if assessment_id is not None: - assessment = get_assessment_by_id( - session=session, - assessment_id=assessment_id, - organization_id=organization_id, - project_id=project_id, - ) - if not assessment: - return [] - runs = get_assessment_runs_for_manager(session=session, assessment=assessment) - return runs[offset : offset + limit] - - assessments = list_assessments( - session=session, - organization_id=organization_id, - project_id=project_id, - limit=limit, - offset=offset, + statement = statement.where(AssessmentRun.assessment_id == assessment_id) + + statement = ( + statement.order_by(AssessmentRun.inserted_at.desc()).limit(limit).offset(offset) ) - runs: list[EvaluationRun] = [] - for assessment in assessments: - runs.extend( - get_assessment_runs_for_manager(session=session, assessment=assessment) - ) - return runs + return list(session.exec(statement).all()) -def get_assessment_run_by_id( +def update_assessment_run_status( session: Session, - run_id: int, - organization_id: int, - project_id: int, -) -> EvaluationRun | None: - """Get a specific assessment evaluation run by ID.""" - statement = ( - select(EvaluationRun) - .where(EvaluationRun.id == run_id) - .where(EvaluationRun.organization_id == organization_id) - .where(EvaluationRun.project_id == project_id) - .where(EvaluationRun.type == ASSESSMENT_TYPE) - ) - return session.exec(statement).first() + run: AssessmentRun, + status: str, + error_message: str | None = None, + batch_job_id: int | None = None, + total_items: int | None = None, + object_store_url: str | None = None, +) -> AssessmentRun: + """Update an assessment run's status and optional fields.""" + run.status = status + run.updated_at = now() + + if error_message is not None: + run.error_message = error_message + if batch_job_id is not None: + run.batch_job_id = batch_job_id + if total_items is not None: + run.total_items = total_items + if object_store_url is not None: + run.object_store_url = object_store_url + + session.add(run) + try: + session.commit() + session.refresh(run) + except Exception as e: + session.rollback() + logger.error(f"[update_assessment_run_status] Failed: {e}", exc_info=True) + raise + + return run + + +# ── Assessment status recomputation ───────────────────────────── def _determine_assessment_status( @@ -216,7 +237,7 @@ def _determine_assessment_status( completed_runs: int, failed_runs: int, ) -> str: - """Compute parent assessment status from child evaluation runs.""" + """Compute parent assessment status from child run counts.""" if total_runs == 0: return "pending" if completed_runs == total_runs: @@ -245,19 +266,16 @@ def recompute_assessment_status( raise ValueError(f"Assessment {assessment_id} not found") statement = ( - select(EvaluationRun) - .where(EvaluationRun.assessment_id == assessment_id) - .where(EvaluationRun.type == ASSESSMENT_TYPE) - .order_by(EvaluationRun.id.asc()) + select(AssessmentRun) + .where(AssessmentRun.assessment_id == assessment_id) + .order_by(AssessmentRun.id.asc()) ) runs = list(session.exec(statement).all()) - pending_runs = sum(1 for run in runs if run.status == "pending") - processing_runs = sum( - 1 for run in runs if run.status in {"processing", "in_progress"} - ) - completed_runs = sum(1 for run in runs if run.status == "completed") - failed_runs = sum(1 for run in runs if run.status == "failed") + pending_runs = sum(1 for r in runs if r.status == "pending") + processing_runs = sum(1 for r in runs if r.status in {"processing", "in_progress"}) + completed_runs = sum(1 for r in runs if r.status == "completed") + failed_runs = sum(1 for r in runs if r.status == "failed") total_runs = len(runs) assessment.total_runs = total_runs @@ -273,21 +291,19 @@ def recompute_assessment_status( failed_runs=failed_runs, ) assessment.error_message = ( - f"{failed_runs} of {total_runs} evaluation run(s) failed" - if failed_runs > 0 - else None + f"{failed_runs} of {total_runs} run(s) failed" if failed_runs > 0 else None ) assessment.run_stats = [ { - "run_id": run.id, - "config_id": str(run.config_id) if run.config_id else None, - "config_version": run.config_version, - "status": run.status, - "total_items": run.total_items, - "error_message": run.error_message, - "updated_at": run.updated_at.isoformat() if run.updated_at else None, + "run_id": r.id, + "config_id": str(r.config_id) if r.config_id else None, + "config_version": r.config_version, + "status": r.status, + "total_items": r.total_items, + "error_message": r.error_message, + "updated_at": r.updated_at.isoformat() if r.updated_at else None, } - for run in runs + for r in runs ] assessment.updated_at = now() @@ -301,37 +317,3 @@ def recompute_assessment_status( raise return assessment - - -def update_assessment_run_status( - session: Session, - eval_run: EvaluationRun, - status: str, - error_message: str | None = None, - batch_job_id: int | None = None, - total_items: int | None = None, - object_store_url: str | None = None, -) -> EvaluationRun: - """Update an assessment run's status and optional fields.""" - eval_run.status = status - eval_run.updated_at = now() - - if error_message is not None: - eval_run.error_message = error_message - if batch_job_id is not None: - eval_run.batch_job_id = batch_job_id - if total_items is not None: - eval_run.total_items = total_items - if object_store_url is not None: - eval_run.object_store_url = object_store_url - - session.add(eval_run) - try: - session.commit() - session.refresh(eval_run) - except Exception as e: - session.rollback() - logger.error(f"[update_assessment_run_status] Failed: {e}", exc_info=True) - raise - - return eval_run diff --git a/backend/app/assessment/models.py b/backend/app/assessment/models.py index 6fad58e10..fc1a717f4 100644 --- a/backend/app/assessment/models.py +++ b/backend/app/assessment/models.py @@ -1,20 +1,23 @@ """Assessment models — DB table, Pydantic schemas, and LLM param wrappers.""" from datetime import datetime -from typing import Any, Literal +from typing import TYPE_CHECKING, Any, Literal, Optional from uuid import UUID from pydantic import BaseModel, Field from sqlalchemy import Column, Index, Text from sqlalchemy.dialects.postgresql import JSONB from sqlmodel import Field as SQLField -from sqlmodel import SQLModel +from sqlmodel import Relationship, SQLModel from app.core.util import now from app.models.llm.request import TextLLMParams +if TYPE_CHECKING: + from app.models.batch_job import BatchJob -# ── Database model ────────────────────────────────────────────── + +# ── Database models ───────────────────────────────────────────── class Assessment(SQLModel, table=True): @@ -128,6 +131,135 @@ class Assessment(SQLModel, table=True): ) +class AssessmentRun(SQLModel, table=True): + """Dedicated table for assessment evaluation runs.""" + + __tablename__ = "assessment_run" + __table_args__ = ( + Index("idx_assessment_run_status_org", "status", "organization_id"), + Index("idx_assessment_run_status_project", "status", "project_id"), + Index("idx_assessment_run_assessment_id", "assessment_id"), + ) + + id: int = SQLField( + default=None, + primary_key=True, + sa_column_kwargs={"comment": "Unique identifier for the assessment run"}, + ) + run_name: str = SQLField( + index=True, + description="Name of the assessment run (matches experiment name)", + sa_column_kwargs={"comment": "Name of the assessment run"}, + ) + assessment_id: int | None = SQLField( + default=None, + foreign_key="assessment.id", + nullable=True, + ondelete="SET NULL", + sa_column_kwargs={"comment": "Reference to parent assessment manager row"}, + ) + dataset_id: int = SQLField( + foreign_key="evaluation_dataset.id", + nullable=False, + ondelete="CASCADE", + sa_column_kwargs={"comment": "Reference to the evaluation dataset"}, + ) + dataset_name: str = SQLField( + nullable=False, + sa_column_kwargs={"comment": "Name of the dataset used"}, + ) + config_id: UUID | None = SQLField( + default=None, + foreign_key="config.id", + nullable=True, + sa_column_kwargs={"comment": "Reference to the stored config used"}, + ) + config_version: int | None = SQLField( + default=None, + nullable=True, + sa_column_kwargs={"comment": "Version of the config used"}, + ) + status: str = SQLField( + default="pending", + sa_column_kwargs={ + "comment": "Run status: pending, processing, completed, failed" + }, + ) + batch_job_id: int | None = SQLField( + default=None, + foreign_key="batch_job.id", + nullable=True, + ondelete="SET NULL", + sa_column_kwargs={"comment": "Reference to the batch job processing this run"}, + ) + total_items: int = SQLField( + default=0, + nullable=False, + sa_column_kwargs={"comment": "Total number of dataset items in this run"}, + ) + input: dict[str, Any] | None = SQLField( + default=None, + sa_column=Column( + JSONB, + nullable=True, + comment="Assessment input config: prompt_template, text_columns, attachments, output_schema", + ), + ) + object_store_url: str | None = SQLField( + default=None, + nullable=True, + sa_column_kwargs={"comment": "S3 URL of processed batch results"}, + ) + error_message: str | None = SQLField( + default=None, + sa_column=Column( + Text, + nullable=True, + comment="Error message if the run failed", + ), + ) + eval_score: dict[str, Any] | None = SQLField( + default=None, + sa_column=Column( + JSONB, + nullable=True, + comment="Evaluation scores (reserved for future use)", + ), + ) + eval_score_trace_url: str | None = SQLField( + default=None, + nullable=True, + sa_column_kwargs={"comment": "S3 URL for evaluation score traces (reserved)"}, + ) + organization_id: int = SQLField( + foreign_key="organization.id", + nullable=False, + ondelete="CASCADE", + sa_column_kwargs={"comment": "Reference to the organization"}, + ) + project_id: int = SQLField( + foreign_key="project.id", + nullable=False, + ondelete="CASCADE", + sa_column_kwargs={"comment": "Reference to the project"}, + ) + inserted_at: datetime = SQLField( + default_factory=now, + nullable=False, + sa_column_kwargs={"comment": "Timestamp when the run was created"}, + ) + updated_at: datetime = SQLField( + default_factory=now, + nullable=False, + sa_column_kwargs={"comment": "Timestamp when the run was last updated"}, + ) + + # Relationships + batch_job: Optional["BatchJob"] = Relationship( + sa_relationship_kwargs={"foreign_keys": "[AssessmentRun.batch_job_id]"} + ) + + class AssessmentPublic(BaseModel): """Public model for assessment manager rows.""" @@ -253,8 +385,9 @@ class AssessmentRunPublic(BaseModel): error_message: str | None organization_id: int project_id: int - assessment_config: dict[str, Any] | None = Field( - None, description="Assessment-specific configuration (prompt, columns, schema)" + input: dict[str, Any] | None = Field( + None, + description="Assessment input config (prompt_template, text_columns, attachments, output_schema)", ) inserted_at: datetime updated_at: datetime diff --git a/backend/app/assessment/processing.py b/backend/app/assessment/processing.py index 66e47c929..88cf3f22d 100644 --- a/backend/app/assessment/processing.py +++ b/backend/app/assessment/processing.py @@ -9,7 +9,6 @@ import logging from typing import Any -from fastapi import HTTPException from sqlmodel import Session from app.assessment.crud import ( @@ -17,6 +16,7 @@ update_assessment_run_status, ) from app.assessment.events import assessment_event_broker +from app.assessment.models import AssessmentRun from app.core.batch import ( BATCH_KEY, OpenAIBatchProvider, @@ -29,7 +29,6 @@ from app.core.batch.client import GeminiClient from app.core.batch.gemini import BatchJobState, extract_text_from_response_dict from app.crud.job import get_batch_job -from app.models.evaluation import EvaluationRun from app.utils import get_openai_client logger = logging.getLogger(__name__) @@ -239,35 +238,35 @@ def parse_assessment_output( async def check_and_process_assessment( - eval_run: EvaluationRun, + run: AssessmentRun, session: Session, ) -> dict[str, Any]: """Check assessment batch status and process if completed. Args: - eval_run: EvaluationRun with type='assessment' + run: AssessmentRun to check session: Database session Returns: Dict with status information """ - log_prefix = f"[assessment={eval_run.id}]" - previous_status = eval_run.status + log_prefix = f"[assessment_run={run.id}]" + previous_status = run.status try: - if not eval_run.batch_job_id: - raise ValueError(f"Assessment run {eval_run.id} has no batch_job_id") + if not run.batch_job_id: + raise ValueError(f"Assessment run {run.id} has no batch_job_id") - batch_job = get_batch_job(session=session, batch_job_id=eval_run.batch_job_id) + batch_job = get_batch_job(session=session, batch_job_id=run.batch_job_id) if not batch_job: - raise ValueError(f"BatchJob {eval_run.batch_job_id} not found") + raise ValueError(f"BatchJob {run.batch_job_id} not found") # Get provider and poll status provider = _get_batch_provider( session=session, provider_name=batch_job.provider, - organization_id=eval_run.organization_id, - project_id=eval_run.project_id, + organization_id=run.organization_id, + project_id=run.project_id, ) status_result = poll_batch_status( session=session, @@ -300,19 +299,19 @@ async def check_and_process_assessment( update_assessment_run_status( session=session, - eval_run=eval_run, + run=run, status="failed", error_message=error_msg, ) - if eval_run.assessment_id is not None: + if run.assessment_id is not None: recompute_assessment_status( - session=session, assessment_id=eval_run.assessment_id + session=session, assessment_id=run.assessment_id ) return { - "run_id": eval_run.id, - "assessment_id": eval_run.assessment_id, - "run_name": eval_run.run_name, + "run_id": run.id, + "assessment_id": run.assessment_id, + "run_name": run.run_name, "previous_status": previous_status, "current_status": "failed", "provider_status": provider_status, @@ -325,22 +324,24 @@ async def check_and_process_assessment( f"batch_job_id={batch_job.id} | provider_status={provider_status}" ) return { - "run_id": eval_run.id, - "assessment_id": eval_run.assessment_id, - "run_name": eval_run.run_name, + "run_id": run.id, + "assessment_id": run.assessment_id, + "run_name": run.run_name, "previous_status": previous_status, - "current_status": eval_run.status, + "current_status": run.status, "provider_status": provider_status, "action": "no_change", } # Emit SSE: results are being prepared - assessment_event_broker.publish({ - "type": "assessment.results_preparing", - "assessment_id": eval_run.assessment_id, - "run_id": eval_run.id, - "message": "Results are being prepared", - }) + assessment_event_broker.publish( + { + "type": "assessment.results_preparing", + "assessment_id": run.assessment_id, + "run_id": run.id, + "message": "Results are being prepared", + } + ) # Download and process results raw_results = download_batch_results(provider=provider, batch_job=batch_job) @@ -373,31 +374,33 @@ async def check_and_process_assessment( update_assessment_run_status( session=session, - eval_run=eval_run, + run=run, status=run_status, error_message=error_msg, object_store_url=object_store_url, ) - if eval_run.assessment_id is not None: + if run.assessment_id is not None: recompute_assessment_status( - session=session, assessment_id=eval_run.assessment_id + session=session, assessment_id=run.assessment_id ) # Emit SSE: results are ready - assessment_event_broker.publish({ - "type": "assessment.results_ready", - "assessment_id": eval_run.assessment_id, - "run_id": eval_run.id, - "status": run_status, - "total_results": len(parsed), - "errors": error_count, - "message": "Results are ready", - }) + assessment_event_broker.publish( + { + "type": "assessment.results_ready", + "assessment_id": run.assessment_id, + "run_id": run.id, + "status": run_status, + "total_results": len(parsed), + "errors": error_count, + "message": "Results are ready", + } + ) return { - "run_id": eval_run.id, - "assessment_id": eval_run.assessment_id, - "run_name": eval_run.run_name, + "run_id": run.id, + "assessment_id": run.assessment_id, + "run_name": run.run_name, "previous_status": previous_status, "current_status": run_status, "provider_status": provider_status, @@ -417,19 +420,19 @@ async def check_and_process_assessment( error_msg = batch_job.error_message or f"Batch {provider_status}" update_assessment_run_status( session=session, - eval_run=eval_run, + run=run, status="failed", error_message=error_msg, ) - if eval_run.assessment_id is not None: + if run.assessment_id is not None: recompute_assessment_status( - session=session, assessment_id=eval_run.assessment_id + session=session, assessment_id=run.assessment_id ) return { - "run_id": eval_run.id, - "assessment_id": eval_run.assessment_id, - "run_name": eval_run.run_name, + "run_id": run.id, + "assessment_id": run.assessment_id, + "run_name": run.run_name, "previous_status": previous_status, "current_status": "failed", "provider_status": provider_status, @@ -440,11 +443,11 @@ async def check_and_process_assessment( else: # Still processing return { - "run_id": eval_run.id, - "assessment_id": eval_run.assessment_id, - "run_name": eval_run.run_name, + "run_id": run.id, + "assessment_id": run.assessment_id, + "run_name": run.run_name, "previous_status": previous_status, - "current_status": eval_run.status, + "current_status": run.status, "provider_status": provider_status, "action": "no_change", } @@ -456,23 +459,23 @@ async def check_and_process_assessment( ) update_assessment_run_status( session=session, - eval_run=eval_run, + run=run, status="failed", - error_message=f"Check failed: {str(e)}", + error_message="Processing failed. Check server logs for details.", ) - if eval_run.assessment_id is not None: + if run.assessment_id is not None: recompute_assessment_status( - session=session, assessment_id=eval_run.assessment_id + session=session, assessment_id=run.assessment_id ) return { - "run_id": eval_run.id, - "assessment_id": eval_run.assessment_id, - "run_name": eval_run.run_name, + "run_id": run.id, + "assessment_id": run.assessment_id, + "run_name": run.run_name, "previous_status": previous_status, "current_status": "failed", "provider_status": "unknown", "action": "failed", - "error": str(e), + "error": "Processing failed", } diff --git a/backend/app/assessment/routes.py b/backend/app/assessment/routes.py index dc794cc10..59eb243fd 100644 --- a/backend/app/assessment/routes.py +++ b/backend/app/assessment/routes.py @@ -18,7 +18,7 @@ from fastapi.responses import StreamingResponse from app.api.deps import AuthContextDep, SessionDep -from app.api.permissions import Permission, require_permission +from app.api.permissions import Permission, require_feature, require_permission from app.assessment.crud import ( get_assessment_by_id, get_assessment_run_by_id, @@ -33,6 +33,7 @@ AssessmentExportRow, AssessmentPublic, AssessmentResponse, + AssessmentRun, AssessmentRunPublic, ) from app.assessment.service import ( @@ -50,16 +51,21 @@ from app.assessment.utils.export import _safe_filename_part from app.assessment.validators import validate_dataset_file from app.core.cloud import get_cloud_storage +from app.core.feature_flags import FeatureFlag from app.core.storage_utils import generate_timestamped_filename from app.crud.evaluations import get_dataset_by_id from app.crud.evaluations import list_datasets as list_evaluation_datasets from app.crud.evaluations.dataset import delete_dataset as delete_dataset_crud -from app.models.evaluation import EvaluationDataset, EvaluationRun +from app.models.evaluation import EvaluationDataset from app.utils import APIResponse, load_description logger = logging.getLogger(__name__) -router = APIRouter(prefix="/assessment", tags=["Assessment"]) +router = APIRouter( + prefix="/assessment", + tags=["Assessment"], + dependencies=[Depends(require_feature(FeatureFlag.ASSESSMENT))], +) # ── Dataset routes ─────────────────────────────────────────────── @@ -464,7 +470,7 @@ def export_assessment_results( ) # Build per-run export data - runs_with_rows: list[tuple[EvaluationRun, list[AssessmentExportRow]]] = [] + runs_with_rows: list[tuple[AssessmentRun, list[AssessmentExportRow]]] = [] all_rows: list[AssessmentExportRow] = [] for run in runs: rows = load_export_rows_for_run(session=session, run=run, assessment=assessment) @@ -495,7 +501,7 @@ def export_assessment_results( if run.config_version else f"run_{run.id}" ) - config_id_short = (run.config_id or "")[:8] + config_id_short = str(run.config_id)[:8] if run.config_id else "" file_base = _safe_filename_part(f"{config_label}_{config_id_short}") file_bytes, _ = serialize_export_rows(rows, export_format) zf.writestr(f"{file_base}.{export_format}", file_bytes) @@ -600,7 +606,7 @@ def list_evaluations( error_message=r.error_message, organization_id=r.organization_id, project_id=r.project_id, - assessment_config=(r.score or {}).get("assessment_config"), + input=r.input, inserted_at=r.inserted_at, updated_at=r.updated_at, ) @@ -647,7 +653,7 @@ def get_evaluation( error_message=run.error_message, organization_id=run.organization_id, project_id=run.project_id, - assessment_config=(run.score or {}).get("assessment_config"), + input=run.input, inserted_at=run.inserted_at, updated_at=run.updated_at, ) diff --git a/backend/app/assessment/service.py b/backend/app/assessment/service.py index ebb84d132..cc82a63b1 100644 --- a/backend/app/assessment/service.py +++ b/backend/app/assessment/service.py @@ -16,15 +16,15 @@ update_assessment_run_status, ) from app.assessment.models import ( + Assessment, AssessmentAttachment, AssessmentConfigRef, AssessmentCreate, AssessmentResponse, + AssessmentRun, AssessmentRunSummary, ) from app.crud.evaluations import get_dataset_by_id -from app.assessment.models import Assessment -from app.models.evaluation import EvaluationRun logger = logging.getLogger(__name__) @@ -33,20 +33,20 @@ def _build_retry_request( *, experiment_name: str, dataset_id: int, - runs: list[EvaluationRun], + runs: list[AssessmentRun], ) -> AssessmentCreate: if not runs: raise HTTPException(status_code=400, detail="No assessment runs found to retry") first_run = runs[0] - assessment_config = (first_run.score or {}).get("assessment_config") - if not isinstance(assessment_config, dict): + assessment_input = first_run.input + if not isinstance(assessment_input, dict): raise HTTPException( status_code=400, - detail="Assessment configuration is missing for retry", + detail="Assessment input configuration is missing for retry", ) - attachments = assessment_config.get("attachments") or [] + attachments = assessment_input.get("attachments") or [] configs: list[AssessmentConfigRef] = [] for run in runs: if not run.config_id or run.config_version is None: @@ -64,10 +64,10 @@ def _build_retry_request( return AssessmentCreate( experiment_name=experiment_name, dataset_id=dataset_id, - prompt_template=assessment_config.get("prompt_template"), - text_columns=list(assessment_config.get("text_columns") or []), + prompt_template=assessment_input.get("prompt_template"), + text_columns=list(assessment_input.get("text_columns") or []), attachments=[AssessmentAttachment.model_validate(item) for item in attachments], - output_schema=assessment_config.get("output_schema"), + output_schema=assessment_input.get("output_schema"), configs=configs, ) @@ -80,20 +80,8 @@ def start_assessment( ) -> AssessmentResponse: """Start an assessment evaluation. - Validates the dataset, resolves each config, creates one EvaluationRun per config, + Validates the dataset, resolves each config, creates one AssessmentRun per config, and kicks off batch processing for each. - - Args: - session: Database session - request: Validated request body - organization_id: Organization ID - project_id: Project ID - - Returns: - AssessmentResponse with created run summaries - - Raises: - HTTPException: If dataset not found or configs invalid """ logger.info( f"[start_assessment] Starting | " @@ -116,14 +104,14 @@ def start_assessment( detail=f"Dataset {request.dataset_id} not found or not accessible", ) - # 2. Build assessment-specific config to store with each run - assessment_config: dict[str, Any] = { + # 2. Build assessment input to store with each run (flat structure) + assessment_input: dict[str, Any] = { "prompt_template": request.prompt_template, "text_columns": request.text_columns, "attachments": [a.model_dump() for a in request.attachments], } if request.output_schema: - assessment_config["output_schema"] = request.output_schema + assessment_input["output_schema"] = request.output_schema # 3. Validate all configs first before creating any runs resolved_configs = [] @@ -152,8 +140,8 @@ def start_assessment( total_runs=len(resolved_configs), ) - # 5. Create one EvaluationRun per config and submit batches - runs: list[EvaluationRun] = [] + # 5. Create one AssessmentRun per config and submit batches + runs: list[AssessmentRun] = [] for cfg, config_blob in resolved_configs: run = create_assessment_run( session=session, @@ -165,24 +153,24 @@ def start_assessment( config_version=cfg.config_version, organization_id=organization_id, project_id=project_id, - assessment_config=assessment_config, + assessment_input=assessment_input, ) # Submit batch for this run try: batch_job = submit_assessment_batch( session=session, - eval_run=run, + run=run, dataset=dataset, config_blob=config_blob, - assessment_config=assessment_config, + assessment_input=assessment_input, organization_id=organization_id, project_id=project_id, ) run = update_assessment_run_status( session=session, - eval_run=run, + run=run, status="processing", batch_job_id=batch_job.id, total_items=batch_job.total_items, @@ -196,9 +184,9 @@ def start_assessment( ) run = update_assessment_run_status( session=session, - eval_run=run, + run=run, status="failed", - error_message=f"Batch submission failed: {str(e)}", + error_message="Batch submission failed. Please try again or contact support.", ) recompute_assessment_status(session=session, assessment_id=assessment.id) @@ -253,7 +241,7 @@ def retry_assessment( def retry_assessment_run( session: Session, - run: EvaluationRun, + run: AssessmentRun, organization_id: int, project_id: int, ) -> AssessmentResponse: diff --git a/backend/app/assessment/utils/export.py b/backend/app/assessment/utils/export.py index 8f7800384..5da399d53 100644 --- a/backend/app/assessment/utils/export.py +++ b/backend/app/assessment/utils/export.py @@ -11,15 +11,14 @@ from fastapi.responses import StreamingResponse from app.assessment.batch import _load_dataset_rows -from app.assessment.models import AssessmentExportRow +from app.assessment.models import Assessment, AssessmentExportRow, AssessmentRun from app.assessment.processing import parse_assessment_output from app.assessment.utils.parsing import parse_stored_results, usage_totals from app.core.cloud import get_cloud_storage from app.core.storage_utils import generate_timestamped_filename from app.crud.job import get_batch_job -from app.assessment.models import Assessment from app.models.batch_job import BatchJob -from app.models.evaluation import EvaluationDataset, EvaluationRun +from app.models.evaluation import EvaluationDataset logger = logging.getLogger(__name__) @@ -101,7 +100,8 @@ def _expand_output_columns( row_payload, input_col_names = _expand_input_columns(row_payload) base_fields = [ - f for f in AssessmentExportRow.model_fields.keys() + f + for f in AssessmentExportRow.model_fields.keys() if f not in ("output", "input_data") ] @@ -207,7 +207,8 @@ def serialize_export_rows( # XLSX shows input columns + output columns only (no metadata fields). metadata_fields = { - f for f in AssessmentExportRow.model_fields.keys() + f + for f in AssessmentExportRow.model_fields.keys() if f not in ("output", "input_data") } excel_fields = [f for f in fieldnames if f not in metadata_fields] @@ -256,7 +257,7 @@ def build_export_response( def _load_parsed_results_for_run( session: Any, - run: EvaluationRun, + run: AssessmentRun, batch_job: BatchJob, ) -> list[dict[str, Any]] | None: """Fetch and parse the stored batch results for a run. @@ -317,7 +318,7 @@ def _load_parsed_results_for_run( def _load_dataset_rows_for_run( session: Any, - run: EvaluationRun, + run: AssessmentRun, ) -> list[dict[str, str]]: """Load original dataset rows for input-output correlation. @@ -343,7 +344,7 @@ def _load_dataset_rows_for_run( def load_export_rows_for_run( session: Any, - run: EvaluationRun, + run: AssessmentRun, assessment: Assessment | None = None, ) -> list[AssessmentExportRow]: """Load flattened export rows for a single child assessment run.""" diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 44a7d7771..541e0c461 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -121,6 +121,11 @@ def AWS_S3_BUCKET(self) -> str: CELERY_ENABLE_UTC: bool = True CELERY_TIMEZONE: str = "UTC" + # Unleash feature flags + UNLEASH_URL: str = "" + UNLEASH_API_KEY: str = "" + UNLEASH_APP_NAME: str = "kaapi-backend" + # callback timeouts and limits CALLBACK_CONNECT_TIMEOUT: int = 3 CALLBACK_READ_TIMEOUT: int = 10 diff --git a/backend/app/core/feature_flags/__init__.py b/backend/app/core/feature_flags/__init__.py new file mode 100644 index 000000000..7678642a5 --- /dev/null +++ b/backend/app/core/feature_flags/__init__.py @@ -0,0 +1,54 @@ +from enum import Enum + +from app.core.feature_flags.client import ( + get_client, + init_unleash, + shutdown_unleash, +) +from app.core.feature_flags.context import build_context + + +class FeatureFlag(str, Enum): + ASSESSMENT = "Assessment" + + +def is_enabled( + flag: FeatureFlag, + organization_id: int, + project_id: int | None = None, + user_id: int | None = None, + default: bool = False, +) -> bool: + """Check whether *flag* is enabled for the given scope.""" + ctx = build_context(organization_id, project_id, user_id) + + def _fallback(_feature_name: str, _context: dict | None) -> bool: + return default + + return get_client().is_enabled(flag.value, ctx, fallback_function=_fallback) + + +def resolve_all_flags( + organization_id: int, + project_id: int | None = None, + user_id: int | None = None, +) -> dict[str, bool]: + """Evaluate every registered flag for the given scope.""" + return { + flag.name: is_enabled( + flag, + organization_id=organization_id, + project_id=project_id, + user_id=user_id, + ) + for flag in FeatureFlag + } + + +__all__ = [ + "FeatureFlag", + "init_unleash", + "is_enabled", + "resolve_all_flags", + "shutdown_unleash", +] diff --git a/backend/app/core/feature_flags/client.py b/backend/app/core/feature_flags/client.py new file mode 100644 index 000000000..2ab2ab2f8 --- /dev/null +++ b/backend/app/core/feature_flags/client.py @@ -0,0 +1,96 @@ +"""Unleash client singleton. + +Initialised once at app startup via ``init_unleash()``. All other code +accesses the client through ``get_client()``. + +When Unleash is not configured (empty URL / key), a no-op stub is used +so the app can run without an Unleash server — every flag defaults to its +fallback value. +""" + +import logging +from typing import Any, Protocol + +from UnleashClient import UnleashClient + +from app.core.config import settings + +logger = logging.getLogger(__name__) + + +class _FeatureFlagClient(Protocol): + """Minimal interface both the real SDK and the stub satisfy.""" + + def is_enabled( + self, + feature_name: str, + context: dict[str, str] | None = None, + fallback_function: Any = None, + ) -> bool: + ... + + def destroy(self) -> None: + ... + + +class _NoOpClient: + """Stub used when Unleash is not configured.""" + + def is_enabled( + self, + feature_name: str, + context: dict[str, str] | None = None, + fallback_function: Any = None, + ) -> bool: + if fallback_function is not None: + return fallback_function(feature_name, context) + return False + + def destroy(self) -> None: + pass + + +_client: _FeatureFlagClient | None = None + + +def init_unleash() -> None: + """Initialise the Unleash SDK (call once at app startup).""" + global _client + + url = settings.UNLEASH_URL.rstrip("/") if settings.UNLEASH_URL else "" + api_key = settings.UNLEASH_API_KEY + app_name = settings.UNLEASH_APP_NAME + + if not url or not api_key: + logger.warning( + "[init_unleash] UNLEASH_URL or UNLEASH_API_KEY not set — " + "using no-op client (all flags default to fallback)" + ) + _client = _NoOpClient() + return + + _client = UnleashClient( + url=url, + app_name=app_name, + custom_headers={"Authorization": api_key}, + ) + _client.initialize_client() + logger.info(f"[init_unleash] Unleash initialised | url={url} | app={app_name}") + + +def get_client() -> _FeatureFlagClient: + """Return the initialised Unleash client.""" + if _client is None: + raise RuntimeError( + "Unleash client not initialised. Call init_unleash() at startup." + ) + return _client + + +def shutdown_unleash() -> None: + """Gracefully shut down the Unleash client.""" + global _client + if _client is not None: + _client.destroy() + _client = None + logger.info("[shutdown_unleash] Unleash client destroyed") diff --git a/backend/app/core/feature_flags/context.py b/backend/app/core/feature_flags/context.py new file mode 100644 index 000000000..ba234b138 --- /dev/null +++ b/backend/app/core/feature_flags/context.py @@ -0,0 +1,21 @@ +"""Build Unleash evaluation context from auth information.""" + + +def build_context( + organization_id: int, + project_id: int | None = None, + user_id: int | None = None, +) -> dict[str, str]: + """Build an Unleash context dict from auth dimensions. + + Unleash strategies use these fields for targeting: + - organizationId → gate at org level + - projectId → drill down to org + project + - userId → drill down to individual user (future) + """ + ctx: dict[str, str] = {"organizationId": str(organization_id)} + if project_id is not None: + ctx["projectId"] = str(project_id) + if user_id is not None: + ctx["userId"] = str(user_id) + return ctx diff --git a/backend/app/main.py b/backend/app/main.py index 47a27f371..10461c2ec 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,3 +1,6 @@ +import logging +from contextlib import asynccontextmanager + import sentry_sdk from fastapi import FastAPI @@ -8,10 +11,13 @@ from app.api.docs.openapi_config import tags_metadata, customize_openapi_schema from app.core.config import settings from app.core.exception_handlers import register_exception_handlers +from app.core.feature_flags import init_unleash, shutdown_unleash from app.core.middleware import http_request_logger from app.load_env import load_environment +logger = logging.getLogger(__name__) + # Load environment variables load_environment() @@ -23,11 +29,23 @@ def custom_generate_unique_id(route: APIRoute) -> str: if settings.SENTRY_DSN and settings.ENVIRONMENT != "development": sentry_sdk.init(dsn=str(settings.SENTRY_DSN), enable_tracing=True) + +@asynccontextmanager +async def lifespan(app: FastAPI): + """App startup / shutdown lifecycle.""" + init_unleash() + logger.info("[lifespan] Application started") + yield + shutdown_unleash() + logger.info("[lifespan] Application shut down") + + app = FastAPI( title=settings.PROJECT_NAME, openapi_url=f"{settings.API_V1_STR}/openapi.json", generate_unique_id_function=custom_generate_unique_id, description="**Responsible AI for the development sector**", + lifespan=lifespan, ) diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index b5cb3f0c6..06b2e55bf 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -86,6 +86,8 @@ EvaluationRunPublic, ) +from app.assessment.models import Assessment, AssessmentRun # noqa: F401 + from .file import File, FilePublic, FileType, AudioUploadResponse from .fine_tuning import ( diff --git a/backend/app/models/evaluation.py b/backend/app/models/evaluation.py index ab667878b..d2d2beecc 100644 --- a/backend/app/models/evaluation.py +++ b/backend/app/models/evaluation.py @@ -255,18 +255,6 @@ class EvaluationRun(SQLModel, table=True): sa_column_kwargs={"comment": "Reference to the evaluation dataset"}, ) - # Assessment reference (set when run belongs to a parent Assessment) - assessment_id: int | None = SQLField( - default=None, - foreign_key="assessment.id", - nullable=True, - ondelete="SET NULL", - description="Reference to parent assessment manager row, if applicable", - sa_column_kwargs={ - "comment": "Reference to parent assessment manager row, if applicable" - }, - ) - # Batch job references batch_job_id: int | None = SQLField( default=None, @@ -403,7 +391,6 @@ class EvaluationRunPublic(SQLModel): config_id: UUID | None config_version: int | None dataset_id: int - assessment_id: int | None batch_job_id: int | None embedding_batch_job_id: int | None status: str diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 395f0120b..df8ce051d 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -44,6 +44,8 @@ dependencies = [ "whisper-normalizer>=0.1.12", "elevenlabs>=2.38.1", "gevent>=25.9.1", + "openpyxl>=3.1.0", + "UnleashClient>=6.0.0", ] [tool.uv] diff --git a/backend/uv.lock b/backend/uv.lock index e8fd3ca4c..d1bc099f2 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -238,6 +238,7 @@ dependencies = [ { name = "numpy" }, { name = "openai" }, { name = "openai-responses" }, + { name = "openpyxl" }, { name = "pandas" }, { name = "passlib", extra = ["bcrypt"] }, { name = "pre-commit" }, @@ -255,6 +256,7 @@ dependencies = [ { name = "sentry-sdk", extra = ["fastapi"] }, { name = "sqlmodel" }, { name = "tenacity" }, + { name = "unleashclient" }, { name = "whisper-normalizer" }, ] @@ -292,6 +294,7 @@ requires-dist = [ { name = "numpy", specifier = ">=1.24.0" }, { name = "openai", specifier = ">=1.100.1" }, { name = "openai-responses" }, + { name = "openpyxl", specifier = ">=3.1.0" }, { name = "pandas", specifier = ">=2.3.2" }, { name = "passlib", extras = ["bcrypt"], specifier = ">=1.7.4,<2.0.0" }, { name = "pre-commit", specifier = ">=3.8.0" }, @@ -309,6 +312,7 @@ requires-dist = [ { name = "sentry-sdk", extras = ["fastapi"], specifier = ">=2.20.0" }, { name = "sqlmodel", specifier = ">=0.0.21,<1.0.0" }, { name = "tenacity", specifier = ">=8.2.3,<9.0.0" }, + { name = "unleashclient", specifier = ">=6.0.0" }, { name = "whisper-normalizer", specifier = ">=0.1.12" }, ] @@ -323,6 +327,27 @@ dev = [ { name = "types-passlib", specifier = ">=1.7.7.20240106,<2.0.0.0" }, ] +[[package]] +name = "appdirs" +version = "1.4.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/d8/05696357e0311f5b5c316d7b95f46c669dd9c15aaeecbb48c7d0aeb88c40/appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", size = 13470, upload-time = "2020-05-11T07:59:51.037Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/00/2344469e2084fb287c2e0b57b72910309874c3245463acd6cf5e3db69324/appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128", size = 9566, upload-time = "2020-05-11T07:59:49.499Z" }, +] + +[[package]] +name = "apscheduler" +version = "3.11.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzlocal" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/12/3e4389e5920b4c1763390c6d371162f3784f86f85cd6d6c1bfe68eef14e2/apscheduler-3.11.2.tar.gz", hash = "sha256:2a9966b052ec805f020c8c4c3ae6e6a06e24b1bf19f2e11d91d8cca0473eef41", size = 108683, upload-time = "2025-12-22T00:39:34.884Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/64/2e54428beba8d9992aa478bb8f6de9e4ecaa5f8f513bcfd567ed7fb0262d/apscheduler-3.11.2-py3-none-any.whl", hash = "sha256:ce005177f741409db4e4dd40a7431b76feb856b9dd69d57e0da49d6715bfd26d", size = 64439, upload-time = "2025-12-22T00:39:33.303Z" }, +] + [[package]] name = "asgi-correlation-id" version = "4.3.4" @@ -903,6 +928,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/55/7e/b648d640d88d31de49e566832aca9cce025c52d6349b0a0fc65e9df1f4c5/emails-0.6-py2.py3-none-any.whl", hash = "sha256:72c1e3198075709cc35f67e1b49e2da1a2bc087e9b444073db61a379adfb7f3c", size = 56250, upload-time = "2020-06-19T11:20:40.466Z" }, ] +[[package]] +name = "et-xmlfile" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/38/af70d7ab1ae9d4da450eeec1fa3918940a5fafb9055e934af8d6eb0c2313/et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54", size = 17234, upload-time = "2024-10-25T17:25:40.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, +] + [[package]] name = "fastapi" version = "0.135.1" @@ -1079,6 +1113,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/f6/da4db31001e854025ffd26bc9ba0740a9cbba2c3259695f7c5834908b336/fastuuid-0.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:df61342889d0f5e7a32f7284e55ef95103f2110fee433c2ae7c2c0956d76ac8a", size = 156457, upload-time = "2025-10-19T22:33:44.579Z" }, ] +[[package]] +name = "fcache" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "appdirs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bf/44/1441c96346698a5f5c2676109911dc4c70b7086755ed15b80164fd4d2f71/fcache-0.5.0.tar.gz", hash = "sha256:bc7a50d1ec2b2b66b1ab0ab2de6076ec1c9e8b524cfcaea5ff07064bdcd784c9", size = 7221, upload-time = "2023-03-19T22:11:53.018Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/7f/de1c875d8ea4850ec90d57c340007fe2f8a8dbbafd278ab96e5ae3b602fe/fcache-0.5.0-py3-none-any.whl", hash = "sha256:2fb3af3482456e264fb271213dd1603de6a583afd7d8cc1f0875f37821f58530", size = 8066, upload-time = "2023-03-19T22:11:54.49Z" }, +] + [[package]] name = "filelock" version = "3.25.2" @@ -1694,6 +1740,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/6b/4d3bdea30ceb3e4cf3ac1a2f104ffc20b6caa636549874262b2fa8cedaec/langfuse-2.60.3-py3-none-any.whl", hash = "sha256:2b866c44f24d5f06b617d7f14f75a2e42577538b530e4e26dc6ad770d6d1399e", size = 275008, upload-time = "2025-04-15T17:01:13.799Z" }, ] +[[package]] +name = "launchdarkly-eventsource" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1a/77/f4ec64fb2e4d72a3a261f741cd3ef3bbd8cb22760c471f04366a1bc98100/launchdarkly_eventsource-1.5.1.tar.gz", hash = "sha256:f122f80b36db6ea1ab20af62c82b8b2668682259b415053c94400dd6c07922a7", size = 25583, upload-time = "2026-01-23T17:35:39.462Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/c6/c0816382050a97ebf0e4590e0adf93a317e7e64333eed4467b1081b261d7/launchdarkly_eventsource-1.5.1-py3-none-any.whl", hash = "sha256:43dfbc14a3962c9bce252320d690cdcbfda0ca00501226d517ae77e60f570d44", size = 33964, upload-time = "2026-01-23T17:35:38.386Z" }, +] + [[package]] name = "librt" version = "0.8.1" @@ -1953,6 +2011,88 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "mmh3" +version = "5.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/91/1a/edb23803a168f070ded7a3014c6d706f63b90c84ccc024f89d794a3b7a6d/mmh3-5.2.1.tar.gz", hash = "sha256:bbea5b775f0ac84945191fb83f845a6fd9a21a03ea7f2e187defac7e401616ad", size = 33775, upload-time = "2026-03-05T15:55:57.716Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/94/bc5c3b573b40a328c4d141c20e399039ada95e5e2a661df3425c5165fd84/mmh3-5.2.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0cc21533878e5586b80d74c281d7f8da7932bc8ace50b8d5f6dbf7e3935f63f1", size = 56087, upload-time = "2026-03-05T15:54:21.92Z" }, + { url = "https://files.pythonhosted.org/packages/f6/80/64a02cc3e95c3af0aaa2590849d9ed24a9f14bb93537addde688e039b7c3/mmh3-5.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4eda76074cfca2787c8cf1bec603eaebdddd8b061ad5502f85cddae998d54f00", size = 40500, upload-time = "2026-03-05T15:54:22.953Z" }, + { url = "https://files.pythonhosted.org/packages/8b/72/e6d6602ce18adf4ddcd0e48f2e13590cc92a536199e52109f46f259d3c46/mmh3-5.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:eee884572b06bbe8a2b54f424dbd996139442cf83c76478e1ec162512e0dd2c7", size = 40034, upload-time = "2026-03-05T15:54:23.943Z" }, + { url = "https://files.pythonhosted.org/packages/59/c2/bf4537a8e58e21886ef16477041238cab5095c836496e19fafc34b7445d2/mmh3-5.2.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:0d0b7e803191db5f714d264044e06189c8ccd3219e936cc184f07106bd17fd7b", size = 97292, upload-time = "2026-03-05T15:54:25.335Z" }, + { url = "https://files.pythonhosted.org/packages/e5/e2/51ed62063b44d10b06d975ac87af287729eeb5e3ed9772f7584a17983e90/mmh3-5.2.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8e6c219e375f6341d0959af814296372d265a8ca1af63825f65e2e87c618f006", size = 103274, upload-time = "2026-03-05T15:54:26.44Z" }, + { url = "https://files.pythonhosted.org/packages/75/ce/12a7524dca59eec92e5b31fdb13ede1e98eda277cf2b786cf73bfbc24e81/mmh3-5.2.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:26fb5b9c3946bf7f1daed7b37e0c03898a6f062149127570f8ede346390a0825", size = 106158, upload-time = "2026-03-05T15:54:28.578Z" }, + { url = "https://files.pythonhosted.org/packages/86/1f/d3ba6dd322d01ab5d44c46c8f0c38ab6bbbf9b5e20e666dfc05bf4a23604/mmh3-5.2.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3c38d142c706201db5b2345166eeef1e7740e3e2422b470b8ba5c8727a9b4c7a", size = 113005, upload-time = "2026-03-05T15:54:29.767Z" }, + { url = "https://files.pythonhosted.org/packages/b6/a9/15d6b6f913294ea41b44d901741298e3718e1cb89ee626b3694625826a43/mmh3-5.2.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50885073e2909251d4718634a191c49ae5f527e5e1736d738e365c3e8be8f22b", size = 120744, upload-time = "2026-03-05T15:54:30.931Z" }, + { url = "https://files.pythonhosted.org/packages/76/b3/70b73923fd0284c439860ff5c871b20210dfdbe9a6b9dd0ee6496d77f174/mmh3-5.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b3f99e1756fc48ad507b95e5d86f2fb21b3d495012ff13e6592ebac14033f166", size = 99111, upload-time = "2026-03-05T15:54:32.353Z" }, + { url = "https://files.pythonhosted.org/packages/dd/38/99f7f75cd27d10d8b899a1caafb9d531f3903e4d54d572220e3d8ac35e89/mmh3-5.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62815d2c67f2dd1be76a253d88af4e1da19aeaa1820146dec52cf8bee2958b16", size = 98623, upload-time = "2026-03-05T15:54:33.801Z" }, + { url = "https://files.pythonhosted.org/packages/fd/68/6e292c0853e204c44d2f03ea5f090be3317a0e2d9417ecb62c9eb27687df/mmh3-5.2.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8f767ba0911602ddef289404e33835a61168314ebd3c729833db2ed685824211", size = 106437, upload-time = "2026-03-05T15:54:35.177Z" }, + { url = "https://files.pythonhosted.org/packages/dd/c6/fedd7284c459cfb58721d461fcf5607a4c1f5d9ab195d113d51d10164d16/mmh3-5.2.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:67e41a497bac88cc1de96eeba56eeb933c39d54bc227352f8455aa87c4ca4000", size = 110002, upload-time = "2026-03-05T15:54:36.673Z" }, + { url = "https://files.pythonhosted.org/packages/3b/ac/ca8e0c19a34f5b71390171d2ff0b9f7f187550d66801a731bb68925126a4/mmh3-5.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3d74a03fb57757ece25aa4b3c1c60157a1cece37a020542785f942e2f827eed5", size = 97507, upload-time = "2026-03-05T15:54:37.804Z" }, + { url = "https://files.pythonhosted.org/packages/df/94/6ebb9094cfc7ac5e7950776b9d13a66bb4a34f83814f32ba2abc9494fc68/mmh3-5.2.1-cp312-cp312-win32.whl", hash = "sha256:7374d6e3ef72afe49697ecd683f3da12f4fc06af2d75433d0580c6746d2fa025", size = 40773, upload-time = "2026-03-05T15:54:40.077Z" }, + { url = "https://files.pythonhosted.org/packages/5b/3c/cd3527198cf159495966551c84a5f36805a10ac17b294f41f67b83f6a4d6/mmh3-5.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:3a9fed49c6ce4ed7e73f13182760c65c816da006debe67f37635580dfb0fae00", size = 41560, upload-time = "2026-03-05T15:54:41.148Z" }, + { url = "https://files.pythonhosted.org/packages/15/96/6fe5ebd0f970a076e3ed5512871ce7569447b962e96c125528a2f9724470/mmh3-5.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:bbfcb95d9a744e6e2827dfc66ad10e1020e0cac255eb7f85652832d5a264c2fc", size = 39313, upload-time = "2026-03-05T15:54:42.171Z" }, + { url = "https://files.pythonhosted.org/packages/25/a5/9daa0508a1569a54130f6198d5462a92deda870043624aa3ea72721aa765/mmh3-5.2.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:723b2681ed4cc07d3401bbea9c201ad4f2a4ca6ba8cddaff6789f715dd2b391e", size = 40832, upload-time = "2026-03-05T15:54:43.212Z" }, + { url = "https://files.pythonhosted.org/packages/0a/6b/3230c6d80c1f4b766dedf280a92c2241e99f87c1504ff74205ec8cebe451/mmh3-5.2.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:3619473a0e0d329fd4aec8075628f8f616be2da41605300696206d6f36920c3d", size = 41964, upload-time = "2026-03-05T15:54:44.204Z" }, + { url = "https://files.pythonhosted.org/packages/62/fb/648bfddb74a872004b6ee751551bfdda783fe6d70d2e9723bad84dbe5311/mmh3-5.2.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:e48d4dbe0f88e53081da605ae68644e5182752803bbc2beb228cca7f1c4454d6", size = 39114, upload-time = "2026-03-05T15:54:45.205Z" }, + { url = "https://files.pythonhosted.org/packages/95/c2/ab7901f87af438468b496728d11264cb397b3574d41506e71b92128e0373/mmh3-5.2.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:a482ac121de6973897c92c2f31defc6bafb11c83825109275cffce54bb64933f", size = 39819, upload-time = "2026-03-05T15:54:46.509Z" }, + { url = "https://files.pythonhosted.org/packages/2f/ed/6f88dda0df67de1612f2e130ffea34cf84aaee5bff5b0aff4dbff2babe34/mmh3-5.2.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:17fbb47f0885ace8327ce1235d0416dc86a211dcd8cc1e703f41523be32cfec8", size = 40330, upload-time = "2026-03-05T15:54:47.864Z" }, + { url = "https://files.pythonhosted.org/packages/3d/66/7516d23f53cdf90f43fce24ab80c28f45e6851d78b46bef8c02084edf583/mmh3-5.2.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d51fde50a77f81330523562e3c2734ffdca9c4c9e9d355478117905e1cfe16c6", size = 56078, upload-time = "2026-03-05T15:54:48.9Z" }, + { url = "https://files.pythonhosted.org/packages/bc/34/4d152fdf4a91a132cb226b671f11c6b796eada9ab78080fb5ce1e95adaab/mmh3-5.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:19bbd3b841174ae6ed588536ab5e1b1fe83d046e668602c20266547298d939a9", size = 40498, upload-time = "2026-03-05T15:54:49.942Z" }, + { url = "https://files.pythonhosted.org/packages/d4/4c/8e3af1b6d85a299767ec97bd923f12b06267089c1472c27c1696870d1175/mmh3-5.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:be77c402d5e882b6fbacfd90823f13da8e0a69658405a39a569c6b58fdb17b03", size = 40033, upload-time = "2026-03-05T15:54:50.994Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f2/966ea560e32578d453c9e9db53d602cbb1d0da27317e232afa7c38ceba11/mmh3-5.2.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:fd96476f04db5ceba1cfa0f21228f67c1f7402296f0e73fee3513aa680ad237b", size = 97320, upload-time = "2026-03-05T15:54:52.072Z" }, + { url = "https://files.pythonhosted.org/packages/bb/0d/2c5f9893b38aeb6b034d1a44ecd55a010148054f6a516abe53b5e4057297/mmh3-5.2.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:707151644085dd0f20fe4f4b573d28e5130c4aaa5f587e95b60989c5926653b5", size = 103299, upload-time = "2026-03-05T15:54:53.569Z" }, + { url = "https://files.pythonhosted.org/packages/1c/fc/2ebaef4a4d4376f89761274dc274035ffd96006ab496b4ee5af9b08f21a9/mmh3-5.2.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3737303ca9ea0f7cb83028781148fcda4f1dac7821db0c47672971dabcf63593", size = 106222, upload-time = "2026-03-05T15:54:55.092Z" }, + { url = "https://files.pythonhosted.org/packages/57/09/ea7ffe126d0ba0406622602a2d05e1e1a6841cc92fc322eb576c95b27fad/mmh3-5.2.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2778fed822d7db23ac5008b181441af0c869455b2e7d001f4019636ac31b6fe4", size = 113048, upload-time = "2026-03-05T15:54:56.305Z" }, + { url = "https://files.pythonhosted.org/packages/85/57/9447032edf93a64aa9bef4d9aa596400b1756f40411890f77a284f6293ca/mmh3-5.2.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d57dea657357230cc780e13920d7fa7db059d58fe721c80020f94476da4ca0a1", size = 120742, upload-time = "2026-03-05T15:54:57.453Z" }, + { url = "https://files.pythonhosted.org/packages/53/82/a86cc87cc88c92e9e1a598fee509f0409435b57879a6129bf3b3e40513c7/mmh3-5.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:169e0d178cb59314456ab30772429a802b25d13227088085b0d49b9fe1533104", size = 99132, upload-time = "2026-03-05T15:54:58.583Z" }, + { url = "https://files.pythonhosted.org/packages/54/f7/6b16eb1b40ee89bb740698735574536bc20d6cdafc65ae702ea235578e05/mmh3-5.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7e4e1f580033335c6f76d1e0d6b56baf009d1a64d6a4816347e4271ba951f46d", size = 98686, upload-time = "2026-03-05T15:55:00.078Z" }, + { url = "https://files.pythonhosted.org/packages/e8/88/a601e9f32ad1410f438a6d0544298ea621f989bd34a0731a7190f7dec799/mmh3-5.2.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:2bd9f19f7f1fcebd74e830f4af0f28adad4975d40d80620be19ffb2b2af56c9f", size = 106479, upload-time = "2026-03-05T15:55:01.532Z" }, + { url = "https://files.pythonhosted.org/packages/d6/5c/ce29ae3dfc4feec4007a437a1b7435fb9507532a25147602cd5b52be86db/mmh3-5.2.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:c88653877aeb514c089d1b3d473451677b8b9a6d1497dbddf1ae7934518b06d2", size = 110030, upload-time = "2026-03-05T15:55:02.934Z" }, + { url = "https://files.pythonhosted.org/packages/13/30/ae444ef2ff87c805d525da4fa63d27cda4fe8a48e77003a036b8461cfd5c/mmh3-5.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fceef7fe67c81e1585198215e42ad3fdba3a25644beda8fbdaf85f4d7b93175a", size = 97536, upload-time = "2026-03-05T15:55:04.135Z" }, + { url = "https://files.pythonhosted.org/packages/4b/f9/dc3787ee5c813cc27fe79f45ad4500d9b5437f23a7402435cc34e07c7718/mmh3-5.2.1-cp313-cp313-win32.whl", hash = "sha256:54b64fb2433bc71488e7a449603bf8bd31fbcf9cb56fbe1eb6d459e90b86c37b", size = 40769, upload-time = "2026-03-05T15:55:05.277Z" }, + { url = "https://files.pythonhosted.org/packages/43/67/850e0b5a1e97799822ebfc4ca0e8c6ece3ed8baf7dcdf64de817dfdda2ca/mmh3-5.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:cae6383181f1e345317742d2ddd88f9e7d2682fa4c9432e3a74e47d92dce0229", size = 41563, upload-time = "2026-03-05T15:55:06.283Z" }, + { url = "https://files.pythonhosted.org/packages/c0/cc/98c90b28e1da5458e19fbfaf4adb5289208d3bfccd45dd14eab216a2f0bb/mmh3-5.2.1-cp313-cp313-win_arm64.whl", hash = "sha256:022aa1a528604e6c83d0a7705fdef0b5355d897a9e0fa3a8d26709ceaa06965d", size = 39310, upload-time = "2026-03-05T15:55:07.323Z" }, + { url = "https://files.pythonhosted.org/packages/63/b4/65bc1fb2bb7f83e91c30865023b1847cf89a5f237165575e8c83aa536584/mmh3-5.2.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:d771f085fcdf4035786adfb1d8db026df1eb4b41dac1c3d070d1e49512843227", size = 40794, upload-time = "2026-03-05T15:55:09.773Z" }, + { url = "https://files.pythonhosted.org/packages/c4/86/7168b3d83be8eb553897b1fac9da8bbb06568e5cfe555ffc329ebb46f59d/mmh3-5.2.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:7f196cd7910d71e9d9860da0ff7a77f64d22c1ad931f1dd18559a06e03109fc0", size = 41923, upload-time = "2026-03-05T15:55:10.924Z" }, + { url = "https://files.pythonhosted.org/packages/bf/9b/b653ab611c9060ce8ff0ba25c0226757755725e789292f3ca138a58082cd/mmh3-5.2.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b1f12bd684887a0a5d55e6363ca87056f361e45451105012d329b86ec19dbe0b", size = 39131, upload-time = "2026-03-05T15:55:11.961Z" }, + { url = "https://files.pythonhosted.org/packages/9b/b4/5a2e0d34ab4d33543f01121e832395ea510132ea8e52cdf63926d9d81754/mmh3-5.2.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:d106493a60dcb4aef35a0fac85105e150a11cf8bc2b0d388f5a33272d756c966", size = 39825, upload-time = "2026-03-05T15:55:13.013Z" }, + { url = "https://files.pythonhosted.org/packages/bd/69/81699a8f39a3f8d368bec6443435c0c392df0d200ad915bf0d222b588e03/mmh3-5.2.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:44983e45310ee5b9f73397350251cdf6e63a466406a105f1d16cb5baa659270b", size = 40344, upload-time = "2026-03-05T15:55:14.026Z" }, + { url = "https://files.pythonhosted.org/packages/0c/b3/71c8c775807606e8fd8acc5c69016e1caf3200d50b50b6dd4b40ce10b76c/mmh3-5.2.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:368625fb01666655985391dbad3860dc0ba7c0d6b9125819f3121ee7292b4ac8", size = 56291, upload-time = "2026-03-05T15:55:15.137Z" }, + { url = "https://files.pythonhosted.org/packages/6f/75/2c24517d4b2ce9e4917362d24f274d3d541346af764430249ddcc4cb3a08/mmh3-5.2.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:72d1cc63bcc91e14933f77d51b3df899d6a07d184ec515ea7f56bff659e124d7", size = 40575, upload-time = "2026-03-05T15:55:16.518Z" }, + { url = "https://files.pythonhosted.org/packages/bf/b9/e4a360164365ac9f07a25f0f7928e3a66eb9ecc989384060747aa170e6aa/mmh3-5.2.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e8b4b5580280b9265af3e0409974fb79c64cf7523632d03fbf11df18f8b0181e", size = 40052, upload-time = "2026-03-05T15:55:17.735Z" }, + { url = "https://files.pythonhosted.org/packages/97/ca/120d92223a7546131bbbc31c9174168ee7a73b1366f5463ffe69d9e691fe/mmh3-5.2.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4cbbde66f1183db040daede83dd86c06d663c5bb2af6de1142b7c8c37923dd74", size = 97311, upload-time = "2026-03-05T15:55:18.959Z" }, + { url = "https://files.pythonhosted.org/packages/b6/71/c1a60c1652b8813ef9de6d289784847355417ee0f2980bca002fe87f4ae5/mmh3-5.2.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8ff038d52ef6aa0f309feeba00c5095c9118d0abf787e8e8454d6048db2037fc", size = 103279, upload-time = "2026-03-05T15:55:20.448Z" }, + { url = "https://files.pythonhosted.org/packages/48/29/ad97f4be1509cdcb28ae32c15593ce7c415db47ace37f8fad35b493faa9a/mmh3-5.2.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a4130d0b9ce5fad6af07421b1aecc7e079519f70d6c05729ab871794eded8617", size = 106290, upload-time = "2026-03-05T15:55:21.6Z" }, + { url = "https://files.pythonhosted.org/packages/77/29/1f86d22e281bd8827ba373600a4a8b0c0eae5ca6aa55b9a8c26d2a34decc/mmh3-5.2.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6e0bfe77d238308839699944164b96a2eeccaf55f2af400f54dc20669d8d5f2", size = 113116, upload-time = "2026-03-05T15:55:22.826Z" }, + { url = "https://files.pythonhosted.org/packages/a7/7c/339971ea7ed4c12d98f421f13db3ea576a9114082ccb59d2d1a0f00ccac1/mmh3-5.2.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f963eafc0a77a6c0562397da004f5876a9bcf7265a7bcc3205e29636bc4a1312", size = 120740, upload-time = "2026-03-05T15:55:24.3Z" }, + { url = "https://files.pythonhosted.org/packages/e4/92/3c7c4bdb8e926bb3c972d1e2907d77960c1c4b250b41e8366cf20c6e4373/mmh3-5.2.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:92883836caf50d5255be03d988d75bc93e3f86ba247b7ca137347c323f731deb", size = 99143, upload-time = "2026-03-05T15:55:25.456Z" }, + { url = "https://files.pythonhosted.org/packages/df/0a/33dd8706e732458c8375eae63c981292de07a406bad4ec03e5269654aa2c/mmh3-5.2.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:57b52603e89355ff318025dd55158f6e71396c0f1f609d548e9ea9c94cc6ce0a", size = 98703, upload-time = "2026-03-05T15:55:26.723Z" }, + { url = "https://files.pythonhosted.org/packages/51/04/76bbce05df76cbc3d396f13b2ea5b1578ef02b6a5187e132c6c33f99d596/mmh3-5.2.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f40a95186a72fa0b67d15fef0f157bfcda00b4f59c8a07cbe5530d41ac35d105", size = 106484, upload-time = "2026-03-05T15:55:28.214Z" }, + { url = "https://files.pythonhosted.org/packages/d3/8f/c6e204a2c70b719c1f62ffd9da27aef2dddcba875ea9c31ca0e87b975a46/mmh3-5.2.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:58370d05d033ee97224c81263af123dea3d931025030fd34b61227a768a8858a", size = 110012, upload-time = "2026-03-05T15:55:29.532Z" }, + { url = "https://files.pythonhosted.org/packages/e3/37/7181efd8e39db386c1ebc3e6b7d1f702a09d7c1197a6f2742ed6b5c16597/mmh3-5.2.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7be6dfb49e48fd0a7d91ff758a2b51336f1cd21f9d44b20f6801f072bd080cdd", size = 97508, upload-time = "2026-03-05T15:55:31.01Z" }, + { url = "https://files.pythonhosted.org/packages/42/0f/afa7ca2615fd85e1469474bb860e381443d0b868c083b62b41cb1d7ca32f/mmh3-5.2.1-cp314-cp314-win32.whl", hash = "sha256:54fe8518abe06a4c3852754bfd498b30cc58e667f376c513eac89a244ce781a4", size = 41387, upload-time = "2026-03-05T15:55:32.403Z" }, + { url = "https://files.pythonhosted.org/packages/71/0d/46d42a260ee1357db3d486e6c7a692e303c017968e14865e00efa10d09fc/mmh3-5.2.1-cp314-cp314-win_amd64.whl", hash = "sha256:3f796b535008708846044c43302719c6956f39ca2d93f2edda5319e79a29efbb", size = 42101, upload-time = "2026-03-05T15:55:33.646Z" }, + { url = "https://files.pythonhosted.org/packages/a4/7b/848a8378059d96501a41159fca90d6a99e89736b0afbe8e8edffeac8c74b/mmh3-5.2.1-cp314-cp314-win_arm64.whl", hash = "sha256:cd471ede0d802dd936b6fab28188302b2d497f68436025857ca72cd3810423fe", size = 39836, upload-time = "2026-03-05T15:55:35.026Z" }, + { url = "https://files.pythonhosted.org/packages/27/61/1dabea76c011ba8547c25d30c91c0ec22544487a8750997a27a0c9e1180b/mmh3-5.2.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:5174a697ce042fa77c407e05efe41e03aa56dae9ec67388055820fb48cf4c3ba", size = 57727, upload-time = "2026-03-05T15:55:36.162Z" }, + { url = "https://files.pythonhosted.org/packages/b7/32/731185950d1cf2d5e28979cc8593016ba1619a295faba10dda664a4931b5/mmh3-5.2.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0a3984146e414684a6be2862d84fcb1035f4984851cb81b26d933bab6119bf00", size = 41308, upload-time = "2026-03-05T15:55:37.254Z" }, + { url = "https://files.pythonhosted.org/packages/76/aa/66c76801c24b8c9418b4edde9b5e57c75e72c94e29c48f707e3962534f18/mmh3-5.2.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:bd6e7d363aa93bd3421b30b6af97064daf47bc96005bddba67c5ffbc6df426b8", size = 40758, upload-time = "2026-03-05T15:55:38.61Z" }, + { url = "https://files.pythonhosted.org/packages/9e/bb/79a1f638a02f0ae389f706d13891e2fbf7d8c0a22ecde67ba828951bb60a/mmh3-5.2.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:113f78e7463a36dbbcea05bfe688efd7fa759d0f0c56e73c974d60dcfec3dfcc", size = 109670, upload-time = "2026-03-05T15:55:40.13Z" }, + { url = "https://files.pythonhosted.org/packages/26/94/8cd0e187a288985bcfc79bf5144d1d712df9dee74365f59d26e3a1865be6/mmh3-5.2.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7e8ec5f606e0809426d2440e0683509fb605a8820a21ebd120dcdba61b74ef7f", size = 117399, upload-time = "2026-03-05T15:55:42.076Z" }, + { url = "https://files.pythonhosted.org/packages/42/94/dfea6059bd5c5beda565f58a4096e43f4858fb6d2862806b8bbd12cbb284/mmh3-5.2.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:22b0f9971ec4e07e8223f2beebe96a6cfc779d940b6f27d26604040dd74d3a44", size = 120386, upload-time = "2026-03-05T15:55:43.481Z" }, + { url = "https://files.pythonhosted.org/packages/47/cb/f9c45e62aaa67220179f487772461d891bb582bb2f9783c944832c60efd9/mmh3-5.2.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85ffc9920ffc39c5eee1e3ac9100c913a0973996fbad5111f939bbda49204bb7", size = 125924, upload-time = "2026-03-05T15:55:44.638Z" }, + { url = "https://files.pythonhosted.org/packages/a5/83/fe54a4a7c11bc9f623dfc1707decd034245602b076dfc1dcc771a4163170/mmh3-5.2.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7aec798c2b01aaa65a55f1124f3405804184373abb318a3091325aece235f67c", size = 135280, upload-time = "2026-03-05T15:55:45.866Z" }, + { url = "https://files.pythonhosted.org/packages/97/67/fe7e9e9c143daddd210cd22aef89cbc425d58ecf238d2b7d9eb0da974105/mmh3-5.2.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:55dbbd8ffbc40d1697d5e2d0375b08599dae8746b0b08dea05eee4ce81648fac", size = 110050, upload-time = "2026-03-05T15:55:47.074Z" }, + { url = "https://files.pythonhosted.org/packages/43/c4/6d4b09fcbef80794de447c9378e39eefc047156b290fa3dd2d5257ca8227/mmh3-5.2.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:6c85c38a279ca9295a69b9b088a2e48aa49737bb1b34e6a9dc6297c110e8d912", size = 111158, upload-time = "2026-03-05T15:55:48.239Z" }, + { url = "https://files.pythonhosted.org/packages/81/a6/ca51c864bdb30524beb055a6d8826db3906af0834ec8c41d097a6e8573d5/mmh3-5.2.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:6290289fa5fb4c70fd7f72016e03633d60388185483ff3b162912c81205ae2cf", size = 116890, upload-time = "2026-03-05T15:55:49.405Z" }, + { url = "https://files.pythonhosted.org/packages/cc/04/5a1fe2e2ad843d03e89af25238cbc4f6840a8bb6c4329a98ab694c71deda/mmh3-5.2.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4fc6cd65dc4d2fdb2625e288939a3566e36127a84811a4913f02f3d5931da52d", size = 123121, upload-time = "2026-03-05T15:55:50.61Z" }, + { url = "https://files.pythonhosted.org/packages/af/4d/3c820c6f4897afd25905270a9f2330a23f77a207ea7356f7aadace7273c0/mmh3-5.2.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:623f938f6a039536cc02b7582a07a080f13fdfd48f87e63201d92d7e34d09a18", size = 110187, upload-time = "2026-03-05T15:55:52.143Z" }, + { url = "https://files.pythonhosted.org/packages/21/54/1d71cd143752361c0aebef16ad3f55926a6faf7b112d355745c1f8a25f7f/mmh3-5.2.1-cp314-cp314t-win32.whl", hash = "sha256:29bc3973676ae334412efdd367fcd11d036b7be3efc1ce2407ef8676dabfeb82", size = 41934, upload-time = "2026-03-05T15:55:53.564Z" }, + { url = "https://files.pythonhosted.org/packages/9d/e4/63a2a88f31d93dea03947cccc2a076946857e799ea4f7acdecbf43b324aa/mmh3-5.2.1-cp314-cp314t-win_amd64.whl", hash = "sha256:28cfab66577000b9505a0d068c731aee7ca85cd26d4d63881fab17857e0fe1fb", size = 43036, upload-time = "2026-03-05T15:55:55.252Z" }, + { url = "https://files.pythonhosted.org/packages/a0/0f/59204bf136d1201f8d7884cfbaf7498c5b4674e87a4c693f9bde63741ce1/mmh3-5.2.1-cp314-cp314t-win_arm64.whl", hash = "sha256:dfd51b4c56b673dfbc43d7d27ef857dd91124801e2806c69bb45585ce0fa019b", size = 40391, upload-time = "2026-03-05T15:55:56.697Z" }, +] + [[package]] name = "more-itertools" version = "10.8.0" @@ -2241,6 +2381,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/08/98/50c755503a55550f170d0211297bc8791b8bf10bf04cb16b4b95ca71d1e3/openai_responses-0.13.1-py3-none-any.whl", hash = "sha256:b5fc7fb15f546b551757864c1dfaeb01b8a4fc0c353961bd6d0d45ff26389721", size = 51887, upload-time = "2025-12-02T21:37:40.15Z" }, ] +[[package]] +name = "openpyxl" +version = "3.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "et-xmlfile" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/f9/88d94a75de065ea32619465d2f77b29a0469500e99012523b91cc4141cd1/openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050", size = 186464, upload-time = "2024-06-28T14:03:44.161Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910, upload-time = "2024-06-28T14:03:41.161Z" }, +] + [[package]] name = "packaging" version = "24.2" @@ -3527,6 +3679,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/39/338d9219c4e87f3e708f18857ecd24d22a0c3094752393319553096b98af/scipy-1.17.1-cp314-cp314t-win_arm64.whl", hash = "sha256:200e1050faffacc162be6a486a984a0497866ec54149a01270adc8a59b7c7d21", size = 25489165, upload-time = "2026-02-23T00:22:29.563Z" }, ] +[[package]] +name = "semver" +version = "3.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/d1/d3159231aec234a59dd7d601e9dd9fe96f3afff15efd33c1070019b26132/semver-3.0.4.tar.gz", hash = "sha256:afc7d8c584a5ed0a11033af086e8af226a9c0b206f313e0301f8dd7b6b589602", size = 269730, upload-time = "2025-01-24T13:19:27.617Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl", hash = "sha256:9c824d87ba7f7ab4a1890799cec8596f15c1241cb473404ea1cb0c55e4b04746", size = 17912, upload-time = "2025-01-24T13:19:24.949Z" }, +] + [[package]] name = "sentry-sdk" version = "2.54.0" @@ -3960,6 +4121,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" }, ] +[[package]] +name = "unleashclient" +version = "6.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "apscheduler" }, + { name = "fcache" }, + { name = "importlib-metadata" }, + { name = "launchdarkly-eventsource" }, + { name = "mmh3" }, + { name = "python-dateutil" }, + { name = "requests" }, + { name = "semver" }, + { name = "yggdrasil-engine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/34/244a4f22640c6c62ff0b9d98fedaf1e2ced416c295c4801c37bdeeaa797f/unleashclient-6.7.0.tar.gz", hash = "sha256:a34649ef2232c3e0e18232b2067c07dd0bf474b977efc67c8806d816418ebe29", size = 51334, upload-time = "2026-03-17T09:39:34.018Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/b5/2f17c404083971c7e01e74df6623d26e14fefa822ca454ef7eb82ee6e8fa/unleashclient-6.7.0-py3-none-any.whl", hash = "sha256:1fbf4bc66ec9b952e0131012f954adfc1d476d54ad320905bc5f4fdea963a9fc", size = 40865, upload-time = "2026-03-17T09:39:32.823Z" }, +] + [[package]] name = "urllib3" version = "2.6.3" @@ -4361,6 +4542,61 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/69/68/c8739671f5699c7dc470580a4f821ef37c32c4cb0b047ce223a7f115757f/yarl-1.23.0-py3-none-any.whl", hash = "sha256:a2df6afe50dea8ae15fa34c9f824a3ee958d785fd5d089063d960bae1daa0a3f", size = 48288, upload-time = "2026-03-01T22:07:51.388Z" }, ] +[[package]] +name = "yggdrasil-engine" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/25/d17a40b177b262498a1825f667a00bef84813a65ed8027904993e3dba0fc/yggdrasil_engine-1.3.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:838880e916da4af97c10ff38e1845ef38ed91201c2868e3275e1372c8979a3ca", size = 3658693, upload-time = "2026-03-16T13:34:56.066Z" }, + { url = "https://files.pythonhosted.org/packages/53/6c/9bb859a290946618f96d79a0bd9ce8c79deb0eb8cbfd2c52f1fea767f56b/yggdrasil_engine-1.3.0-cp310-abi3-macosx_11_0_x86_64.whl", hash = "sha256:8d23e865a82b47bac7a59376fd4f0e000ae8b3b4e25fc27a5554f00093f2ff75", size = 3828862, upload-time = "2026-03-16T13:34:57.341Z" }, + { url = "https://files.pythonhosted.org/packages/7b/bc/691c080744f7a554e9bc27a1390be5e27ca8f2cd768557695ccc2c3bcb58/yggdrasil_engine-1.3.0-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:bf78418d4b66c4ec888fa9444061c65358df013105f85d7ddcd64fe78e23f38d", size = 3228816, upload-time = "2026-03-16T13:34:58.68Z" }, + { url = "https://files.pythonhosted.org/packages/3b/f5/4efdcb6d779c79d37a1c34da78d7203b2f37a2ee6c5ef7d02f55098545de/yggdrasil_engine-1.3.0-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:8a244b6084dfd1bb15dd862cecd82f3a69d26c233e3b60d63c4b31b224d92257", size = 3615783, upload-time = "2026-03-16T13:35:00.236Z" }, + { url = "https://files.pythonhosted.org/packages/69/9a/3c1224ccc524a88fa292b73d9cf25d6a89ea1a824bfe038813ed37cb4710/yggdrasil_engine-1.3.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1a332da70a8efaaed9b603676efa5dac7ed6e4f4cb4524bc3712bd4ea90c4186", size = 3226168, upload-time = "2026-03-16T13:35:01.454Z" }, + { url = "https://files.pythonhosted.org/packages/42/90/fe20ecdf83b739bb9b3d82f9049f91ced13fc1bd6966d0a70079a3bb7958/yggdrasil_engine-1.3.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:48a027b7c54e2837988e07a44c77ddd6e2d68aac98e8289b0d75a6a4e4f6f0da", size = 3612247, upload-time = "2026-03-16T13:35:03.013Z" }, + { url = "https://files.pythonhosted.org/packages/72/c2/1186bf1c70a5526933ef21a85c00daff2c579cb58e6297e8dc644a4ba402/yggdrasil_engine-1.3.0-cp310-abi3-win_amd64.whl", hash = "sha256:afa6c9a454b79126048a62fced3862c0608d74a466f172625924bb28108b5ff4", size = 6788103, upload-time = "2026-03-16T13:35:04.183Z" }, + { url = "https://files.pythonhosted.org/packages/0d/d4/325701dc756206f9ffd44ff7bfff056f904a909d7d3dd952bfe46d6e8f13/yggdrasil_engine-1.3.0-cp310-abi3-win_arm64.whl", hash = "sha256:bf2abdd70976ff95b8f72cc4e8a63968702cadbdac035f7924dcffb314042482", size = 2856830, upload-time = "2026-03-16T13:35:05.413Z" }, + { url = "https://files.pythonhosted.org/packages/c3/35/4aee684c59b7afe179cbfc5ec5e98b99fad411081ba31a72d5495dcd31c4/yggdrasil_engine-1.3.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:80aba2cd6bf06434bbe996e384434c4fa1f5e7371a46ef5deeb1c8c34afe7fe4", size = 3658693, upload-time = "2026-03-16T13:35:06.848Z" }, + { url = "https://files.pythonhosted.org/packages/0d/4b/c9c4b23850f138c3ac305da1f8d20436b51925908366a06cf7d864742f80/yggdrasil_engine-1.3.0-cp311-abi3-macosx_11_0_x86_64.whl", hash = "sha256:ac81c0a4e6120bee6e3c3049420ab7b44415d9fd876d1ae549206fb582d82a4d", size = 3828862, upload-time = "2026-03-16T13:35:08.166Z" }, + { url = "https://files.pythonhosted.org/packages/35/02/3f331713a0e96f6619884551f836bbfe353b3246495a7483b7f860bf64c2/yggdrasil_engine-1.3.0-cp311-abi3-manylinux2014_aarch64.whl", hash = "sha256:8946d23a0883012a5e3404643f210e8cf654030af87722262ad28e92fb07aa76", size = 3228816, upload-time = "2026-03-16T13:35:09.592Z" }, + { url = "https://files.pythonhosted.org/packages/48/fe/d9c1f806ff4ab7116eb7030adc93eb483c3aa95fc6bc2fc249b79468acf9/yggdrasil_engine-1.3.0-cp311-abi3-manylinux2014_x86_64.whl", hash = "sha256:66c1d03edb846a0fc1a622685ee80ae9552fd1ddd21d41524466dc8fe914b37c", size = 3615783, upload-time = "2026-03-16T13:35:10.701Z" }, + { url = "https://files.pythonhosted.org/packages/fe/73/ab452f8193443d5f9984e4c4ef5ca66318979454a07c911c5c28ae3ef867/yggdrasil_engine-1.3.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:928230be698325d9fdc8a01b477c5948b86e979897a54f6fa6030f4a1d334bc5", size = 3226168, upload-time = "2026-03-16T13:35:12.098Z" }, + { url = "https://files.pythonhosted.org/packages/54/9c/e66c8d8cd757c591d9b63abb54055d84e4f62cc4ce765b23eed462fa569d/yggdrasil_engine-1.3.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f632266983af11bc572d9cf29e8baeeae4c96f736a764034ca82dea7e6640cbc", size = 3612247, upload-time = "2026-03-16T13:35:13.388Z" }, + { url = "https://files.pythonhosted.org/packages/01/36/1cd3be83b34afcdbab0eb9988d3596aae152fc895c063bc055603cf0e43b/yggdrasil_engine-1.3.0-cp311-abi3-win_amd64.whl", hash = "sha256:8096fdf4674987d9c1d3f72b8a86e81121cbed25f74d66d4413460556f56ea3c", size = 6788103, upload-time = "2026-03-16T13:35:14.575Z" }, + { url = "https://files.pythonhosted.org/packages/9e/eb/11465ed190a83f0e5286c451658200d3d635a17d57b0eea24bb4daf9b134/yggdrasil_engine-1.3.0-cp311-abi3-win_arm64.whl", hash = "sha256:3b0df9e6ab138f0dfc4cf2d88ae060b360060ef7fc239ab64a808ec56e450567", size = 2856830, upload-time = "2026-03-16T13:35:15.877Z" }, + { url = "https://files.pythonhosted.org/packages/a0/e3/74d15e6f3d72cfa014514c3a1f83ca99386bce44c4fefef57453eb379f1b/yggdrasil_engine-1.3.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:19d1136cab1963f7cc3d1926f6cbcfc42de1f5c8c0e4d8d6e68765768d8ed238", size = 3658693, upload-time = "2026-03-16T13:35:17.331Z" }, + { url = "https://files.pythonhosted.org/packages/80/2d/7587707cee582894502e916228178491dddffe826668451143f794ba580e/yggdrasil_engine-1.3.0-cp312-abi3-macosx_11_0_x86_64.whl", hash = "sha256:6f0b7aa3be0aaf28cee7b7f3428922d4735f72533052aad6e4187f7814ba56f0", size = 3828862, upload-time = "2026-03-16T13:35:18.734Z" }, + { url = "https://files.pythonhosted.org/packages/9e/94/6310786d27afca9d681fc52e9584cd7e111b8e6d76b7f2d89318feb04685/yggdrasil_engine-1.3.0-cp312-abi3-manylinux2014_aarch64.whl", hash = "sha256:28dde2312ccd778d32c94103b2c8332a72c077b834b55d8408a334a736460f53", size = 3228816, upload-time = "2026-03-16T13:35:20.109Z" }, + { url = "https://files.pythonhosted.org/packages/cb/37/d6f3ba356f097ccfa63dd5354a4d2d892e7e5cb8feed3810ab890fbfb6ad/yggdrasil_engine-1.3.0-cp312-abi3-manylinux2014_x86_64.whl", hash = "sha256:b13a49c29d676b4674ee9362eedb86491332d4d2da07afb9ea2140a1b05e8a90", size = 3615783, upload-time = "2026-03-16T13:35:21.589Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6e/b0a7309a82d2114e295155f500ba0ac1c446b0d952a8629687b6ab087985/yggdrasil_engine-1.3.0-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ca0d5493e365ee2ccc9b69434c34986d5616e8277ed8609a9e307a101c4e0a23", size = 3226168, upload-time = "2026-03-16T13:35:22.783Z" }, + { url = "https://files.pythonhosted.org/packages/5d/9a/0c32456c2c3ee0ae2b6f430ead2436725adde7756558802fd51c0dfb05df/yggdrasil_engine-1.3.0-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b21708fb0389d2fc240b35377c7821571ecba93ef4e0a01a8d15e53f016d4971", size = 3612247, upload-time = "2026-03-16T13:35:23.927Z" }, + { url = "https://files.pythonhosted.org/packages/2e/5d/d9ffe1a87ce1a35a54b546a91b5001d98e8c22158324028549a3866612e7/yggdrasil_engine-1.3.0-cp312-abi3-win_amd64.whl", hash = "sha256:ab3a3f024c7c2d692bbdc8e1467bd16aa98b5d30ed33938fbde61ed0f3e4ea97", size = 6788103, upload-time = "2026-03-16T13:35:25.162Z" }, + { url = "https://files.pythonhosted.org/packages/80/15/6346328f065147aa03ff6ecc7731d15146b23da9296d6001e9e623a45d30/yggdrasil_engine-1.3.0-cp312-abi3-win_arm64.whl", hash = "sha256:8da87fcd93d6ccbab5d3a2d2aaf54b447265be61b19f65ba6044986f35798f99", size = 2856830, upload-time = "2026-03-16T13:35:26.621Z" }, + { url = "https://files.pythonhosted.org/packages/9d/f3/c3f7e33b655f006e32f9116b5dc396b972cc06df0f74c443df0d5a8773e0/yggdrasil_engine-1.3.0-cp313-abi3-macosx_11_0_arm64.whl", hash = "sha256:4a700c1711e0db80d57bd5e5fe32f537d7cf999e92e0d1c5b086ce5a17e69f98", size = 3658693, upload-time = "2026-03-16T13:35:27.872Z" }, + { url = "https://files.pythonhosted.org/packages/db/4b/edfb8617eb5b120114ae6ba54adfeb99643ba55639a5689714d0969f6751/yggdrasil_engine-1.3.0-cp313-abi3-macosx_11_0_x86_64.whl", hash = "sha256:a69049ca69abae3945a0d9811e4f8aa6b94e5ab12af1ff32cdf9b2bd30e41e67", size = 3828862, upload-time = "2026-03-16T13:35:29.069Z" }, + { url = "https://files.pythonhosted.org/packages/77/34/9c789b3354d03919b574c22ad0a528a8c46aeb0285636781824e7065c4a4/yggdrasil_engine-1.3.0-cp313-abi3-manylinux2014_aarch64.whl", hash = "sha256:b07d0ecd58a36d7c25c3708526900a34d5f4ccb77cd822ec27f17a6952f6278a", size = 3228816, upload-time = "2026-03-16T13:35:30.52Z" }, + { url = "https://files.pythonhosted.org/packages/ec/04/50987e3e1ba65309bdd01e3808678eaecf9b90b6b980984059a172f85c14/yggdrasil_engine-1.3.0-cp313-abi3-manylinux2014_x86_64.whl", hash = "sha256:a2cacdf5ee7eb942e0b09e0a76e6d28460cd528c961126b01c58d2dcd4b14ea3", size = 3615783, upload-time = "2026-03-16T13:35:31.997Z" }, + { url = "https://files.pythonhosted.org/packages/79/1b/e1de4db806a886d3b1c6b52616159063a4a5e51d91678986313f0bbfc5d6/yggdrasil_engine-1.3.0-cp313-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:91d28e7d8838b14722f6165cd19b3ae8cea711a43485f2260f988d922c2b84ea", size = 3226168, upload-time = "2026-03-16T13:35:33.376Z" }, + { url = "https://files.pythonhosted.org/packages/46/27/c5227985b51a794f3fee6446f53f3f8d9c539cde1c1ce208a144adf2a9a4/yggdrasil_engine-1.3.0-cp313-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:18a493f539408ec262be315d40f58e1da899fea6b1be18695516469352c42ac7", size = 3612247, upload-time = "2026-03-16T13:35:34.741Z" }, + { url = "https://files.pythonhosted.org/packages/f5/8c/9a5cd9e5dbd8c3760488d9c89ba8d2468a191ab365a3e21f826a98c2612c/yggdrasil_engine-1.3.0-cp313-abi3-win_amd64.whl", hash = "sha256:3ec38c4be410618d9826749f275f451cc07dce426413e3576ef7b6a6db1474e2", size = 6788103, upload-time = "2026-03-16T13:35:36.201Z" }, + { url = "https://files.pythonhosted.org/packages/85/11/129ac02c0d9d50e74221a1305262594dd7563f60857f4d21733b9839846a/yggdrasil_engine-1.3.0-cp313-abi3-win_arm64.whl", hash = "sha256:1704c458dfe3dbc4a8655a5271b2bef7264d8d7b392f7e02194ff3171d3c7538", size = 2856830, upload-time = "2026-03-16T13:35:37.406Z" }, + { url = "https://files.pythonhosted.org/packages/d0/e0/77ae708a01952b370841569626d73b8a666a8b07c48ba4192ef3cf92e255/yggdrasil_engine-1.3.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:e073a0ecf15fee1315db1bd02baadd0c1309b67188a16bf23a50456b2f7be951", size = 3658693, upload-time = "2026-03-16T13:35:38.983Z" }, + { url = "https://files.pythonhosted.org/packages/ef/73/11cd527b8a5f93d06fabd3b712cf53193e90f4cb1808ff86dbbc2d57652b/yggdrasil_engine-1.3.0-cp38-abi3-macosx_11_0_x86_64.whl", hash = "sha256:037eff18de77e9b357829d3dd5cc4ee5bd5aaf0eb949cd87e163790bd319cc27", size = 3828862, upload-time = "2026-03-16T13:35:40.176Z" }, + { url = "https://files.pythonhosted.org/packages/32/33/5e3c6fcadb813eef326b651d9c83970056a585b4f7c756d4daffee51ee87/yggdrasil_engine-1.3.0-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:d005998f66048e637b17b6a169bc009d55d9c26d10dfaada2e7fe296eadfd599", size = 3228816, upload-time = "2026-03-16T13:35:41.718Z" }, + { url = "https://files.pythonhosted.org/packages/19/82/87f47dc3bbba7493fa89e6782293e4e7c98e804972f346dbbdf98cce96b1/yggdrasil_engine-1.3.0-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:84f6168452c062df2386e7ebd858066f8a82562531c4be37b5024634a6ece9eb", size = 3615783, upload-time = "2026-03-16T13:35:43.233Z" }, + { url = "https://files.pythonhosted.org/packages/e4/c9/db659d67bb5d611bf0e1362f8f8b12e11eded7208fb4cda151d6af5a1d8c/yggdrasil_engine-1.3.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1ab1b8f457e55b49eaa86dd1c42ab8ff45d0e6f5ec7be31cfed6c698f0c44e75", size = 3226168, upload-time = "2026-03-16T13:35:44.41Z" }, + { url = "https://files.pythonhosted.org/packages/d9/33/9462f70d27b4d27e3306181d926ba58462abac71c5c0d56924f97efadc7c/yggdrasil_engine-1.3.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a5f7da76799391a4ae1462223229092b91bef863626ce3f662027a0321513a27", size = 3612247, upload-time = "2026-03-16T13:35:45.701Z" }, + { url = "https://files.pythonhosted.org/packages/ed/c2/60ffb224815bede077e2a1595a4fd0d7dfd159e9c72d12d6f00f6c2fb2f9/yggdrasil_engine-1.3.0-cp38-abi3-win_amd64.whl", hash = "sha256:0113ba088babf7cced92f88c64ec9ea3e28692c705fedab5b3262f3bcfc492ba", size = 6788103, upload-time = "2026-03-16T13:35:46.903Z" }, + { url = "https://files.pythonhosted.org/packages/03/9a/78ead94008923463a325b1c832b80bd0249e5d6f39750d4df0bfc53940f7/yggdrasil_engine-1.3.0-cp38-abi3-win_arm64.whl", hash = "sha256:02a6cbe73892b5a574fd3962ff692e00e6ed51af5ba4f8f1ea6745392cfd5fac", size = 2856830, upload-time = "2026-03-16T13:35:48.307Z" }, + { url = "https://files.pythonhosted.org/packages/55/01/70a6c7ca2410b520cc651eb3107c886c006733529e345ee9d8b904daba86/yggdrasil_engine-1.3.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:de45a251ceeed07818735ce894c781d7a142a60718fbff76419eae66b9859a94", size = 3658693, upload-time = "2026-03-16T13:35:49.527Z" }, + { url = "https://files.pythonhosted.org/packages/c0/10/b7167f3a1541d5710f36fd3341efe6abba06a650637185a8a679f0ee837c/yggdrasil_engine-1.3.0-cp39-abi3-macosx_11_0_x86_64.whl", hash = "sha256:b1996338ff1a1dca2dfb337ee111487f4c92351cb31d94e3badb8a97524be4c3", size = 3828862, upload-time = "2026-03-16T13:35:50.703Z" }, + { url = "https://files.pythonhosted.org/packages/e7/e8/5cbfdcf84515c11747fc62fe787d4b791052ccdf8458fef4bdc04b1ff575/yggdrasil_engine-1.3.0-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:93e9fa8639c1a887e0175823316eb986ee58ec94999ebadbf0cc4bc84b085410", size = 3228816, upload-time = "2026-03-16T13:35:51.888Z" }, + { url = "https://files.pythonhosted.org/packages/1e/b2/9d1fe86cbf5e52a8418cc027ea86373a146204d3deba4ba2cbbd0d3d271f/yggdrasil_engine-1.3.0-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:049b407a23fabacface173abd802340922246cea6c9b9842af5926742fbf5572", size = 3615783, upload-time = "2026-03-16T13:35:53.409Z" }, + { url = "https://files.pythonhosted.org/packages/a7/23/78a6ef98ec968912d49960ba69f0eddb3a10a61829b964aeb78e6dfb6f65/yggdrasil_engine-1.3.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:778c15dc202b83472407f83622e52612a58ae979c2c8240aec1277dab6f3d407", size = 3226168, upload-time = "2026-03-16T13:35:54.651Z" }, + { url = "https://files.pythonhosted.org/packages/06/de/8c7654cf60ac98a9c90b755476c46cd56d8140e5ac03a5ac823cbd2ef8e1/yggdrasil_engine-1.3.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:fe2f98703fbb8b04aac05de5491a85d6e5263c1944c66dbb9253d1395d6e9935", size = 3612247, upload-time = "2026-03-16T13:35:55.751Z" }, + { url = "https://files.pythonhosted.org/packages/87/b3/879f3caecd6ab7d0fd3b8b120044094edc1f19433ea0a4579cecfef6c433/yggdrasil_engine-1.3.0-cp39-abi3-win_amd64.whl", hash = "sha256:324f69da1314aaa1e81fbf7dd4ca6b2e241f2b1ce8bc2e018cf17b4193f6523f", size = 6788103, upload-time = "2026-03-16T13:35:56.946Z" }, + { url = "https://files.pythonhosted.org/packages/6b/fe/7ed9c9532843f69f50c325c8e94cc3552923dd6b50582a4d896ab6c81ec0/yggdrasil_engine-1.3.0-cp39-abi3-win_arm64.whl", hash = "sha256:ecbc7db34359958bcf30b9ad5ab1f68c1114d19cb47b14f2c1a341a1468cd73c", size = 2856830, upload-time = "2026-03-16T13:35:58.508Z" }, +] + [[package]] name = "zipp" version = "3.23.0"