Assessment: Pipeline

.env.example

@@ -85,6 +85,9 @@ OPENAI_API_KEY=""
 KAAPI_GUARDRAILS_AUTH=""
 KAAPI_GUARDRAILS_URL=""
 
+# Unleash feature flags
+UNLEASH_URL=
+UNLEASH_API_KEY=
 OTEL_ENABLED=true
 OTEL_SERVICE_NAME=kaapi-backend
 

.gitignore

@@ -21,3 +21,6 @@ ENV/
 
 #
 /backend/app/logs
+
+# OpenObserve local data
+/backend/data/

backend/app/alembic/versions/050_add_assessment_manager_table.py

@@ -0,0 +1,357 @@
+"""add assessment and assessment_run tables
+
+Revision ID: 050
+Revises: 049
+Create Date: 2026-03-26 23:30:00.000000
+
+"""
+
+import sqlalchemy as sa
+import sqlmodel.sql.sqltypes
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "050"
+down_revision = "049"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "assessment",
+        sa.Column(
+            "id",
+            sa.Integer(),
+            nullable=False,
+            comment="Unique identifier for the assessment",
+        ),
+        sa.Column(
+            "experiment_name",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=False,
+            comment="Experiment name shared by child config runs",
+        ),
+        sa.Column(
+            "dataset_id",
+            sa.Integer(),
+            nullable=False,
+            comment="Reference to the evaluation dataset",
+        ),
+        sa.Column(
+            "dataset_name",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=False,
+            comment="Name of the dataset used by this assessment",
+        ),
+        sa.Column(
+            "status",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=False,
+            server_default="pending",
+            comment="Overall assessment status across all child evaluation runs",
+        ),
+        sa.Column(
+            "total_runs",
+            sa.Integer(),
+            nullable=False,
+            server_default="0",
+            comment="Total number of child evaluation runs",
+        ),
+        sa.Column(
+            "pending_runs",
+            sa.Integer(),
+            nullable=False,
+            server_default="0",
+            comment="Number of child runs in pending state",
+        ),
+        sa.Column(
+            "processing_runs",
+            sa.Integer(),
+            nullable=False,
+            server_default="0",
+            comment="Number of child runs in processing state",
+        ),
+        sa.Column(
+            "completed_runs",
+            sa.Integer(),
+            nullable=False,
+            server_default="0",
+            comment="Number of child runs in completed state",
+        ),
+        sa.Column(
+            "failed_runs",
+            sa.Integer(),
+            nullable=False,
+            server_default="0",
+            comment="Number of child runs in failed state",
+        ),
+        sa.Column(
+            "run_stats",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=False,
+            server_default=sa.text("'[]'::jsonb"),
+            comment="Cached status snapshot for child evaluation runs",
+        ),
+        sa.Column(
+            "error_message",
+            sa.Text(),
+            nullable=True,
+            comment="Aggregated error message for child run failures",
+        ),
+        sa.Column(
+            "callback_url",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=True,
+            comment="Optional frontend callback URL for status updates",
+        ),
+        sa.Column(
+            "organization_id",
+            sa.Integer(),
+            nullable=False,
+            comment="Reference to the organization",
+        ),
+        sa.Column(
+            "project_id",
+            sa.Integer(),
+            nullable=False,
+            comment="Reference to the project",
+        ),
+        sa.Column(
+            "inserted_at",
+            sa.DateTime(),
+            nullable=False,
+            comment="Timestamp when the assessment was created",
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(),
+            nullable=False,
+            comment="Timestamp when the assessment was last updated",
+        ),
+        sa.ForeignKeyConstraint(
+            ["dataset_id"],
+            ["evaluation_dataset.id"],
+            name="fk_assessment_dataset_id",
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["organization_id"],
+            ["organization.id"],
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["project_id"],
+            ["project.id"],
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_assessment_experiment_name"),
+        "assessment",
+        ["experiment_name"],
+        unique=False,
+    )
+    op.create_index(
+        "idx_assessment_status_org",
+        "assessment",
+        ["status", "organization_id"],
+        unique=False,
+    )
+    op.create_index(
+        "idx_assessment_status_project",
+        "assessment",
+        ["status", "project_id"],
+        unique=False,
+    )
+    op.create_table(
+        "assessment_run",
+        sa.Column(
+            "id",
+            sa.Integer(),
+            nullable=False,
+            comment="Unique identifier for the assessment run",
+        ),
+        sa.Column(
+            "run_name",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=False,
+            comment="Name of the assessment run",
+        ),
+        sa.Column(
+            "assessment_id",
+            sa.Integer(),
+            nullable=True,
+            comment="Reference to parent assessment manager row",
+        ),
+        sa.Column(
+            "dataset_id",
+            sa.Integer(),
+            nullable=False,
+            comment="Reference to the evaluation dataset",
+        ),
+        sa.Column(
+            "dataset_name",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=False,
+            comment="Name of the dataset used",
+        ),
+        sa.Column(
+            "config_id",
+            sa.Uuid(),
+            nullable=True,
+            comment="Reference to the stored config used",
+        ),
+        sa.Column(
+            "config_version",
+            sa.Integer(),
+            nullable=True,
+            comment="Version of the config used",
+        ),
+        sa.Column(
+            "status",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=False,
+            server_default="pending",
+            comment="Run status: pending, processing, completed, failed",
+        ),
+        sa.Column(
+            "batch_job_id",
+            sa.Integer(),
+            nullable=True,
+            comment="Reference to the batch job processing this run",
+        ),
+        sa.Column(
+            "total_items",
+            sa.Integer(),
+            nullable=False,
+            server_default="0",
+            comment="Total number of dataset items in this run",
+        ),
+        sa.Column(
+            "input",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=True,
+            comment="Assessment input config: prompt_template, text_columns, attachments, output_schema",
+        ),
+        sa.Column(
+            "object_store_url",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=True,
+            comment="S3 URL of processed batch results",
+        ),
+        sa.Column(
+            "error_message",
+            sa.Text(),
+            nullable=True,
+            comment="Error message if the run failed",
+        ),
+        sa.Column(
+            "eval_score",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=True,
+            comment="Evaluation scores (reserved for future use)",
+        ),
+        sa.Column(
+            "eval_score_trace_url",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=True,
+            comment="S3 URL for evaluation score traces (reserved)",
+        ),
+        sa.Column(
+            "organization_id",
+            sa.Integer(),
+            nullable=False,
+            comment="Reference to the organization",
+        ),
+        sa.Column(
+            "project_id",
+            sa.Integer(),
+            nullable=False,
+            comment="Reference to the project",
+        ),
+        sa.Column(
+            "inserted_at",
+            sa.DateTime(),
+            nullable=False,
+            comment="Timestamp when the run was created",
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(),
+            nullable=False,
+            comment="Timestamp when the run was last updated",
+        ),
+        sa.ForeignKeyConstraint(
+            ["assessment_id"],
+            ["assessment.id"],
+            name="fk_assessment_run_assessment_id",
+            ondelete="SET NULL",
+        ),
+        sa.ForeignKeyConstraint(
+            ["dataset_id"],
+            ["evaluation_dataset.id"],
+            name="fk_assessment_run_dataset_id",
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["config_id"],
+            ["config.id"],
+            name="fk_assessment_run_config_id",
+        ),
+        sa.ForeignKeyConstraint(
+            ["batch_job_id"],
+            ["batch_job.id"],
+            name="fk_assessment_run_batch_job_id",
+            ondelete="SET NULL",
+        ),
+        sa.ForeignKeyConstraint(
+            ["organization_id"],
+            ["organization.id"],
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["project_id"],
+            ["project.id"],
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_assessment_run_run_name"),
+        "assessment_run",
+        ["run_name"],
+        unique=False,
+    )
+    op.create_index(
+        "idx_assessment_run_status_org",
+        "assessment_run",
+        ["status", "organization_id"],
+        unique=False,
+    )
+    op.create_index(
+        "idx_assessment_run_status_project",
+        "assessment_run",
+        ["status", "project_id"],
+        unique=False,
+    )
+    op.create_index(
+        "idx_assessment_run_assessment_id",
+        "assessment_run",
+        ["assessment_id"],
+        unique=False,
+    )
+
+
+def downgrade():
+    op.drop_index("idx_assessment_run_assessment_id", table_name="assessment_run")
+    op.drop_index("idx_assessment_run_status_project", table_name="assessment_run")
+    op.drop_index("idx_assessment_run_status_org", table_name="assessment_run")
+    op.drop_index(op.f("ix_assessment_run_run_name"), table_name="assessment_run")
+    op.drop_table("assessment_run")
+    op.drop_index("idx_assessment_status_project", table_name="assessment")
+    op.drop_index("idx_assessment_status_org", table_name="assessment")
+    op.drop_index(op.f("ix_assessment_experiment_name"), table_name="assessment")
+    op.drop_table("assessment")

backend/app/api/main.py

@@ -29,7 +29,8 @@
     collection_job,
     model_config,
 )
-from app.api.routes import evaluations
+from app.api.routes import evaluations, features
+from app.assessment import routes as assessment_routes
 from app.core.config import settings
 
 api_router = APIRouter()
@@ -57,8 +58,10 @@
 api_router.include_router(user_project.router)
 api_router.include_router(users.router)
 api_router.include_router(utils.router)
+api_router.include_router(features.router)
 api_router.include_router(fine_tuning.router)
 api_router.include_router(model_evaluation.router)
+api_router.include_router(assessment_routes.router)
 api_router.include_router(model_config.router)
 
 if settings.ENVIRONMENT in ["development", "testing"]: