From 7826b2356f8696368b5bea4f454219d0dd3b07b5 Mon Sep 17 00:00:00 2001 From: Javiershenbc Date: Wed, 29 Apr 2026 12:09:07 +0200 Subject: [PATCH 1/2] feat: add rows processed and data owner Added (with migration) rows processed and data owner to file uploads scheema and file uploads table --- ...5a6_add_upload_row_stats_and_data_owner.py | 31 +++++++++++++++++++ api/data_ingestion/models/file_upload.py | 4 +++ api/data_ingestion/routers/upload.py | 12 +++++-- api/data_ingestion/schemas/upload.py | 4 +++ .../check-file-uploads/UploadsTable.tsx | 20 ++++++++++++ ui/src/types/upload.ts | 8 +++++ 6 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 api/data_ingestion/migrations/versions/2026_04_28_1200_b1c2d3e4f5a6_add_upload_row_stats_and_data_owner.py diff --git a/api/data_ingestion/migrations/versions/2026_04_28_1200_b1c2d3e4f5a6_add_upload_row_stats_and_data_owner.py b/api/data_ingestion/migrations/versions/2026_04_28_1200_b1c2d3e4f5a6_add_upload_row_stats_and_data_owner.py new file mode 100644 index 00000000..f8289bd6 --- /dev/null +++ b/api/data_ingestion/migrations/versions/2026_04_28_1200_b1c2d3e4f5a6_add_upload_row_stats_and_data_owner.py @@ -0,0 +1,31 @@ +"""add upload row stats and data owner + +Revision ID: b1c2d3e4f5a6 +Revises: a3c6ab14b3f8 +Create Date: 2026-04-28 12:00:00.000000 + +""" +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "b1c2d3e4f5a6" +down_revision: str | None = "a3c6ab14b3f8" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.add_column("file_uploads", sa.Column("data_owner", sa.String(), nullable=True)) + op.add_column("file_uploads", sa.Column("rows", sa.Integer(), nullable=True)) + op.add_column("file_uploads", sa.Column("rows_passed", sa.Integer(), nullable=True)) + op.add_column("file_uploads", sa.Column("rows_failed", sa.Integer(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("file_uploads", "rows_failed") + op.drop_column("file_uploads", "rows_passed") + op.drop_column("file_uploads", "rows") + op.drop_column("file_uploads", "data_owner") diff --git a/api/data_ingestion/models/file_upload.py b/api/data_ingestion/models/file_upload.py index 9a131b29..bf8a6209 100644 --- a/api/data_ingestion/models/file_upload.py +++ b/api/data_ingestion/models/file_upload.py @@ -37,6 +37,10 @@ class FileUpload(BaseModel): ) metadata_json_path: Mapped[str] = mapped_column(nullable=True) bronze_path: Mapped[str] = mapped_column(nullable=True, default=None) + data_owner: Mapped[str] = mapped_column(nullable=True, default=None) + rows: Mapped[int] = mapped_column(nullable=True, default=None) + rows_passed: Mapped[int] = mapped_column(nullable=True, default=None) + rows_failed: Mapped[int] = mapped_column(nullable=True, default=None) is_processed_in_staging: Mapped[bool] = mapped_column(nullable=False, default=False) country: Mapped[str] = mapped_column(VARCHAR(3), nullable=False) dataset: Mapped[str] = mapped_column(nullable=False) diff --git a/api/data_ingestion/routers/upload.py b/api/data_ingestion/routers/upload.py index 8e63ad04..61328d46 100644 --- a/api/data_ingestion/routers/upload.py +++ b/api/data_ingestion/routers/upload.py @@ -375,6 +375,7 @@ async def upload_file( select(DatabaseUser).where(DatabaseUser.email == email) ) + metadata_fields = orjson.loads(form.metadata) file_upload = FileUpload( uploader_id=database_user.id, uploader_email=database_user.email, @@ -384,6 +385,7 @@ async def upload_file( original_filename=file.filename, column_to_schema_mapping=orjson.loads(form.column_to_schema_mapping), column_license=orjson.loads(form.column_license), + data_owner=metadata_fields.get("data_owner"), ) db.add(file_upload) @@ -400,7 +402,7 @@ async def upload_file( try: metadata = { - **{str(k): str(v) for k, v in orjson.loads(form.metadata).items()}, + **{str(k): str(v) for k, v in metadata_fields.items()}, "country": form.country, "uploader_email": email, } @@ -500,6 +502,7 @@ async def upload_unstructured( # noqa: C901 select(DatabaseUser).where(DatabaseUser.email == email) ) + unstructured_metadata_fields = orjson.loads(form.metadata) file_upload = FileUpload( uploader_id=database_user.id, uploader_email=database_user.email, @@ -509,6 +512,7 @@ async def upload_unstructured( # noqa: C901 column_to_schema_mapping={}, column_license={}, dq_status=DQStatusEnum.SKIPPED, + data_owner=unstructured_metadata_fields.get("data_owner"), ) db.add(file_upload) await db.commit() @@ -517,7 +521,7 @@ async def upload_unstructured( # noqa: C901 try: metadata = { - **{str(k): str(v) for k, v in orjson.loads(form.metadata).items()}, + **{str(k): str(v) for k, v in unstructured_metadata_fields.items()}, "country": form.country, "uploader_email": email, } @@ -604,6 +608,7 @@ async def upload_structured( # noqa: C901 select(DatabaseUser).where(DatabaseUser.email == email) ) + structured_metadata_fields = orjson.loads(form.metadata) file_upload = FileUpload( uploader_id=database_user.id, uploader_email=database_user.email, @@ -613,6 +618,7 @@ async def upload_structured( # noqa: C901 column_to_schema_mapping={}, column_license={}, dq_status=DQStatusEnum.SKIPPED, + data_owner=structured_metadata_fields.get("data_owner"), ) db.add(file_upload) await db.commit() @@ -621,7 +627,7 @@ async def upload_structured( # noqa: C901 try: metadata = { - **{str(k): str(v) for k, v in orjson.loads(form.metadata).items()}, + **{str(k): str(v) for k, v in structured_metadata_fields.items()}, "country": form.country, "uploader_email": email, "dataset_type": "structured", diff --git a/api/data_ingestion/schemas/upload.py b/api/data_ingestion/schemas/upload.py index 781b7f94..181529ef 100644 --- a/api/data_ingestion/schemas/upload.py +++ b/api/data_ingestion/schemas/upload.py @@ -25,6 +25,10 @@ class FileUpload(BaseModel): column_to_schema_mapping: dict[str, str] column_license: dict[str, str] upload_path: str + data_owner: str | None + rows: int | None + rows_passed: int | None + rows_failed: int | None model_config = ConfigDict(from_attributes=True) diff --git a/ui/src/components/check-file-uploads/UploadsTable.tsx b/ui/src/components/check-file-uploads/UploadsTable.tsx index 3ee39ba1..b59070bf 100644 --- a/ui/src/components/check-file-uploads/UploadsTable.tsx +++ b/ui/src/components/check-file-uploads/UploadsTable.tsx @@ -58,6 +58,22 @@ const columns: DataTableHeader[] = [ key: "country", header: "Country", }, + { + key: "rows", + header: "Tot. entries", + }, + { + key: "rows_passed", + header: "Passed", + }, + { + key: "rows_failed", + header: "Rejected", + }, + { + key: "data_owner", + header: "Data owner", + }, { key: "status", header: "DQ check status", @@ -135,6 +151,10 @@ function UploadsTable({ )} ), + rows: upload.rows?.toLocaleString() ?? "—", + rows_passed: upload.rows_passed?.toLocaleString() ?? "—", + rows_failed: upload.rows_failed?.toLocaleString() ?? "—", + data_owner: upload.data_owner ?? "—", status: ( Date: Thu, 30 Apr 2026 15:04:47 +0200 Subject: [PATCH 2/2] feat: table column selecting Table column selecting modal --- .../ColumnSelectorModal.tsx | 175 ++++++++++++++++++ .../check-file-uploads/UploadsTable.tsx | 20 +- ui/src/components/upload/UploadLanding.tsx | 57 ++++-- 3 files changed, 238 insertions(+), 14 deletions(-) create mode 100644 ui/src/components/check-file-uploads/ColumnSelectorModal.tsx diff --git a/ui/src/components/check-file-uploads/ColumnSelectorModal.tsx b/ui/src/components/check-file-uploads/ColumnSelectorModal.tsx new file mode 100644 index 00000000..c00b5a9e --- /dev/null +++ b/ui/src/components/check-file-uploads/ColumnSelectorModal.tsx @@ -0,0 +1,175 @@ +import { useEffect, useState } from "react"; + +import { Close } from "@carbon/icons-react"; +import { Button, Checkbox } from "@carbon/react"; + +export interface ColumnConfig { + key: string; + label: string; + alwaysVisible?: boolean; +} + +export const ALL_COLUMN_CONFIGS: ColumnConfig[] = [ + { key: "id", label: "Upload ID", alwaysVisible: true }, + { key: "created", label: "Upload date" }, + { key: "uploader_email", label: "Uploaded by" }, + { key: "dataset", label: "Dataset" }, + { key: "country", label: "Country" }, + { key: "rows", label: "Total entries" }, + { key: "rows_passed", label: "Records Passed" }, + { key: "rows_failed", label: "Records Rejected" }, + { key: "data_owner", label: "Data Owner" }, + { key: "status", label: "DQ status" }, +]; + +export const DEFAULT_VISIBLE_COLUMNS = new Set([ + "id", + "created", + "uploader_email", + "dataset", + "country", + "rows", + "rows_failed", + "status", +]); + +const MAX_VISIBLE = 9; +const STORAGE_KEY = "uploads_table_visible_columns"; + +export function loadVisibleColumns(): Set { + try { + const stored = localStorage.getItem(STORAGE_KEY); + if (stored) return new Set(JSON.parse(stored)); + } catch { + // ignore + } + return new Set(DEFAULT_VISIBLE_COLUMNS); +} + +export function saveVisibleColumns(cols: Set) { + localStorage.setItem(STORAGE_KEY, JSON.stringify([...cols])); +} + +interface ColumnSelectorModalProps { + open: boolean; + onClose: () => void; + visibleColumns: Set; + onSave: (cols: Set) => void; +} + +export default function ColumnSelectorModal({ + open, + onClose, + visibleColumns, + onSave, +}: ColumnSelectorModalProps) { + const [draft, setDraft] = useState>(new Set(visibleColumns)); + + useEffect(() => { + if (open) setDraft(new Set(visibleColumns)); + }, [open, visibleColumns]); + + const toggleableColumns = ALL_COLUMN_CONFIGS.filter(c => !c.alwaysVisible); + const selectedCount = draft.size; + const atMax = selectedCount >= MAX_VISIBLE; + + function toggle(key: string, checked: boolean) { + const next = new Set(draft); + if (checked) next.add(key); + else next.delete(key); + setDraft(next); + } + + function handleSave() { + saveVisibleColumns(draft); + onSave(draft); + onClose(); + } + + function handleReset() { + setDraft(new Set(DEFAULT_VISIBLE_COLUMNS)); + } + + if (!open) return null; + + return ( + <> + {/* Backdrop */} +