Feat/tech 8707 improve uploads table

api/data_ingestion/migrations/versions/2026_04_28_1200_b1c2d3e4f5a6_add_upload_row_stats_and_data_owner.py

@@ -0,0 +1,31 @@
+"""add upload row stats and data owner
+
+Revision ID: b1c2d3e4f5a6
+Revises: a3c6ab14b3f8
+Create Date: 2026-04-28 12:00:00.000000
+
+"""
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "b1c2d3e4f5a6"
+down_revision: str | None = "a3c6ab14b3f8"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.add_column("file_uploads", sa.Column("data_owner", sa.String(), nullable=True))
+    op.add_column("file_uploads", sa.Column("rows", sa.Integer(), nullable=True))
+    op.add_column("file_uploads", sa.Column("rows_passed", sa.Integer(), nullable=True))
+    op.add_column("file_uploads", sa.Column("rows_failed", sa.Integer(), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("file_uploads", "rows_failed")
+    op.drop_column("file_uploads", "rows_passed")
+    op.drop_column("file_uploads", "rows")
+    op.drop_column("file_uploads", "data_owner")

api/data_ingestion/models/file_upload.py

@@ -37,6 +37,10 @@ class FileUpload(BaseModel):
     )
     metadata_json_path: Mapped[str] = mapped_column(nullable=True)
     bronze_path: Mapped[str] = mapped_column(nullable=True, default=None)
+    data_owner: Mapped[str] = mapped_column(nullable=True, default=None)
+    rows: Mapped[int] = mapped_column(nullable=True, default=None)
+    rows_passed: Mapped[int] = mapped_column(nullable=True, default=None)
+    rows_failed: Mapped[int] = mapped_column(nullable=True, default=None)
     is_processed_in_staging: Mapped[bool] = mapped_column(nullable=False, default=False)
     country: Mapped[str] = mapped_column(VARCHAR(3), nullable=False)
     dataset: Mapped[str] = mapped_column(nullable=False)

api/data_ingestion/routers/upload.py

@@ -375,6 +375,7 @@ async def upload_file(
         select(DatabaseUser).where(DatabaseUser.email == email)
     )
 
+    metadata_fields = orjson.loads(form.metadata)
     file_upload = FileUpload(
         uploader_id=database_user.id,
         uploader_email=database_user.email,
@@ -384,6 +385,7 @@ async def upload_file(
         original_filename=file.filename,
         column_to_schema_mapping=orjson.loads(form.column_to_schema_mapping),
         column_license=orjson.loads(form.column_license),
+        data_owner=metadata_fields.get("data_owner"),
     )
 
     db.add(file_upload)
@@ -400,7 +402,7 @@ async def upload_file(
 
     try:
         metadata = {
-            **{str(k): str(v) for k, v in orjson.loads(form.metadata).items()},
+            **{str(k): str(v) for k, v in metadata_fields.items()},
             "country": form.country,
             "uploader_email": email,
         }
@@ -500,6 +502,7 @@ async def upload_unstructured(  # noqa: C901
         select(DatabaseUser).where(DatabaseUser.email == email)
     )
 
+    unstructured_metadata_fields = orjson.loads(form.metadata)
     file_upload = FileUpload(
         uploader_id=database_user.id,
         uploader_email=database_user.email,
@@ -509,6 +512,7 @@ async def upload_unstructured(  # noqa: C901
         column_to_schema_mapping={},
         column_license={},
         dq_status=DQStatusEnum.SKIPPED,
+        data_owner=unstructured_metadata_fields.get("data_owner"),
     )
     db.add(file_upload)
     await db.commit()
@@ -517,7 +521,7 @@ async def upload_unstructured(  # noqa: C901
 
     try:
         metadata = {
-            **{str(k): str(v) for k, v in orjson.loads(form.metadata).items()},
+            **{str(k): str(v) for k, v in unstructured_metadata_fields.items()},
             "country": form.country,
             "uploader_email": email,
         }
@@ -604,6 +608,7 @@ async def upload_structured(  # noqa: C901
         select(DatabaseUser).where(DatabaseUser.email == email)
     )
 
+    structured_metadata_fields = orjson.loads(form.metadata)
     file_upload = FileUpload(
         uploader_id=database_user.id,
         uploader_email=database_user.email,
@@ -613,6 +618,7 @@ async def upload_structured(  # noqa: C901
         column_to_schema_mapping={},
         column_license={},
         dq_status=DQStatusEnum.SKIPPED,
+        data_owner=structured_metadata_fields.get("data_owner"),
     )
     db.add(file_upload)
     await db.commit()
@@ -621,7 +627,7 @@ async def upload_structured(  # noqa: C901
 
     try:
         metadata = {
-            **{str(k): str(v) for k, v in orjson.loads(form.metadata).items()},
+            **{str(k): str(v) for k, v in structured_metadata_fields.items()},
             "country": form.country,
             "uploader_email": email,
             "dataset_type": "structured",

api/data_ingestion/schemas/upload.py

@@ -25,6 +25,10 @@ class FileUpload(BaseModel):
     column_to_schema_mapping: dict[str, str]
     column_license: dict[str, str]
     upload_path: str
+    data_owner: str | None
+    rows: int | None
+    rows_passed: int | None
+    rows_failed: int | None
 
     model_config = ConfigDict(from_attributes=True)
 

ui/src/components/check-file-uploads/ColumnSelectorModal.tsx

@@ -0,0 +1,175 @@
+import { useEffect, useState } from "react";
+
+import { Close } from "@carbon/icons-react";
+import { Button, Checkbox } from "@carbon/react";
+
+export interface ColumnConfig {
+  key: string;
+  label: string;
+  alwaysVisible?: boolean;
+}
+
+export const ALL_COLUMN_CONFIGS: ColumnConfig[] = [
+  { key: "id", label: "Upload ID", alwaysVisible: true },
+  { key: "created", label: "Upload date" },
+  { key: "uploader_email", label: "Uploaded by" },
+  { key: "dataset", label: "Dataset" },
+  { key: "country", label: "Country" },
+  { key: "rows", label: "Total entries" },
+  { key: "rows_passed", label: "Records Passed" },
+  { key: "rows_failed", label: "Records Rejected" },
+  { key: "data_owner", label: "Data Owner" },
+  { key: "status", label: "DQ status" },
+];
+
+export const DEFAULT_VISIBLE_COLUMNS = new Set([
+  "id",
+  "created",
+  "uploader_email",
+  "dataset",
+  "country",
+  "rows",
+  "rows_failed",
+  "status",
+]);
+
+const MAX_VISIBLE = 9;
+const STORAGE_KEY = "uploads_table_visible_columns";
+
+export function loadVisibleColumns(): Set<string> {
+  try {
+    const stored = localStorage.getItem(STORAGE_KEY);
+    if (stored) return new Set(JSON.parse(stored));
+  } catch {
+    // ignore
+  }
+  return new Set(DEFAULT_VISIBLE_COLUMNS);
+}
+
+export function saveVisibleColumns(cols: Set<string>) {
+  localStorage.setItem(STORAGE_KEY, JSON.stringify([...cols]));
+}
+
+interface ColumnSelectorModalProps {
+  open: boolean;
+  onClose: () => void;
+  visibleColumns: Set<string>;
+  onSave: (cols: Set<string>) => void;
+}
+
+export default function ColumnSelectorModal({
+  open,
+  onClose,
+  visibleColumns,
+  onSave,
+}: ColumnSelectorModalProps) {
+  const [draft, setDraft] = useState<Set<string>>(new Set(visibleColumns));
+
+  useEffect(() => {
+    if (open) setDraft(new Set(visibleColumns));
+  }, [open, visibleColumns]);
+
+  const toggleableColumns = ALL_COLUMN_CONFIGS.filter(c => !c.alwaysVisible);
+  const selectedCount = draft.size;
+  const atMax = selectedCount >= MAX_VISIBLE;
+
+  function toggle(key: string, checked: boolean) {
+    const next = new Set(draft);
+    if (checked) next.add(key);
+    else next.delete(key);
+    setDraft(next);
+  }
+
+  function handleSave() {
+    saveVisibleColumns(draft);
+    onSave(draft);
+    onClose();
+  }
+
+  function handleReset() {
+    setDraft(new Set(DEFAULT_VISIBLE_COLUMNS));
+  }
+
+  if (!open) return null;
+
+  return (
+    <>
+      {/* Backdrop */}
+      <div
+        className="fixed inset-0 z-[8999] bg-black/40"
+        onClick={onClose}
+        aria-hidden="true"
+      />
+
+      {/* Side panel */}
+      <div
+        role="dialog"
+        aria-label="Column selector"
+        className="fixed right-0 top-0 z-[9000] flex h-full w-80 flex-col bg-white shadow-2xl"
+      >
+        {/* Header */}
+        <div className="flex items-start justify-between border-b border-[var(--cds-border-subtle)] px-4 py-4">
+          <div>
+            <h3 className="text-base font-semibold text-[var(--cds-text-primary)]">
+              Columns
+            </h3>
+            <p className="mt-0.5 text-xs text-[var(--cds-text-secondary)]">
+              {selectedCount} selected (max. {MAX_VISIBLE})
+            </p>
+          </div>
+          <button
+            onClick={onClose}
+            className="ml-4 text-[var(--cds-icon-primary)] hover:text-[var(--cds-icon-secondary)]"
+            aria-label="Close"
+          >
+            <Close size={20} />
+          </button>
+        </div>
+
+        {/* Column list */}
+        <div className="flex-1 overflow-y-auto">
+          {/* Always-visible column */}
+          <div className="flex items-center border-b border-[var(--cds-border-subtle)] px-4 py-3 opacity-40">
+            <div className="flex-1">
+              <Checkbox id="col-id" labelText="Upload ID" checked disabled />
+            </div>
+          </div>
+
+          {/* Toggleable columns */}
+          {toggleableColumns.map(col => {
+            const isChecked = draft.has(col.key);
+            const isDisabled = !isChecked && atMax;
+            return (
+              <div
+                key={col.key}
+                className="flex items-center border-b border-[var(--cds-border-subtle)] px-4 py-3"
+              >
+                <div className="flex-1">
+                  <Checkbox
+                    id={`col-${col.key}`}
+                    labelText={col.label}
+                    checked={isChecked}
+                    disabled={isDisabled}
+                    onChange={(_: unknown, { checked }: { checked: boolean }) =>
+                      toggle(col.key, checked)
+                    }
+                  />
+                </div>
+              </div>
+            );
+          })}
+        </div>
+
+        {/* Footer */}
+        <div className="flex border-t border-[var(--cds-border-subtle)]">
+          <Button kind="ghost" className="flex-1" onClick={handleReset}>
+            Reset
+          </Button>
+          <Button kind="primary" className="flex-1" onClick={handleSave}>
+            Save
+          </Button>
+        </div>
+      </div>
+    </>
+  );
+}

ui/src/components/check-file-uploads/UploadsTable.tsx

@@ -29,7 +29,9 @@ import {
   UploadResponse,
 } from "@/types/upload.ts";
 
-const columns: DataTableHeader[] = [
+import { ALL_COLUMN_CONFIGS } from "./ColumnSelectorModal";
+
+const ALL_COLUMNS: DataTableHeader[] = [
   {
     key: "id",
     header: "Upload ID",
@@ -58,6 +60,22 @@ const columns: DataTableHeader[] = [
     key: "country",
     header: "Country",
   },
+  {
+    key: "rows",
+    header: "Tot. entries",
+  },
+  {
+    key: "rows_passed",
+    header: "Passed",
+  },
+  {
+    key: "rows_failed",
+    header: "Rejected",
+  },
+  {
+    key: "data_owner",
+    header: "Data owner",
+  },
   {
     key: "status",
     header: "DQ check status",
@@ -85,6 +103,7 @@ interface UploadsTableProps {
   }) => void;
   source?: string | null;
   dataset?: string | null;
+  visibleColumns: Set<string>;
 }
 
 function UploadsTable({
@@ -93,6 +112,7 @@ function UploadsTable({
   handlePaginationChange,
   source,
   dataset,
+  visibleColumns,
 }: UploadsTableProps) {
   const { data: uploadsQuery, isLoading } = useSuspenseQuery({
     queryFn: () =>
@@ -105,6 +125,16 @@ function UploadsTable({
     queryKey: ["uploads", page, pageSize, source, dataset],
   });
 
+  const activeColumns = useMemo(() => {
+    const selectableKeys = new Set(ALL_COLUMN_CONFIGS.map(c => c.key));
+    return ALL_COLUMNS.filter(
+      col =>
+        col.key === "actions" ||
+        !selectableKeys.has(col.key) ||
+        visibleColumns.has(col.key),
+    );
+  }, [visibleColumns]);
+
   const renderUploads = useMemo<PagedResponse<TableUpload>>(() => {
     const uploads = uploadsQuery?.data ?? {
       data: [],
@@ -135,6 +165,10 @@ function UploadsTable({
             )}
           </>
         ),
+        rows: upload.rows?.toLocaleString() ?? "—",
+        rows_passed: upload.rows_passed?.toLocaleString() ?? "—",
+        rows_failed: upload.rows_failed?.toLocaleString() ?? "—",
+        data_owner: upload.data_owner ?? "—",
         status: (
           <Tag
             type={DQStatusTagMapping[upload.dq_status]}
@@ -162,9 +196,9 @@ function UploadsTable({
   }, [page, pageSize, uploadsQuery?.data]);
 
   return isLoading ? (
-    <DataTableSkeleton headers={columns} />
+    <DataTableSkeleton headers={activeColumns} />
   ) : (
-    <DataTable headers={columns} rows={renderUploads.data}>
+    <DataTable headers={activeColumns} rows={renderUploads.data}>
       {({ rows, headers, getHeaderProps, getRowProps, getTableProps }) => (
         <TableContainer>
           <Table {...getTableProps()}>