Update staging (#245)

* feat: readable labels (#232)

* feat: readable labels

Readable labels.

Splitting dq-readable-checks branch into two, one for labels, another one for future modals.

* fix: add search for assertionLabel

Upgrading the dq search with the assertionLabel to work with old checks names (internal) and new description

* fix: UI arrow position fix

Checks table overview arrow position fix

* feat: import values from nocodb

Import dq label values from nocoDB

* fix: TECH-9140 file validation (#239)

* fix: TECH-9140 file validation

TECH-9140 file type validation

* fix: add xls format

add xls format

* fix: only uploaders filter list (#244)

Created a endpoint for retrieving the user list for the filter modal to only show the users with uploads

Author: Javiershenbc

api/data_ingestion/routers/upload.py

@@ -42,13 +42,20 @@
 from data_ingestion.permissions.permissions import IsPrivileged
 from data_ingestion.schemas.core import PagedResponseSchema
 from data_ingestion.schemas.upload import (
+    DataQualityCheckLabel,
     FileUpload as FileUploadSchema,
     FileUploadRequest,
     UnstructuredFileUploadRequest,
     ValidateFuzzyRequest,
 )
 from data_ingestion.utils.data_quality import get_metadata_path
 from data_ingestion.utils.fuzzy_matching import run_fuzzy_matching
+from data_ingestion.utils.nocodb import (
+    get_nocodb_table_id_from_name,
+    get_nocodb_table_rows,
+)
+
+DQ_CHECK_LABELS_TABLE_NAME = "SchoolGeolocationMasterDQChecksTest"
 
 router = APIRouter(
     prefix="/api/upload",
@@ -57,6 +64,81 @@
 )
 
 
+def _parse_bool(value) -> bool:
+    if isinstance(value, bool):
+        return value
+    if value is None:
+        return True
+    return str(value).strip().lower() in {"true", "1", "yes", "y"}
+
+
+def _parse_sort_order(value) -> int | None:
+    if value in (None, ""):
+        return None
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _parse_dq_table_column_name(value: str) -> tuple[str, str]:
+    key = value.removeprefix("dq_")
+    assertion, _, column_key = key.partition("-")
+    return assertion, column_key
+
+
+def _normalize_dq_check_label(row: dict) -> DataQualityCheckLabel | None:
+    dq_table_column_name = row.get("DQ Table Column Name") or ""
+    parsed_assertion, parsed_column_key = _parse_dq_table_column_name(
+        dq_table_column_name
+    )
+    assertion = row.get("Assertion") or parsed_assertion
+
+    if not assertion:
+        return None
+
+    active = _parse_bool(row.get("Active"))
+    if not active:
+        return None
+
+    ui_error_description = (
+        row.get("UI Error Description")
+        or row.get("Human Readable Name")
+        or assertion.replace("_", " ")
+    )
+
+    return DataQualityCheckLabel(
+        assertion=assertion,
+        column_key=row.get("Column Key") or parsed_column_key,
+        ui_error_description=ui_error_description,
+        dq_table_column_name=dq_table_column_name or None,
+        dq_check_category=row.get("DQ Check Category"),
+        column_checked=row.get("Column Checked"),
+        human_readable_name=row.get("Human Readable Name"),
+        active=active,
+        sort_order=_parse_sort_order(row.get("Sort Order")),
+    )
+
+
+@router.get("/data_quality_check_labels", response_model=list[DataQualityCheckLabel])
+async def list_data_quality_check_labels():
+    table_id = get_nocodb_table_id_from_name(DQ_CHECK_LABELS_TABLE_NAME)
+    rows = get_nocodb_table_rows(table_id)
+    labels = [
+        label for row in rows if (label := _normalize_dq_check_label(row)) is not None
+    ]
+
+    return sorted(
+        labels,
+        key=lambda label: (
+            label.sort_order is None,
+            label.sort_order or 0,
+            label.assertion,
+            label.column_key,
+        ),
+    )
+
+
 @router.get("/basic_check/{dataset}")
 async def list_basic_checks(
     dataset: str = "geolocation",

api/data_ingestion/routers/users.py

@@ -11,7 +11,7 @@
 from data_ingestion.internal.auth import azure_scheme
 from data_ingestion.internal.groups import GroupsApi
 from data_ingestion.internal.users import UsersApi
-from data_ingestion.models import Role, User
+from data_ingestion.models import FileUpload, Role, User
 from data_ingestion.permissions.permissions import IsPrivileged
 from data_ingestion.schemas.group import ModifyUserAccessRequest
 from data_ingestion.schemas.invitation import (
@@ -48,6 +48,20 @@ async def list_users(db: AsyncSession = Depends(get_db)):
     )
 
 
+@router.get(
+    "/uploaders",
+    response_model=list[DatabaseUser],
+    dependencies=[Security(IsPrivileged())],
+)
+async def list_uploaders(db: AsyncSession = Depends(get_db)):
+    return await db.scalars(
+        select(User)
+        .join(FileUpload, FileUpload.uploader_id == User.id)
+        .distinct()
+        .order_by(User.given_name, User.surname, User.email)
+    )
+
+
 @router.post("", response_model=DatabaseUser, dependencies=[Security(IsPrivileged())])
 async def create_user(
     body: DatabaseUserCreateRequest,

api/data_ingestion/schemas/upload.py

@@ -67,3 +67,15 @@ class UnstructuredFileUploadRequest:
 class ValidateFuzzyRequest:
     file: UploadFile = Form(...)
     column_to_schema_mapping: str = Form(...)
+
+
+class DataQualityCheckLabel(BaseModel):
+    assertion: str
+    column_key: str = ""
+    ui_error_description: str
+    dq_table_column_name: str | None = None
+    dq_check_category: str | None = None
+    column_checked: str | None = None
+    human_readable_name: str | None = None
+    active: bool = True
+    sort_order: int | None = None

ui/src/api/routers/uploads.ts

@@ -4,6 +4,7 @@ import { PagedResponse } from "@/types/api.ts";
 import {
   BasicChecks,
   DataQualityCheck,
+  DataQualityCheckLabel,
   FuzzyValidationParams,
   FuzzyValidationResponse,
   UploadParams,
@@ -19,6 +20,11 @@ export default function routes(axi: AxiosInstance) {
     ): Promise<AxiosResponse<DataQualityCheck>> => {
       return axi.get(`upload/data_quality_check/${upload_id}`);
     },
+    list_data_quality_check_labels: (): Promise<
+      AxiosResponse<DataQualityCheckLabel[]>
+    > => {
+      return axi.get("upload/data_quality_check_labels");
+    },
     list_uploads: (params?: {
       page?: number;
       page_size?: number;

ui/src/api/routers/users.ts

@@ -11,6 +11,9 @@ export default function routes(axi: AxiosInstance) {
     list: (): Promise<AxiosResponse<DatabaseUserWithRoles[]>> => {
       return axi.get("/users");
     },
+    listUploaders: (): Promise<AxiosResponse<DatabaseUser[]>> => {
+      return axi.get("/users/uploaders");
+    },
     get: (id: string): Promise<AxiosResponse<DatabaseUserWithRoles>> => {
       return axi.get(`/users/${id}`);
     },

ui/src/components/check-file-uploads/ColumnChecks.tsx

@@ -13,11 +13,35 @@ import {
   TableHeader,
   TableRow,
 } from "@carbon/react";
+import { useQuery } from "@tanstack/react-query";
 
+import { api } from "@/api";
 import { cn } from "@/lib/utils.ts";
-import { Check } from "@/types/upload";
+import { Check, DataQualityCheckLabel } from "@/types/upload";
 import { commaNumber } from "@/utils/number.ts";
 
+const getLabelKey = (assertion: string, columnKey = "") =>
+  `${assertion}-${columnKey}`;
+
+const buildAssertionLabelMap = (labels: DataQualityCheckLabel[]) =>
+  labels.reduce<Record<string, string>>((acc, label) => {
+    acc[getLabelKey(label.assertion, label.column_key)] =
+      label.ui_error_description;
+    return acc;
+  }, {});
+
+export const formatAssertion = (
+  assertion: string,
+  columnKey: string,
+  assertionLabels: Record<string, string>,
+) => {
+  return (
+    assertionLabels[getLabelKey(assertion, columnKey)] ??
+    assertionLabels[getLabelKey(assertion)] ??
+    assertion.replace(/_/g, " ")
+  );
+};
+
 interface ExtendedDataTableHeader extends DataTableHeader {
   sortable?: boolean;
 }
@@ -33,6 +57,16 @@ const DataQualityChecks = ({ data }: DataQualityChecksProps) => {
     direction: "ascending" | "descending";
   }>({ key: "", direction: "ascending" });
 
+  const { data: dataQualityCheckLabelsQuery } = useQuery({
+    queryKey: ["data_quality_check_labels"],
+    queryFn: api.uploads.list_data_quality_check_labels,
+  });
+
+  const assertionLabels = useMemo(
+    () => buildAssertionLabelMap(dataQualityCheckLabelsQuery?.data ?? []),
+    [dataQualityCheckLabelsQuery],
+  );
+
   const handleUpSort = (key: string) => {
     setSortConfig({ key, direction: "ascending" });
   };
@@ -44,9 +78,19 @@ const DataQualityChecks = ({ data }: DataQualityChecksProps) => {
   const filteredAndSortedRows = useMemo(() => {
     const result = data.filter(check => {
       const searchString = searchTerm.toLowerCase();
+      const columnKey = check.column === "" ? "NO_COLUMN" : check.column;
+      const columnDisplay =
+        columnKey === "NO_COLUMN" ? "Entire row" : columnKey;
+      const assertionLabel = formatAssertion(
+        check.assertion,
+        check.column,
+        assertionLabels,
+      );
       return (
         check.column.toLowerCase().includes(searchString) ||
-        check.assertion.toLowerCase().includes(searchString)
+        check.assertion.toLowerCase().includes(searchString) ||
+        columnDisplay.toLowerCase().includes(searchString) ||
+        assertionLabel.toLowerCase().includes(searchString)
       );
     });
 
@@ -78,13 +122,13 @@ const DataQualityChecks = ({ data }: DataQualityChecksProps) => {
     }
 
     return result;
-  }, [data, searchTerm, sortConfig]);
+  }, [assertionLabels, data, searchTerm, sortConfig]);
 
   const renderSortControls = (key: string) => {
     const isActive = sortConfig.key === key;
 
     return (
-      <div className="absolute right-1 top-1/2 flex -translate-y-1/2 flex-col">
+      <div className="ml-2 flex shrink-0 flex-col">
         <ChevronUp
           className={cn(
             "cursor-pointer transition-colors duration-150",
@@ -112,18 +156,19 @@ const DataQualityChecks = ({ data }: DataQualityChecksProps) => {
   const rows = filteredAndSortedRows.map(check => {
     const {
       assertion,
-      column = "NO_COLUMN",
+      column = "",
       count_failed,
       count_passed,
       percent_passed,
     } = check;
 
     const columnKey = column === "" ? "NO_COLUMN" : column;
+    const columnDisplay = columnKey === "NO_COLUMN" ? "Entire row" : columnKey;
 
     return {
       id: `${assertion}-${columnKey}`,
-      column: columnKey,
-      assertion,
+      column: columnDisplay,
+      assertion: formatAssertion(assertion, column, assertionLabels),
       result_with_errors: (
         <div className="flex items-center">
           {count_failed > 0 ? (
@@ -152,12 +197,12 @@ const DataQualityChecks = ({ data }: DataQualityChecksProps) => {
   const dqResultHeaders: ExtendedDataTableHeader[] = [
     {
       key: "column",
-      header: "Column(s)",
+      header: "Column",
       sortable: false,
     },
     {
       key: "assertion",
-      header: "Validation Rule",
+      header: "Check Description",
       sortable: false,
     },
     {
@@ -185,7 +230,7 @@ const DataQualityChecks = ({ data }: DataQualityChecksProps) => {
       <div className="rounded-lg border bg-white shadow-sm">
         <div className="border-b px-4 py-3">
           <h3 className="text-lg font-semibold text-gray-800">
-            Overview of all fields sorted by type
+            Overview of all checks sorted by type
           </h3>
         </div>
 
@@ -199,9 +244,9 @@ const DataQualityChecks = ({ data }: DataQualityChecksProps) => {
                       <TableHeader
                         key={header.key}
                         isSortable={false}
-                        className={cn("relative bg-blue-50 text-gray-700")}
+                        className={cn("bg-blue-50 text-gray-700")}
                       >
-                        <div className="flex w-full items-center justify-between">
+                        <div className="flex w-full items-center justify-between gap-2">
                           <span>{header.header}</span>
                           {(header as ExtendedDataTableHeader).sortable &&
                             renderSortControls(header.key)}

ui/src/components/upload/FilterModal.tsx

@@ -59,8 +59,8 @@ function FilterModal({
   }, [open]);
 
   const { data: usersData } = useQuery({
-    queryKey: ["users"],
-    queryFn: api.users.list,
+    queryKey: ["uploaders"],
+    queryFn: api.users.listUploaders,
     enabled: isPrivileged,
   });