diff --git a/api/data_ingestion/constants.py b/api/data_ingestion/constants.py index 9e51ee47..cf142245 100644 --- a/api/data_ingestion/constants.py +++ b/api/data_ingestion/constants.py @@ -14,6 +14,8 @@ class Constants(BaseSettings): UPLOAD_FILE_SIZE_LIMIT_MB: int | float = 100 UPLOAD_PATH_PREFIX: str = "raw/uploads" UPLOAD_METADATA_PATH_PREFIX: str = "raw/upload_metadata" + HEALTH_UPLOAD_PATH_PREFIX: str = "updated_master_schema/health-master" + HEALTH_UPLOAD_METADATA_PATH_PREFIX: str = "raw/upload_metadata/health-master" API_INGESTION_SCHEMA_UPLOAD_PATH: str = "schemas/qos/school-connectivity" VALID_UPLOAD_TYPES: dict[str, list[str]] = { diff --git a/api/data_ingestion/models/file_upload.py b/api/data_ingestion/models/file_upload.py index 9a131b29..e085b5b4 100644 --- a/api/data_ingestion/models/file_upload.py +++ b/api/data_ingestion/models/file_upload.py @@ -60,10 +60,13 @@ def filename(self) -> str: country = self.country if self.dataset == "structured": - # For structured datasets, use original filename with upload ID original_name = Path(self.original_filename).stem filename = f"{original_name}_{self.id}" return f"{filename}{ext}" + if self.dataset == "health": + # {ISO3}_{original_stem}_{timestamp}.csv under health-master// + stem = Path(self.original_filename).stem or "health_upload" + return f"{country}_{stem}_{timestamp}{ext}" else: filename_elements = [self.id, country, self.dataset] if self.source is not None and self.dataset != "geolocation": @@ -80,6 +83,15 @@ def upload_path(self) -> str: return f"{settings.LAKEHOUSE_PATH}/raw/custom-dataset/{self.filename}" + if self.dataset == "health": + # CSV: updated_master_schema/health-master//__.csv + # Metadata sidecar: raw/upload_metadata/health-master//... (see get_metadata_path) + country_segment = "$NA" if self.country == "N/A" else self.country + return ( + f"updated_master_schema/health-master/" + f"{country_segment}/{self.filename}" + ) + # For other datasets, use the uploads path if self.dataset == "unstructured": dataset_path = "unstructured" diff --git a/api/data_ingestion/routers/upload.py b/api/data_ingestion/routers/upload.py index 15c92fd2..d0c71c39 100644 --- a/api/data_ingestion/routers/upload.py +++ b/api/data_ingestion/routers/upload.py @@ -564,6 +564,13 @@ async def upload_unstructured( # noqa: C901 ) db.add(file_upload) await db.commit() + await db.refresh(file_upload) + + # Keep parity with school uploads: persist a sidecar JSON metadata path. + metadata_file_path = get_metadata_path(file_upload.upload_path) + file_upload.metadata_json_path = metadata_file_path + db.add(file_upload) + await db.commit() client = storage_client.get_blob_client(file_upload.upload_path) @@ -583,6 +590,11 @@ async def upload_unstructured( # noqa: C901 metadata=metadata, content_settings=ContentSettings(content_type=file_type), ) + metadata_blob_client = storage_client.get_blob_client( + file_upload.metadata_json_path + ) + metadata_json_bytes = json.dumps(metadata, indent=2).encode() + metadata_blob_client.upload_blob(metadata_json_bytes, overwrite=True) response.status_code = status.HTTP_201_CREATED except HttpResponseError as err: raise HTTPException( @@ -643,6 +655,17 @@ async def upload_structured( # noqa: C901 detail="File extension must be .csv for structured datasets.", ) + portal_ds = (form.portal_dataset or "").strip().lower() + if portal_ds == "health": + dataset_label = "health" + elif portal_ds == "": + dataset_label = "structured" + else: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid portal_dataset value.", + ) + # For structured datasets, always use "N/A" as country if form.country == "Global Dataset": country_code = "N/A" @@ -660,7 +683,7 @@ async def upload_structured( # noqa: C901 uploader_id=database_user.id, uploader_email=database_user.email, country=country_code, - dataset="structured", + dataset=dataset_label, original_filename=file.filename, column_to_schema_mapping={}, column_license={}, @@ -668,6 +691,12 @@ async def upload_structured( # noqa: C901 ) db.add(file_upload) await db.commit() + await db.refresh(file_upload) + + metadata_file_path = get_metadata_path(file_upload.upload_path) + file_upload.metadata_json_path = metadata_file_path + db.add(file_upload) + await db.commit() client = storage_client.get_blob_client(file_upload.upload_path) @@ -676,7 +705,7 @@ async def upload_structured( # noqa: C901 **{str(k): str(v) for k, v in orjson.loads(form.metadata).items()}, "country": form.country, "uploader_email": email, - "dataset_type": "structured", + "dataset_type": dataset_label, } if form.source is not None: @@ -688,6 +717,11 @@ async def upload_structured( # noqa: C901 metadata=metadata, content_settings=ContentSettings(content_type=file_type), ) + metadata_blob_client = storage_client.get_blob_client( + file_upload.metadata_json_path + ) + metadata_json_bytes = json.dumps(metadata, indent=2).encode() + metadata_blob_client.upload_blob(metadata_json_bytes, overwrite=True) response.status_code = status.HTTP_201_CREATED except HttpResponseError as err: raise HTTPException( diff --git a/api/data_ingestion/schemas/upload.py b/api/data_ingestion/schemas/upload.py index 781b7f94..d9965867 100644 --- a/api/data_ingestion/schemas/upload.py +++ b/api/data_ingestion/schemas/upload.py @@ -61,6 +61,8 @@ class UnstructuredFileUploadRequest: country: str = Form(...) metadata: str = Form(...) source: str | None = Form(None) + # When "health", stores dataset=health and uses the health raw path (see upload_structured). + portal_dataset: str | None = Form(None) @dataclass diff --git a/api/data_ingestion/utils/data_quality.py b/api/data_ingestion/utils/data_quality.py index a1e9fab5..3b45f36c 100644 --- a/api/data_ingestion/utils/data_quality.py +++ b/api/data_ingestion/utils/data_quality.py @@ -40,16 +40,25 @@ def process_n_columns(column_name: str, df: pd.DataFrame, rows: int) -> dict | N def get_metadata_path(filepath: str) -> str: # Normalize paths by stripping leading slashes for comparison normalized_filepath = filepath.lstrip("/") - normalized_prefix = constants.UPLOAD_PATH_PREFIX.lstrip("/") - normalized_metadata_prefix = constants.UPLOAD_METADATA_PATH_PREFIX.lstrip("/") - if normalized_filepath.startswith(normalized_prefix): - return ( - normalized_filepath.replace( - normalized_prefix, normalized_metadata_prefix, 1 + path_prefix_pairs = ( + (constants.UPLOAD_PATH_PREFIX, constants.UPLOAD_METADATA_PATH_PREFIX), + ( + constants.HEALTH_UPLOAD_PATH_PREFIX, + constants.HEALTH_UPLOAD_METADATA_PATH_PREFIX, + ), + ) + + for upload_prefix, metadata_prefix in path_prefix_pairs: + normalized_upload_prefix = upload_prefix.lstrip("/") + normalized_metadata_prefix = metadata_prefix.lstrip("/") + if normalized_filepath.startswith(normalized_upload_prefix): + return ( + normalized_filepath.replace( + normalized_upload_prefix, normalized_metadata_prefix, 1 + ) + + ".metadata.json" ) - + ".metadata.json" - ) file_path = Path(filepath) metadata_file_path = f"{file_path.stem}.metadata.json" diff --git a/api/tests/test_metadata_path.py b/api/tests/test_metadata_path.py new file mode 100644 index 00000000..bdde7513 --- /dev/null +++ b/api/tests/test_metadata_path.py @@ -0,0 +1,19 @@ +from data_ingestion.utils.data_quality import get_metadata_path + + +def test_get_metadata_path_for_school_upload(): + upload_path = "raw/uploads/school-geolocation/FJI/abc.csv" + assert ( + get_metadata_path(upload_path) + == "raw/upload_metadata/school-geolocation/FJI/abc.csv.metadata.json" + ) + + +def test_get_metadata_path_for_health_upload(): + upload_path = ( + "updated_master_schema/health-master/FJI/FJI_fiji_health_20260525-160000.csv" + ) + assert ( + get_metadata_path(upload_path) == "raw/upload_metadata/health-master/FJI/" + "FJI_fiji_health_20260525-160000.csv.metadata.json" + ) diff --git a/ui/src/components/upload/Health.tsx b/ui/src/components/upload/Health.tsx new file mode 100644 index 00000000..c1790b6a --- /dev/null +++ b/ui/src/components/upload/Health.tsx @@ -0,0 +1,30 @@ +import { + BaseUploadMetadataForm, + UploadMetadataFormProps, +} from "@/components/upload/uploadMetadataFormBase.tsx"; +import { + health, + healthMetadataDatasetSection, + healthMetadataNationalSection, +} from "@/constants/metadata"; + +const HEALTH_INTRO = { + title: "Add health metadata", + paragraphs: [ + "Provide context for this health dataset: who compiled or uploaded it, what period it covers, and how it was collected.", + "Required fields include country, health dataset description, focal point (person uploading or responsible), data owner, and the year the data refers to.", + ], +}; + +export function Health(props: UploadMetadataFormProps) { + return ( + + ); +} diff --git a/ui/src/components/upload/UploadLanding.tsx b/ui/src/components/upload/UploadLanding.tsx index 9b78adb0..8cfe43c7 100644 --- a/ui/src/components/upload/UploadLanding.tsx +++ b/ui/src/components/upload/UploadLanding.tsx @@ -54,7 +54,8 @@ function UploadLanding(props: UploadLandingProps) { const { hasCoverage, hasGeolocation, isAdmin } = useRoles(); // Tab 0 = Geolocation (source gigasync), 1 = API (source api), - // 2 = Coverage (dataset coverage), 3 = Schemaless (dataset structured) + // 2 = Coverage (dataset coverage), 3 = Schemaless (dataset structured), + // 4 = Health (dataset health) const tabFilter = (() => { switch (selectedTab) { case 0: @@ -65,6 +66,8 @@ function UploadLanding(props: UploadLandingProps) { return { source: null, dataset: "coverage" as const }; case 3: return { source: null, dataset: "structured" as const }; + case 4: + return { source: null, dataset: "health" as const }; default: return { source: null, dataset: null }; } @@ -114,7 +117,7 @@ function UploadLanding(props: UploadLandingProps) { shared and in which context.

-
+
{(hasGeolocation || isAdmin) && ( +
@@ -172,6 +188,7 @@ function UploadLanding(props: UploadLandingProps) { API Coverage Schemaless + Health + + + {isUploadError && ( +
+ Error occurred during file upload. Please try again +
+ )} + {isNullFile && ( +
+ File is missing at this step, please upload the file again +
+ )} + + + + + ); +} diff --git a/ui/src/constants/metadata.ts b/ui/src/constants/metadata.ts index db670a36..58fd5203 100644 --- a/ui/src/constants/metadata.ts +++ b/ui/src/constants/metadata.ts @@ -36,12 +36,33 @@ const schoolIdTypeOptions = [ "Unknown", ] as const; +const healthIdTypeOptions = [ + "", + "dhis2_id", + "hims_id", + "hfml_id", + "Other", + "Unknown", +] as const; + const yesNoUnknownOptions = ["", "Yes", "No", "Unknown"] as const; const requiredFieldErrorMessage = "This field is required"; const notInRangeErrorMessage = "Selection is not within the provided range"; +/** Section headings used as keys on `metadataMapping` and for school metadata grid layout. */ +export const schoolMetadataDatasetSection = + "Information about the school dataset"; +export const schoolMetadataNationalSection = + "Information about national school data collection practices"; + +/** Section headings used as keys on `health` and for health metadata grid layout. */ +export const healthMetadataDatasetSection = + "Information about the health dataset"; +export const healthMetadataNationalSection = + "Information about national health data collection practices"; + export const metadataMapping: Record = { "": [ { @@ -61,7 +82,7 @@ export const metadataMapping: Record = { validator: z.string().min(1, { message: requiredFieldErrorMessage }), }, ], - "Information about the school dataset": [ + [schoolMetadataDatasetSection]: [ { name: "focal_point_name", label: "Focal point name", @@ -128,7 +149,7 @@ export const metadataMapping: Record = { validator: z.string().optional(), }, ], - "Information about national school data collection practices": [ + [schoolMetadataNationalSection]: [ { name: "frequency_of_school_data_collection", label: "Frequency of data collection or update", @@ -186,6 +207,152 @@ export const metadataMapping: Record = { ], }; +/** Same field `name`s as school mapping (Zod / API JSON keys); health-specific labels and section titles. */ +export const health: Record = { + "": [ + { + name: "country", + label: "Country", + helperText: "", + type: "text", + required: true, + validator: z.string().min(1, { message: requiredFieldErrorMessage }), + }, + { + name: "description", + label: "Health dataset description", + helperText: "e.g. indicator set, time period, collection purpose", + type: "text", + required: true, + validator: z.string().min(1, { message: requiredFieldErrorMessage }), + }, + ], + [healthMetadataDatasetSection]: [ + { + name: "focal_point_name", + label: "Person uploading / focal point name", + helperText: "Name of the person submitting or compiling this dataset", + type: "text", + required: true, + validator: z.string().min(1, { message: requiredFieldErrorMessage }), + }, + { + name: "focal_point_contact", + label: "Focal point email", + helperText: "Email of the person responsible for this upload", + type: "text", + required: true, + validator: z.string().email(), + }, + { + name: "data_owner", + label: "Health data owner/s", + helperText: + "e.g. Ministry of Health, national statistics office, implementing partner", + type: "text", + required: true, + validator: z.string().min(1, { message: requiredFieldErrorMessage }), + }, + { + name: "year_of_data_collection", + label: "Year the data refers to", + helperText: "Select year", + type: "year", + required: true, + validator: z.union([ + z.string().max(0), + z.coerce + .number() + .min(unicefFoundingYear, notInRangeErrorMessage) + .max(currentYear, notInRangeErrorMessage), + ]), + }, + { + name: "modality_of_data_collection", + label: "Modality of health data collection", + helperText: "Select an option", + type: "enum", + enum: modalityCollectionOptions, + required: false, + validator: z.enum(modalityCollectionOptions).optional(), + }, + { + name: "school_ids_type", + label: "Primary record / facility ID type", + helperText: "How records are keyed in this file (if applicable)", + type: "enum", + enum: healthIdTypeOptions, + required: false, + validator: z.enum(healthIdTypeOptions).optional(), + }, + { + name: "data_quality_issues", + label: "Health data gaps / quality issues", + helperText: + "Describe missing fields, coverage limits, known biases, or linkage issues", + type: "text", + required: false, + validator: z.string().optional(), + }, + ], + [healthMetadataNationalSection]: [ + { + name: "frequency_of_school_data_collection", + label: "Frequency of health data collection or update", + helperText: "Select an option", + type: "enum", + enum: frequencyCollectionOptions, + required: false, + validator: z.enum(frequencyCollectionOptions).optional(), + }, + { + name: "next_school_data_collection", + label: "Date of the next scheduled health data collection", + helperText: "MM / YYYY", + type: "text", + required: false, + validator: z + .string() + .optional() + .refine( + val => !val?.trim() || /^(0[1-9]|1[0-2])\/\d{4}$/.test(val.trim()), + "Use MM/YYYY format (e.g. 01/2025)", + ) + .refine(val => { + if (!val?.trim()) return true; + const match = val.trim().match(/^(0[1-9]|1[0-2])\/(\d{4})$/); + if (!match) return true; + const month = parseInt(match[1], 10); + const year = parseInt(match[2], 10); + return ( + year > currentYear || + (year === currentYear && month >= currentMonth) + ); + }, "Date must be in the current month or in the future"), + }, + { + name: "emis_system", + label: + "Is there a functioning national health information system (or equivalent EMR/registry)?", + helperText: "Select an option", + type: "enum", + enum: yesNoUnknownOptions, + required: false, + validator: z.enum(yesNoUnknownOptions).optional(), + }, + { + name: "school_contacts", + label: + "Does the data owner have access to facility or respondent contact details (e.g. phone, email)?", + helperText: "Select an option", + type: "enum", + enum: yesNoUnknownOptions, + required: false, + validator: z.enum(yesNoUnknownOptions).optional(), + }, + ], +}; + export const yearList = [ "", ...Array(currentYear - unicefFoundingYear + 1) diff --git a/ui/src/routes/upload/$uploadGroup/$uploadType.tsx b/ui/src/routes/upload/$uploadGroup/$uploadType.tsx index 757dca8b..1cd598ad 100644 --- a/ui/src/routes/upload/$uploadGroup/$uploadType.tsx +++ b/ui/src/routes/upload/$uploadGroup/$uploadType.tsx @@ -41,7 +41,9 @@ export const Route = createFileRoute("/upload/$uploadGroup/$uploadType")({ if ( uploadGroup === "other" && - !["unstructured", "structured", "schemaless"].includes(uploadType) + !["unstructured", "structured", "schemaless", "health"].includes( + uploadType, + ) ) { throw doRedirect; } @@ -89,6 +91,7 @@ function Layout() { const isCoverage = uploadType === "coverage"; const isGeolocation = uploadType === "geolocation"; + const isHealth = uploadGroup === "other" && uploadType === "health"; const title = uploadType.replace(/-/g, " "); const isUnstructured = @@ -109,7 +112,7 @@ function Layout() { (isCoverage && hasCoverage) || (isGeolocation && hasGeolocation) || isAdmin || - ((isSchemaless || isSchemalessOptions) && hasAccess); + ((isSchemaless || isSchemalessOptions || isHealth) && hasAccess); useEffect(() => { return resetUploadSliceState; @@ -158,6 +161,20 @@ function Layout() { ); + const HEALTH_DESCRIPTION = ( + <> +

+ Upload a health dataset CSV (up to the file size limit shown on the next + step). Please add health metadata—including who uploaded the file, the + period the data refers to, and a dataset description—before submitting. +

+

+ Health facility data following the health schema, once uploaded, will be + available for querying in Superset and Health Master. +

+ + ); + const COVERAGE_DESCRIPTION = ( <>

@@ -186,9 +203,13 @@ function Layout() { -

{title}

+

+ {isHealth ? "Health dataset" : title} +

- {title === "geolocation" + {isHealth + ? HEALTH_DESCRIPTION + : title === "geolocation" ? GEOLOCATION_DESCRIPTION : title === "coverage" ? COVERAGE_DESCRIPTION @@ -198,7 +219,25 @@ function Layout() {
- {isUnstructured ? ( + {isHealth ? ( + + + + + + ) : isUnstructured ? ( { + const geo = countryDatasets["School Geolocation"] ?? []; + const cov = countryDatasets["School Coverage"] ?? []; + return [...new Set([...geo, ...cov])]; + }, [countryDatasets]); + + const allCountryNamesForHealth = useMemo(() => { + const allGroups = healthGroupsQuery?.data ?? []; + const allGroupNames = allGroups.map(group => group.name); + return [ + ...new Set( + allGroupNames + .map(name => name.split("-School")) + .filter(split => split.length > 1) + .map(split => split[0]), + ), + ]; + }, [healthGroupsQuery?.data]); + + const healthMetadataCountryOptions = useMemo(() => { + if (!isHealth) return []; + const base = isPrivileged ? allCountryNamesForHealth : healthCountryPool; + return ["N/A", ...base]; + }, [isHealth, isPrivileged, allCountryNamesForHealth, healthCountryPool]); + + const [isHealthUploading, setIsHealthUploading] = useState(false); + const [isHealthUploadError, setIsHealthUploadError] = useState(false); + const [isHealthNullFile, setIsHealthNullFile] = useState(false); + const { register, watch } = useForm<{ source: string | null; mode: typeof storeMode; @@ -140,15 +194,22 @@ export default function Index() { const { data: schemaQuery, isFetching: isSchemaFetching } = useQuery({ queryFn: () => api.schema.get(metaschemaName, mode === "Update"), queryKey: ["schema", metaschemaName, mode, false], - enabled: isCoverage - ? !!source - : isGeolocation - ? !!mode - : !isUnstructured && !isStructured, + enabled: + !isHealth && + (isCoverage + ? !!source + : isGeolocation + ? !!mode + : !isUnstructured && !isStructured), }); const schema = schemaQuery?.data ?? []; + useEffect(() => { + if (!isHealth) return; + setStepIndex(healthStep); + }, [healthStep, isHealth, setStepIndex]); + useEffect(() => { const { file } = uploadSlice; if (schema.length && file) { @@ -284,6 +345,197 @@ export default function Index() { } } + function handleHealthAddFiles(addedFiles: File[]) { + const nextFile = addedFiles.at(0) ?? null; + if (!nextFile) return; + + setHealthFileError(""); + const looksCsv = + nextFile.type in validHealthCsvTypes || + nextFile.name.toLowerCase().endsWith(".csv"); + + if (!looksCsv) { + setHealthFileError("Only CSV files are accepted."); + return; + } + + if (nextFile.size > MAX_UPLOAD_FILE_SIZE_MB * 1024 * 1024) { + setHealthFileError( + `File size exceeds ${MAX_UPLOAD_FILE_SIZE_MB} MB limit`, + ); + return; + } + + setUploadSliceState({ + uploadSlice: { + ...uploadSlice, + fuzzyCorrections: [], + fuzzyValidationRequestKey: null, + fuzzyValidationResult: null, + file: nextFile, + timeStamp: new Date(), + }, + }); + } + + function resetHealthFlow() { + resetUploadSliceState(); + setHealthStep(0); + setHealthFileError(""); + setIsHealthUploadError(false); + setIsHealthNullFile(false); + } + + const onHealthMetadataSubmit: SubmitHandler = async data => { + if (uploadSlice.file === null) { + setIsHealthNullFile(true); + return; + } + + setIsHealthUploading(true); + setIsHealthUploadError(false); + setIsHealthNullFile(false); + + const metadata = { ...data }; + const country = metadata.country; + delete metadata.country; + + Object.keys(metadata).forEach(key => { + const k = key as keyof typeof metadata; + if (metadata[k] === "") { + (metadata as Record)[key] = null; + } + }); + + try { + await uploadStructuredFile.mutateAsync({ + country, + file: uploadSlice.file, + source: storeSource, + metadata: JSON.stringify({ ...metadata, mode: uploadSlice.mode }), + portal_dataset: "health", + }); + setUploadDate(uploadSlice.timeStamp); + setStepIndex(2); + void navigate({ to: "./success" }); + } catch (err) { + console.error("Health upload failed:", err); + setIsHealthUploadError(true); + } finally { + setIsHealthUploading(false); + } + }; + + if (isHealth) { + const csvFormats = [ + ...new Set(Object.values(validHealthCsvTypes).flat()), + ].join(", "); + + return ( + + {healthStep === 0 && ( + <> +
+
+

+ Upload file +

+

+ File formats: {csvFormats} up to {MAX_UPLOAD_FILE_SIZE_MB}MB +

+
+ {hasUploadedFile && file ? ( + { + handleRemoveFile(); + setHealthFileError(""); + }} + iconDescription="Remove file" + aria-label={`Remove ${file.name}`} + /> + ) : ( + + handleHealthAddFiles(addedFiles) + } + /> + )} +
+ {healthFileError ? ( +

{healthFileError}

+ ) : null} +
+
+ + + + + + )} + + {healthStep === 1 && ( + <> +

+ Selected file:{" "} + {file?.name} +

+ { + setHealthStep(0); + setIsHealthUploadError(false); + setIsHealthNullFile(false); + }} + > + Back + + } + /> + + )} +
+ ); + } + return ( {isCoverage && ( diff --git a/ui/src/routes/upload/$uploadGroup/$uploadType/metadata.tsx b/ui/src/routes/upload/$uploadGroup/$uploadType/metadata.tsx index af8ff47d..e103c162 100644 --- a/ui/src/routes/upload/$uploadGroup/$uploadType/metadata.tsx +++ b/ui/src/routes/upload/$uploadGroup/$uploadType/metadata.tsx @@ -36,7 +36,12 @@ import { SelectFromArray, SelectFromEnum, } from "@/components/upload/MetadataInputs.tsx"; -import { metadataMapping, yearList } from "@/constants/metadata"; +import { + metadataMapping, + schoolMetadataDatasetSection, + schoolMetadataNationalSection, + yearList, +} from "@/constants/metadata"; import { useStore } from "@/context/store"; import useRoles from "@/hooks/useRoles.ts"; import { MetadataFormMapping } from "@/types/metadata.ts"; @@ -88,7 +93,7 @@ function getFormRows( if (groupKey === "") { return [[formItems[0], null], [formItems[1]]]; } - if (groupKey === "Information about the school dataset") { + if (groupKey === schoolMetadataDatasetSection) { const pairs: MetadataFormMapping[][] = []; for (let i = 0; i < 6 && i < formItems.length; i += 2) { pairs.push(formItems.slice(i, i + 2)); @@ -98,9 +103,7 @@ function getFormRows( } return pairs; } - if ( - groupKey === "Information about national school data collection practices" - ) { + if (groupKey === schoolMetadataNationalSection) { const pairs: MetadataFormMapping[][] = []; for (let i = 0; i < formItems.length; i += 2) { pairs.push(formItems.slice(i, i + 2)); @@ -116,7 +119,7 @@ const RenderFormItem = ({ register, }: { formItem: MetadataFormMapping; - errors: FieldErrors; + errors: FieldErrors; register: UseFormRegister; }) => { switch (formItem.type) { @@ -257,7 +260,6 @@ function Metadata() { } if (Object.keys(errors).length > 0) { - // form has errors, don't submit return; } @@ -430,10 +432,6 @@ function Metadata() { )} - - {/* - - */} ); diff --git a/ui/src/routes/upload/$uploadGroup/$uploadType/success.tsx b/ui/src/routes/upload/$uploadGroup/$uploadType/success.tsx index 1d5633e6..b5d1951f 100644 --- a/ui/src/routes/upload/$uploadGroup/$uploadType/success.tsx +++ b/ui/src/routes/upload/$uploadGroup/$uploadType/success.tsx @@ -50,11 +50,12 @@ export const Route = createFileRoute( const isUnstructured = uploadGroup === "other" && uploadType === "unstructured"; const isStructured = uploadGroup === "other" && uploadType === "structured"; + const isHealth = uploadGroup === "other" && uploadType === "health"; if (isUnstructured) { return; - } else if (isStructured) { - // For structured datasets, allow access even without file since upload is already complete + } else if (isStructured || isHealth) { + // Structured / health CSV: upload already finished before this screen return; } else if ( !file || @@ -119,18 +120,19 @@ function Success() { const isUnstructured = uploadGroup === "other" && uploadType === "unstructured"; const isStructured = uploadGroup === "other" && uploadType === "structured"; + const isHealth = uploadGroup === "other" && uploadType === "health"; const { data: basicCheckQuery, isFetching: isBasicCheckFetching } = useQuery({ queryFn: () => api.uploads.list_basic_checks(uploadType, source), queryKey: ["basic_checks", uploadType, source], - enabled: !isStructured, // Don't query for structured datasets + enabled: !isStructured && !isHealth, }); const basicCheck = basicCheckQuery?.data ?? []; const { data: uploadQuery } = useQuery({ queryKey: ["upload", uploadId], queryFn: () => api.uploads.get_upload(uploadId), - enabled: !isStructured && !!uploadId, // Don't query for structured datasets + enabled: !isStructured && !isHealth && !!uploadId, }); const uploadData = useMemo( () => uploadQuery?.data ?? initialUploadResponse, @@ -150,7 +152,7 @@ function Success() { queryKey: ["dq_check", uploadId], queryFn: () => api.uploads.get_data_quality_check(uploadId), refetchInterval: 7000, - enabled: !isUnstructured && !isStructured, // Don't query for structured datasets + enabled: !isUnstructured && !isStructured && !isHealth, }); const dqResult = useMemo( @@ -202,6 +204,9 @@ function Success() { const structuredMessage = "Your file has been uploaded and will be made available for query on Superset within 5 minutes."; + const healthMessage = + "Your health dataset file and metadata have been uploaded. Downstream staging in Azure Data Lake and Dagster will pick this up in a later step."; + type TagColors = ComponentProps["type"]; const statusTagMap: Record = { @@ -257,6 +262,13 @@ function Success() { Back to Home + ) : isHealth ? ( + <> + {healthMessage} + + ) : (
diff --git a/ui/src/types/upload.ts b/ui/src/types/upload.ts index 53901ecc..43816d00 100644 --- a/ui/src/types/upload.ts +++ b/ui/src/types/upload.ts @@ -82,6 +82,8 @@ export interface UploadStructuredParams { file: File; source?: string | null; metadata: string; + /** Routes CSV to dataset `health` in blob storage when set to `"health"`. */ + portal_dataset?: string; } export interface FuzzyValueMapping {