diff --git a/cypress/e2e/ocotillo/well-inventory-bulk.cy.ts b/cypress/e2e/ocotillo/well-inventory-bulk.cy.ts new file mode 100644 index 00000000..b38cc659 --- /dev/null +++ b/cypress/e2e/ocotillo/well-inventory-bulk.cy.ts @@ -0,0 +1,51 @@ +/// + +describe('Well Inventory Bulk Import Page', () => { + beforeEach(() => { + cy.login() + cy.visit('/ocotillo/well-inventory-bulk-import') + }) + + it('should render the well inventory bulk import page UI without errors', () => { + cy.get('input').should('have.length.at.least', 1) + cy.contains('Well Inventory Bulk Import').should('exist') + cy.get('label').contains(/upload csv/i).should('exist') + cy.get('button').contains(/submit/i).should('exist') + }) + + it('should preserve CSV data through parse and unparse cycle', () => { + cy.readFile('cypress/fixtures/well-inventory.csv').then((originalCsv) => { + cy.intercept('POST', '**/well-inventory-csv', (req) => { + // For data sent as multipart string, check body contains CSV content + const body = req.body as string + const originalLines = originalCsv.split('\n') + + // Check that key data from original CSV is in the submitted body + originalLines.forEach(line => { + if (line.trim()) { + //check all original csv fields for exact match in body + const fields = line.split(',') + fields.forEach(field => { + console.log("checking field: ", field.trim()) + expect(body).to.include(field.trim()) + }) + } + }) + + req.reply({ + statusCode: 200, + body: { + summary: { total_rows_processed: 1, total_rows_imported: 1, validation_errors_or_warnings: 0 }, + wells: [], + validation_errors: [] + } + }) + }).as('submitCSV') + + cy.get('input[type="file"]').selectFile('cypress/fixtures/well-inventory.csv', { force: true }) + cy.wait(1500) + cy.get('button').contains(/submit/i).click() + cy.wait('@submitCSV') + }) + }) +}) \ No newline at end of file diff --git a/cypress/fixtures/well-inventory.csv b/cypress/fixtures/well-inventory.csv new file mode 100644 index 00000000..d39ac291 --- /dev/null +++ b/cypress/fixtures/well-inventory.csv @@ -0,0 +1,2 @@ +project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,measuring_point_description +Test Project_2,WELL-002,SITE-001,2024-01-15 10:30:00,John Smith,352342,4040485,13N,5500.5,Survey-grade GPS,2.5,Top of casing \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 67a5fc9a..6fe5a554 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,6 +36,7 @@ "@tiptap/react": "^2.9.1", "@tiptap/starter-kit": "^2.9.1", "@turf/turf": "^7.2.0", + "@types/papaparse": "^5.5.0", "axios-auth-refresh": "^3.3.6", "d3-polygon": "^3.0.1", "echarts": "^5.6.0", @@ -46,6 +47,7 @@ "mapbox-gl": "^3.0.0", "mapbox-gl-style-switcher": "^1.0.11", "pako": "^2.1.0", + "papaparse": "^5.5.3", "proj4": "^2.15.0", "react": "^18.0.0", "react-dom": "^18.0.0", @@ -1744,9 +1746,9 @@ "license": "Python-2.0" }, "node_modules/@hey-api/json-schema-ref-parser/node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", "dev": true, "license": "MIT", "dependencies": { @@ -7848,6 +7850,15 @@ "integrity": "sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==", "license": "MIT" }, + "node_modules/@types/papaparse": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/@types/papaparse/-/papaparse-5.5.0.tgz", + "integrity": "sha512-GVs5iMQmUr54BAZYYkByv8zPofFxmyxUpISPb2oh8sayR3+1zbxasrOvoKiHJ/nnoq/uULuPsu1Lze1EkagVFg==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/parse-json": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz", @@ -12187,9 +12198,9 @@ "license": "MIT" }, "node_modules/glob": { - "version": "10.4.5", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", - "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", "license": "ISC", "dependencies": { "foreground-child": "^3.1.0", @@ -13649,9 +13660,9 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", + "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", "license": "MIT", "dependencies": { "argparse": "^1.0.7", @@ -15912,9 +15923,9 @@ "license": "(MIT AND Zlib)" }, "node_modules/papaparse": { - "version": "5.4.1", - "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.4.1.tgz", - "integrity": "sha512-HipMsgJkZu8br23pW15uvo6sib6wne/4woLZPlFf3rpDyMe9ywEXUsuD7+6K9PRkJlVT51j/sCOYDKGGS3ZJrw==", + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.5.3.tgz", + "integrity": "sha512-5QvjGxYVjxO59MGU2lHVYpRWBBtKHnlIAcSe1uNFCkkptUh63NFRj0FJQm7nR67puEruUci/ZkjmEFrjCAyP4A==", "license": "MIT" }, "node_modules/parent-module": { @@ -20110,9 +20121,9 @@ } }, "node_modules/vite": { - "version": "6.3.6", - "resolved": "https://registry.npmjs.org/vite/-/vite-6.3.6.tgz", - "integrity": "sha512-0msEVHJEScQbhkbVTb/4iHZdJ6SXp/AvxL2sjwYQFfBqleHtnCqv1J3sa9zbWz/6kW1m9Tfzn92vW+kZ1WV6QA==", + "version": "6.4.1", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.1.tgz", + "integrity": "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==", "dev": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index ae6a287d..59272eef 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "@tiptap/react": "^2.9.1", "@tiptap/starter-kit": "^2.9.1", "@turf/turf": "^7.2.0", + "@types/papaparse": "^5.5.0", "axios-auth-refresh": "^3.3.6", "d3-polygon": "^3.0.1", "echarts": "^5.6.0", @@ -68,6 +69,7 @@ "mapbox-gl": "^3.0.0", "mapbox-gl-style-switcher": "^1.0.11", "pako": "^2.1.0", + "papaparse": "^5.5.3", "proj4": "^2.15.0", "react": "^18.0.0", "react-dom": "^18.0.0", diff --git a/src/pages/ocotillo/well-inventory-bulk-import/grid-defs.tsx b/src/pages/ocotillo/well-inventory-bulk-import/grid-defs.tsx new file mode 100644 index 00000000..4b6622f4 --- /dev/null +++ b/src/pages/ocotillo/well-inventory-bulk-import/grid-defs.tsx @@ -0,0 +1,57 @@ +import { GridColDef } from '@mui/x-data-grid' +import { IconButton } from '@mui/material' +import DeleteIcon from '@mui/icons-material/Delete' +import { allFieldNames, requiredFields, numericFields, booleanFields } from './utils' +import type { TableRow } from './index' + +export function createGridColumns( + getCellError: (rowId: number, fieldName: string) => boolean, + handleDeleteRow: (id: number) => void +): GridColDef[] { + // Create columns for all fields from schema + const dataColumns: GridColDef[] = allFieldNames.map((fieldName) => { + const isRequired = requiredFields.includes(fieldName) + const isNumeric = numericFields.includes(fieldName) + const isBoolean = booleanFields.includes(fieldName) + + const baseColumn: GridColDef = { + field: fieldName, + headerName: fieldName, + width: 150, + editable: true, + cellClassName: (params) => { + return getCellError(params.row.id, fieldName) ? 'error-cell' : '' + }, + } + + if (isNumeric) { + baseColumn.type = 'number' + baseColumn.width = 130 + } else if (isBoolean) { + baseColumn.type = 'boolean' + baseColumn.width = 120 + } + + return baseColumn + }) + + // Add actions column at the beginning + const actionsColumn: GridColDef = { + field: 'actions', + headerName: 'Actions', + width: 100, + sortable: false, + renderCell: (params) => ( + handleDeleteRow(params.row.id)} + color="error" + > + + + ), + } + + return [actionsColumn, ...dataColumns] +} + diff --git a/src/pages/ocotillo/well-inventory-bulk-import/index.tsx b/src/pages/ocotillo/well-inventory-bulk-import/index.tsx new file mode 100644 index 00000000..8251ced1 --- /dev/null +++ b/src/pages/ocotillo/well-inventory-bulk-import/index.tsx @@ -0,0 +1,587 @@ +import { + Box, + Button, + Stack, + Typography, + Card, + Alert, + Chip, + List, + ListItem, + ListItemText, + Divider, + Link, +} from '@mui/material' +import { Create } from '@refinedev/mui' +import { useNotification, useDataProvider } from '@refinedev/core' +import { useState, useMemo, useCallback } from 'react' +import FileUploadIcon from '@mui/icons-material/FileUpload' +import InfoIcon from '@mui/icons-material/Info' +import { DataGrid, type GridRowModel } from '@mui/x-data-grid' +import Papa from 'papaparse' +import { parseCSV } from '@/utils/ParseCSV' +import { validateAllRows, allFieldNames, mapValidationErrors, mapApiErrors, type ErrorMap, type FieldErrorMap, type ApiValidationError } from './utils' +import { wellInventoryRowSchema, type WellInventoryRow } from './schema' +import { createGridColumns } from './grid-defs' +import type { FetchValidationError } from '@/interfaces/FetchValidationError' + +interface UploadResult { + validation_errors: ApiValidationError[] + summary: { + total_rows_processed: number + total_rows_imported: number + validation_errors_or_warnings: number + } + wells: string[] +} + +export interface TableRow extends Omit { + id: number + _errors?: string[] + utm_easting?: number | string + utm_northing?: number | string + utm_zone?: string + elevation_ft?: number | string + measuring_point_height_ft?: number | string +} + +export const WellInventoryBulkImport: React.FC = () => { + const [rows, setRows] = useState([]) + const [isSubmitting, setIsSubmitting] = useState(false) + const [uploadResult, setUploadResult] = useState(null) + const [validationErrors, setValidationErrors] = useState(new Map()) + const [fieldErrors, setFieldErrors] = useState(new Map()) + const { open: openNotification } = useNotification() + const dataProvider = useDataProvider() + const provider = dataProvider('ocotillo') + + // Clear errors for a specific row + const clearRowErrors = useCallback((rowId: number) => { + setValidationErrors(prev => { + const next = new Map(prev) + next.delete(rowId) + return next + }) + setFieldErrors(prev => { + const next = new Map(prev) + Array.from(next.keys()) + .filter(key => key.startsWith(`${rowId}-`)) + .forEach(key => next.delete(key)) + return next + }) + }, []) + + const handleCSVImport = async (event: React.ChangeEvent) => { + //If multiple file upload is attempted (should not be possible), notify the user + if (event.target.files?.length && event.target.files.length > 1) { + openNotification({ + message: 'Only one file can be uploaded at a time', + description: 'Please upload only one file at a time.', + type: 'error', + }) + return + } + //get file + const file = event.target.files?.[0] + if (!file) return + + //parse the file + try { + const parsedRows: Awaited = await parseCSV(file, allFieldNames) + const newRows: TableRow[] = parsedRows.map((row, index) => ({ + ...row, + id: Date.now() + index, + })) + setRows(newRows) + + const errors = validateAllRows(newRows) + const [errorMap, fieldErrorMap] = mapValidationErrors(errors, newRows) + setValidationErrors(errorMap) + setFieldErrors(fieldErrorMap) + + openNotification({ + message: 'CSV data added to table', + description: `Added ${newRows.length} row(s) to the table from the CSV file. Please review and fix any errors before submitting.`, + type: 'success', + }) + } catch (error) { + const message = error instanceof Error ? error.message : 'Failed to parse CSV file' + openNotification({ + message: 'CSV import failed', + description: message, + type: 'error', + }) + } finally { + event.target.value = '' + } + } + + const handleDeleteRow = useCallback((id: number) => { + setRows(prev => prev.filter(row => row.id !== id)) + clearRowErrors(id) + }, [clearRowErrors]) + + const processRowUpdate = useCallback((newRow: GridRowModel): GridRowModel => { + setRows(prev => prev.map(row => row.id === newRow.id ? (newRow as TableRow) : row)) + + const validation = wellInventoryRowSchema.safeParse(newRow) + + if (!validation.success) { + const errors: string[] = [] + const fieldErrorMap = new Map>() + + validation.error.issues.forEach(err => { + const field = err.path.join('.') + errors.push(`${field}: ${err.message}`) + + const key = `${newRow.id}-${field}` + if (!fieldErrorMap.has(key)) { + fieldErrorMap.set(key, new Set()) + } + fieldErrorMap.get(key)!.add(err.message) + }) + + setValidationErrors(prev => { + const next = new Map(prev) + next.set(newRow.id, errors) + return next + }) + setFieldErrors(prev => { + const next = new Map(prev) + // Clear existing errors for this row + Array.from(next.keys()) + .filter(key => key.startsWith(`${newRow.id}-`)) + .forEach(key => next.delete(key)) + // Add new errors + fieldErrorMap.forEach((value, key) => next.set(key, value)) + return next + }) + } else { + clearRowErrors(newRow.id) + } + + return newRow + }, [clearRowErrors]) + + const handleSubmit = async () => { + if (rows.length === 0) { + openNotification({ + message: 'No data to submit', + description: 'Please add rows to the table or import a CSV file.', + type: 'error', + }) + return + } + + const errors = validateAllRows(rows) + if (errors.length > 0) { + const [errorMap] = mapValidationErrors(errors, rows) + setValidationErrors(errorMap) + openNotification({ + message: 'Validation errors found', + description: `Please fix ${errors.length} validation error(s) before submitting.`, + type: 'error', + }) + return + } + + setIsSubmitting(true) + + try { + const csvContent = convertRowsToCSV(rows) + const blob = new Blob([csvContent], { type: 'text/csv' }) + const file = new File([blob], 'well-inventory.csv', { type: 'text/csv' }) + const formData = new FormData() + formData.append('file', file) + + const result = await provider.custom({ + url: 'well-inventory-csv', + method: 'post', + payload: formData, + headers: {}, + }) + + if (result?.data) { + setUploadResult(result.data as UploadResult) + } + + openNotification({ + message: 'Upload successful', + description: 'The well inventory file has been imported successfully.', + type: 'success', + }) + } catch (error) { + const fetchError = error as FetchValidationError & { data?: UploadResult } + + if (fetchError.status === 422 && fetchError.data && 'validation_errors' in fetchError.data) { + const apiErrors = fetchError.data + const errorCount = apiErrors.summary?.validation_errors_or_warnings ?? 0 + + if (apiErrors.validation_errors) { + const [errorMap, fieldErrorMap] = mapApiErrors(apiErrors.validation_errors, rows) + setValidationErrors(errorMap) + setFieldErrors(fieldErrorMap) + } + + openNotification({ + message: 'Import failed - Validation Errors', + description: `${errorCount} validation error(s) found. Please fix the errors in the table and try again.`, + type: 'error', + }) + } else { + const message = fetchError.message || 'An error occurred while uploading the file.' + openNotification({ + message: 'Import failed', + description: message, + type: 'error', + }) + } + } finally { + setIsSubmitting(false) + } + } + + const handleReset = () => { + setRows([]) + setUploadResult(null) + setValidationErrors(new Map()) + const fileInput = document.getElementById('csv-input') as HTMLInputElement + if (fileInput) { + fileInput.value = '' + } + } + + const convertRowsToCSV = (rows: TableRow[]): string => { + // Convert TableRow to WellInventoryRow (remove id and _errors) + const csvData = rows.map((row) => { + const csvRow: any = {} + allFieldNames.forEach((fieldName) => { + let value = row[fieldName as keyof WellInventoryRow] + + // Convert undefined/null to empty string (Papa.unparse would output "undefined"/"null" otherwise) + csvRow[fieldName] = value == null ? '' : String(value) + }) + return csvRow + }) + + return Papa.unparse(csvData, { + columns: allFieldNames, + }) + } + + const columns = useMemo(() => { + const getCellError = (rowId: number, fieldName: string): boolean => { + return fieldErrors.has(`${rowId}-${fieldName}`) + } + + return createGridColumns(getCellError, handleDeleteRow) + }, [fieldErrors, handleDeleteRow]) + + const hasValidationErrors = validationErrors.size > 0 + const errorCount = validationErrors.size + + return ( + + Well Inventory Bulk Import + + } + saveButtonProps={{ + children: 'Submit', + onClick: handleSubmit, + disabled: rows.length === 0 || isSubmitting || hasValidationErrors, + loading: isSubmitting, + sx: { display: 'none'} + }} + > + + {!uploadResult && ( + <> + + + Upload a CSV file to validate your well inventory data and submit in bulk. + + } sx={{ mb: 2 }}> + + All fields are editable. Scroll horizontally to see additional columns. Data will not be saved until you submit. +
+ If validation fails on submission, the table will be updated to show the errors and you will be able to fix the errors and submit again. +
+ Well screens and well attachments are not supported in this bulk import. + + + + + + + + + + + {rows.length > 0 && ( + + )} + {hasValidationErrors && ( + + )} + {rows.length > 0 && !hasValidationErrors && ( + + )} + + + {hasValidationErrors && ( + }> + + Validation Errors Found + + + Please fix the errors in the table before submitting. Errors are highlighted in red. + + + {Array.from(validationErrors.entries()).map(([rowId, errors]) => { + const row = rows.find(r => r.id === rowId) + const rowNumber = rows.findIndex(r => r.id === rowId) + 1 + return ( + + + Row {rowNumber} - {(row as TableRow)?.well_name_point_id || 'Unnamed'} + + {errors.map((error, index) => ( + + • {error} + + ))} + + ) + })} + + + )} + + {rows.length >= 0 && ( + + + { + openNotification({ + message: 'Row update failed', + description: error instanceof Error ? error.message : 'Failed to update row', + type: 'error', + }) + }} + getRowClassName={(params) => { + return validationErrors.has(params.row.id) ? 'error-row' : '' + }} + disableRowSelectionOnClick + slots={{ + noRowsOverlay: () => ( + + No rows to display. Import a CSV file to get started. + + ) + }} + sx={{ + '& .error-row': { + backgroundColor: 'rgba(211, 47, 47, 0.08)', + '&:hover': { + backgroundColor: 'rgba(211, 47, 47, 0.12)', + }, + }, + '& .error-cell': { + backgroundColor: 'rgba(211, 47, 47, 0.2) !important', + border: '2px solid #d32f2f', + '&:focus': { + backgroundColor: 'rgba(211, 47, 47, 0.3) !important', + }, + }, + }} + /> + + + + Note: All fields are editable. Scroll horizontally to see additional columns. + + + + )} + + + )} + + {uploadResult && ( + + + + {uploadResult.summary.total_rows_imported > 0 ? ( + <> + + Import Completed Successfully! + + + The well inventory data has been imported successfully. + + + ) : ( + <> + + Import Failed - Validation Errors + + + No wells were imported due to validation errors. Please review the errors below and fix your CSV file. + + + )} + + + + + Import Summary + + + 0 ? "success" : "default"} + variant="outlined" + /> + 0 ? "success" : "error"} + variant="outlined" + /> + {uploadResult.summary.validation_errors_or_warnings > 0 && ( + + )} + + + + {uploadResult.wells && uploadResult.wells.length > 0 && ( + + + Imported Wells ({uploadResult.wells.length}) + + + {uploadResult.wells.map((well, index) => ( +
+ + + + {index < uploadResult.wells.length - 1 && } +
+ ))} + + + )} + + {uploadResult.validation_errors && + uploadResult.validation_errors.length > 0 && ( + + 0 ? "warning" : "error"} + icon={} + > + + Validation Errors + + + {uploadResult.validation_errors.map((error, index) => ( + + + Row {error.row} - {error.field} + + } + secondary={ + + {error.error} + {error.value !== undefined && error.value !== '' && ( + (Value: "{error.value}") + )} + + } + /> + {index < uploadResult.validation_errors.length - 1 && } + + ))} + + + + )} + + + + + + + )} + + + ) +} + diff --git a/src/pages/ocotillo/well-inventory-bulk-import/schema.ts b/src/pages/ocotillo/well-inventory-bulk-import/schema.ts new file mode 100644 index 00000000..90394a5e --- /dev/null +++ b/src/pages/ocotillo/well-inventory-bulk-import/schema.ts @@ -0,0 +1,212 @@ +import { z } from 'zod' +import { optionalNumber } from '@/utils/zod/optionalNumber' +import { requiredNumber } from '@/utils/zod/requiredNumber' +import { optionalBoolean } from '@/utils/zod/optionalBoolean' + +export const wellInventoryRowSchema = z.object({ + // Required fields + project: z.string().min(1, 'Project is required'), + well_name_point_id: z.string().min(1, 'Well name/point ID is required'), + site_name: z.string().min(1, 'Site name is required'), + date_time: z.string().min(1, 'Date/Time is required'), + field_staff: z.string().min(1, 'Field staff is required'), + utm_easting: requiredNumber('UTM Easting is required and must be a number'), + utm_northing: requiredNumber('UTM Northing is required and must be a number'), + utm_zone: z.string().min(1, 'UTM Zone is required'), + elevation_ft: requiredNumber('Elevation (ft) is required and must be a number'), + elevation_method: z.string().min(1, 'Elevation method is required'), + measuring_point_height_ft: requiredNumber('Measuring point height (ft) is required and must be a number'), + + // Optional field staff + field_staff_2: z.string().optional(), + field_staff_3: z.string().optional(), + + // Contact 1 fields + contact_1_name: z.string().optional(), + contact_1_organization: z.string().optional(), + contact_1_role: z.string().optional(), + contact_1_type: z.string().optional(), + contact_1_phone_1: z.string().optional(), + contact_1_phone_1_type: z.string().optional(), + contact_1_phone_2: z.string().optional(), + contact_1_phone_2_type: z.string().optional(), + contact_1_email_1: z.string().optional(), + contact_1_email_1_type: z.string().optional(), + contact_1_email_2: z.string().optional(), + contact_1_email_2_type: z.string().optional(), + contact_1_address_1_line_1: z.string().optional(), + contact_1_address_1_line_2: z.string().optional(), + contact_1_address_1_type: z.string().optional(), + contact_1_address_1_state: z.string().optional(), + contact_1_address_1_city: z.string().optional(), + contact_1_address_1_postal_code: z.string().optional(), + contact_1_address_2_line_1: z.string().optional(), + contact_1_address_2_line_2: z.string().optional(), + contact_1_address_2_type: z.string().optional(), + contact_1_address_2_state: z.string().optional(), + contact_1_address_2_city: z.string().optional(), + contact_1_address_2_postal_code: z.string().optional(), + + // Contact 2 fields + contact_2_name: z.string().optional(), + contact_2_organization: z.string().optional(), + contact_2_role: z.string().optional(), + contact_2_type: z.string().optional(), + contact_2_phone_1: z.string().optional(), + contact_2_phone_1_type: z.string().optional(), + contact_2_phone_2: z.string().optional(), + contact_2_phone_2_type: z.string().optional(), + contact_2_email_1: z.string().optional(), + contact_2_email_1_type: z.string().optional(), + contact_2_email_2: z.string().optional(), + contact_2_email_2_type: z.string().optional(), + contact_2_address_1_line_1: z.string().optional(), + contact_2_address_1_line_2: z.string().optional(), + contact_2_address_1_type: z.string().optional(), + contact_2_address_1_state: z.string().optional(), + contact_2_address_1_city: z.string().optional(), + contact_2_address_1_postal_code: z.string().optional(), + contact_2_address_2_line_1: z.string().optional(), + contact_2_address_2_line_2: z.string().optional(), + contact_2_address_2_type: z.string().optional(), + contact_2_address_2_state: z.string().optional(), + contact_2_address_2_city: z.string().optional(), + contact_2_address_2_postal_code: z.string().optional(), + + // Site information + directions_to_site: z.string().optional(), + specific_location_of_well: z.string().optional(), + repeat_measurement_permission: optionalBoolean, + sampling_permission: optionalBoolean, + datalogger_installation_permission: optionalBoolean, + public_availability_acknowledgement: optionalBoolean, + special_requests: z.string().optional(), + result_communication_preference: z.string().optional(), + contact_special_requests_notes: z.string().optional(), + + // Well details + ose_well_record_id: z.string().optional(), + date_drilled: z.string().optional(), + completion_source: z.string().optional(), + total_well_depth_ft: optionalNumber, + historic_depth_to_water_ft: optionalNumber, + depth_source: z.string().optional(), + well_pump_type: z.string().optional(), + well_pump_depth_ft: optionalNumber, + is_open: optionalBoolean, + datalogger_possible: optionalBoolean, + casing_diameter_ft: optionalNumber, + measuring_point_description: z.string().optional(), + well_purpose: z.string().optional(), + well_purpose_2: z.string().optional(), + well_hole_status: z.string().optional(), + monitoring_frequency: z.string().optional(), + sampling_scenario_notes: z.string().optional(), + well_measuring_notes: z.string().optional(), + sample_possible: optionalBoolean, +}).superRefine((data, ctx) => { + // Contact validation: if name exists, role and type must exist + for (const contactNum of [1, 2]) { + const name = data[`contact_${contactNum}_name` as keyof typeof data] as string | undefined + const role = data[`contact_${contactNum}_role` as keyof typeof data] as string | undefined + const type = data[`contact_${contactNum}_type` as keyof typeof data] as string | undefined + + if (name) { + if (!role) { + ctx.addIssue({ + code: 'custom', + message: `contact_${contactNum}_role must be provided if name is provided`, + path: [`contact_${contactNum}_role`] + }) + } + if (!type) { + ctx.addIssue({ + code: 'custom', + message: `contact_${contactNum}_type must be provided if name is provided`, + path: [`contact_${contactNum}_type`] + }) + } + } + + // Phone validation: if phone exists, phone_type must exist + for (const phoneNum of [1, 2]) { + const phone = data[`contact_${contactNum}_phone_${phoneNum}` as keyof typeof data] as string | undefined + const phoneType = data[`contact_${contactNum}_phone_${phoneNum}_type` as keyof typeof data] as string | undefined + if (phone && !phoneType) { + ctx.addIssue({ + code: 'custom', + message: `contact_${contactNum}_phone_${phoneNum}_type must be provided if phone number is provided`, + path: [`contact_${contactNum}_phone_${phoneNum}_type`] + }) + } + } + + // Email validation: if email exists, email_type must exist + for (const emailNum of [1, 2]) { + const email = data[`contact_${contactNum}_email_${emailNum}` as keyof typeof data] as string | undefined + const emailType = data[`contact_${contactNum}_email_${emailNum}_type` as keyof typeof data] as string | undefined + if (email && !emailType) { + ctx.addIssue({ + code: 'custom', + message: `contact_${contactNum}_email_${emailNum}_type must be provided if email is provided`, + path: [`contact_${contactNum}_email_${emailNum}_type`] + }) + } + } + + // Address validation: if any address field exists, all required fields must exist + for (const addrNum of [1, 2]) { + const line1 = data[`contact_${contactNum}_address_${addrNum}_line_1` as keyof typeof data] as string | undefined + const type = data[`contact_${contactNum}_address_${addrNum}_type` as keyof typeof data] as string | undefined + const state = data[`contact_${contactNum}_address_${addrNum}_state` as keyof typeof data] as string | undefined + const city = data[`contact_${contactNum}_address_${addrNum}_city` as keyof typeof data] as string | undefined + const postalCode = data[`contact_${contactNum}_address_${addrNum}_postal_code` as keyof typeof data] as string | undefined + const line2 = data[`contact_${contactNum}_address_${addrNum}_line_2` as keyof typeof data] as string | undefined + + // Check if any address field is filled + const hasAnyAddressField = line1 || type || state || city || postalCode || line2 + + if (hasAnyAddressField) { + if (!line1) { + ctx.addIssue({ + code: 'custom', + message: `contact_${contactNum}_address_${addrNum}_line_1 is required when address fields are provided`, + path: [`contact_${contactNum}_address_${addrNum}_line_1`] + }) + } + if (!type) { + ctx.addIssue({ + code: 'custom', + message: `contact_${contactNum}_address_${addrNum}_type is required when address fields are provided`, + path: [`contact_${contactNum}_address_${addrNum}_type`] + }) + } + if (!state) { + ctx.addIssue({ + code: 'custom', + message: `contact_${contactNum}_address_${addrNum}_state is required when address fields are provided`, + path: [`contact_${contactNum}_address_${addrNum}_state`] + }) + } + if (!city) { + ctx.addIssue({ + code: 'custom', + message: `contact_${contactNum}_address_${addrNum}_city is required when address fields are provided`, + path: [`contact_${contactNum}_address_${addrNum}_city`] + }) + } + if (!postalCode) { + ctx.addIssue({ + code: 'custom', + message: `contact_${contactNum}_address_${addrNum}_postal_code is required when address fields are provided`, + path: [`contact_${contactNum}_address_${addrNum}_postal_code`] + }) + } + } + } + } +}) + +export type WellInventoryRow = z.infer + +export const wellInventoryRowsSchema = z.array(wellInventoryRowSchema) diff --git a/src/pages/ocotillo/well-inventory-bulk-import/utils.ts b/src/pages/ocotillo/well-inventory-bulk-import/utils.ts new file mode 100644 index 00000000..1d438dc3 --- /dev/null +++ b/src/pages/ocotillo/well-inventory-bulk-import/utils.ts @@ -0,0 +1,121 @@ +import { wellInventoryRowSchema } from './schema' +import type { WellInventoryRow } from './schema' + +export type { WellInventoryRow } + +// Derive field names from the zod schema +export const allFieldNames: string[] = Object.keys(wellInventoryRowSchema.shape) + +// Field type definitions (used for grid column configuration) +const requiredNumericFields = ['utm_easting', 'utm_northing', 'elevation_ft', 'measuring_point_height_ft'] +const requiredStringFields = ['utm_zone'] +const optionalNumericFields = ['total_well_depth_ft', 'historic_depth_to_water_ft', 'well_pump_depth_ft', 'casing_diameter_ft'] +const otherRequiredStringFields = ['project', 'well_name_point_id', 'site_name', 'date_time', 'field_staff', 'elevation_method'] + +export const numericFields = [...requiredNumericFields, ...optionalNumericFields] +export const booleanFields = ['is_open', 'datalogger_possible', 'sample_possible'] +export const requiredFields = [...otherRequiredStringFields, ...requiredStringFields, ...requiredNumericFields] + +// Parse a single row using the schema and return the errors +export function validateRow(row: any, rowIndex: number): { isValid: boolean; errors: string[] } { + const result = wellInventoryRowSchema.safeParse(row) + + if (result.success) { + return { isValid: true, errors: [] } + } + + const errors = result.error.issues.map(err => { + const field = err.path.join('.') + return `${field}: ${err.message}` + }) + + return { isValid: false, errors } +} + +// Validate all rows and return the errors +export function validateAllRows(rows: any[]): Array<{ rowIndex: number; errors: string[] }> { + const validationErrors: Array<{ rowIndex: number; errors: string[] }> = [] + + rows.forEach((row, index) => { + const validation = validateRow(row, index) + const errors = [...validation.errors] + + if (errors.length > 0) { + validationErrors.push({ + rowIndex: index + 1, // 1-based for display due to table header row + errors, + }) + } + }) + + return validationErrors +} + +// Error mapping types +export type ErrorMap = Map +export type FieldErrorMap = Map> + +// API error types +export interface ApiValidationError { + row: number + field: string + error: string + value?: string +} + +// Map validation errors to error maps +export function mapValidationErrors( + errors: Array<{ rowIndex: number; errors: string[] }>, + rows: T[] +): [ErrorMap, FieldErrorMap] { + const errorMap = new Map() + const fieldErrorMap = new Map>() + + errors.forEach(({ rowIndex, errors }) => { + const tableRow = rows[rowIndex - 1] + if (!tableRow) return + + errorMap.set(tableRow.id, errors) + + errors.forEach(error => { + const match = error.match(/^([^:]+):\s*(.+)$/) + if (match) { + const [, fieldName, errorMessage] = match + const key = `${tableRow.id}-${fieldName.trim()}` + if (!fieldErrorMap.has(key)) { + fieldErrorMap.set(key, new Set()) + } + fieldErrorMap.get(key)!.add(errorMessage.trim()) + } + }) + }) + + return [errorMap, fieldErrorMap] +} + +// Map API validation errors to error maps +export function mapApiErrors( + apiErrors: ApiValidationError[], + rows: T[] +): [ErrorMap, FieldErrorMap] { + const errorMap = new Map() + const fieldErrorMap = new Map>() + + apiErrors.forEach(apiError => { + const tableRow = rows[apiError.row - 1] + if (!tableRow) return + + const errorMsg = `${apiError.field}: ${apiError.error}` + const existingErrors = errorMap.get(tableRow.id) || [] + errorMap.set(tableRow.id, [...existingErrors, errorMsg]) + + const key = `${tableRow.id}-${apiError.field}` + if (!fieldErrorMap.has(key)) { + fieldErrorMap.set(key, new Set()) + } + fieldErrorMap.get(key)!.add(apiError.error) + }) + + return [errorMap, fieldErrorMap] +} + diff --git a/src/providers/access-control-provider.ts b/src/providers/access-control-provider.ts index 0fbe3678..6b29a248 100644 --- a/src/providers/access-control-provider.ts +++ b/src/providers/access-control-provider.ts @@ -62,6 +62,7 @@ const defineUserAbility = (groups: string[]) => { can('list', 'ocotillo.apps') can('list', 'ocotillo.water-chemistry-import') + can('list', 'ocotillo.well-inventory-bulk-import') can('list', 'ocotillo.hydrograph-corrector') } diff --git a/src/providers/ocotillo-data-provider.ts b/src/providers/ocotillo-data-provider.ts index 57463e7f..fedfda0b 100644 --- a/src/providers/ocotillo-data-provider.ts +++ b/src/providers/ocotillo-data-provider.ts @@ -213,11 +213,13 @@ export const ocotilloDataProvider: DataProvider = { } }, custom: async ({ url, method, payload, headers }) => { + const isFormData = payload instanceof FormData + const config: AxiosRequestConfig = { url: `${API_URL}/${url}`, method: method || 'GET', headers: { - 'Content-Type': 'application/json', + ...(isFormData ? {} : { 'Content-Type': 'application/json' }), ...headers, }, } @@ -226,12 +228,28 @@ export const ocotilloDataProvider: DataProvider = { config.data = payload } - const response = await axiosInstance(config) + try { + const response = await axiosInstance(config) - if (response.status < 200 || response.status > 299) throw response + if (response.status < 200 || response.status > 299) throw response - return { data: response.data } + return { data: response.data } + } catch (error: any) { + /** + * 422 Error handling for bulk well inventory import based on API Pydantic validation errors + */ + if (error.response?.status === 422 && error.response?.data?.validation_errors) { + const transformedError = new Error('Validation errors occurred during import') + ;(transformedError as any).status = error.response.status + ;(transformedError as any).data = error.response.data + ;(transformedError as any).message = 'Validation errors occurred during import' + throw transformedError + } else { + throw error + } + } }, + update: async ({ resource, id, variables }) => { resource = cleanResourceName(resource) diff --git a/src/resources/ocotillo.tsx b/src/resources/ocotillo.tsx index 2e71e466..d08795d2 100644 --- a/src/resources/ocotillo.tsx +++ b/src/resources/ocotillo.tsx @@ -15,6 +15,7 @@ import { Workspaces, MoreVertOutlined, LibraryBooksOutlined, + UploadFile, } from '@mui/icons-material' let tables: { @@ -319,6 +320,16 @@ let ocotillo = [ label: 'Apps', }, }, + { + name: 'well-inventory-bulk-import', + list: '/ocotillo/well-inventory-bulk-import', + meta: { + label: 'Well Inventory Bulk Import', + parent: 'ocotillo.apps', + nestedLevel: 2, + icon: , + }, + }, { name: 'hydrograph-corrector', list: '/ocotillo/hydrograph-corrector', diff --git a/src/routes/ocotillo.tsx b/src/routes/ocotillo.tsx index 3814d9a3..8d72236f 100644 --- a/src/routes/ocotillo.tsx +++ b/src/routes/ocotillo.tsx @@ -69,6 +69,7 @@ import { GroundwaterLevelForm } from '@/pages/ocotillo/groundwater-level-form/st import { WellInventoryForm } from '@/pages/ocotillo/well-inventory-form' import { LexiconList } from '@/pages/ocotillo/lexicon' import { WaterChemistryApp } from '@/pages/ocotillo/water-chemistry-app' +import { WellInventoryBulkImport } from '@/pages/ocotillo/well-inventory-bulk-import' import { WellScreenCreate, WellScreenEdit, @@ -161,6 +162,9 @@ export const OcotilloRoutes = () => { } /> + + } /> + // Forms } /> diff --git a/src/utils/ParseCSV.ts b/src/utils/ParseCSV.ts new file mode 100644 index 00000000..7ef05577 --- /dev/null +++ b/src/utils/ParseCSV.ts @@ -0,0 +1,41 @@ +import Papa from 'papaparse' + +export function parseCSV>( + file: File, + fieldNames: string[] +): Promise { + return new Promise((resolve, reject) => { + Papa.parse(file, { + header: true, + skipEmptyLines: true, + transformHeader: (header) => header.trim().toLowerCase(), + complete: (results) => { + try { + if (!results.data || results.data.length === 0) { + reject(new Error('CSV must have at least a header row and one data row')) + return + } + + // Map CSV data to expected field names + const rows: T[] = results.data.map((rawRow: any) => { + const row: any = {} + fieldNames.forEach((fieldName) => { + const headerKey = fieldName.toLowerCase() + const value = rawRow[headerKey] + row[fieldName] = value ? String(value).trim() : '' + }) + return row + }) + + resolve(rows) + } catch (error) { + reject(error) + } + }, + error: (error) => { + reject(new Error(`Failed to parse CSV: ${error.message}`)) + }, + }) + }) +} + diff --git a/src/utils/index.ts b/src/utils/index.ts index d9cab181..e9052b35 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -5,8 +5,9 @@ export * from './FallbackWithDefault' export * from './FetchLookupTable' export * from './GetFieldPathsFromLoc' export * from './GetFormattedDate' -export * from './RemoveEmptyFields' +export * from './ParseCSV' export * from './ParseWktPoint' +export * from './RemoveEmptyFields' export * from './Transform' export * from './UpdateMapView' export * from './UtmToLonLat' diff --git a/src/utils/zod/optionalBoolean.ts b/src/utils/zod/optionalBoolean.ts new file mode 100644 index 00000000..b665adc7 --- /dev/null +++ b/src/utils/zod/optionalBoolean.ts @@ -0,0 +1,11 @@ +import { z } from 'zod' + +export const optionalBoolean = z.preprocess( + (val) => { + if (val === '' || val === null || val === undefined) return undefined + if (typeof val === 'boolean') return val + const str = String(val).toLowerCase() + return str === 'true' || str === '1' || str === 'yes' + }, + z.boolean().optional() + ) \ No newline at end of file diff --git a/src/utils/zod/optionalNumber.ts b/src/utils/zod/optionalNumber.ts new file mode 100644 index 00000000..81e8a8b8 --- /dev/null +++ b/src/utils/zod/optionalNumber.ts @@ -0,0 +1,10 @@ +import { z } from 'zod' + +export const optionalNumber = z.preprocess( + (val) => { + if (val === '' || val === null || val === undefined) return undefined + const num = Number(val) + return isNaN(num) ? undefined : num + }, + z.number().optional() + ) \ No newline at end of file diff --git a/src/utils/zod/requiredNumber.ts b/src/utils/zod/requiredNumber.ts new file mode 100644 index 00000000..761d835b --- /dev/null +++ b/src/utils/zod/requiredNumber.ts @@ -0,0 +1,10 @@ +import { z } from 'zod' + +export const requiredNumber = (message: string) => + z.preprocess( + (val) => { + if (val === '' || val === null || val === undefined) return NaN + return val + }, + z.coerce.number().refine((val) => !isNaN(val), { message }) + ) \ No newline at end of file