Skip to content

Commit e1497ec

Browse files
William-HillWilliam-Hill
committed
fix(upload): resolve review findings — batch SQL, wire schema override, derive state
- Export CONFIDENT_THRESHOLD/TENTATIVE_THRESHOLD constants from upload-schemas.ts and replace all hardcoded 0.6/0.3/0.59 values with them - Remove selectedSchemaLabel state; derive it from SCHEMAS.find() alongside selectedSchemaObj so ColumnMapper receives the real schema object instead of null - Wire the "Wrong? Change type" button via showSchemaOverride boolean state; renders schema picker inline below green banner when toggled; resets on resetWizard - Pre-compute matchedCount/unmappedCount once and replace all inline columns.filter() calls in JSX - Refactor upsertRows to build SQL template once outside the loop and execute one multi-row INSERT per 500-row batch (N+1 → N/BATCH_SIZE); per-row fallback fires only on batch failure to identify bad rows - Remove stale "// Log to upload_history" comment
1 parent 21f4831 commit e1497ec

3 files changed

Lines changed: 133 additions & 71 deletions

File tree

‎codebenders-dashboard/app/admin/upload/page.tsx‎

Lines changed: 52 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import { DataPreview } from "@/components/upload/data-preview"
88
import { UploadSummary } from "@/components/upload/upload-summary"
99
import { Button } from "@/components/ui/button"
1010
import { AlertCircle, CheckCircle, Loader2 } from "lucide-react"
11-
import type { ColumnMapping } from "@/lib/upload-schemas"
11+
import { SCHEMAS, CONFIDENT_THRESHOLD, type ColumnMapping } from "@/lib/upload-schemas"
1212

1313
type Step = "upload" | "preview" | "complete"
1414

@@ -47,7 +47,7 @@ export default function UploadPage() {
4747
const [preview, setPreview] = useState<PreviewData | null>(null)
4848
const [columns, setColumns] = useState<ColumnMapping[]>([])
4949
const [selectedSchema, setSelectedSchema] = useState<string | null>(null)
50-
const [selectedSchemaLabel, setSelectedSchemaLabel] = useState<string | null>(null)
50+
const [showSchemaOverride, setShowSchemaOverride] = useState(false)
5151
const [commitResult, setCommitResult] = useState<CommitResult | null>(null)
5252
const [loading, setLoading] = useState(false)
5353
const [error, setError] = useState<string | null>(null)
@@ -85,7 +85,6 @@ export default function UploadPage() {
8585
setPreview(data)
8686
setColumns(data.columns)
8787
setSelectedSchema(data.detectedSchema)
88-
setSelectedSchemaLabel(data.detectedSchemaLabel)
8988
setStep("preview")
9089
} catch (err) {
9190
setError((err as Error).message)
@@ -95,9 +94,8 @@ export default function UploadPage() {
9594
}, [])
9695

9796
const handleSchemaOverride = useCallback(
98-
(schemaId: string, label: string) => {
97+
(schemaId: string) => {
9998
setSelectedSchema(schemaId)
100-
setSelectedSchemaLabel(label)
10199
},
102100
[]
103101
)
@@ -141,7 +139,7 @@ export default function UploadPage() {
141139
setPreview(null)
142140
setColumns([])
143141
setSelectedSchema(null)
144-
setSelectedSchemaLabel(null)
142+
setShowSchemaOverride(false)
145143
setCommitResult(null)
146144
setError(null)
147145
}, [])
@@ -150,6 +148,10 @@ export default function UploadPage() {
150148
const hasRequiredErrors = (preview?.errors?.length ?? 0) > 0
151149
const stepLabels = ["Upload", "Preview & Map", "Complete"]
152150
const stepIndex = step === "upload" ? 0 : step === "preview" ? 1 : 2
151+
const selectedSchemaObj = SCHEMAS.find((s) => s.id === selectedSchema) ?? null
152+
const selectedSchemaLabel = selectedSchemaObj?.label ?? null
153+
const matchedCount = columns.filter((c) => c.status === "matched").length
154+
const unmappedCount = columns.filter((c) => c.status === "unmapped").length
153155

154156
return (
155157
<div className="container mx-auto px-4 py-6 max-w-4xl">
@@ -230,24 +232,46 @@ export default function UploadPage() {
230232
{/* Step 2: Preview & Map */}
231233
{step === "preview" && preview && (
232234
<div className="space-y-5">
233-
{preview.confidence >= 0.6 ? (
234-
<div className="bg-green-50 border border-green-200 rounded-lg p-3 flex items-center justify-between">
235-
<div className="flex items-center gap-2 text-sm">
236-
<CheckCircle className="h-4 w-4 text-green-600" />
237-
<span className="font-semibold">{selectedSchemaLabel}</span>
238-
<span className="text-muted-foreground">
239-
— {file?.name} — {preview.totalRows} rows,{" "}
240-
{columns.filter((c) => c.status === "matched").length}/
241-
{columns.length} columns matched
242-
</span>
235+
{preview.confidence >= CONFIDENT_THRESHOLD ? (
236+
<>
237+
<div className="bg-green-50 border border-green-200 rounded-lg p-3 flex items-center justify-between">
238+
<div className="flex items-center gap-2 text-sm">
239+
<CheckCircle className="h-4 w-4 text-green-600" />
240+
<span className="font-semibold">{selectedSchemaLabel}</span>
241+
<span className="text-muted-foreground">
242+
— {file?.name} — {preview.totalRows} rows,{" "}
243+
{matchedCount}/
244+
{columns.length} columns matched
245+
</span>
246+
</div>
247+
<button
248+
className="text-xs text-muted-foreground border px-2 py-0.5 rounded hover:bg-muted"
249+
onClick={() => setShowSchemaOverride(true)}
250+
>
251+
Wrong? Change type
252+
</button>
243253
</div>
244-
<button
245-
className="text-xs text-muted-foreground border px-2 py-0.5 rounded hover:bg-muted"
246-
onClick={() => {}}
247-
>
248-
Wrong? Change type
249-
</button>
250-
</div>
254+
{showSchemaOverride && (
255+
<div className="flex gap-2 flex-wrap mt-2">
256+
{preview.scores.filter((s) => s.score > 0.1).map((s) => (
257+
<button
258+
key={s.schemaId}
259+
className={`text-xs px-3 py-1.5 rounded border ${
260+
selectedSchema === s.schemaId
261+
? "border-purple-600 bg-purple-50 font-semibold"
262+
: "border-muted hover:bg-muted/50"
263+
}`}
264+
onClick={() => {
265+
setSelectedSchema(s.schemaId)
266+
setShowSchemaOverride(false)
267+
}}
268+
>
269+
{s.label} <span className="text-muted-foreground">({Math.round(s.score * 100)}%)</span>
270+
</button>
271+
))}
272+
</div>
273+
)}
274+
</>
251275
) : (
252276
<div className="bg-amber-50 border border-amber-200 rounded-lg p-4">
253277
<div className="flex items-center gap-2 text-sm mb-3">
@@ -271,7 +295,7 @@ export default function UploadPage() {
271295
? "border-purple-600 bg-purple-50 font-semibold"
272296
: "border-muted hover:bg-muted/50"
273297
}`}
274-
onClick={() => handleSchemaOverride(s.schemaId, s.label)}
298+
onClick={() => handleSchemaOverride(s.schemaId)}
275299
>
276300
{s.label}{" "}
277301
<span className="text-muted-foreground">
@@ -287,7 +311,7 @@ export default function UploadPage() {
287311
<h3 className="text-sm font-semibold mb-2">Column Mapping</h3>
288312
<ColumnMapper
289313
columns={columns}
290-
schema={null}
314+
schema={selectedSchemaObj}
291315
onMappingChange={setColumns}
292316
/>
293317
</div>
@@ -300,12 +324,12 @@ export default function UploadPage() {
300324
📊 <strong>{preview.totalRows}</strong> rows
301325
</span>
302326
<span className="text-green-700">
303-
✓ {columns.filter((c) => c.status === "matched").length} matched
327+
✓ {matchedCount} matched
304328
</span>
305-
{columns.filter((c) => c.status === "unmapped").length > 0 && (
329+
{unmappedCount > 0 && (
306330
<span className="text-amber-700">
307331
⚠{" "}
308-
{columns.filter((c) => c.status === "unmapped").length}{" "}
332+
{unmappedCount}{" "}
309333
unmapped
310334
</span>
311335
)}

‎codebenders-dashboard/app/api/admin/upload/commit/route.ts‎

Lines changed: 75 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@ export async function POST(request: NextRequest) {
4343

4444
const result = await upsertRows(rows, columnMapping, schema)
4545

46-
// Log to upload_history
4746
const pool = getPool()
4847
const status =
4948
result.errors.length > 0 && result.inserted === 0
@@ -114,15 +113,70 @@ async function upsertRows(
114113
}
115114
if (errors.length > 0) return { inserted: 0, skipped: 0, errors }
116115

117-
// Process in batches
116+
// Build SQL template once — columns are determined by the mapping, not per-row
117+
const cols = Array.from(headerToDb.values())
118+
const conflictClause = schema.upsertKey.join(", ")
119+
const updateSet = cols
120+
.filter((c) => !schema.upsertKey.includes(c))
121+
.map((c) => `${c} = EXCLUDED.${c}`)
122+
.join(", ")
123+
124+
// Cache required column names
125+
const requiredDbCols = schema.columns
126+
.filter((c) => c.required)
127+
.map((c) => c.name)
128+
129+
// Process in batches — one INSERT per batch (N+1 → N/BATCH_SIZE queries)
118130
for (let i = 0; i < rows.length; i += BATCH_SIZE) {
119131
const batch = rows.slice(i, i + BATCH_SIZE)
132+
const batchValues: string[] = []
133+
const batchParams: string[] = []
134+
let paramIdx = 1
120135

121136
for (let j = 0; j < batch.length; j++) {
122137
const row = batch[j]
123-
const rowIndex = i + j + 1 // 1-based for user display
138+
const rowIndex = i + j + 1
139+
140+
const dbRow: Record<string, string> = {}
141+
for (const [header, dbCol] of headerToDb) {
142+
let value = row[header] ?? ""
143+
const transform = transforms.get(dbCol)
144+
if (transform) value = transform(value)
145+
dbRow[dbCol] = value
146+
}
147+
148+
// Check required fields have values
149+
const missing = requiredDbCols.find((c) => !dbRow[c] || dbRow[c].trim() === "")
150+
if (missing) {
151+
skipped++
152+
errors.push({ row: rowIndex, column: missing, message: `Empty required field: ${missing}` })
153+
continue
154+
}
155+
156+
const rowPlaceholders = cols.map(() => `$${paramIdx++}`)
157+
batchValues.push(`(${rowPlaceholders.join(", ")})`)
158+
batchParams.push(...cols.map((c) => dbRow[c]))
159+
}
160+
161+
if (batchValues.length === 0) continue
162+
163+
try {
164+
const batchSql = updateSet
165+
? `INSERT INTO ${schema.targetTable} (${cols.join(", ")})
166+
VALUES ${batchValues.join(", ")}
167+
ON CONFLICT (${conflictClause}) DO UPDATE SET ${updateSet}`
168+
: `INSERT INTO ${schema.targetTable} (${cols.join(", ")})
169+
VALUES ${batchValues.join(", ")}
170+
ON CONFLICT (${conflictClause}) DO NOTHING`
171+
172+
const result = await pool.query(batchSql, batchParams)
173+
inserted += result.rowCount ?? batchValues.length
174+
} catch (err) {
175+
// If batch fails, fall back to per-row to identify the bad row(s)
176+
for (let j = 0; j < batch.length; j++) {
177+
const row = batch[j]
178+
const rowIndex = i + j + 1
124179

125-
try {
126180
const dbRow: Record<string, string> = {}
127181
for (const [header, dbCol] of headerToDb) {
128182
let value = row[header] ?? ""
@@ -131,44 +185,25 @@ async function upsertRows(
131185
dbRow[dbCol] = value
132186
}
133187

134-
// Check required fields have values
135-
const missingRequired = schema.columns
136-
.filter((c) => c.required && (!dbRow[c.name] || dbRow[c.name].trim() === ""))
137-
if (missingRequired.length > 0) {
188+
const missing = requiredDbCols.find((c) => !dbRow[c] || dbRow[c].trim() === "")
189+
if (missing) continue // already counted above
190+
191+
try {
192+
const singlePlaceholders = cols.map((_, idx) => `$${idx + 1}`)
193+
const singleSql = updateSet
194+
? `INSERT INTO ${schema.targetTable} (${cols.join(", ")})
195+
VALUES (${singlePlaceholders.join(", ")})
196+
ON CONFLICT (${conflictClause}) DO UPDATE SET ${updateSet}`
197+
: `INSERT INTO ${schema.targetTable} (${cols.join(", ")})
198+
VALUES (${singlePlaceholders.join(", ")})
199+
ON CONFLICT (${conflictClause}) DO NOTHING`
200+
201+
await pool.query(singleSql, cols.map((c) => dbRow[c]))
202+
inserted++
203+
} catch (rowErr) {
138204
skipped++
139-
errors.push({
140-
row: rowIndex,
141-
column: missingRequired[0].name,
142-
message: `Empty required field: ${missingRequired[0].name}`,
143-
})
144-
continue
205+
errors.push({ row: rowIndex, message: (rowErr as Error).message })
145206
}
146-
147-
const cols = Object.keys(dbRow)
148-
const vals = Object.values(dbRow)
149-
const placeholders = cols.map((_, idx) => `$${idx + 1}`)
150-
const updateSet = cols
151-
.filter((c) => !schema.upsertKey.includes(c))
152-
.map((c) => `${c} = EXCLUDED.${c}`)
153-
.join(", ")
154-
155-
const conflictClause = schema.upsertKey.join(", ")
156-
const sql = updateSet
157-
? `INSERT INTO ${schema.targetTable} (${cols.join(", ")})
158-
VALUES (${placeholders.join(", ")})
159-
ON CONFLICT (${conflictClause}) DO UPDATE SET ${updateSet}`
160-
: `INSERT INTO ${schema.targetTable} (${cols.join(", ")})
161-
VALUES (${placeholders.join(", ")})
162-
ON CONFLICT (${conflictClause}) DO NOTHING`
163-
164-
await pool.query(sql, vals)
165-
inserted++
166-
} catch (err) {
167-
skipped++
168-
errors.push({
169-
row: rowIndex,
170-
message: (err as Error).message,
171-
})
172207
}
173208
}
174209
}

‎codebenders-dashboard/lib/upload-schemas.ts‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,9 @@ export interface ColumnMapping {
2929
status: "matched" | "unmapped"
3030
}
3131

32+
export const CONFIDENT_THRESHOLD = 0.6
33+
export const TENTATIVE_THRESHOLD = 0.3
34+
3235
// ── Value Transforms ─────────────────────────────────────────────────────────
3336

3437
const ENROLLMENT_TYPE_MAP: Record<string, string> = {
@@ -249,7 +252,7 @@ export function detectSchema(headers: string[]): DetectionResult {
249252
// Strip internal columnCount before returning
250253
const publicScores = scores.map(({ schemaId, label, score }) => ({ schemaId, label, score }))
251254

252-
if (best.score >= 0.6) {
255+
if (best.score >= CONFIDENT_THRESHOLD) {
253256
// Required-column gate: all required columns must be present to confirm
254257
// high confidence. Without this, a 3-column subset could score 1.0 recall
255258
// against a 24-column schema and be incorrectly auto-accepted.
@@ -266,10 +269,10 @@ export function detectSchema(headers: string[]): DetectionResult {
266269
return { schema: bestSchema, confidence: best.score, scores: publicScores }
267270
}
268271
// Cap to tentative band when required columns are missing
269-
return { schema: bestSchema, confidence: Math.min(best.score, 0.59), scores: publicScores }
272+
return { schema: bestSchema, confidence: Math.min(best.score, CONFIDENT_THRESHOLD - 0.01), scores: publicScores }
270273
}
271274

272-
if (best.score >= 0.3) {
275+
if (best.score >= TENTATIVE_THRESHOLD) {
273276
return { schema: schemaMap.get(best.schemaId) ?? null, confidence: best.score, scores: publicScores }
274277
}
275278

0 commit comments

Comments
 (0)