feat: Add validation and improve error messaging for CSV import

src/permit/permit.service.ts

@@ -566,7 +566,7 @@ export async function searchPermitByAddress(address: string) {
     .limit(1);
 }
 
-export async function bulkImport(csvData: Object[], user: AuthenticatedUser) {
+export async function bulkImport(csvData: any[], user: AuthenticatedUser) {
   const allowedFields = [
     "Permit Number",
     "County",
@@ -584,53 +584,84 @@ export async function bulkImport(csvData: Object[], user: AuthenticatedUser) {
   const failed = [];
   const created = [];
 
+  const clientCache = {};
+  const countyCache = {};
+
+  // Validation run
   for (const [index, record] of csvData.entries()) {
-    try {
+    const errors = [];
-      if (!record["Permit Number"]) {
+
-        failed.push(index);
+    if (!record["Permit Number"]) {
-        continue;
+      errors.push("Permit Number is empty");
+    } else if (!record["County"]) {
+      errors.push("County is empty");
+    } else if (!record["Client"]) {
+      errors.push("Client is empty");
+    } else if (!record["Address"]) {
+      errors.push("Address is empty");
+    }
+
+    if (record["County"] && record["Client"]) {
+      let clientData = clientCache[record["Client"]];
+      if (!clientData) {
+        const clientInDb = await orgModel.findOne({ name: record["Client"] });
+        if (clientInDb) {
+          clientData = {
+            id: clientInDb._id,
+            pid: clientInDb.pid,
+            licenseNumber: clientInDb.licenseNumber,
+            name: clientInDb.name,
+            avatar: clientInDb.avatar,
+          };
+
+          clientCache[record["Client"]] = clientData;
+        } else {
+          errors.push("Client not found");
+        }
       }
 
+      csvData[index].clientData = clientData;
+
+      let countyData = countyCache[record["County"]];
+      if (!countyData) {
+        const countyInDb = await orgModel.findOne({ name: record["County"] });
+        if (countyInDb) {
+          countyData = {
+            id: countyInDb._id,
+            pid: countyInDb.pid,
+            name: countyInDb.name,
+            avatar: countyInDb.avatar,
+          };
+
+          countyCache[record["County"]] = countyData;
+        } else {
+          errors.push("County not found");
+        }
+      }
+
+      csvData[index].countyData = countyData;
+    }
+
+    if (errors.length > 0) failed.push({ rowId: index + 2, errors });
+  }
+
+  if (failed.length > 0) return { created, failed, allowedFields };
+
+  // Main run
+  for (const [index, record] of csvData.entries()) {
+    try {
       const permitInDb = await permitModel.findOne({
         permitNumber: record["Permit Number"],
       });
 
       if (!permitInDb) {
-        let clientData = null;
-        let countyData = null;
-
-        if (record["Client"]) {
-          const clientInDb = await orgModel.findOne({ name: record["Client"] });
-          if (clientInDb) {
-            clientData = {
-              id: clientInDb._id,
-              pid: clientInDb.pid,
-              licenseNumber: clientInDb.licenseNumber,
-              name: clientInDb.name,
-              avatar: clientInDb.avatar,
-            };
-          }
-        }
-
-        if (record["County"]) {
-          const countyInDb = await orgModel.findOne({ name: record["County"] });
-          if (countyInDb) {
-            countyData = {
-              id: countyInDb._id,
-              pid: countyInDb.pid,
-              name: countyInDb.name,
-              avatar: countyInDb.avatar,
-            };
-          }
-        }
-
         const newPermit = await permitModel.create({
           tenantId: user.tenantId,
           pid: generateId(),
           permitNumber: record["Permit Number"],
-          county: countyData,
+          county: record.countyData,
-          client: clientData?.id,
+          client: record.clientData?.id,
-          clientData: clientData,
+          clientData: record.clientData,
           cleanStatus: record["County Status"],
           address: record["Address"],
           recordType: record["Record Type"],
@@ -649,10 +680,17 @@ export async function bulkImport(csvData: Object[], user: AuthenticatedUser) {
         });
 
         created.push(populatedPermit);
+      } else {
+        failed.push({
+          rowId: index + 2,
+          errors: [
+            `Permit with this number: ${record["Permit Number"]} already exists`,
+          ],
+        });
       }
     } catch (err) {
       console.log(err);
-      failed.push(index + 2);
+      failed.push({ rowId: index + 2, errors: ["Internal Error"] });
     }
   }