From f1b6f48a51b80f1c864f62d1afc745bd5e2469d8 Mon Sep 17 00:00:00 2001 From: Akhil Meka Date: Fri, 21 Nov 2025 16:55:01 +0530 Subject: [PATCH] feat: add bulk import endpoint --- package.json | 1 + pnpm-lock.yaml | 24 +++++++++ src/permit/permit.controller.ts | 19 +++++++ src/permit/permit.route.ts | 10 ++++ src/permit/permit.schema.ts | 1 + src/permit/permit.service.ts | 93 +++++++++++++++++++++++++++++++++ 6 files changed, 148 insertions(+) diff --git a/package.json b/package.json index d256de2..ab1901c 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "fastify": "^5.2.0", "fastify-type-provider-zod": "^4.0.2", "fastify-zod": "^1.4.0", + "json-2-csv": "^5.5.10", "lru-cache": "^11.0.2", "mongoose": "^8.9.0", "openai": "^5.19.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bcbd5f6..7e0b620 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -53,6 +53,9 @@ importers: fastify-zod: specifier: ^1.4.0 version: 1.4.0(fastify@5.2.0) + json-2-csv: + specifier: ^5.5.10 + version: 5.5.10 lru-cache: specifier: ^11.0.2 version: 11.0.2 @@ -691,6 +694,10 @@ packages: supports-color: optional: true + deeks@3.1.0: + resolution: {integrity: sha512-e7oWH1LzIdv/prMQ7pmlDlaVoL64glqzvNgkgQNgyec9ORPHrT2jaOqMtRyqJuwWjtfb6v+2rk9pmaHj+F137A==} + engines: {node: '>= 16'} + delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} @@ -699,6 +706,10 @@ packages: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} + doc-path@4.1.1: + resolution: {integrity: sha512-h1ErTglQAVv2gCnOpD3sFS6uolDbOKHDU1BZq+Kl3npPqroU3dYL42lUgMfd5UimlwtRgp7C9dLGwqQ5D2HYgQ==} + engines: {node: '>=16'} + dot-case@3.0.4: resolution: {integrity: sha512-Kv5nKlh6yRrdrGvxeJ2e5y2eRUpkUosIW4A2AS38zwSz27zu7ufDwQPi5Jhs3XAlGNetl3bmnGhQsMtkKJnj3w==} @@ -855,6 +866,10 @@ packages: resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==} hasBin: true + json-2-csv@5.5.10: + resolution: {integrity: sha512-Dep8wO3Fr5wNjQevO2Z8Y7yeee/nYSGRsi7q6zJDKEVHxXkXT+v21vxHmDX923UzmCXXkSo62HaTz6eTWzFLaw==} + engines: {node: '>= 16'} + json-schema-ref-resolver@1.0.1: resolution: {integrity: sha512-EJAj1pgHc1hxF6vo2Z3s69fMjO1INq6eGHXZ8Z6wCQeldCuwxGK9Sxf4/cScGn3FZubCVUehfWtcDM/PLteCQw==} @@ -2371,10 +2386,14 @@ snapshots: dependencies: ms: 2.1.3 + deeks@3.1.0: {} + delayed-stream@1.0.0: {} depd@2.0.0: {} + doc-path@4.1.1: {} + dot-case@3.0.4: dependencies: no-case: 3.0.4 @@ -2572,6 +2591,11 @@ snapshots: dependencies: argparse: 2.0.1 + json-2-csv@5.5.10: + dependencies: + deeks: 3.1.0 + doc-path: 4.1.1 + json-schema-ref-resolver@1.0.1: dependencies: fast-deep-equal: 3.1.3 diff --git a/src/permit/permit.controller.ts b/src/permit/permit.controller.ts index cc1323f..fd42cf2 100644 --- a/src/permit/permit.controller.ts +++ b/src/permit/permit.controller.ts @@ -1,6 +1,7 @@ import { FastifyReply, FastifyRequest } from "fastify"; import { CreatePermitInput, UpdatePermitInput } from "./permit.schema"; import { + bulkImport, createPermit, deletePermit, getPermit, @@ -10,6 +11,7 @@ import { updatePermit, } from "./permit.service"; import { PageQueryParams } from "../pagination"; +import { csv2json } from "json-2-csv"; export async function createPermitHandler( req: FastifyRequest, @@ -117,3 +119,20 @@ export async function searchPermitByAddressHandler( return err; } } + +export async function bulkImportHandler( + req: FastifyRequest, + res: FastifyReply +) { + try { + const data = await req.file(); + const csvString = (await data.toBuffer()).toString("utf-8"); + + const parsedCSV = csv2json(csvString, { delimiter: { eol: "\r\n" } }); + const result = await bulkImport(parsedCSV, req.user); + + return res.code(200).send(result); + } catch (err) { + return err; + } +} diff --git a/src/permit/permit.route.ts b/src/permit/permit.route.ts index f028927..ea07f20 100644 --- a/src/permit/permit.route.ts +++ b/src/permit/permit.route.ts @@ -1,5 +1,6 @@ import { FastifyInstance } from "fastify"; import { + bulkImportHandler, createPermitHandler, deletePermitHandler, getPermitHandler, @@ -138,6 +139,15 @@ export async function permitRoutes(fastify: FastifyInstance) { } ); + fastify.post( + "/bulkImport", + { + config: { requiredClaims: ["permit:write"] }, + preHandler: [fastify.authorize], + }, + bulkImportHandler + ); + await noteRoutes(fastify); fastify.addHook("onSend", hideFields("permits")); diff --git a/src/permit/permit.schema.ts b/src/permit/permit.schema.ts index 27d9ffd..c490676 100644 --- a/src/permit/permit.schema.ts +++ b/src/permit/permit.schema.ts @@ -110,6 +110,7 @@ const permitSchema = new mongoose.Schema({ relationship: String, type_text: String, }, + importFlag: Boolean, }); permitSchema.index({ tenantId: 1, permitNumber: 1 }, { unique: true }); diff --git a/src/permit/permit.service.ts b/src/permit/permit.service.ts index dde115f..14b2486 100644 --- a/src/permit/permit.service.ts +++ b/src/permit/permit.service.ts @@ -518,3 +518,96 @@ export async function searchPermitByAddress(address: string) { .sort({ score: { $meta: "textScore" } }) .limit(1); } + +export async function bulkImport(csvData: Object[], user: AuthenticatedUser) { + const allowedFields = [ + "Permit Number", + "County", + "Client", + "Address", + "Open Date", + "County Status", + "Record Type", + "Lot", + "Block", + "Job Number", + "Community Name", + ]; + + const failed = []; + const created = []; + + for (const [index, record] of csvData.entries()) { + try { + if (!record["Permit Number"]) { + failed.push(index); + continue; + } + + const permitInDb = await permitModel.findOne({ + permitNumber: record["Permit Number"], + }); + + if (!permitInDb) { + let clientData = null; + let countyData = null; + + if (record["Client"]) { + const clientInDb = await orgModel.findOne({ name: record["Client"] }); + if (clientInDb) { + clientData = { + id: clientInDb._id, + pid: clientInDb.pid, + licenseNumber: clientInDb.licenseNumber, + name: clientInDb.name, + avatar: clientInDb.avatar, + }; + } + } + + if (record["County"]) { + const countyInDb = await orgModel.findOne({ name: record["County"] }); + if (countyInDb) { + countyData = { + id: countyInDb._id, + pid: countyInDb.pid, + name: countyInDb.name, + avatar: countyInDb.avatar, + }; + } + } + + const newPermit = await permitModel.create({ + tenantId: user.tenantId, + pid: generateId(), + permitNumber: record["Permit Number"], + county: countyData, + client: clientData?.id, + clientData: clientData, + cleanStatus: record["County Status"], + address: record["Address"], + recordType: record["Record Type"], + lot: record["Lot"], + block: record["Block"], + jobNumber: record["Job Number"], + communityName: record["Community Name"], + createdAt: new Date(), + createdBy: user.userId, + importFlag: true, + }); + + const populatedPermit = await newPermit.populate({ + path: "assignedTo createdBy", + select: "pid name avatar", + }); + + created.push(populatedPermit); + } + } catch (err) { + console.log(err); + failed.push(index + 2); + } + } + + return { created, failed, allowedFields }; +}