diff --git a/src/index.ts b/src/index.ts index acbdcc8..d308068 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,21 +1,30 @@ import { promises as fs ,existsSync} from "fs"; import {createIfNot} from "./utils/fileUtils.js" import * as path from "node:path" -async function writeSQL(statement: string, saveFileAs = "") { +async function writeSQL(statement: string, saveFileAs = "", isAppend: boolean = false) { try { const destinationFile = process.argv[2] || saveFileAs; if (!destinationFile) { throw new Error("Missing saveFileAs parameter"); } createIfNot(path.resolve(`./sql/${destinationFile}.sql`)) - await fs.writeFile(`sql/${process.argv[2]}.sql`, statement); + if(isAppend){ + await fs.appendFile(`sql/${process.argv[2]}.sql`, statement); + }else{ + await fs.writeFile(`sql/${process.argv[2]}.sql`, statement); + } } catch (err) { console.log(err); } } -async function readCSV(csvFileName = "") { + +async function readCSV(csvFileName = "", batchSize: number = 0) { try { const fileAndTableName = process.argv[2] || csvFileName; + + batchSize = parseInt(process.argv[3]) || batchSize || 500; + let isAppend: boolean = false; + if (!fileAndTableName) { throw new Error("Missing csvFileName parameter"); } @@ -32,7 +41,7 @@ async function readCSV(csvFileName = "") { columnNames.forEach((name) => (beginSQLInsert += `${name}, `)); beginSQLInsert = beginSQLInsert.slice(0, -2) + ")\nVALUES\n"; let values = ""; - linesArray.forEach((line) => { + linesArray.forEach((line, index) => { // Parses each line of CSV into field values array const arr = line.split(/,(?=(?:(?:[^"]*"){2})*[^"]*$)/); if (arr.length > columnNames.length) { @@ -42,6 +51,19 @@ async function readCSV(csvFileName = "") { console.log(arr); throw new Error("Too Few Values in row"); } + + // Check batch size (rows per batch) + if(index > 0 && index % batchSize == 0){ + values = values.slice(0, -2) + ";\n\n"; + + const sqlStatement = beginSQLInsert + values; + + // Write File + writeSQL(sqlStatement, fileAndTableName, isAppend); + values = ""; + isAppend = true; + } + let valueLine = "\t("; arr.forEach((value) => { // Matches NULL values, Numbers, @@ -64,7 +86,7 @@ async function readCSV(csvFileName = "") { values = values.slice(0, -2) + ";"; const sqlStatement = beginSQLInsert + values; // Write File - writeSQL(sqlStatement); + writeSQL(sqlStatement, fileAndTableName, isAppend); } catch (err) { console.log(err); }