fix: improve CSV export and import handling for proper data formatting

Author: ivictbor

custom/ExportCsv.vue

@@ -54,12 +54,9 @@ async function exportCsv() {
       throw new Error(resp.error);
     }
 
-    // Parse the CSV data to ensure proper formatting
-    const parsedData = Papa.parse(resp.data).data;
-    
     // Generate properly formatted CSV
-    const csvContent = '\ufeff' + Papa.unparse(parsedData, {
-      quotes: true, // Force quotes around all fields
+    const csvContent = '\ufeff' + Papa.unparse(resp.data, {
+      quotes: resp.columnsToForceQuote, // Force quotes only certain columns (!!!not all this breaks BI/Excel tasks)
       quoteChar: '"',
       escapeChar: '"',
     });

custom/ImportCsv.vue

@@ -165,6 +165,7 @@ async function importCsv() {
         Papa.parse(text, {
           header: true,
           skipEmptyLines: true,
+          // dynamicTyping: true, - bad option becaue it tries to parse "1" -> int even if the column is string 
 
           complete: async (results) => {
             if (results.errors.length > 0) {
@@ -214,12 +215,7 @@ async function importCsv() {
     reader.readAsText(file);
   };
 }
-function handleImportClick() {
-  console.log('handleImportClick', checkProgress.value);
-  if (!checkProgress.value) {
-    importCsv();
-  }
-}
+
 
 function click() {
   importCsv();

index.ts

@@ -66,42 +66,27 @@ export default class ImportExport extends AdminForthPlugin {
           getTotals: true,
         });
 
-        // csv export
+        // prepare data for PapaParse unparse
         const columns = this.resourceConfig.columns.filter((col) => !col.virtual);
-        
-        const escapeCSV = (value: any, column?: AdminForthResourceColumn) => {
-          if (value === null || value === undefined) {
-            return '""';
-          }
-          if (column?.type === AdminForthDataTypes.FLOAT 
-              || column?.type === AdminForthDataTypes.INTEGER) {
-            // no quotes for numbers
-            return String(value);
-          }
-          if (column?.type === AdminForthDataTypes.BOOLEAN) {
-            // no quotes for boolean values
-            if (value === true) {
-              return 'true';
-            } else if (value === false) {
-              return 'false';
-            }
-          }
-          const str = String(value);
-          if (str.includes(',') || str.includes('"') || str.includes('\n')) {
-            return `"${str.replace(/"/g, '""')}"`;
-          }
-          return `"${str}"`;
-        };
 
-        let csv = data.data.map((row) => {
-          return columns.map((col) => escapeCSV(row[col.name], col)).join(',');
-        }).join('\n');
+        const columnsToForceQuote = columns.map(col => {
+          return col.type !== AdminForthDataTypes.FLOAT 
+            && col.type !== AdminForthDataTypes.INTEGER 
+            && col.type !== AdminForthDataTypes.BOOLEAN;
+        })
 
-        // add headers
-        const headers = columns.map((col) => escapeCSV(col.name)).join(',');
-        csv = `${headers}\n${csv}`;
+        const fields = columns.map((col) => col.name);
+
+        const rows = data.data.map((row) => {
+          return columns.map((col) => row[col.name]);
+        });
 
-        return { data: csv, exportedCount: data.total, ok: true };
+        return { 
+          data: { fields, data: rows }, 
+          columnsToForceQuote, 
+          exportedCount: data.total, 
+          ok: true 
+        };
       }
     });
 
@@ -130,15 +115,31 @@ export default class ImportExport extends AdminForthPlugin {
 
         const primaryKeyColumn = this.resourceConfig.columns.find(col => col.primaryKey);
 
+        const resourceColumns = columns.map(colName => this.resourceConfig.columns.find(c => c.name === colName));
+
         const columnValues: any[] = Object.values(data);
         for (let i = 0; i < columnValues[0].length; i++) {
           const row = {};
           for (let j = 0; j < columns.length; j++) {
-            row[columns[j]] = columnValues[j][i];
+            const val = columnValues[j][i];
+            const resourceCol = resourceColumns[j];
+
+            if ( (resourceCol.type === AdminForthDataTypes.INTEGER 
+                || resourceCol.type === AdminForthDataTypes.FLOAT) && val !== ''
+              ) {
+              // convert empty strings to null for numeric fields
+              row[columns[j]] = +val;
+            } else if (resourceCol.type === AdminForthDataTypes.BOOLEAN && val !== '') {
+              row[columns[j]] = (val.toLowerCase() === 'true' || val === '1' || val === 1);
+            } else {
+              row[columns[j]] = val;
+            }
           }
           rows.push(row);
         }
 
+        console.log('Prepared rows for import:', rows);
+
         let importedCount = 0;
         let updatedCount = 0;
 
@@ -190,11 +191,24 @@ export default class ImportExport extends AdminForthPlugin {
         }
 
         const primaryKeyColumn = this.resourceConfig.columns.find(col => col.primaryKey);
+        const resourceColumns = columns.map(colName => this.resourceConfig.columns.find(c => c.name === colName));
         const columnValues: any[] = Object.values(data);
         for (let i = 0; i < columnValues[0].length; i++) {
           const row = {};
           for (let j = 0; j < columns.length; j++) {
-            row[columns[j]] = columnValues[j][i];
+            const val = columnValues[j][i];
+            const resourceCol = resourceColumns[j];
+
+            if ( (resourceCol.type === AdminForthDataTypes.INTEGER 
+                || resourceCol.type === AdminForthDataTypes.FLOAT) && val !== ''
+              ) {
+              // convert empty strings to null for numeric fields
+              row[columns[j]] = +val;
+            } else if (resourceCol.type === AdminForthDataTypes.BOOLEAN && val !== '') {
+              row[columns[j]] = (val.toLowerCase() === 'true' || val === '1' || val === 1);
+            } else {
+              row[columns[j]] = val;
+            }
           }
           rows.push(row);
         }