diff --git a/README.md b/README.md index 4197667..5816ed2 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,45 @@ CSV2J COPY ( SELECT 1 AS ID, 10.99 AS AMOUNT, TIMESTAMP '2022-01-31 23:59:58.987' AS DAT_CREATION UNION ALL SELECT 2 AS ID, 7.50 AS AMOUNT, TIMESTAMP '2023-01-31 21:59:58.987' AS DAT_CREATION -) TO '/tmp/file.csv' WITH CSV HEADER +) TO '/tmp/file.csv' WITH CSV [ZIP|GZIP] HEADER ``` +### Formatting Date,Time,Datetime and Numbers +#### Selecting language +``` +LANGUAGE '' -- en-US pt-BR fr-FR +``` +Default is System Language
+More language tag see [Supported Locales]("https://www.oracle.com/java/technologies/javase/jdk8-jre8-suported-locales.html#util-text") + +#### Using custom templates +``` +DATETIMEFORMAT 'yyyy-MM-dd HH:mm:ss.SSS' +DATEFORMAT 'yyyy-MM-dd' +TIMEFORMAT 'HH:mm:ss.SSS' +NUMBERFORMAT '#00' +DECIMALFORMAT '#00.00' +``` +More patterns details see +[SimpleDateFormat](https://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html) +and [DecimalFormat](https://docs.oracle.com/javase/7/docs/api/java/text/DecimalFormat.html) + +### Full Example +```sql +CSV2J COPY ( + SELECT 1234567890 AS ID, 1234567890.995 AS AMOUNT, TIMESTAMP '2022-01-31 23:59:58.987' AS DAT_CREATION, DATE '2023-03-11' as DAT_BORN, TIME '21:47:01.001' as TIME_BORN + UNION ALL + SELECT 1 AS ID, 1.12345 AS AMOUNT, TIMESTAMP '2022-01-31 23:59:58' AS DAT_CREATION, DATE '1978-04-16' as DAT_BORN, TIME '23:35:00' as TIME_BORN +) TO '/tmp/file.csv' WITH CSV GZIP HEADER +LANGUAGE 'pt-BR' +DATETIMEFORMAT 'yyyy-MM-dd HH:mm:ss.SSS' +DATEFORMAT 'yyyy-MM-dd' +TIMEFORMAT 'HH:mm:ss.SSS' +NUMBERFORMAT '#00' +DECIMALFORMAT '#00.00' +``` + + ## Using ### SQL client software application diff --git a/build.gradle b/build.gradle index 3f3546f..b6a8f76 100644 --- a/build.gradle +++ b/build.gradle @@ -41,6 +41,8 @@ dependencies { testImplementation group: 'org.postgresql', name: 'postgresql', version: '9.4.1212' testImplementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0' + testImplementation group: 'org.apache.commons', name: 'commons-compress', version: '1.22' + } tasks.named('test') { diff --git a/gradlew.bat b/gradlew.bat index 107acd3..f80d9f9 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -33,7 +33,7 @@ set APP_HOME=%DIRNAME% for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" "-Djavax.net.ssl.trustStorePassword=changeit" @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome diff --git a/src/main/antlr/PostgreSQLLexer.g4 b/src/main/antlr/PostgreSQLLexer.g4 index 77ae258..4933102 100644 --- a/src/main/antlr/PostgreSQLLexer.g4 +++ b/src/main/antlr/PostgreSQLLexer.g4 @@ -2192,6 +2192,41 @@ LOOP OPEN : 'OPEN' ; + +ZIP + : 'ZIP' + ; + +GZIP + : 'GZIP' + ; + +BZIP2 + : 'BZIP2' + ; + +DATEFORMAT + : 'DATEFORMAT' + ; + +TIMEFORMAT + : 'TIMEFORMAT' + ; + +DATETIMEFORMAT + : 'DATETIMEFORMAT' + ; + +NUMBERFORMAT + : 'NUMBERFORMAT' + ; + +DECIMALFORMAT + : 'DECIMALFORMAT' + ; + + + // // IDENTIFIERS (4.1.1) diff --git a/src/main/antlr/PostgreSQLParser.g4 b/src/main/antlr/PostgreSQLParser.g4 index b2564c4..eab8546 100644 --- a/src/main/antlr/PostgreSQLParser.g4 +++ b/src/main/antlr/PostgreSQLParser.g4 @@ -660,9 +660,18 @@ copy2csv_opt_list copy2csv_opt_item : DELIMITER sconst | CSV + | GZIP + | ZIP + | BZIP2 | HEADER_P | CREATE_TABLE_P | ENCODING sconst + | LANGUAGE sconst + | DATEFORMAT sconst + | TIMEFORMAT sconst + | DATETIMEFORMAT sconst + | NUMBERFORMAT sconst + | DECIMALFORMAT sconst ; createstmt diff --git a/src/main/java/com/mageddo/csv2jdbc/CopyCsvStatement.java b/src/main/java/com/mageddo/csv2jdbc/CopyCsvStatement.java index f19bc94..74538ed 100644 --- a/src/main/java/com/mageddo/csv2jdbc/CopyCsvStatement.java +++ b/src/main/java/com/mageddo/csv2jdbc/CopyCsvStatement.java @@ -83,6 +83,49 @@ public CopyCsvStatement validateIsCsv() { return this; } + public boolean isZIP() { + return this.options.containsKey(Option.ZIP); + } + + public boolean isGZIP() { + return this.options.containsKey(Option.GZIP); + } + + public boolean isBZIP2() { + return this.options.containsKey(Option.BZIP2); + } + + public String getCompression() { + if( isGZIP() ) return Option.GZIP; + if( isZIP() ) return Option.ZIP; + if( isBZIP2() ) return Option.BZIP2; + return ""; + } + + protected void setFile(Path file) { + this.file = file; + } + + public String getLanguage() { + return this.options.getOrDefault(Option.LANGUAGE,Option.DEFAULT_LANGUAGE).getValue(); + } + + public String getDateFormat() { + return this.options.getOrDefault("DATEFORMAT",Option.DEFAULT_NULL).getValue(); + } + public String getTimeFormat() { + return this.options.getOrDefault("TIMEFORMAT",Option.DEFAULT_NULL).getValue(); + } + public String getDateTimeFormat() { + return this.options.getOrDefault("DATETIMEFORMAT",Option.DEFAULT_NULL).getValue(); + } + public String getNumberFormat() { + return this.options.getOrDefault("NUMBERFORMAT",Option.DEFAULT_NULL).getValue(); + } + public String getDecimalFormat() { + return this.options.getOrDefault("DECIMALFORMAT",Option.DEFAULT_NULL).getValue(); + } + @Getter @ToString @EqualsAndHashCode(of = "name") @@ -98,6 +141,17 @@ public static class Option { public static final String ENCODING = "ENCODING"; + public static final String ZIP = "ZIP"; + + public static final String GZIP = "GZIP"; + + public static final String BZIP2 = "BZIP2"; + + public static final String LANGUAGE = "LANGUAGE"; + + + public static final Option DEFAULT_NULL = new Option(null); + public static final Option DEFAULT_CSV = new Option(CSV); public static final Option DEFAULT_HEADER = new Option(HEADER); @@ -107,6 +161,12 @@ public static class Option { public static final Option DEFAULT_ENCODING = new Option(ENCODING, "utf-8"); + public static final Option DEFAULT_ZIP = new Option(ZIP); + + public static final Option DEFAULT_GZIP = new Option(GZIP); + + public static final Option DEFAULT_LANGUAGE = new Option(LANGUAGE); + private final String name; private final String value; diff --git a/src/main/java/com/mageddo/csv2jdbc/Csv2JdbcDriver.java b/src/main/java/com/mageddo/csv2jdbc/Csv2JdbcDriver.java index 668edd5..1c9e60e 100644 --- a/src/main/java/com/mageddo/csv2jdbc/Csv2JdbcDriver.java +++ b/src/main/java/com/mageddo/csv2jdbc/Csv2JdbcDriver.java @@ -45,7 +45,7 @@ public Connection connect(String url, Properties info) throws SQLException { final String delegateDriverClassName = getOrDefault( params, PROP_DELEGATE_DRIVER_CLASSNAME, - "org.h2.Driver" + info.getProperty(PROP_DELEGATE_DRIVER_CLASSNAME,"org.h2.Driver") ); this.delegate = Reflections.createInstance(delegateDriverClassName); final String delegateUrl = toDelegateUrl(url); diff --git a/src/main/java/com/mageddo/csv2jdbc/Csv2JdbcExecutor.java b/src/main/java/com/mageddo/csv2jdbc/Csv2JdbcExecutor.java index c50b824..75aa3ab 100644 --- a/src/main/java/com/mageddo/csv2jdbc/Csv2JdbcExecutor.java +++ b/src/main/java/com/mageddo/csv2jdbc/Csv2JdbcExecutor.java @@ -2,9 +2,18 @@ import java.io.BufferedWriter; import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.nio.charset.Charset; +import java.nio.file.Path; +import java.nio.file.Paths; import java.sql.Connection; import java.sql.SQLException; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.zip.GZIPOutputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; @@ -12,7 +21,7 @@ public class Csv2JdbcExecutor { - public static final int HEADER_COUNT = 1; + //public static final int HEADER_COUNT = 1; private final Connection connection; private final CopyCsvStatement csvStm; @@ -40,40 +49,66 @@ public int execute() throws SQLException { } private int extractQueryToCsv() throws SQLException { + CsvExtractor extractor = new CsvExtractor( this.csvStm.getFile().toString(), + this.csvStm.hasHeader(), this.csvStm.getDelimiter(), this.csvStm.getCharset(), + this.csvStm.getCompression() ); + try { + CsvTableDaos.streamSelect(this.connection, this.csvStm.getExtractSql(), + o -> extractor.accept( new FormatterAndCounterResultSet(o, this.csvStm) ) ); + } catch (Exception e) { + throw new SQLException(e); + } + if( extractor.getFiles().size() == 1 ) { + this.csvStm.setFile( Paths.get( extractor.getFiles().iterator().next() ) ); + } + Log.log("status=csvWritten"); + Log.log("status=linesCount, lines={}", extractor.getRowCount()); + return extractor.getRowCount(); + } + + private int extractQueryToCsv1() throws SQLException { + final AtomicInteger rowCount = new AtomicInteger(); try { CsvTableDaos.streamSelect(this.connection, this.csvStm.getExtractSql(), (rs) -> { try (final CSVPrinter printer = this.createCsvPrinter()) { Log.log("status=printingRecords"); - printer.printRecords(rs, true); + final FormatterAndCounterResultSet prs = new FormatterAndCounterResultSet(rs, this.csvStm); + printer.printRecords(prs, this.csvStm.hasHeader()); + rowCount.set( prs.getRow() - 1 ); } }); } catch (Exception e) { throw new SQLException(e); } Log.log("status=csvWritten"); - try { + Log.log("status=linesCount, lines={}", rowCount.get()); + return rowCount.get(); + /*try { final int lines = Files.countLines(this.csvStm.getFile()) - HEADER_COUNT; Log.log("status=linesCount, lines={}", lines); return lines; } catch (IOException e) { throw new SQLException(e); - } + }/**/ } private int extractQueryToCsv0() throws SQLException { + final AtomicInteger rowCount = new AtomicInteger(0); try { final BufferedWriter out = java.nio.file.Files.newBufferedWriter(this.csvStm.getFile()); CsvTableDaos.streamSelect(this.connection, this.csvStm.getExtractSql(), (rs) -> { // try (final CSVPrinter printer = this.createCsvPrinter()) { - final int columns = rs + final FormatterAndCounterResultSet prs = new FormatterAndCounterResultSet(rs, this.csvStm); + final int columns = prs .getMetaData() .getColumnCount(); - while (rs.next()){ + while (prs.next()){ for (int i = 1; i <= columns; i++) { - out.write(rs.getString(i)); + out.write( prs.getObject(i).toString() ); out.write(", "); } out.write('\n'); + rowCount.incrementAndGet(); } // printer.printRecords(rs, true); // } @@ -81,11 +116,12 @@ private int extractQueryToCsv0() throws SQLException { } catch (Exception e) { throw new SQLException(e); } - try { + return rowCount.get(); + /*try { return Files.countLines(this.csvStm.getFile()) - HEADER_COUNT; } catch (IOException e) { throw new SQLException(e); - } + }/**/ } private int loadCsvIntoTable() throws SQLException { @@ -118,10 +154,33 @@ CSVParser createCsvParser() throws IOException { } private CSVPrinter createCsvPrinter() throws IOException { + final Appendable appendableOut; + if( this.csvStm.isGZIP() ) { + Path file = this.csvStm.getFile(); + file = file.resolveSibling( file.getFileName() + ".gzip" ); + this.csvStm.setFile(file); + final OutputStream outs = java.nio.file.Files.newOutputStream( file ); + final GZIPOutputStream gzipout = new GZIPOutputStream( outs ); + appendableOut = new OutputStreamWriter(gzipout, this.csvStm.getCharset() ); + } else if( this.csvStm.isZIP() ) { + final String fileNameInsideZip = this.csvStm.getFile().getFileName().toString(); + Path file = this.csvStm.getFile(); + file = file.resolveSibling( file.getFileName() + ".zip" ); + this.csvStm.setFile(file); + final OutputStream outs = java.nio.file.Files.newOutputStream( file ); + final ZipOutputStream zipout = new ZipOutputStream( outs ); + final ZipEntry zeCsv = new ZipEntry( fileNameInsideZip ); + zipout.putNextEntry(zeCsv); + appendableOut = new OutputStreamWriter(zipout, this.csvStm.getCharset() ); + } else { + appendableOut = java.nio.file.Files.newBufferedWriter( this.csvStm.getFile(), + this.csvStm.getCharset() ); + } + return this.getCsvFormat() .builder() .build() - .print(this.csvStm.getFile(), this.csvStm.getCharset()) + .print( appendableOut ) ; } diff --git a/src/main/java/com/mageddo/csv2jdbc/CsvExtractor.java b/src/main/java/com/mageddo/csv2jdbc/CsvExtractor.java new file mode 100644 index 0000000..bb72594 --- /dev/null +++ b/src/main/java/com/mageddo/csv2jdbc/CsvExtractor.java @@ -0,0 +1,176 @@ +package com.mageddo.csv2jdbc; + +import lombok.Getter; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.lang.reflect.Constructor; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Clob; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.zip.GZIPOutputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import static com.mageddo.csv2jdbc.CopyCsvStatement.Option.GZIP; +import static com.mageddo.csv2jdbc.CopyCsvStatement.Option.ZIP; +import static com.mageddo.csv2jdbc.CopyCsvStatement.Option.BZIP2; + +public class CsvExtractor implements Csv2JdbcPreparedStatement.Consumer { + + private static final Pattern PATTERN = Pattern.compile("\\{([^\\}]+)\\}"); + private final HashMap csvPrinterOutput = new HashMap<>(1); + private final HashMap fileReplaceColumns = new HashMap<>(); + + private final Collection lUsedOutputStream = new ArrayList<>(); + + private final String filePathAndPattern; + private final boolean printHeader; + private final char delimiter; + private final String compression; + private final Charset charset; + + @Getter + private int rowCount = 0; + + public CsvExtractor(String filePathAndPattern, boolean printHeader ) throws SQLException { + this( filePathAndPattern, printHeader, ';', Charset.defaultCharset(), null ); + } + public CsvExtractor(String filePathAndPattern, boolean printHeader, char delimiter, + Charset charset, String compression ) throws SQLException { + try { + this.filePathAndPattern = String.format(filePathAndPattern, Calendar.getInstance()); + } catch(Exception e) { + throw new SQLException(e); + } + this.printHeader = printHeader; + this.delimiter = delimiter; + this.compression = compression; + this.charset = charset; + Matcher m = PATTERN.matcher(filePathAndPattern); + while (m.find()) { + fileReplaceColumns.put( m.group(1), Pattern.quote( m.group(0) ) ); + } + } + + private CSVPrinter createCSVPrinter(String fileName) throws Exception { + final Path file = Paths.get( fileName ); + if( !Files.exists( file.getParent() ) ) + Files.createDirectories( file.getParent() ); + final Appendable appendableOut; + if( this.compression.equalsIgnoreCase(GZIP) ) { + Path gzFile = file.resolveSibling( fileName + ".gzip" ); + final OutputStream outs = java.nio.file.Files.newOutputStream( gzFile ); + final GZIPOutputStream gzipout = new GZIPOutputStream( outs ); + appendableOut = new OutputStreamWriter(gzipout, this.charset ); + } else if( this.compression.equalsIgnoreCase(ZIP) ) { + Path zFile = file.resolveSibling( fileName + ".zip" ); + //this.csvStm.setFile(file); + final OutputStream outs = java.nio.file.Files.newOutputStream( zFile ); + //lUsedOutputStream.add(outs); + final ZipOutputStream zipout = new ZipOutputStream( outs ); + final ZipEntry zeCsv = new ZipEntry( file.getFileName().toString() ); + zipout.putNextEntry(zeCsv); + appendableOut = new OutputStreamWriter(zipout, this.charset ); + } else if( this.compression.equalsIgnoreCase(BZIP2) ) { + Constructor constructor = Class.forName("org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream").getConstructor(OutputStream.class); + Path bz2File = file.resolveSibling( fileName + ".bz2" ); + final OutputStream outs = java.nio.file.Files.newOutputStream( bz2File ); + final OutputStream bz2out = (OutputStream) constructor.newInstance( outs ); + appendableOut = new OutputStreamWriter(bz2out, this.charset ); + } else { + appendableOut = java.nio.file.Files.newBufferedWriter( file, this.charset ); + } + + final CSVPrinter newCSVPrinter = CSVFormat.Builder + .create(CSVFormat.DEFAULT) + .setSkipHeaderRecord(true) + .setDelimiter(this.delimiter) + .build() + .print( appendableOut ) + ; + this.csvPrinterOutput.put( fileName, newCSVPrinter ); + return newCSVPrinter; + } + + private CSVPrinter getCSVPrinter(String fileName, ResultSet rs) throws Exception { + if( !this.csvPrinterOutput.containsKey(fileName) ) { + final CSVPrinter newCSVPrinter = createCSVPrinter(fileName); + if( this.printHeader ) + newCSVPrinter.printHeaders(rs); + return newCSVPrinter; + } + return this.csvPrinterOutput.get( fileName ); + } + + @Override + public void accept(ResultSet rs) throws Exception { + try { + if( fileReplaceColumns.isEmpty() ) { + final CSVPrinter printer = createCSVPrinter(this.filePathAndPattern); + printer.printRecords(rs, this.printHeader); + this.rowCount = rs.getRow(); + } else { + final int columnCount = rs.getMetaData().getColumnCount(); + while (rs.next()) { + String finalFilePath = this.filePathAndPattern; + for(Map.Entry col : fileReplaceColumns.entrySet() ) { + finalFilePath = finalFilePath.replaceAll( col.getValue(), + rs.getObject(col.getKey()).toString() ); + } + CSVPrinter currentCSVPrinter = getCSVPrinter(finalFilePath, rs); + for (int i = 1; i <= columnCount; i++) { + final Object object = rs.getObject(i); + currentCSVPrinter.print(object instanceof Clob ? ((Clob) object).getCharacterStream() : + object); + } + currentCSVPrinter.println(); + rowCount++; + } + } + } finally { + close(); + } + + + } + + private void close() { + for(CSVPrinter printer : this.csvPrinterOutput.values() ) { + try { + printer.close(); + } catch (IOException e) { + Log.log(e.getMessage(), e); + } + System.out.println( printer.getOut().toString() + "\t\t" + printer.getOut().getClass() ); + } + for(OutputStream output : this.lUsedOutputStream ) { + try { + output.close(); + } catch (IOException e) { + Log.log(e.getMessage(), e); + } + } + this.csvPrinterOutput.clear(); + this.lUsedOutputStream.clear(); + } + + public Collection getFiles() { + return this.csvPrinterOutput.keySet(); + } + +} diff --git a/src/main/java/com/mageddo/csv2jdbc/DelegateResultSet.java b/src/main/java/com/mageddo/csv2jdbc/DelegateResultSet.java new file mode 100644 index 0000000..e86c4f0 --- /dev/null +++ b/src/main/java/com/mageddo/csv2jdbc/DelegateResultSet.java @@ -0,0 +1,16 @@ +package com.mageddo.csv2jdbc; + +import lombok.experimental.Delegate; + +import java.sql.ResultSet; + +public class DelegateResultSet implements ResultSet { + + @Delegate(types = ResultSet.class) + final private ResultSet delegate; + + public DelegateResultSet(ResultSet delegate) { + this.delegate = delegate; + } + +} diff --git a/src/main/java/com/mageddo/csv2jdbc/FormatterAndCounterResultSet.java b/src/main/java/com/mageddo/csv2jdbc/FormatterAndCounterResultSet.java new file mode 100644 index 0000000..4572bb9 --- /dev/null +++ b/src/main/java/com/mageddo/csv2jdbc/FormatterAndCounterResultSet.java @@ -0,0 +1,94 @@ +package com.mageddo.csv2jdbc; + +import java.math.BigDecimal; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Time; +import java.text.DateFormat; +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; +import java.text.NumberFormat; +import java.text.SimpleDateFormat; +import java.util.Locale; + +public class FormatterAndCounterResultSet extends DelegateResultSet { + private int count = 0; + + private NumberFormat numberFormat; + private NumberFormat decimalFormat; + + private DateFormat dateTimeFormat; + private DateFormat dateFormat; + private DateFormat timeFormat; + + public FormatterAndCounterResultSet(ResultSet delegate, CopyCsvStatement csvStmt) { + super(delegate); + configureFormatters( csvStmt.getLanguage(), csvStmt.getNumberFormat(), + csvStmt.getDecimalFormat(), csvStmt.getDateFormat(), csvStmt.getTimeFormat(), + csvStmt.getDateTimeFormat() ); + } + + private void configureFormatters(String sLanguage, String sNumberFormat, String sDecimalFormat, + String sDateFormat, String sTimeFormat, String sDateTimeFormat) { + final Locale locale = sLanguage != null ? Locale.forLanguageTag(sLanguage) : Locale.getDefault(); + if( sNumberFormat != null ) { + numberFormat = new DecimalFormat(sNumberFormat, DecimalFormatSymbols.getInstance(locale) ); + } else if( sLanguage!=null ) { + numberFormat = NumberFormat.getInstance(locale); + } + if( sDecimalFormat != null ) { + decimalFormat = new DecimalFormat(sDecimalFormat, DecimalFormatSymbols.getInstance(locale) ); + } else if( sLanguage!=null ) { + decimalFormat = NumberFormat.getNumberInstance(locale); + } + + if( sDateFormat != null ) { + dateFormat = new SimpleDateFormat( sDateFormat ); + } else if( sLanguage!=null ) { + dateFormat = DateFormat.getDateInstance(DateFormat.SHORT, locale); + } + if( sTimeFormat != null ) { + timeFormat = new SimpleDateFormat( sTimeFormat ); + } else if( sLanguage!=null ) { + timeFormat = DateFormat.getTimeInstance(DateFormat.MEDIUM, locale); + } + if( sDateTimeFormat != null ) { + dateTimeFormat = new SimpleDateFormat( sDateTimeFormat ); + } else if( sLanguage!=null ) { + dateTimeFormat = DateFormat.getDateTimeInstance(DateFormat.SHORT, DateFormat.MEDIUM, locale); + } + } + + @Override + public boolean next() throws SQLException { + if( super.next() ) { + count++; + return true; + } + return false; + } + + @Override + public int getRow() throws SQLException { + return count; + } + + @Override + public Object getObject(int columnIndex) throws SQLException { + final Object obj = super.getObject(columnIndex); + if( obj == null ) { + return null; + } else if( obj instanceof Time && timeFormat != null ) { + return timeFormat.format( obj ); + } else if( obj instanceof java.sql.Date && dateFormat != null ) { + return dateFormat.format( obj ); + } else if( obj instanceof java.util.Date && dateTimeFormat != null ) { + return dateTimeFormat.format( obj ); + } else if( (obj instanceof Integer || obj instanceof Long) && numberFormat != null ) { + return numberFormat.format( obj ); + } else if( (obj instanceof Double || obj instanceof Float || obj instanceof BigDecimal ) && decimalFormat != null ) { + return decimalFormat.format( obj ); + } + return obj; + } +} diff --git a/src/test/java/com/mageddo/csv2jdbc/Csv2JdbcConverterTest.java b/src/test/java/com/mageddo/csv2jdbc/Csv2JdbcConverterTest.java index e2113d0..e519167 100644 --- a/src/test/java/com/mageddo/csv2jdbc/Csv2JdbcConverterTest.java +++ b/src/test/java/com/mageddo/csv2jdbc/Csv2JdbcConverterTest.java @@ -11,6 +11,8 @@ class Csv2JdbcConverterTest { + final String FRUITFILEPATH = System.getProperty("os.name").toLowerCase().startsWith("win") ? "\\tmp\\fruit.csv" : "/tmp/fruit.csv"; + @Test void mustParseCsvLoadToTableStmt(){ @@ -28,7 +30,7 @@ void mustParseCsvLoadToTableStmt(){ assertEquals(';', csvStatement.getDelimiter()); assertEquals("FRUIT_TABLE", csvStatement.getTableName()); assertEquals("[]", csvStatement.getCols().toString()); - assertEquals("/tmp/fruit.csv", String.valueOf(csvStatement.getFile())); + assertEquals(FRUITFILEPATH, String.valueOf(csvStatement.getFile())); assertEquals("UTF-8", csvStatement.getCharset().displayName()); assertTrue(csvStatement.hasHeader()); @@ -84,7 +86,7 @@ CSV2J COPY ( assertNull(csvStatement.getTableName()); assertFalse(csvStatement.mustCreateTable()); assertEquals("[]", csvStatement.getCols().toString()); - assertEquals("/tmp/fruit.csv", String.valueOf(csvStatement.getFile())); + assertEquals(FRUITFILEPATH, String.valueOf(csvStatement.getFile())); assertEquals("UTF-8", csvStatement.getCharset().displayName()); assertTrue(csvStatement.hasHeader()); } diff --git a/src/test/java/com/mageddo/csv2jdbc/Csv2JdbcDriverDynamicFileTest.java b/src/test/java/com/mageddo/csv2jdbc/Csv2JdbcDriverDynamicFileTest.java new file mode 100644 index 0000000..78263d1 --- /dev/null +++ b/src/test/java/com/mageddo/csv2jdbc/Csv2JdbcDriverDynamicFileTest.java @@ -0,0 +1,122 @@ +package com.mageddo.csv2jdbc; + +import org.jdbi.v3.core.Jdbi; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Calendar; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class Csv2JdbcDriverDynamicFileTest { + + public static final String JDBC_URL = "jdbc:csv2jdbc:h2:mem:testdb;" + "DB_CLOSE_DELAY=-1" + + "?delegateDriverClassName=org.h2.Driver"; + + static { + try { + Class.forName(Csv2JdbcDriver.class.getName()); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + @Test + void mustCreatePath(@TempDir Path tempDir) throws Exception { + final var jdbi = Jdbi.create(JDBC_URL, "SA", ""); + final var csvFile = tempDir.resolve("tmp/csv.csv"); + + // act + // assert + jdbi.useHandle(h -> { + final var updated = h.createUpdate(String.format(""" + CSV2J COPY ( + SELECT 1 AS ID, 10.99 AS AMOUNT, TIMESTAMP '2022-01-31 23:59:58.987' AS DAT_CREATION + UNION ALL + SELECT 2 AS ID, 7.50 AS AMOUNT, TIMESTAMP '2023-01-31 21:59:58.987' AS DAT_CREATION + ) TO '%s' WITH CSV HEADER + """, csvFile)) + .execute(); + assertEquals(2, updated); + }); + + assertTrue(Files.isDirectory( tempDir.resolve("tmp") ) ); + + assertEquals(""" + ID,AMOUNT,DAT_CREATION + 1,10.99,2022-01-31 23:59:58.987 + 2,7.50,2023-01-31 21:59:58.987 + """.replaceAll("\n", "\r\n"), Files.readString(csvFile)); + + } + + @Test + void mustCreatePathWithCurrentDate(@TempDir Path tempDir) throws Exception { + final var jdbi = Jdbi.create(JDBC_URL, "SA", ""); + final var csvFile = tempDir.resolve("%1$tY%1$tm%1$td%1$tH%1$tM%1$tS/csv.csv"); + final var checkPath = String.format( "%1$tY%1$tm%1$td%1$tH%1$tM%1$tS", Calendar.getInstance() ); + + // act + // assert + jdbi.useHandle(h -> { + final var updated = h.createUpdate(String.format(""" + CSV2J COPY ( + SELECT 1 AS ID, 10.99 AS AMOUNT, TIMESTAMP '2022-01-31 23:59:58.987' AS DAT_CREATION + UNION ALL + SELECT 2 AS ID, 7.50 AS AMOUNT, TIMESTAMP '2023-01-31 21:59:58.987' AS DAT_CREATION + ) TO '%s' WITH CSV HEADER + """, csvFile)) + .execute(); + assertEquals(2, updated); + }); + + assertTrue(Files.isDirectory( tempDir.resolve(checkPath) ) ); + + final var csvFileCheck = tempDir.resolve(checkPath + "/csv.csv"); + + assertEquals(""" + ID,AMOUNT,DAT_CREATION + 1,10.99,2022-01-31 23:59:58.987 + 2,7.50,2023-01-31 21:59:58.987 + """.replaceAll("\n", "\r\n"), Files.readString(csvFileCheck)); + + } + + @Test + void mustFileNameByColumn(@TempDir Path tempDir) throws Exception { + final var jdbi = Jdbi.create(JDBC_URL, "SA", ""); + final var csvFile = tempDir.resolve("csv_{ID}.csv"); + + // act + // assert + jdbi.useHandle(h -> { + final var updated = h.createUpdate(String.format(""" + CSV2J COPY ( + SELECT 1 AS ID, 10.99 AS AMOUNT, TIMESTAMP '2022-01-31 23:59:58.987' AS DAT_CREATION + UNION ALL + SELECT 2 AS ID, 7.50 AS AMOUNT, TIMESTAMP '2023-01-31 21:59:58.987' AS DAT_CREATION + ) TO '%s' WITH CSV HEADER + """, csvFile)) + .execute(); + assertEquals(2, updated); + }); + + assertEquals(2, Files.list( tempDir ).count() ); + + final var csvFileFirst = tempDir.resolve("csv_1.csv"); + assertEquals(""" + ID,AMOUNT,DAT_CREATION + 1,10.99,2022-01-31 23:59:58.987 + """.replaceAll("\n", "\r\n"), Files.readString(csvFileFirst)); + + final var csvFileSecond = tempDir.resolve("csv_2.csv"); + assertEquals(""" + ID,AMOUNT,DAT_CREATION + 2,7.50,2023-01-31 21:59:58.987 + """.replaceAll("\n", "\r\n"), Files.readString(csvFileSecond)); + } + +} diff --git a/src/test/java/com/mageddo/csv2jdbc/Csv2JdbcDriverTest.java b/src/test/java/com/mageddo/csv2jdbc/Csv2JdbcDriverTest.java index 0aebbb9..58d3977 100644 --- a/src/test/java/com/mageddo/csv2jdbc/Csv2JdbcDriverTest.java +++ b/src/test/java/com/mageddo/csv2jdbc/Csv2JdbcDriverTest.java @@ -1,12 +1,18 @@ package com.mageddo.csv2jdbc; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.sql.DriverManager; +import java.util.zip.GZIPInputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import org.antlr.v4.runtime.CharStreams; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.h2.util.IOUtils; import org.jdbi.v3.core.Jdbi; import org.junit.jupiter.api.Test; @@ -147,6 +153,223 @@ CSV2J COPY ( """.replaceAll("\n", "\r\n"), Files.readString(csvFile)); } + @Test + void mustExtractQueryToCsvWithOutHeader(@TempDir Path tempDir) throws Exception { + + // arrange + final var jdbi = Jdbi.create(JDBC_URL, "SA", ""); + final var csvFile = tempDir.resolve("csv.csv"); + + // act + // assert + jdbi.useHandle(h -> { + final var updated = h.createUpdate(String.format(""" + CSV2J COPY ( + SELECT 1 AS ID, 10.99 AS AMOUNT, TIMESTAMP '2022-01-31 23:59:58.987' AS DAT_CREATION + UNION ALL + SELECT 2 AS ID, 7.50 AS AMOUNT, TIMESTAMP '2023-01-31 21:59:58.987' AS DAT_CREATION + ) TO '%s' WITH CSV + """, csvFile)) + .execute(); + assertEquals(2, updated); + }); + + assertEquals(""" + 1,10.99,2022-01-31 23:59:58.987 + 2,7.50,2023-01-31 21:59:58.987 + """.replaceAll("\n", "\r\n"), Files.readString(csvFile)); + } + + @Test + void mustExtractQueryToCsvZIP(@TempDir Path tempDir) throws Exception { + + // arrange + final var jdbi = Jdbi.create(JDBC_URL, "SA", ""); + final var csvFile = tempDir.resolve("csv.csv"); + + // act + // assert + jdbi.useHandle(h -> { + final var updated = h.createUpdate(String.format(""" + CSV2J COPY ( + SELECT 1 AS ID, 10.99 AS AMOUNT, TIMESTAMP '2022-01-31 23:59:58.987' AS DAT_CREATION + UNION ALL + SELECT 2 AS ID, 7.50 AS AMOUNT, TIMESTAMP '2023-01-31 21:59:58.987' AS DAT_CREATION + ) TO '%s' WITH CSV HEADER ZIP + """, csvFile)) + .execute(); + assertEquals(2, updated); + }); + + ZipFile zip = new ZipFile( csvFile + ".zip" ); + ZipEntry entry = zip.getEntry(csvFile.getFileName().toString() ); + String readFromZip = CharStreams.fromStream( zip.getInputStream( entry ) ).toString(); + zip.close(); + + assertEquals(""" + ID,AMOUNT,DAT_CREATION + 1,10.99,2022-01-31 23:59:58.987 + 2,7.50,2023-01-31 21:59:58.987 + """.replaceAll("\n", "\r\n"), readFromZip ); + } + + @Test + void mustExtractQueryToCsvGZIP(@TempDir Path tempDir) throws Exception { + + // arrange + final var jdbi = Jdbi.create(JDBC_URL, "SA", ""); + final var csvFile = tempDir.resolve("csv.csv"); + + // act + // assert + jdbi.useHandle(h -> { + final var updated = h.createUpdate(String.format(""" + CSV2J COPY ( + SELECT 1 AS ID, 10.99 AS AMOUNT, TIMESTAMP '2022-01-31 23:59:58.987' AS DAT_CREATION + UNION ALL + SELECT 2 AS ID, 7.50 AS AMOUNT, TIMESTAMP '2023-01-31 21:59:58.987' AS DAT_CREATION + ) TO '%s' WITH CSV HEADER GZIP + """, csvFile)) + .execute(); + assertEquals(2, updated); + }); + + InputStream fileStream = new FileInputStream(csvFile + ".gzip"); + InputStream gzipStream = new GZIPInputStream(fileStream); + String readFromGZip = CharStreams.fromStream( gzipStream ).toString(); + fileStream.close(); + + assertEquals(""" + ID,AMOUNT,DAT_CREATION + 1,10.99,2022-01-31 23:59:58.987 + 2,7.50,2023-01-31 21:59:58.987 + """.replaceAll("\n", "\r\n"), readFromGZip ); + } + + @Test + void mustExtractQueryToCsvUsingPT_BR(@TempDir Path tempDir) throws Exception { + + // arrange + final var jdbi = Jdbi.create(JDBC_URL, "SA", ""); + final var csvFile = tempDir.resolve("csv.csv"); + + // act + // assert + jdbi.useHandle(h -> { + final var updated = h.createUpdate(String.format(""" + CSV2J COPY ( + SELECT 1234567890 AS ID, 1234567890.99 AS AMOUNT, TIMESTAMP '2022-01-31 23:59:58.987' AS DAT_CREATION, DATE '2023-03-11' as DAT_BORN, TIME '21:47:01' as TIME_BORN + UNION ALL + SELECT 1000000 AS ID, 1000.99 AS AMOUNT, TIMESTAMP '2022-01-31 23:59:58.987' AS DAT_CREATION, DATE '1978-04-16' as DAT_BORN, TIME '23:35:00' as TIME_BORN + ) TO '%s' WITH CSV HEADER LANGUAGE 'pt-BR' + """, csvFile)) + .execute(); + assertEquals(2, updated); + }); + + assertEquals(""" + ID,AMOUNT,DAT_CREATION,DAT_BORN,TIME_BORN + 1.234.567.890,"1.234.567.890,99",31/01/2022 23:59:58,11/03/2023,21:47:01 + 1.000.000,"1.000,99",31/01/2022 23:59:58,16/04/1978,23:35:00 + """.replaceAll("\n", "\r\n"), Files.readString(csvFile)); + } + + @Test + void mustExtractQueryToCsvUsingEN_US(@TempDir Path tempDir) throws Exception { + + // arrange + final var jdbi = Jdbi.create(JDBC_URL, "SA", ""); + final var csvFile = tempDir.resolve("csv.csv"); + + // act + // assert + jdbi.useHandle(h -> { + final var updated = h.createUpdate(String.format(""" + CSV2J COPY ( + SELECT 1234567890 AS ID, 1234567890.99 AS AMOUNT, TIMESTAMP '2022-01-31 23:59:58.987' AS DAT_CREATION, DATE '2023-03-11' as DAT_BORN, TIME '21:47:01' as TIME_BORN + UNION ALL + SELECT 1000000 AS ID, 1000.99 AS AMOUNT, TIMESTAMP '2022-01-31 23:59:58.987' AS DAT_CREATION, DATE '1978-04-16' as DAT_BORN, TIME '23:35:00' as TIME_BORN + ) TO '%s' WITH CSV HEADER LANGUAGE 'en-US' + """, csvFile)) + .execute(); + assertEquals(2, updated); + }); + + assertEquals(""" + ID,AMOUNT,DAT_CREATION,DAT_BORN,TIME_BORN + "1,234,567,890","1,234,567,890.99","1/31/22, 11:59:58 PM",3/11/23,9:47:01 PM + "1,000,000","1,000.99","1/31/22, 11:59:58 PM",4/16/78,11:35:00 PM + """.replaceAll("\n", "\r\n"), Files.readString(csvFile)); + } + + + @Test + void mustExtractQueryToCsvUsingPT_BR_with_CustomFormat(@TempDir Path tempDir) throws Exception { + + // arrange + final var jdbi = Jdbi.create(JDBC_URL, "SA", ""); + final var csvFile = tempDir.resolve("csv.csv"); + + // act + // assert + jdbi.useHandle(h -> { + final var updated = h.createUpdate(String.format(""" + CSV2J COPY ( + SELECT 1234567890 AS ID, 1234567890.995 AS AMOUNT, TIMESTAMP '2022-01-31 23:59:58.987' AS DAT_CREATION, DATE '2023-03-11' as DAT_BORN, TIME '21:47:01.001' as TIME_BORN + UNION ALL + SELECT 1 AS ID, 1.12345 AS AMOUNT, TIMESTAMP '2022-01-31 23:59:58' AS DAT_CREATION, DATE '1978-04-16' as DAT_BORN, TIME '23:35:00' as TIME_BORN + ) TO '%s' WITH CSV HEADER LANGUAGE 'pt-BR' + DATETIMEFORMAT 'yyyy-MM-dd HH:mm:ss.SSSXXX' + DATEFORMAT 'yyyy-MM-dd' + TIMEFORMAT 'HH:mm:ss.SSSXXX' + NUMBERFORMAT '#00' + DECIMALFORMAT '#00.00' + """, csvFile)) + .execute(); + assertEquals(2, updated); + }); + + assertEquals(""" + ID,AMOUNT,DAT_CREATION,DAT_BORN,TIME_BORN + 1234567890,"1234567891,00",2022-01-31 23:59:58.987-03:00,2023-03-11,21:47:01.001-03:00 + 01,"01,12",2022-01-31 23:59:58.000-03:00,1978-04-16,23:35:00.000-03:00 + """.replaceAll("\n", "\r\n"), Files.readString(csvFile)); + } + + @Test + void mustExtractQueryToCsvBZIP2(@TempDir Path tempDir) throws Exception { + + // arrange + final var jdbi = Jdbi.create(JDBC_URL, "SA", ""); + final var csvFile = tempDir.resolve("csv.csv"); + + // act + // assert + jdbi.useHandle(h -> { + final var updated = h.createUpdate(String.format(""" + CSV2J COPY ( + SELECT 1 AS ID, 10.99 AS AMOUNT, TIMESTAMP '2022-01-31 23:59:58.987' AS DAT_CREATION + UNION ALL + SELECT 2 AS ID, 7.50 AS AMOUNT, TIMESTAMP '2023-01-31 21:59:58.987' AS DAT_CREATION + ) TO '%s' WITH CSV HEADER BZIP2 + """, csvFile)) + .execute(); + assertEquals(2, updated); + }); + + InputStream fileStream = new FileInputStream(csvFile + ".bz2"); + BZip2CompressorInputStream bzip2Stream = new BZip2CompressorInputStream(fileStream); + String readFromBZip2 = CharStreams.fromStream( bzip2Stream ).toString(); + fileStream.close(); + + assertEquals(""" + ID,AMOUNT,DAT_CREATION + 1,10.99,2022-01-31 23:59:58.987 + 2,7.50,2023-01-31 21:59:58.987 + """.replaceAll("\n", "\r\n"), readFromBZip2 ); + } + + private void copy(String source, Path target) throws IOException { final InputStream in = getClass().getResourceAsStream(source); final OutputStream out = Files.newOutputStream(target);