Examples with CsvWriter org.jumpmind.symmetric.csv.CsvWriter used on opensource projects

Example 1 with CsvWriter

use of org.jumpmind.symmetric.csv.CsvWriter in project symmetric-ds by JumpMind.

the class AbstractDataLoaderServiceTest method test10MultipleBatch.

@Test
public void test10MultipleBatch() throws Exception {
    Level old = setLoggingLevelForTest(Level.OFF);
    String[] values = { getNextId(), "string", "string not null2", "char2", "char not null2", "2007-01-02 00:00:00.000", "2007-02-03 04:05:06.000", "0", "47", "67.89", "0.474" };
    String[] values2 = { getNextId(), "This string is too large and will cause the statement to fail", "string not null2", "char2", "char not null2", "Not a date", "2007-02-03 04:05:06.000", "0", "47", "123456789.00", "0.474" };
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    CsvWriter writer = getWriter(out);
    writer.writeRecord(new String[] { CsvConstants.NODEID, TestConstants.TEST_CLIENT_EXTERNAL_ID });
    writer.writeRecord(new String[] { CsvConstants.CHANNEL, TestConstants.TEST_CHANNEL_ID });
    String nextBatchId = getNextBatchId();
    writer.writeRecord(new String[] { CsvConstants.BATCH, nextBatchId });
    writeTable(writer, TEST_TABLE, TEST_KEYS, TEST_COLUMNS);
    writer.write(CsvConstants.INSERT);
    writer.writeRecord(values, true);
    writer.writeRecord(new String[] { CsvConstants.COMMIT, nextBatchId });
    String nextBatchId2 = getNextBatchId();
    writer.writeRecord(new String[] { CsvConstants.BATCH, nextBatchId2 });
    writeTable(writer, TEST_TABLE, TEST_KEYS, TEST_COLUMNS);
    writer.write(CsvConstants.INSERT);
    writer.writeRecord(values2, true);
    writer.writeRecord(new String[] { CsvConstants.COMMIT, nextBatchId2 });
    writer.close();
    load(out);
    assertTestTableEquals(values[0], values);
    assertTestTableEquals(values2[0], null);
    assertEquals(findIncomingBatchStatus(Integer.parseInt(nextBatchId), TestConstants.TEST_CLIENT_EXTERNAL_ID), IncomingBatch.Status.OK, "Wrong status. " + printDatabase());
    assertEquals(findIncomingBatchStatus(Integer.parseInt(nextBatchId2), TestConstants.TEST_CLIENT_EXTERNAL_ID), IncomingBatch.Status.ER, "Wrong status. " + printDatabase());
    setLoggingLevelForTest(old);
}

Example 2 with CsvWriter

use of org.jumpmind.symmetric.csv.CsvWriter in project symmetric-ds by JumpMind.

the class MySqlBulkDatabaseWriter method write.

public void write(CsvData data) {
    DataEventType dataEventType = data.getDataEventType();
    switch(dataEventType) {
        case INSERT:
            statistics.get(batch).increment(DataWriterStatisticConstants.STATEMENTCOUNT);
            statistics.get(batch).increment(DataWriterStatisticConstants.LINENUMBER);
            statistics.get(batch).startTimer(DataWriterStatisticConstants.DATABASEMILLIS);
            try {
                String[] parsedData = data.getParsedData(CsvData.ROW_DATA);
                byte[] byteData = null;
                if (needsBinaryConversion) {
                    ByteArrayOutputStream out = new ByteArrayOutputStream();
                    CsvWriter writer = new CsvWriter(new OutputStreamWriter(out), ',');
                    writer.setEscapeMode(CsvWriter.ESCAPE_MODE_BACKSLASH);
                    writer.setRecordDelimiter('\n');
                    writer.setTextQualifier('"');
                    writer.setUseTextQualifier(true);
                    writer.setForceQualifier(true);
                    writer.setNullString("\\N");
                    Column[] columns = targetTable.getColumns();
                    for (int i = 0; i < columns.length; i++) {
                        if (columns[i].isOfBinaryType() && parsedData[i] != null) {
                            if (i > 0) {
                                out.write(',');
                            }
                            out.write('"');
                            if (batch.getBinaryEncoding().equals(BinaryEncoding.HEX)) {
                                out.write(escape(Hex.decodeHex(parsedData[i].toCharArray())));
                            } else if (batch.getBinaryEncoding().equals(BinaryEncoding.BASE64)) {
                                out.write(new String(Hex.encodeHex(Base64.decodeBase64(parsedData[i].getBytes()))).getBytes());
                            }
                            out.write('"');
                        } else {
                            writer.write(parsedData[i], true);
                            writer.flush();
                        }
                    }
                    writer.endRecord();
                    writer.close();
                    byteData = out.toByteArray();
                } else {
                    String formattedData = CsvUtils.escapeCsvData(parsedData, '\n', '"', CsvWriter.ESCAPE_MODE_BACKSLASH, "\\N");
                    byteData = formattedData.getBytes();
                }
                this.stagedInputFile.getOutputStream().write(byteData);
                loadedRows++;
                loadedBytes += byteData.length;
            } catch (Exception ex) {
                throw getPlatform().getSqlTemplate().translate(ex);
            } finally {
                statistics.get(batch).stopTimer(DataWriterStatisticConstants.DATABASEMILLIS);
            }
            break;
        case UPDATE:
        case DELETE:
        default:
            flush();
            super.write(data);
            break;
    }
    if (loadedRows >= maxRowsBeforeFlush || loadedBytes >= maxBytesBeforeFlush) {
        flush();
    }
}

Example 3 with CsvWriter

use of org.jumpmind.symmetric.csv.CsvWriter in project symmetric-ds by JumpMind.

the class AbstractDataLoaderServiceTest method test01IncomingBatch.

@Test
public void test01IncomingBatch() throws Exception {
    String[] insertValues = new String[TEST_COLUMNS.length];
    insertValues[2] = insertValues[4] = "incoming test";
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    CsvWriter writer = getWriter(out);
    writer.writeRecord(new String[] { CsvConstants.NODEID, TestConstants.TEST_CLIENT_EXTERNAL_ID });
    writer.writeRecord(new String[] { CsvConstants.CHANNEL, TestConstants.TEST_CHANNEL_ID });
    String nextBatchId = getNextBatchId();
    writer.writeRecord(new String[] { CsvConstants.BATCH, nextBatchId });
    writeTable(writer, TEST_TABLE, TEST_KEYS, TEST_COLUMNS);
    insertValues[0] = getNextId();
    writer.write(CsvConstants.INSERT);
    writer.writeRecord(insertValues, true);
    writer.writeRecord(new String[] { CsvConstants.COMMIT, nextBatchId });
    writer.close();
    load(out);
    IncomingBatch batch = getIncomingBatchService().findIncomingBatch(batchId, TestConstants.TEST_CLIENT_EXTERNAL_ID);
    assertEquals(batch.getStatus(), IncomingBatch.Status.OK, "Wrong status. " + printDatabase());
    assertEquals(batch.getChannelId(), TestConstants.TEST_CHANNEL_ID, "Wrong channel. " + printDatabase());
}

Example 4 with CsvWriter

use of org.jumpmind.symmetric.csv.CsvWriter in project symmetric-ds by JumpMind.

the class AbstractDataLoaderServiceTest method test06ErrorWhileSkip.

@Test
public void test06ErrorWhileSkip() throws Exception {
    Level old = setLoggingLevelForTest(Level.OFF);
    String[] values = { getNextId(), "string2", "string not null2", "char2", "char not null2", "2007-01-02 00:00:00.000", "2007-02-03 04:05:06.000", "0", "47", "67.89", "0.474" };
    testSimple(CsvConstants.INSERT, values, values);
    assertEquals(findIncomingBatchStatus(batchId, TestConstants.TEST_CLIENT_EXTERNAL_ID), IncomingBatch.Status.OK, "Wrong status");
    IncomingBatch batch = getIncomingBatchService().findIncomingBatch(batchId, TestConstants.TEST_CLIENT_EXTERNAL_ID);
    assertNotNull(batch);
    assertEquals(batch.getStatus(), IncomingBatch.Status.OK, "Wrong status");
    assertEquals(batch.getFailedRowNumber(), 0l, "Wrong failed row number");
    assertEquals(batch.getStatementCount(), 1l, "Wrong statement count");
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    CsvWriter writer = getWriter(out);
    writer.writeRecord(new String[] { CsvConstants.NODEID, TestConstants.TEST_CLIENT_EXTERNAL_ID });
    writer.writeRecord(new String[] { CsvConstants.CHANNEL, TestConstants.TEST_CHANNEL_ID });
    writer.writeRecord(new String[] { CsvConstants.BATCH, getBatchId() });
    writer.write(CsvConstants.KEYS);
    writer.writeRecord(TEST_KEYS);
    writer.writeRecord(new String[] { CsvConstants.COMMIT, getBatchId() });
    writer.close();
    // Pause a moment to guarantee our batch comes back in time order
    Thread.sleep(10);
    load(out);
    assertEquals(findIncomingBatchStatus(batchId, TestConstants.TEST_CLIENT_EXTERNAL_ID), IncomingBatch.Status.OK, "Wrong status");
    setLoggingLevelForTest(old);
}

Example 5 with CsvWriter

use of org.jumpmind.symmetric.csv.CsvWriter in project symmetric-ds by JumpMind.

the class AbstractDataLoaderServiceTest method test03UpdateCollision.

@Test
public void test03UpdateCollision() throws Exception {
    Level old = setLoggingLevelForTest(Level.OFF);
    String[] insertValues = new String[TEST_COLUMNS.length];
    insertValues[0] = getNextId();
    insertValues[2] = insertValues[4] = "inserted row for testUpdateCollision";
    String[] updateValues = new String[TEST_COLUMNS.length + 1];
    updateValues[0] = getId();
    updateValues[TEST_COLUMNS.length] = getNextId();
    updateValues[2] = updateValues[4] = "update will become an insert that violates PK";
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    CsvWriter writer = getWriter(out);
    writer.writeRecord(new String[] { CsvConstants.NODEID, TestConstants.TEST_CLIENT_EXTERNAL_ID });
    writer.writeRecord(new String[] { CsvConstants.CHANNEL, TestConstants.TEST_CHANNEL_ID });
    String nextBatchId = getNextBatchId();
    writer.writeRecord(new String[] { CsvConstants.BATCH, nextBatchId });
    writeTable(writer, TEST_TABLE, TEST_KEYS, TEST_COLUMNS);
    // This insert will be OK
    writer.write(CsvConstants.INSERT);
    writer.writeRecord(insertValues, true);
    // Update becomes fallback insert, and then violate the primary key
    writer.write(CsvConstants.UPDATE);
    writer.writeRecord(updateValues, true);
    writer.writeRecord(new String[] { CsvConstants.COMMIT, nextBatchId });
    writer.close();
    load(out);
    IncomingBatch batch = getIncomingBatchService().findIncomingBatch(batchId, TestConstants.TEST_CLIENT_EXTERNAL_ID);
    assertNotNull(batch);
    load(out);
    assertEquals(batch.getStatus(), IncomingBatch.Status.OK, "Wrong status");
    batch = getIncomingBatchService().findIncomingBatch(batchId, TestConstants.TEST_CLIENT_EXTERNAL_ID);
    assertNotNull(batch);
    assertEquals(batch.getStatus(), IncomingBatch.Status.OK, "Wrong status");
    setLoggingLevelForTest(old);
}