Examples with CsvData org.jumpmind.symmetric.io.data.CsvData used on opensource projects

Example 31 with CsvData

use of org.jumpmind.symmetric.io.data.CsvData in project symmetric-ds by JumpMind.

the class SymmetricPushClient method update.

public void update(Table table, String[] data, String[] pkData) {
    writer.start(table);
    writer.write(new CsvData(DataEventType.INSERT, pkData, data));
    writer.end(table);
}

Example 32 with CsvData

use of org.jumpmind.symmetric.io.data.CsvData in project symmetric-ds by JumpMind.

the class TransformWriter method write.

public void write(CsvData data) {
    DataEventType eventType = data.getDataEventType();
    if (activeTransforms != null && activeTransforms.size() > 0 && isTransformable(eventType)) {
        if (data.requiresTable() && sourceTable == null && context.getLastParsedTable() != null) {
            // if we cross batches and the table isn't specified, then
            // use the last table we used
            start(context.getLastParsedTable());
        }
        long ts = System.currentTimeMillis();
        Map<String, String> sourceValues = data.toColumnNameValuePairs(this.sourceTable.getColumnNames(), CsvData.ROW_DATA);
        Map<String, String> oldSourceValues = null;
        if (data.contains(CsvData.OLD_DATA)) {
            oldSourceValues = data.toColumnNameValuePairs(this.sourceTable.getColumnNames(), CsvData.OLD_DATA);
        }
        Map<String, String> sourceKeyValues = null;
        if (data.contains(CsvData.PK_DATA)) {
            sourceKeyValues = data.toKeyColumnValuePairs(this.sourceTable);
        }
        if (eventType == DataEventType.DELETE) {
            sourceValues = oldSourceValues;
            if (sourceValues == null || sourceValues.size() == 0) {
                sourceValues = sourceKeyValues;
            }
        }
        if (log.isDebugEnabled()) {
            log.debug("{} transformation(s) started because of {} on {}.  The original row data was: {}", new Object[] { activeTransforms.size(), eventType.toString(), this.sourceTable.getFullyQualifiedTableName(), sourceValues });
        }
        List<TransformedData> dataThatHasBeenTransformed = new ArrayList<TransformedData>();
        TransformTable[] transformTables = activeTransforms.toArray(new TransformTable[activeTransforms.size()]);
        if (eventType == DataEventType.DELETE) {
            CollectionUtils.reverseArray(transformTables);
        }
        for (TransformTable transformation : transformTables) {
            if (eventType == DataEventType.INSERT && transformation.isUpdateFirst()) {
                eventType = DataEventType.UPDATE;
            }
            dataThatHasBeenTransformed.addAll(transform(eventType, context, transformation, sourceKeyValues, oldSourceValues, sourceValues));
        }
        for (TransformedData transformedData : dataThatHasBeenTransformed) {
            Table transformedTable = transformedData.buildTargetTable();
            CsvData csvData = transformedData.buildTargetCsvData();
            long transformTimeInMs = System.currentTimeMillis() - ts;
            boolean processData = true;
            if (lastTransformedTable == null || !lastTransformedTable.equals(transformedTable)) {
                if (lastTransformedTable != null) {
                    this.nestedWriter.end(lastTransformedTable);
                }
                processData = this.nestedWriter.start(transformedTable);
                if (!processData) {
                    lastTransformedTable = null;
                } else {
                    lastTransformedTable = transformedTable;
                }
            }
            if (processData || !csvData.requiresTable()) {
                this.nestedWriter.write(csvData);
            }
            Statistics stats = this.nestedWriter.getStatistics().get(batch);
            if (stats != null) {
                stats.increment(DataWriterStatisticConstants.TRANSFORMMILLIS, transformTimeInMs);
            }
            ts = System.currentTimeMillis();
        }
    } else {
        if (sourceTable != null) {
            super.start(sourceTable);
        }
        super.write(data);
        if (sourceTable != null) {
            super.end(sourceTable);
        }
    }
}

Example 33 with CsvData

use of org.jumpmind.symmetric.io.data.CsvData in project symmetric-ds by JumpMind.

the class AbstractWriterTest method writeData.

protected long writeData(IDataWriter writer, TableCsvData... datas) {
    this.lastDataWriterUsed = writer;
    DataContext context = new DataContext();
    writer.open(context);
    try {
        for (TableCsvData tableCsvData : datas) {
            Batch batch = new Batch(BatchType.LOAD, getNextBatchId(), "default", BinaryEncoding.BASE64, "00000", "00001", false);
            try {
                writer.start(batch);
                if (writer.start(tableCsvData.table)) {
                    for (CsvData d : tableCsvData.data) {
                        writer.write(d);
                    }
                    writer.end(tableCsvData.table);
                }
                writer.end(batch, false);
            } catch (IgnoreBatchException ex) {
                writer.end(batch, false);
            } catch (Exception ex) {
                writer.end(batch, true);
                if (!isErrorExpected()) {
                    if (ex instanceof RuntimeException) {
                        throw (RuntimeException) ex;
                    } else {
                        throw new RuntimeException(ex);
                    }
                }
            }
        }
    } finally {
        writer.close();
    }
    long statementCount = 0;
    Collection<Statistics> stats = writer.getStatistics().values();
    for (Statistics statistics : stats) {
        statementCount += statistics.get(DataWriterStatisticConstants.STATEMENTCOUNT);
    }
    return statementCount;
}

Example 34 with CsvData

use of org.jumpmind.symmetric.io.data.CsvData in project symmetric-ds by JumpMind.

the class OracleBulkDatabaseWriterTest method testInsertTimestampTZ_timestampWithLocalTimeZone.

@Test
public void testInsertTimestampTZ_timestampWithLocalTimeZone() throws Exception {
    if (platform != null && platform instanceof OracleDatabasePlatform) {
        NativeJdbcExtractor jdbcExtractor = new CommonsDbcpNativeJdbcExtractor();
        platform.getSqlTemplate().update("truncate table test_bulkload_table_1");
        List<CsvData> datas = new ArrayList<CsvData>();
        String id = getNextId();
        String[] values = { id, "string2", "string not null2", "char2", "char not null2", "2007-01-02 03:20:10.000", "2007-02-03 04:05:06.000", "0", "47", "67.89", "-0.0747663", null, null, null, null, "2007-01-02 03:20:10.123456789 -08:00" };
        CsvData data = new CsvData(DataEventType.INSERT, values);
        datas.add(data);
        long count = writeData(new TableCsvData(platform.getTableFromCache("test_bulkload_table_1", false), datas));
        Map<String, Object> rowData = queryForRow(id);
        DataSource datasource = (DataSource) platform.getDataSource();
        Connection connection = datasource.getConnection();
        Connection oracleConnection = jdbcExtractor.getNativeConnection(connection);
        checkTimestampLTZ(rowData.get("TIMESTAMPLTZ9_VALUE"), oracleConnection, new String[] { "2007-01-02 03:20:10.123456789 America/New_York", "2007-01-02 03:20:10.123456789 US/Eastern" });
        Assert.assertEquals(count, countRows("test_bulkload_table_1"));
    }
}

Example 35 with CsvData

use of org.jumpmind.symmetric.io.data.CsvData in project symmetric-ds by JumpMind.

the class OracleBulkDatabaseWriterTest method testInsertCollision.

@Test
public void testInsertCollision() {
    if (platform != null && platform instanceof OracleDatabasePlatform) {
        platform.getSqlTemplate().update("truncate table test_bulkload_table_1");
        String id = getNextId();
        String[] values = { id, "string2", "string not null2", "char2", "char not null2", "2007-01-02 03:20:10.000", "2007-02-03 04:05:06.000", "0", "47", "67.89", "-0.0747663" };
        CsvData data = new CsvData(DataEventType.INSERT, values);
        writeData(data, values);
        Assert.assertEquals(1, countRows("test_bulkload_table_1"));
        try {
            setErrorExpected(true);
            List<CsvData> datas = new ArrayList<CsvData>();
            datas.add(data);
            for (int i = 0; i < 10; i++) {
                values = new String[] { id, "string2", "string not null2", "char2", "char not null2", "2007-01-02 03:20:10.000", "2007-02-03 04:05:06.000", "0", "47", "67.89", "-0.0747663" };
                data = new CsvData(DataEventType.INSERT, values);
                datas.add(data);
            }
            // we should collide and rollback
            writeData(new TableCsvData(platform.getTableFromCache("test_bulkload_table_1", false), datas));
            Assert.assertEquals(1, countRows("test_bulkload_table_1"));
        } finally {
            setErrorExpected(false);
        }
    }
}