use of org.jumpmind.symmetric.io.data.CsvData in project symmetric-ds by JumpMind.
the class SymmetricPushClient method update.
public void update(Table table, String[] data, String[] pkData) {
writer.start(table);
writer.write(new CsvData(DataEventType.INSERT, pkData, data));
writer.end(table);
}
use of org.jumpmind.symmetric.io.data.CsvData in project symmetric-ds by JumpMind.
the class TransformWriter method write.
public void write(CsvData data) {
DataEventType eventType = data.getDataEventType();
if (activeTransforms != null && activeTransforms.size() > 0 && isTransformable(eventType)) {
if (data.requiresTable() && sourceTable == null && context.getLastParsedTable() != null) {
// if we cross batches and the table isn't specified, then
// use the last table we used
start(context.getLastParsedTable());
}
long ts = System.currentTimeMillis();
Map<String, String> sourceValues = data.toColumnNameValuePairs(this.sourceTable.getColumnNames(), CsvData.ROW_DATA);
Map<String, String> oldSourceValues = null;
if (data.contains(CsvData.OLD_DATA)) {
oldSourceValues = data.toColumnNameValuePairs(this.sourceTable.getColumnNames(), CsvData.OLD_DATA);
}
Map<String, String> sourceKeyValues = null;
if (data.contains(CsvData.PK_DATA)) {
sourceKeyValues = data.toKeyColumnValuePairs(this.sourceTable);
}
if (eventType == DataEventType.DELETE) {
sourceValues = oldSourceValues;
if (sourceValues == null || sourceValues.size() == 0) {
sourceValues = sourceKeyValues;
}
}
if (log.isDebugEnabled()) {
log.debug("{} transformation(s) started because of {} on {}. The original row data was: {}", new Object[] { activeTransforms.size(), eventType.toString(), this.sourceTable.getFullyQualifiedTableName(), sourceValues });
}
List<TransformedData> dataThatHasBeenTransformed = new ArrayList<TransformedData>();
TransformTable[] transformTables = activeTransforms.toArray(new TransformTable[activeTransforms.size()]);
if (eventType == DataEventType.DELETE) {
CollectionUtils.reverseArray(transformTables);
}
for (TransformTable transformation : transformTables) {
if (eventType == DataEventType.INSERT && transformation.isUpdateFirst()) {
eventType = DataEventType.UPDATE;
}
dataThatHasBeenTransformed.addAll(transform(eventType, context, transformation, sourceKeyValues, oldSourceValues, sourceValues));
}
for (TransformedData transformedData : dataThatHasBeenTransformed) {
Table transformedTable = transformedData.buildTargetTable();
CsvData csvData = transformedData.buildTargetCsvData();
long transformTimeInMs = System.currentTimeMillis() - ts;
boolean processData = true;
if (lastTransformedTable == null || !lastTransformedTable.equals(transformedTable)) {
if (lastTransformedTable != null) {
this.nestedWriter.end(lastTransformedTable);
}
processData = this.nestedWriter.start(transformedTable);
if (!processData) {
lastTransformedTable = null;
} else {
lastTransformedTable = transformedTable;
}
}
if (processData || !csvData.requiresTable()) {
this.nestedWriter.write(csvData);
}
Statistics stats = this.nestedWriter.getStatistics().get(batch);
if (stats != null) {
stats.increment(DataWriterStatisticConstants.TRANSFORMMILLIS, transformTimeInMs);
}
ts = System.currentTimeMillis();
}
} else {
if (sourceTable != null) {
super.start(sourceTable);
}
super.write(data);
if (sourceTable != null) {
super.end(sourceTable);
}
}
}
use of org.jumpmind.symmetric.io.data.CsvData in project symmetric-ds by JumpMind.
the class AbstractWriterTest method writeData.
protected long writeData(IDataWriter writer, TableCsvData... datas) {
this.lastDataWriterUsed = writer;
DataContext context = new DataContext();
writer.open(context);
try {
for (TableCsvData tableCsvData : datas) {
Batch batch = new Batch(BatchType.LOAD, getNextBatchId(), "default", BinaryEncoding.BASE64, "00000", "00001", false);
try {
writer.start(batch);
if (writer.start(tableCsvData.table)) {
for (CsvData d : tableCsvData.data) {
writer.write(d);
}
writer.end(tableCsvData.table);
}
writer.end(batch, false);
} catch (IgnoreBatchException ex) {
writer.end(batch, false);
} catch (Exception ex) {
writer.end(batch, true);
if (!isErrorExpected()) {
if (ex instanceof RuntimeException) {
throw (RuntimeException) ex;
} else {
throw new RuntimeException(ex);
}
}
}
}
} finally {
writer.close();
}
long statementCount = 0;
Collection<Statistics> stats = writer.getStatistics().values();
for (Statistics statistics : stats) {
statementCount += statistics.get(DataWriterStatisticConstants.STATEMENTCOUNT);
}
return statementCount;
}
use of org.jumpmind.symmetric.io.data.CsvData in project symmetric-ds by JumpMind.
the class OracleBulkDatabaseWriterTest method testInsertTimestampTZ_timestampWithLocalTimeZone.
@Test
public void testInsertTimestampTZ_timestampWithLocalTimeZone() throws Exception {
if (platform != null && platform instanceof OracleDatabasePlatform) {
NativeJdbcExtractor jdbcExtractor = new CommonsDbcpNativeJdbcExtractor();
platform.getSqlTemplate().update("truncate table test_bulkload_table_1");
List<CsvData> datas = new ArrayList<CsvData>();
String id = getNextId();
String[] values = { id, "string2", "string not null2", "char2", "char not null2", "2007-01-02 03:20:10.000", "2007-02-03 04:05:06.000", "0", "47", "67.89", "-0.0747663", null, null, null, null, "2007-01-02 03:20:10.123456789 -08:00" };
CsvData data = new CsvData(DataEventType.INSERT, values);
datas.add(data);
long count = writeData(new TableCsvData(platform.getTableFromCache("test_bulkload_table_1", false), datas));
Map<String, Object> rowData = queryForRow(id);
DataSource datasource = (DataSource) platform.getDataSource();
Connection connection = datasource.getConnection();
Connection oracleConnection = jdbcExtractor.getNativeConnection(connection);
checkTimestampLTZ(rowData.get("TIMESTAMPLTZ9_VALUE"), oracleConnection, new String[] { "2007-01-02 03:20:10.123456789 America/New_York", "2007-01-02 03:20:10.123456789 US/Eastern" });
Assert.assertEquals(count, countRows("test_bulkload_table_1"));
}
}
use of org.jumpmind.symmetric.io.data.CsvData in project symmetric-ds by JumpMind.
the class OracleBulkDatabaseWriterTest method testInsertCollision.
@Test
public void testInsertCollision() {
if (platform != null && platform instanceof OracleDatabasePlatform) {
platform.getSqlTemplate().update("truncate table test_bulkload_table_1");
String id = getNextId();
String[] values = { id, "string2", "string not null2", "char2", "char not null2", "2007-01-02 03:20:10.000", "2007-02-03 04:05:06.000", "0", "47", "67.89", "-0.0747663" };
CsvData data = new CsvData(DataEventType.INSERT, values);
writeData(data, values);
Assert.assertEquals(1, countRows("test_bulkload_table_1"));
try {
setErrorExpected(true);
List<CsvData> datas = new ArrayList<CsvData>();
datas.add(data);
for (int i = 0; i < 10; i++) {
values = new String[] { id, "string2", "string not null2", "char2", "char not null2", "2007-01-02 03:20:10.000", "2007-02-03 04:05:06.000", "0", "47", "67.89", "-0.0747663" };
data = new CsvData(DataEventType.INSERT, values);
datas.add(data);
}
// we should collide and rollback
writeData(new TableCsvData(platform.getTableFromCache("test_bulkload_table_1", false), datas));
Assert.assertEquals(1, countRows("test_bulkload_table_1"));
} finally {
setErrorExpected(false);
}
}
}
Aggregations