use of org.talend.components.jdbc.tjdbcoutput.TJDBCOutputDefinition in project components by Talend.
the class JDBCRollbackTestIT method testRollback.
@SuppressWarnings("rawtypes")
@Test
public void testRollback() throws IOException, ClassNotFoundException, SQLException {
// connection part
TJDBCConnectionDefinition connectionDefinition = new TJDBCConnectionDefinition();
TJDBCConnectionProperties connectionProperties = DBTestUtils.createCommonJDBCConnectionProperties(allSetting, connectionDefinition);
JDBCSourceOrSink sourceOrSink = new JDBCSourceOrSink();
sourceOrSink.initialize(null, connectionProperties);
ValidationResult result = sourceOrSink.validate(container);
assertTrue(result.getStatus() == ValidationResult.Result.OK);
// output part
TJDBCOutputDefinition outputDefinition = new TJDBCOutputDefinition();
TJDBCOutputProperties outputProperties = (TJDBCOutputProperties) outputDefinition.createRuntimeProperties();
outputProperties.main.schema.setValue(DBTestUtils.createTestSchema(tablename));
outputProperties.updateOutputSchemas();
outputProperties.tableSelection.tablename.setValue(tablename);
outputProperties.dataAction.setValue(DataAction.INSERT);
outputProperties.referencedComponent.componentInstanceId.setValue(refComponentId);
outputProperties.referencedComponent.setReference(connectionProperties);
JDBCSink sink = new JDBCSink();
sink.initialize(container, outputProperties);
WriteOperation writerOperation = sink.createWriteOperation();
writerOperation.initialize(container);
JDBCOutputInsertWriter writer = (JDBCOutputInsertWriter) writerOperation.createWriter(container);
try {
writer.open("wid");
IndexedRecord r1 = new GenericData.Record(outputProperties.main.schema.getValue());
r1.put(0, 4);
r1.put(1, "xiaoming");
writer.write(r1);
DBTestUtils.assertSuccessRecord(writer, r1);
IndexedRecord r2 = new GenericData.Record(outputProperties.main.schema.getValue());
r2.put(0, 5);
r2.put(1, "xiaobai");
writer.write(r2);
DBTestUtils.assertSuccessRecord(writer, r2);
writer.close();
} finally {
writer.close();
}
// rollback part
TJDBCRollbackDefinition rollbackDefinition = new TJDBCRollbackDefinition();
TJDBCRollbackProperties rollbackProperties = (TJDBCRollbackProperties) rollbackDefinition.createRuntimeProperties();
rollbackProperties.referencedComponent.componentInstanceId.setValue(refComponentId);
rollbackProperties.closeConnection.setValue(false);
JDBCRollbackSourceOrSink rollbackSourceOrSink = new JDBCRollbackSourceOrSink();
rollbackSourceOrSink.initialize(container, rollbackProperties);
rollbackSourceOrSink.validate(container);
// create another session and check if the data is inserted
int count = -1;
try (Connection conn = JdbcRuntimeUtils.createConnection(allSetting);
Statement statement = conn.createStatement();
ResultSet resultset = statement.executeQuery("select count(*) from " + tablename)) {
if (resultset.next()) {
count = resultset.getInt(1);
}
}
Assert.assertEquals(3, count);
try (java.sql.Connection refConnection = (java.sql.Connection) container.getComponentData(ComponentConstants.CONNECTION_KEY, refComponentId)) {
assertTrue(refConnection != null);
Assert.assertTrue(!refConnection.isClosed());
}
}
use of org.talend.components.jdbc.tjdbcoutput.TJDBCOutputDefinition in project components by Talend.
the class JDBCTypeMappingTestIT method doWriteWithAllType.
@SuppressWarnings({ "rawtypes", "unchecked" })
private void doWriteWithAllType(boolean nullableForAnyColumn) throws IOException {
TJDBCOutputDefinition definition = new TJDBCOutputDefinition();
TJDBCOutputProperties properties = DBTestUtils.createCommonJDBCOutputProperties(allSetting, definition);
Schema schema = DBTestUtils.createTestSchema3(nullableForAnyColumn, tablename);
properties.main.schema.setValue(schema);
properties.updateOutputSchemas();
properties.tableSelection.tablename.setValue(tablename);
properties.dataAction.setValue(DataAction.INSERT);
JDBCOutputWriter writer = DBTestUtils.createCommonJDBCOutputWriter(definition, properties);
try {
writer.open("wid");
List<IndexedRecord> inputRecords = DBTestUtils.prepareIndexRecords(nullableForAnyColumn, tablename);
for (IndexedRecord inputRecord : inputRecords) {
writer.write(inputRecord);
DBTestUtils.assertSuccessRecord(writer, inputRecord);
}
writer.close();
} finally {
writer.close();
}
// read the inserted data from the target table by the reader
Reader reader = null;
try {
TJDBCInputDefinition definition1 = new TJDBCInputDefinition();
TJDBCInputProperties properties1 = DBTestUtils.createCommonJDBCInputProperties(allSetting, definition1);
properties1.main.schema.setValue(DBTestUtils.createTestSchema3(nullableForAnyColumn, tablename));
properties1.tableSelection.tablename.setValue(tablename);
properties1.sql.setValue(DBTestUtils.getSQL(tablename));
reader = DBTestUtils.createCommonJDBCInputReader(properties1);
reader.start();
int i = 0;
while ((i++) < 5) {
// skip the 5 rows at the head
reader.advance();
}
IndexedRecord row = (IndexedRecord) reader.getCurrent();
Integer c1 = (Integer) row.get(0);
Short c2 = (Short) row.get(1);
Long c3 = (Long) row.get(2);
Float c4 = (Float) row.get(3);
Double c5 = (Double) row.get(4);
Float c6 = (Float) row.get(5);
BigDecimal c7 = (BigDecimal) row.get(6);
BigDecimal c8 = (BigDecimal) row.get(7);
Boolean c9 = (Boolean) row.get(8);
String c10 = (String) row.get(9);
Long c11 = (Long) row.get(10);
Long c12 = (Long) row.get(11);
Long c13 = (Long) row.get(12);
String c14 = (String) row.get(13);
String c15 = (String) row.get(14);
assertEquals(1, c1.intValue());
assertEquals(2, c2.intValue());
assertEquals(3, c3.intValue());
Assert.assertNotNull(c4);
Assert.assertNotNull(c5);
Assert.assertNotNull(c6);
assertEquals(new BigDecimal("7.01"), c7);
assertEquals(new BigDecimal("8.01"), c8);
assertEquals(true, c9);
assertEquals("content : 1", c10.trim());
Assert.assertNotNull(c11);
Assert.assertNotNull(c12);
Assert.assertNotNull(c13);
assertEquals("wangwei", c14);
assertEquals("long content : 1", c15);
reader.advance();
row = (IndexedRecord) reader.getCurrent();
c1 = (Integer) row.get(0);
c2 = (Short) row.get(1);
c3 = (Long) row.get(2);
c4 = (Float) row.get(3);
c5 = (Double) row.get(4);
c6 = (Float) row.get(5);
c7 = (BigDecimal) row.get(6);
c8 = (BigDecimal) row.get(7);
c9 = (Boolean) row.get(8);
c10 = (String) row.get(9);
c11 = (Long) row.get(10);
c12 = (Long) row.get(11);
c13 = (Long) row.get(12);
c14 = (String) row.get(13);
c15 = (String) row.get(14);
assertEquals(1, c1.intValue());
assertEquals(2, c2.intValue());
assertEquals(3, c3.intValue());
Assert.assertNotNull(c4);
Assert.assertNotNull(c5);
Assert.assertNotNull(c6);
assertEquals(new BigDecimal("7.01"), c7);
assertEquals(new BigDecimal("8.01"), c8);
assertEquals(true, c9);
assertEquals("content : 2", c10.trim());
Assert.assertNotNull(c11);
Assert.assertNotNull(c12);
Assert.assertNotNull(c13);
assertEquals("gaoyan", c14);
assertEquals("long content : 2", c15);
reader.advance();
row = (IndexedRecord) reader.getCurrent();
c1 = (Integer) row.get(0);
c2 = (Short) row.get(1);
c3 = (Long) row.get(2);
c4 = (Float) row.get(3);
c5 = (Double) row.get(4);
c6 = (Float) row.get(5);
c7 = (BigDecimal) row.get(6);
c8 = (BigDecimal) row.get(7);
c9 = (Boolean) row.get(8);
c10 = (String) row.get(9);
c11 = (Long) row.get(10);
c12 = (Long) row.get(11);
c13 = (Long) row.get(12);
c14 = (String) row.get(13);
c15 = (String) row.get(14);
assertEquals(1, c1.intValue());
assertEquals(2, c2.intValue());
assertEquals(3, c3.intValue());
Assert.assertNotNull(c4);
Assert.assertNotNull(c5);
Assert.assertNotNull(c6);
assertEquals(new BigDecimal("7.01"), c7);
assertEquals(new BigDecimal("8.01"), c8);
assertEquals(true, c9);
assertEquals("content : 3", c10.trim());
Assert.assertNotNull(c11);
Assert.assertNotNull(c12);
Assert.assertNotNull(c13);
assertEquals("dabao", c14);
assertEquals("long content : 3", c15);
reader.advance();
row = (IndexedRecord) reader.getCurrent();
c1 = (Integer) row.get(0);
c2 = (Short) row.get(1);
c3 = (Long) row.get(2);
c4 = (Float) row.get(3);
c5 = (Double) row.get(4);
c6 = (Float) row.get(5);
c7 = (BigDecimal) row.get(6);
c8 = (BigDecimal) row.get(7);
c9 = (Boolean) row.get(8);
c10 = (String) row.get(9);
c11 = (Long) row.get(10);
c12 = (Long) row.get(11);
c13 = (Long) row.get(12);
c14 = (String) row.get(13);
c15 = (String) row.get(14);
assertEquals(1, c1.intValue());
Assert.assertNull(c2);
Assert.assertNull(c3);
Assert.assertNull(c4);
Assert.assertNull(c5);
Assert.assertNull(c6);
Assert.assertNull(c7);
Assert.assertNull(c8);
Assert.assertNull(c9);
Assert.assertNull(c10);
Assert.assertNull(c11);
Assert.assertNull(c12);
// some database set default value for this column as default
// Assert.assertNull(c13);
Assert.assertNull(c14);
Assert.assertNull(c15);
reader.advance();
row = (IndexedRecord) reader.getCurrent();
c1 = (Integer) row.get(0);
c2 = (Short) row.get(1);
c3 = (Long) row.get(2);
c4 = (Float) row.get(3);
c5 = (Double) row.get(4);
c6 = (Float) row.get(5);
c7 = (BigDecimal) row.get(6);
c8 = (BigDecimal) row.get(7);
c9 = (Boolean) row.get(8);
c10 = (String) row.get(9);
c11 = (Long) row.get(10);
c12 = (Long) row.get(11);
c13 = (Long) row.get(12);
c14 = (String) row.get(13);
c15 = (String) row.get(14);
Assert.assertNull(c1);
Assert.assertNull(c2);
Assert.assertNull(c3);
Assert.assertNull(c4);
Assert.assertNull(c5);
Assert.assertNull(c6);
Assert.assertNull(c7);
Assert.assertNull(c8);
Assert.assertNull(c9);
Assert.assertNull(c10);
Assert.assertNull(c11);
Assert.assertNull(c12);
// some database set default value for this column as default
// Assert.assertNull(c13);
Assert.assertEquals("good luck", c14);
Assert.assertNull(c15);
reader.close();
Map<String, Object> returnMap = reader.getReturnValues();
Assert.assertEquals(10, returnMap.get(ComponentDefinition.RETURN_TOTAL_RECORD_COUNT));
} catch (Exception e) {
Assert.fail(e.getMessage());
} finally {
if (reader != null) {
try {
reader.close();
} catch (IOException e) {
Assert.fail(e.getMessage());
}
}
}
}
use of org.talend.components.jdbc.tjdbcoutput.TJDBCOutputDefinition in project components by Talend.
the class JDBCOutputTestIT method testInsertWithDataSupplier.
@Test
public void testInsertWithDataSupplier() throws Exception {
TJDBCOutputDefinition definition = new TJDBCOutputDefinition();
TJDBCOutputProperties properties = DBTestUtils.createCommonJDBCOutputProperties(allSetting, definition);
Schema schema = DBTestUtils.createTestSchema(tablename);
properties.main.schema.setValue(schema);
properties.updateOutputSchemas();
properties.tableSelection.tablename.setValue(tablename);
properties.dataAction.setValue(DataAction.INSERT);
properties.dieOnError.setValue(true);
randomBatchAndCommit(properties);
JDBCSink sink = new JDBCSink();
sink.initialize(null, properties);
WriteOperation writerOperation = sink.createWriteOperation();
Supplier<IndexedRecord> indexRecordSupplier = createDataSupplier(properties.main.schema.getValue());
new WriterDataSupplier<>(writerOperation, indexRecordSupplier, null).writeData();
;
TJDBCInputDefinition definition1 = new TJDBCInputDefinition();
TJDBCInputProperties properties1 = DBTestUtils.createCommonJDBCInputProperties(allSetting, definition1);
List<IndexedRecord> records = DBTestUtils.fetchDataByReaderFromTable(tablename, schema, definition1, properties1);
assertThat(records, hasSize(5));
Assert.assertEquals(new Integer(4), records.get(3).get(0));
Assert.assertEquals("xiaoming", records.get(3).get(1));
Assert.assertEquals(new Integer(5), records.get(4).get(0));
Assert.assertEquals("xiaobai", records.get(4).get(1));
}
use of org.talend.components.jdbc.tjdbcoutput.TJDBCOutputDefinition in project components by Talend.
the class JDBCOutputTestIT method testInsertReject.
@Test
public void testInsertReject() throws Exception {
TJDBCOutputDefinition definition = new TJDBCOutputDefinition();
TJDBCOutputProperties properties = DBTestUtils.createCommonJDBCOutputProperties(allSetting, definition);
Schema schema = DBTestUtils.createTestSchema(tablename);
properties.main.schema.setValue(schema);
properties.updateOutputSchemas();
properties.tableSelection.tablename.setValue(tablename);
properties.dataAction.setValue(DataAction.INSERT);
// reject function can't work with batch function
properties.useBatch.setValue(false);
properties.commitEvery.setValue(DBTestUtils.randomInt());
JDBCOutputWriter writer = DBTestUtils.createCommonJDBCOutputWriter(definition, properties);
try {
writer.open("wid");
IndexedRecord r1 = new GenericData.Record(properties.main.schema.getValue());
r1.put(0, 4);
r1.put(1, "wangwei");
writer.write(r1);
DBTestUtils.assertSuccessRecord(writer, r1);
IndexedRecord r2 = new GenericData.Record(properties.main.schema.getValue());
r2.put(0, 5);
r2.put(1, "the line should be rejected as it's too long");
writer.write(r2);
DBTestUtils.assertRejectRecord(writer);
IndexedRecord r3 = new GenericData.Record(properties.main.schema.getValue());
r3.put(0, 6);
r3.put(1, "gaoyan");
writer.write(r3);
DBTestUtils.assertSuccessRecord(writer, r3);
IndexedRecord r4 = new GenericData.Record(properties.main.schema.getValue());
r4.put(0, 7);
r4.put(1, "the line should be rejected as it's too long");
writer.write(r4);
DBTestUtils.assertRejectRecord(writer);
IndexedRecord r5 = new GenericData.Record(properties.main.schema.getValue());
r5.put(0, 8);
r5.put(1, "dabao");
writer.write(r5);
DBTestUtils.assertSuccessRecord(writer, r5);
writer.close();
} finally {
writer.close();
}
TJDBCInputDefinition definition1 = new TJDBCInputDefinition();
TJDBCInputProperties properties1 = DBTestUtils.createCommonJDBCInputProperties(allSetting, definition1);
List<IndexedRecord> records = DBTestUtils.fetchDataByReaderFromTable(tablename, schema, definition1, properties1);
assertThat(records, hasSize(6));
Assert.assertEquals(new Integer(4), records.get(3).get(0));
Assert.assertEquals("wangwei", records.get(3).get(1));
Assert.assertEquals(new Integer(6), records.get(4).get(0));
Assert.assertEquals("gaoyan", records.get(4).get(1));
Assert.assertEquals(new Integer(8), records.get(5).get(0));
Assert.assertEquals("dabao", records.get(5).get(1));
}
use of org.talend.components.jdbc.tjdbcoutput.TJDBCOutputDefinition in project components by Talend.
the class JDBCOutputTestIT method testUpdateOrInsert.
@Test
public void testUpdateOrInsert() throws Exception {
TJDBCOutputDefinition definition = new TJDBCOutputDefinition();
TJDBCOutputProperties properties = DBTestUtils.createCommonJDBCOutputProperties(allSetting, definition);
Schema schema = DBTestUtils.createTestSchema2(tablename);
properties.main.schema.setValue(schema);
properties.updateOutputSchemas();
properties.tableSelection.tablename.setValue(tablename);
properties.dataAction.setValue(DataAction.UPDATE_OR_INSERT);
properties.dieOnError.setValue(true);
properties.commitEvery.setValue(DBTestUtils.randomInt());
JDBCOutputWriter writer = DBTestUtils.createCommonJDBCOutputWriter(definition, properties);
try {
writer.open("wid");
IndexedRecord r1 = new GenericData.Record(properties.main.schema.getValue());
r1.put(0, 1);
r1.put(1, "wangwei1");
writer.write(r1);
DBTestUtils.assertSuccessRecord(writer, r1);
IndexedRecord r2 = new GenericData.Record(properties.main.schema.getValue());
r2.put(0, 2);
r2.put(1, "gaoyan1");
writer.write(r2);
DBTestUtils.assertSuccessRecord(writer, r2);
IndexedRecord r3 = new GenericData.Record(properties.main.schema.getValue());
r3.put(0, 4);
r3.put(1, "new one");
writer.write(r3);
DBTestUtils.assertSuccessRecord(writer, r3);
writer.close();
} finally {
writer.close();
}
TJDBCInputDefinition definition1 = new TJDBCInputDefinition();
TJDBCInputProperties properties1 = DBTestUtils.createCommonJDBCInputProperties(allSetting, definition1);
List<IndexedRecord> records = DBTestUtils.fetchDataByReaderFromTable(tablename, schema, definition1, properties1);
assertThat(records, hasSize(4));
Assert.assertEquals(new Integer(1), records.get(0).get(0));
Assert.assertEquals("wangwei1", records.get(0).get(1));
Assert.assertEquals(new Integer(2), records.get(1).get(0));
Assert.assertEquals("gaoyan1", records.get(1).get(1));
Assert.assertEquals(new Integer(3), records.get(2).get(0));
Assert.assertEquals("dabao", records.get(2).get(1));
Assert.assertEquals(new Integer(4), records.get(3).get(0));
Assert.assertEquals("new one", records.get(3).get(1));
}
Aggregations