use of org.talend.components.api.component.runtime.WriteOperation in project components by Talend.
the class RuntimeControllerImpl method writeData.
@Override
public void writeData(InputStream rawPayload) throws IOException {
// 1) Read payload (with data as a stream of course)
DatasetWritePayload payload = DatasetWritePayload.readData(rawPayload, mapper);
String definitionName = payload.getConfiguration().getDefinitionName();
// 2) Create properties
Properties properties = propertiesHelpers.propertiesFromDto(payload.getConfiguration());
if (properties instanceof ComponentProperties) {
ComponentProperties componentProperties = (ComponentProperties) properties;
// 3) Retrieve component definition to be able to create the runtime
final ComponentDefinition definition = propertiesHelpers.getDefinition(ComponentDefinition.class, definitionName);
// 4) Get the execution engine
ExecutionEngine executionEngine;
if (definition.isSupportingExecutionEngines(DI)) {
executionEngine = DI;
// 5) Create the sandbox
try (SandboxedInstance instance = RuntimeUtil.createRuntimeClass(definition.getRuntimeInfo(executionEngine, componentProperties, INCOMING), definition.getClass().getClassLoader())) {
Sink datasetRuntimeInstance = (Sink) instance.getInstance();
datasetRuntimeInstance.initialize(null, componentProperties);
Iterator<IndexedRecord> data = payload.getData();
WriteOperation writeOperation = datasetRuntimeInstance.createWriteOperation();
// Supplier return null to signify end of data stream => see WriterDataSupplier.writeData
WriterDataSupplier<?, IndexedRecord> stringWriterDataSupplier = new WriterDataSupplier<Object, IndexedRecord>(writeOperation, () -> data.hasNext() ? data.next() : null, null);
stringWriterDataSupplier.writeData();
}
} else if (definition.isSupportingExecutionEngines(BEAM)) {
throw new UnsupportedOperationException("Beam runtime is not available for dataset write through HTTP API.");
} else {
throw new TalendRuntimeException(CommonErrorCodes.UNREGISTERED_DEFINITION);
}
} else if (properties instanceof DatasetProperties) {
throw new UnsupportedOperationException("HTTP API is only able to write using component implementations. Not " + properties.getClass());
}
}
use of org.talend.components.api.component.runtime.WriteOperation in project components by Talend.
the class JDBCRollbackTestIT method testRollback.
@SuppressWarnings("rawtypes")
@Test
public void testRollback() throws IOException, ClassNotFoundException, SQLException {
// connection part
TJDBCConnectionDefinition connectionDefinition = new TJDBCConnectionDefinition();
TJDBCConnectionProperties connectionProperties = DBTestUtils.createCommonJDBCConnectionProperties(allSetting, connectionDefinition);
JDBCSourceOrSink sourceOrSink = new JDBCSourceOrSink();
sourceOrSink.initialize(null, connectionProperties);
ValidationResult result = sourceOrSink.validate(container);
assertTrue(result.getStatus() == ValidationResult.Result.OK);
// output part
TJDBCOutputDefinition outputDefinition = new TJDBCOutputDefinition();
TJDBCOutputProperties outputProperties = (TJDBCOutputProperties) outputDefinition.createRuntimeProperties();
outputProperties.main.schema.setValue(DBTestUtils.createTestSchema(tablename));
outputProperties.updateOutputSchemas();
outputProperties.tableSelection.tablename.setValue(tablename);
outputProperties.dataAction.setValue(DataAction.INSERT);
outputProperties.referencedComponent.componentInstanceId.setValue(refComponentId);
outputProperties.referencedComponent.setReference(connectionProperties);
JDBCSink sink = new JDBCSink();
sink.initialize(container, outputProperties);
WriteOperation writerOperation = sink.createWriteOperation();
writerOperation.initialize(container);
JDBCOutputInsertWriter writer = (JDBCOutputInsertWriter) writerOperation.createWriter(container);
try {
writer.open("wid");
IndexedRecord r1 = new GenericData.Record(outputProperties.main.schema.getValue());
r1.put(0, 4);
r1.put(1, "xiaoming");
writer.write(r1);
DBTestUtils.assertSuccessRecord(writer, r1);
IndexedRecord r2 = new GenericData.Record(outputProperties.main.schema.getValue());
r2.put(0, 5);
r2.put(1, "xiaobai");
writer.write(r2);
DBTestUtils.assertSuccessRecord(writer, r2);
writer.close();
} finally {
writer.close();
}
// rollback part
TJDBCRollbackDefinition rollbackDefinition = new TJDBCRollbackDefinition();
TJDBCRollbackProperties rollbackProperties = (TJDBCRollbackProperties) rollbackDefinition.createRuntimeProperties();
rollbackProperties.referencedComponent.componentInstanceId.setValue(refComponentId);
rollbackProperties.closeConnection.setValue(false);
JDBCRollbackSourceOrSink rollbackSourceOrSink = new JDBCRollbackSourceOrSink();
rollbackSourceOrSink.initialize(container, rollbackProperties);
rollbackSourceOrSink.validate(container);
// create another session and check if the data is inserted
int count = -1;
try (Connection conn = JdbcRuntimeUtils.createConnection(allSetting);
Statement statement = conn.createStatement();
ResultSet resultset = statement.executeQuery("select count(*) from " + tablename)) {
if (resultset.next()) {
count = resultset.getInt(1);
}
}
Assert.assertEquals(3, count);
try (java.sql.Connection refConnection = (java.sql.Connection) container.getComponentData(ComponentConstants.CONNECTION_KEY, refComponentId)) {
assertTrue(refConnection != null);
Assert.assertTrue(!refConnection.isClosed());
}
}
use of org.talend.components.api.component.runtime.WriteOperation in project components by Talend.
the class JDBCSPTestIT method test_basic_as_output_and_no_input.
@SuppressWarnings("rawtypes")
@Test
public void test_basic_as_output_and_no_input() throws Exception {
TJDBCSPDefinition definition = new TJDBCSPDefinition();
TJDBCSPProperties properties = DBTestUtils.createCommonJDBCSPProperties(allSetting, definition);
properties.spName.setValue("SYSCS_UTIL.SYSCS_DISABLE_LOG_ARCHIVE_MODE");
properties.main.schema.setValue(DBTestUtils.createSPSchema1(tablename));
properties.spParameterTable.parameterTypes.setValue(Arrays.asList(SPParameterTable.ParameterType.IN.name()));
properties.spParameterTable.schemaColumns.setValue(Arrays.asList("PARAMETER"));
JDBCSPSink sink = new JDBCSPSink();
sink.initialize(null, properties);
ValidationResult result = sink.validate(null);
Assert.assertTrue(result.getStatus() == ValidationResult.Result.OK);
WriteOperation operation = sink.createWriteOperation();
JDBCSPWriter writer = (JDBCSPWriter) operation.createWriter(null);
try {
writer.open("wid");
IndexedRecord r1 = new GenericData.Record(properties.main.schema.getValue());
r1.put(0, 0);
writer.write(r1);
} finally {
writer.close();
}
}
use of org.talend.components.api.component.runtime.WriteOperation in project components by Talend.
the class JDBCOutputTestIT method testInsertWithDataSupplier.
@Test
public void testInsertWithDataSupplier() throws Exception {
TJDBCOutputDefinition definition = new TJDBCOutputDefinition();
TJDBCOutputProperties properties = DBTestUtils.createCommonJDBCOutputProperties(allSetting, definition);
Schema schema = DBTestUtils.createTestSchema(tablename);
properties.main.schema.setValue(schema);
properties.updateOutputSchemas();
properties.tableSelection.tablename.setValue(tablename);
properties.dataAction.setValue(DataAction.INSERT);
properties.dieOnError.setValue(true);
randomBatchAndCommit(properties);
JDBCSink sink = new JDBCSink();
sink.initialize(null, properties);
WriteOperation writerOperation = sink.createWriteOperation();
Supplier<IndexedRecord> indexRecordSupplier = createDataSupplier(properties.main.schema.getValue());
new WriterDataSupplier<>(writerOperation, indexRecordSupplier, null).writeData();
;
TJDBCInputDefinition definition1 = new TJDBCInputDefinition();
TJDBCInputProperties properties1 = DBTestUtils.createCommonJDBCInputProperties(allSetting, definition1);
List<IndexedRecord> records = DBTestUtils.fetchDataByReaderFromTable(tablename, schema, definition1, properties1);
assertThat(records, hasSize(5));
Assert.assertEquals(new Integer(4), records.get(3).get(0));
Assert.assertEquals("xiaoming", records.get(3).get(1));
Assert.assertEquals(new Integer(5), records.get(4).get(0));
Assert.assertEquals("xiaobai", records.get(4).get(1));
}
use of org.talend.components.api.component.runtime.WriteOperation in project components by Talend.
the class JdbcRowTestIT method test_die_on_error_as_output.
@SuppressWarnings("rawtypes")
@Test
public void test_die_on_error_as_output() throws Exception {
TJDBCRowDefinition definition = new TJDBCRowDefinition();
TJDBCRowProperties properties = DBTestUtils.createCommonJDBCRowProperties(allSetting, definition);
Schema schema = DBTestUtils.createTestSchema(tablename);
properties.main.schema.setValue(schema);
properties.updateOutputSchemas();
properties.tableSelection.tablename.setValue(tablename);
properties.sql.setValue("insert into " + tablename + " values(?,?)");
properties.dieOnError.setValue(true);
randomCommit(properties);
properties.usePreparedStatement.setValue(true);
properties.preparedStatementTable.indexs.setValue(Arrays.asList(1, 2));
properties.preparedStatementTable.types.setValue(Arrays.asList(PreparedStatementTable.Type.Int.name(), PreparedStatementTable.Type.String.name()));
properties.preparedStatementTable.values.setValue(Arrays.<Object>asList(4, "a too long value"));
try (SandboxedInstance sandboxedInstance = RuntimeUtil.createRuntimeClass(definition.getRuntimeInfo(ExecutionEngine.DI, properties, ConnectorTopology.INCOMING_AND_OUTGOING), properties.getClass().getClassLoader())) {
JDBCRowSink sink = (JDBCRowSink) sandboxedInstance.getInstance();
sink.initialize(null, properties);
ValidationResult result = sink.validate(null);
Assert.assertTrue(result.getStatus() == ValidationResult.Result.OK);
WriteOperation operation = sink.createWriteOperation();
JDBCRowWriter writer = (JDBCRowWriter) operation.createWriter(null);
try {
writer.open("wid");
IndexedRecord r1 = new GenericData.Record(properties.main.schema.getValue());
r1.put(0, 4);
r1.put(1, "xiaoming");
writer.write(r1);
writer.close();
} catch (ComponentException e) {
Assert.assertNotNull(e.getCause());
} finally {
writer.close();
}
}
}
Aggregations