Examples with Sink org.talend.components.api.component.runtime.Sink used on opensource projects

Example 1 with Sink

use of org.talend.components.api.component.runtime.Sink in project components by Talend.

the class TFilterRowDefinitionTest method testGetRuntime.

/**
 * Check {@link TFilterRowDefinition#getRuntime()} returns instance of {@link JiraSource}
 */
@Test
public void testGetRuntime() {
    TFilterRowDefinition definition = new TFilterRowDefinition();
    RuntimeInfo runtimeInfo = definition.getRuntimeInfo(ExecutionEngine.DI, null, ConnectorTopology.INCOMING_AND_OUTGOING);
    SandboxedInstance sandboxedInstance = RuntimeUtil.createRuntimeClass(runtimeInfo, definition.getClass().getClassLoader());
    Sink source = (Sink) sandboxedInstance.getInstance();
    assertThat(source, is(instanceOf(TFilterRowSink.class)));
}

Example 2 with Sink

use of org.talend.components.api.component.runtime.Sink in project components by Talend.

the class SalesforceRuntimeTestUtil method initWriter.

private Writer<?> initWriter(TSalesforceOutputBulkDefinition definition, TSalesforceOutputBulkProperties modelProperties) throws IOException {
    // simulate to generate the runtime code
    TSalesforceOutputBulkProperties runtimeProperties = (TSalesforceOutputBulkProperties) definition.createRuntimeProperties();
    // pass all the value from the ui model
    runtimeProperties.schema.schema.setValue(modelProperties.schema.schema.getValue());
    runtimeProperties.bulkFilePath.setValue(modelProperties.bulkFilePath.getStringValue());
    runtimeProperties.append.setValue(modelProperties.append.getValue());
    runtimeProperties.ignoreNull.setValue(modelProperties.ignoreNull.getValue());
    Object obj = modelProperties.upsertRelationTable.columnName.getValue();
    if (obj != null && obj instanceof List && !((List<?>) obj).isEmpty()) {
        runtimeProperties.upsertRelationTable.columnName.setValue(modelProperties.upsertRelationTable.columnName.getValue());
        runtimeProperties.upsertRelationTable.lookupFieldExternalIdName.setValue(modelProperties.upsertRelationTable.lookupFieldExternalIdName.getValue());
        runtimeProperties.upsertRelationTable.lookupFieldModuleName.setValue(modelProperties.upsertRelationTable.lookupFieldModuleName.getValue());
        runtimeProperties.upsertRelationTable.lookupRelationshipFieldName.setValue(modelProperties.upsertRelationTable.lookupRelationshipFieldName.getValue());
        runtimeProperties.upsertRelationTable.polymorphic.setValue(modelProperties.upsertRelationTable.polymorphic.getValue());
    }
    SourceOrSink source_sink = new SalesforceBulkFileSink();
    source_sink.initialize(null, runtimeProperties);
    ValidationResult result = source_sink.validate(null);
    Assert.assertTrue(result.getStatus() == ValidationResult.Result.OK);
    Sink sink = (Sink) source_sink;
    WriteOperation<?> writeOperation = sink.createWriteOperation();
    writeOperation.initialize(null);
    Writer<?> writer = writeOperation.createWriter(null);
    writer.open("component_instance_id");
    return writer;
}

Example 3 with Sink

use of org.talend.components.api.component.runtime.Sink in project components by Talend.

the class RuntimeControllerImpl method writeData.

@Override
public void writeData(InputStream rawPayload) throws IOException {
    // 1) Read payload (with data as a stream of course)
    DatasetWritePayload payload = DatasetWritePayload.readData(rawPayload, mapper);
    String definitionName = payload.getConfiguration().getDefinitionName();
    // 2) Create properties
    Properties properties = propertiesHelpers.propertiesFromDto(payload.getConfiguration());
    if (properties instanceof ComponentProperties) {
        ComponentProperties componentProperties = (ComponentProperties) properties;
        // 3) Retrieve component definition to be able to create the runtime
        final ComponentDefinition definition = propertiesHelpers.getDefinition(ComponentDefinition.class, definitionName);
        // 4) Get the execution engine
        ExecutionEngine executionEngine;
        if (definition.isSupportingExecutionEngines(DI)) {
            executionEngine = DI;
            // 5) Create the sandbox
            try (SandboxedInstance instance = RuntimeUtil.createRuntimeClass(definition.getRuntimeInfo(executionEngine, componentProperties, INCOMING), definition.getClass().getClassLoader())) {
                Sink datasetRuntimeInstance = (Sink) instance.getInstance();
                datasetRuntimeInstance.initialize(null, componentProperties);
                Iterator<IndexedRecord> data = payload.getData();
                WriteOperation writeOperation = datasetRuntimeInstance.createWriteOperation();
                // Supplier return null to signify end of data stream => see WriterDataSupplier.writeData
                WriterDataSupplier<?, IndexedRecord> stringWriterDataSupplier = new WriterDataSupplier<Object, IndexedRecord>(writeOperation, () -> data.hasNext() ? data.next() : null, null);
                stringWriterDataSupplier.writeData();
            }
        } else if (definition.isSupportingExecutionEngines(BEAM)) {
            throw new UnsupportedOperationException("Beam runtime is not available for dataset write through HTTP API.");
        } else {
            throw new TalendRuntimeException(CommonErrorCodes.UNREGISTERED_DEFINITION);
        }
    } else if (properties instanceof DatasetProperties) {
        throw new UnsupportedOperationException("HTTP API is only able to write using component implementations. Not " + properties.getClass());
    }
}

Example 4 with Sink

use of org.talend.components.api.component.runtime.Sink in project components by Talend.

the class TFilterRowTest method testTFilterRowRuntime.

@Test
public void testTFilterRowRuntime() throws Exception {
    TFilterRowProperties props = (TFilterRowProperties) getComponentService().getComponentProperties("tFilterRow_POC");
    props.schemaListener.afterSchema();
    assertThat(props.conditionsTable.isEditable(), is(false));
    Schema schema = SchemaBuilder.builder().record("testRecord").fields().name("field1").type().stringType().noDefault().endRecord();
    props.schemaMain.schema.setValue(schema);
    final List<Schema.Field> additionalRejectFields = new ArrayList<Schema.Field>();
    Schema.Field field = null;
    field = new Schema.Field(TFilterRowProperties.FIELD_ERROR_MESSAGE, Schema.create(Schema.Type.STRING), null, (Object) null);
    field.addProp(SchemaConstants.TALEND_IS_LOCKED, "false");
    field.addProp(SchemaConstants.TALEND_FIELD_GENERATED, "true");
    field.addProp(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255");
    additionalRejectFields.add(field);
    Schema rejectSchema = newSchema(schema, "rejectOutput", additionalRejectFields);
    props.schemaListener.afterSchema();
    assertThat(props.schemaReject.schema.getValue(), is((Schema) rejectSchema));
    assertThat(props.schemaFlow.schema.getValue(), is((Schema) schema));
    assertThat(props.conditionsTable.isEditable(), is(true));
    props.conditionsTable.columnName.setValue(Arrays.asList("field1"));
    props.conditionsTable.function.setValue(Arrays.asList(FunctionType.EMPTY));
    props.conditionsTable.operator.setValue(Arrays.asList(OperatorType.EQUALS));
    props.conditionsTable.value.setValue(Arrays.asList((Object) "test"));
    props.logicalOperator.setValue(LogicalOperator.Or);
    Sink sink = new TFilterRowSink();
    sink.initialize(null, props);
    WriteOperation<?> writeOperation = sink.createWriteOperation();
    writeOperation.initialize(null);
    WriterWithFeedback<?, ?, ?> writer = (WriterWithFeedback<?, ?, ?>) writeOperation.createWriter(null);
    writer.open("uId");
    IndexedRecord record = new GenericData.Record(schema);
    record.put(schema.getField("field1").pos(), "test");
    writer.write(record);
    Iterable<?> success = writer.getSuccessfulWrites();
    Iterator<?> successIterator = success.iterator();
    assertThat(successIterator.hasNext(), is(true));
    IndexedRecord successWrite = (IndexedRecord) successIterator.next();
    assertThat(successWrite.getSchema(), is(schema));
    assertThat(successWrite, is(record));
    writer.cleanWrites();
    record = new GenericData.Record(schema);
    record.put(schema.getField("field1").pos(), "test123");
    writer.write(record);
    success = writer.getSuccessfulWrites();
    successIterator = success.iterator();
    assertThat(successIterator.hasNext(), is(false));
    Iterable<?> reject = writer.getRejectedWrites();
    Iterator<?> rejectIterator = reject.iterator();
    assertThat(rejectIterator.hasNext(), is(true));
    IndexedRecord rejectWrite = (IndexedRecord) rejectIterator.next();
    assertThat(rejectWrite.getSchema(), is(rejectSchema));
    Result result = (Result) writer.close();
    assertThat(result.totalCount, is(2));
    assertThat(result.successCount, is(1));
    assertThat(result.rejectCount, is(1));
}

Example 5 with Sink

use of org.talend.components.api.component.runtime.Sink in project components by Talend.

the class SalesforceRuntimeTestUtil method initWriter.

private Writer initWriter(TSalesforceOutputBulkDefinition definition, TSalesforceOutputBulkProperties modelProperties) throws IOException {
    // simulate to generate the runtime code
    TSalesforceOutputBulkProperties runtimeProperties = (TSalesforceOutputBulkProperties) definition.createRuntimeProperties();
    // pass all the value from the ui model
    runtimeProperties.schema.schema.setValue(modelProperties.schema.schema.getValue());
    runtimeProperties.bulkFilePath.setValue(modelProperties.bulkFilePath.getStringValue());
    runtimeProperties.append.setValue(modelProperties.append.getValue());
    runtimeProperties.ignoreNull.setValue(modelProperties.ignoreNull.getValue());
    Object obj = modelProperties.upsertRelationTable.columnName.getValue();
    if (obj != null && obj instanceof List && !((List) obj).isEmpty()) {
        runtimeProperties.upsertRelationTable.columnName.setValue(modelProperties.upsertRelationTable.columnName.getValue());
        runtimeProperties.upsertRelationTable.lookupFieldExternalIdName.setValue(modelProperties.upsertRelationTable.lookupFieldExternalIdName.getValue());
        runtimeProperties.upsertRelationTable.lookupFieldModuleName.setValue(modelProperties.upsertRelationTable.lookupFieldModuleName.getValue());
        runtimeProperties.upsertRelationTable.lookupRelationshipFieldName.setValue(modelProperties.upsertRelationTable.lookupRelationshipFieldName.getValue());
        runtimeProperties.upsertRelationTable.polymorphic.setValue(modelProperties.upsertRelationTable.polymorphic.getValue());
    }
    SourceOrSink source_sink = new SalesforceBulkFileSink();
    source_sink.initialize(null, runtimeProperties);
    ValidationResult result = source_sink.validate(null);
    Assert.assertTrue(result.getStatus() == ValidationResult.Result.OK);
    Sink sink = (Sink) source_sink;
    WriteOperation writeOperation = sink.createWriteOperation();
    writeOperation.initialize(null);
    Writer writer = writeOperation.createWriter(null);
    writer.open("component_instance_id");
    return writer;
}