Examples with ComponentDefinition org.talend.components.api.component.ComponentDefinition used on opensource projects

Example 11 with ComponentDefinition

use of org.talend.components.api.component.ComponentDefinition in project components by Talend.

the class SalesforceInputReaderTestIT method testBulkQueryNullValue.

protected void testBulkQueryNullValue(SalesforceConnectionModuleProperties props, String random) throws Throwable {
    ComponentDefinition sfInputDef = new TSalesforceInputDefinition();
    TSalesforceInputProperties sfInputProps = (TSalesforceInputProperties) sfInputDef.createRuntimeProperties();
    sfInputProps.copyValuesFrom(props);
    sfInputProps.manualQuery.setValue(false);
    sfInputProps.module.main.schema.setValue(SCHEMA_QUERY_ACCOUNT);
    sfInputProps.queryMode.setValue(TSalesforceInputProperties.QueryMode.Bulk);
    sfInputProps.condition.setValue("BillingPostalCode = '" + random + "'");
    List<IndexedRecord> inpuRecords = readRows(sfInputProps);
    for (IndexedRecord record : inpuRecords) {
        assertNull(record.get(5));
        assertNull(record.get(6));
    }
}

Example 12 with ComponentDefinition

use of org.talend.components.api.component.ComponentDefinition in project components by Talend.

the class SalesforceWriterTestIT method testNillableBase64Field.

@Test
public void testNillableBase64Field() throws Throwable {
    String moduleName = "StaticResource";
    ComponentDefinition sfDef = new TSalesforceOutputDefinition();
    TSalesforceOutputProperties sfProps = (TSalesforceOutputProperties) sfDef.createRuntimeProperties();
    SalesforceTestBase.setupProps(sfProps.connection, false);
    sfProps.module.setValue("moduleName", moduleName);
    sfProps.module.main.schema.setValue(SCHEMA_STATIC_RESOURCE);
    sfProps.ceaseForError.setValue(true);
    sfProps.module.schemaListener.afterSchema();
    List records = new ArrayList<IndexedRecord>();
    String randomName = getRandomString(10);
    IndexedRecord record = new GenericData.Record(SCHEMA_STATIC_RESOURCE);
    record.put(0, randomName);
    record.put(1, "text/plain");
    record.put(2, "dGhpcyBpcyBiYXNlNjQgIHRlc3QgZmlsZS4g");
    record.put(3, "this is base64 test file.");
    records.add(record);
    SalesforceSink salesforceSink = new SalesforceSink();
    salesforceSink.initialize(adaptor, sfProps);
    salesforceSink.validate(adaptor);
    Writer<Result> batchWriter = salesforceSink.createWriteOperation().createWriter(adaptor);
    writeRows(batchWriter, records);
    TSalesforceInputProperties sfInputProps = getSalesforceInputProperties();
    sfInputProps.copyValuesFrom(sfProps);
    sfInputProps.condition.setValue("Name = '" + randomName + "'");
    sfProps.module.setValue("moduleName", moduleName);
    sfProps.module.main.schema.setValue(SCHEMA_STATIC_RESOURCE);
    List<IndexedRecord> inpuRecords = readRows(sfInputProps);
    try {
        assertEquals(1, inpuRecords.size());
        IndexedRecord r = inpuRecords.get(0);
        assertEquals(randomName, r.get(0));
        assertEquals("text/plain", r.get(1));
        assertEquals("dGhpcyBpcyBiYXNlNjQgIHRlc3QgZmlsZS4g", r.get(2));
        assertEquals("this is base64 test file.", r.get(3));
        assertNotNull(r.get(4));
    } finally {
        deleteRows(inpuRecords, sfInputProps);
    }
}

Example 13 with ComponentDefinition

use of org.talend.components.api.component.ComponentDefinition in project components by Talend.

the class SalesforceWriterTestIT method testWriteDynamicWithFeedback.

/**
 * Test write dynamic with feedback
 */
@Test
public void testWriteDynamicWithFeedback() throws Exception {
    // Component framework objects.
    ComponentDefinition sfDef = new TSalesforceOutputDefinition();
    TSalesforceOutputProperties sfProps = (TSalesforceOutputProperties) sfDef.createRuntimeProperties();
    SalesforceTestBase.setupProps(sfProps.connection, false);
    sfProps.module.setValue("moduleName", "Account");
    Schema dynamicSchema = // 
    SchemaBuilder.builder().record("Schema").prop(SchemaConstants.INCLUDE_ALL_FIELDS, // 
    "true").prop("di.dynamic.column.position", "1").fields().name("Name").type().stringType().noDefault().endRecord();
    Schema dynamicRuntimeSchema = // 
    SchemaBuilder.builder().record("Schema").prop(SchemaConstants.INCLUDE_ALL_FIELDS, // 
    "true").prop("di.dynamic.column.position", "1").fields().name("Name").type().stringType().noDefault().name("BillingStreet").type().stringType().noDefault().name("BillingCity").type().stringType().noDefault().name("BillingState").type().stringType().noDefault().endRecord();
    sfProps.module.main.schema.setValue(dynamicSchema);
    sfProps.extendInsert.setValue(false);
    sfProps.ceaseForError.setValue(false);
    // Automatically generate the out schemas.
    sfProps.module.schemaListener.afterSchema();
    DefaultComponentRuntimeContainerImpl container = new DefaultComponentRuntimeContainerImpl();
    // Initialize the Sink, WriteOperation and Writer
    SalesforceSink sfSink = new SalesforceSink();
    sfSink.initialize(container, sfProps);
    sfSink.validate(container);
    SalesforceWriteOperation sfWriteOp = sfSink.createWriteOperation();
    sfWriteOp.initialize(container);
    SalesforceWriter sfWriter = sfSink.createWriteOperation().createWriter(container);
    sfWriter.open("uid1");
    // Write one record, which should fail for missing name.
    IndexedRecord r1 = new GenericData.Record(dynamicRuntimeSchema);
    r1.put(0, "test");
    r1.put(1, "deleteme");
    r1.put(2, "deleteme");
    r1.put(3, "deleteme");
    sfWriter.write(r1);
    // Check success
    assertThat(sfWriter.getSuccessfulWrites(), hasSize(1));
    IndexedRecord success = sfWriter.getSuccessfulWrites().get(0);
    assertThat(success.getSchema().getFields(), hasSize(4));
    // Check the values copied from the incoming record.
    for (int i = 0; i < r1.getSchema().getFields().size(); i++) {
        assertThat(success.getSchema().getFields().get(i), is(r1.getSchema().getFields().get(i)));
        assertThat(success.get(i), is(r1.get(i)));
    }
    // The success fields.
    assertThat(success.getSchema().getFields().get(0).name(), is("Name"));
    assertThat(success.getSchema().getFields().get(1).name(), is("BillingStreet"));
    assertThat(success.getSchema().getFields().get(2).name(), is("BillingCity"));
    assertThat(success.getSchema().getFields().get(3).name(), is("BillingState"));
    assertThat(success.get(0), is((Object) "test"));
    assertThat(success.get(1), is((Object) "deleteme"));
    assertThat(success.get(2), is((Object) "deleteme"));
    assertThat(success.get(3), is((Object) "deleteme"));
    // Check reject
    IndexedRecord r2 = new GenericData.Record(dynamicRuntimeSchema);
    r2.put(1, "deleteme");
    r2.put(2, "deleteme");
    r2.put(3, "deleteme");
    sfWriter.write(r2);
    assertThat(sfWriter.getRejectedWrites(), hasSize(1));
    IndexedRecord rejected = sfWriter.getRejectedWrites().get(0);
    assertThat(rejected.getSchema().getFields(), hasSize(7));
    // The rejected fields.
    assertThat(rejected.getSchema().getFields().get(0).name(), is("Name"));
    assertThat(rejected.getSchema().getFields().get(1).name(), is("BillingStreet"));
    assertThat(rejected.getSchema().getFields().get(2).name(), is("BillingCity"));
    assertThat(rejected.getSchema().getFields().get(3).name(), is("BillingState"));
    assertThat(rejected.getSchema().getFields().get(4).name(), is(TSalesforceOutputProperties.FIELD_ERROR_CODE));
    assertThat(rejected.getSchema().getFields().get(5).name(), is(TSalesforceOutputProperties.FIELD_ERROR_FIELDS));
    assertThat(rejected.getSchema().getFields().get(6).name(), is(TSalesforceOutputProperties.FIELD_ERROR_MESSAGE));
    assertNull(rejected.get(0));
    assertThat(rejected.get(1), is((Object) "deleteme"));
    assertThat(rejected.get(2), is((Object) "deleteme"));
    assertThat(rejected.get(3), is((Object) "deleteme"));
    assertNotNull(rejected.get(4));
    assertNotNull(rejected.get(5));
    assertNotNull(rejected.get(6));
    // Finish the Writer, WriteOperation and Sink.
    Result wr1 = sfWriter.close();
    sfWriteOp.finalize(Arrays.asList(wr1), container);
}

Example 14 with ComponentDefinition

use of org.talend.components.api.component.ComponentDefinition in project components by Talend.

the class SalesforceWriterTestIT method testOutputUpsert.

@Ignore("Need to add some custom modules in salesforce account for this test")
@Test
public void testOutputUpsert() throws Throwable {
    Schema CUSTOM_LOOKUP_MODULE_SCHEMA = // 
    SchemaBuilder.builder().record("Schema").fields().name("ExternalID__c").type().stringType().noDefault().name("Name").type().stringType().noDefault().name("Id").type().stringType().noDefault().endRecord();
    Schema CUSTOM_TEST_MODULE_SCHEMA = // 
    SchemaBuilder.builder().record("Schema").fields().name("ExternalID__c").type().stringType().noDefault().name("LookupModuleExternalId").type().stringType().noDefault().name("Name").type().stringType().noDefault().name("Id").type().stringType().noDefault().endRecord();
    // Component framework objects.
    ComponentDefinition sfDef = new TSalesforceOutputDefinition();
    // Prepare the lookup module data
    TSalesforceOutputProperties sfLookupProps = (TSalesforceOutputProperties) sfDef.createRuntimeProperties();
    SalesforceTestBase.setupProps(sfLookupProps.connection, false);
    sfLookupProps.module.setValue("moduleName", "TestLookupModule__c");
    sfLookupProps.module.main.schema.setValue(CUSTOM_LOOKUP_MODULE_SCHEMA);
    sfLookupProps.ceaseForError.setValue(true);
    // Automatically generate the out schemas.
    sfLookupProps.module.schemaListener.afterSchema();
    List<IndexedRecord> records = new ArrayList<>();
    IndexedRecord r1 = new GenericData.Record(CUSTOM_LOOKUP_MODULE_SCHEMA);
    r1.put(0, "EXTERNAL_ID_" + UNIQUE_ID);
    r1.put(1, UNIQUE_NAME + "_" + UNIQUE_ID);
    records.add(r1);
    SalesforceSink salesforceSink = new SalesforceSink();
    salesforceSink.initialize(adaptor, sfLookupProps);
    salesforceSink.validate(adaptor);
    Writer<Result> batchWriter = salesforceSink.createWriteOperation().createWriter(adaptor);
    writeRows(batchWriter, records);
    List<IndexedRecord> successRecords = ((SalesforceWriter) batchWriter).getSuccessfulWrites();
    assertEquals(1, successRecords.size());
    // 2. Upsert "TestModule__c" with upsert relation table
    TSalesforceOutputProperties sfTestLookupProps = (TSalesforceOutputProperties) sfDef.createRuntimeProperties().init();
    SalesforceTestBase.setupProps(sfTestLookupProps.connection, false);
    sfTestLookupProps.module.setValue("moduleName", "TestModule__c");
    sfTestLookupProps.module.main.schema.setValue(CUSTOM_TEST_MODULE_SCHEMA);
    // Automatically generate the out schemas.
    sfTestLookupProps.module.schemaListener.afterSchema();
    sfTestLookupProps.outputAction.setValue(OutputAction.UPSERT);
    sfTestLookupProps.afterOutputAction();
    assertEquals(4, sfTestLookupProps.upsertKeyColumn.getPossibleValues().size());
    sfTestLookupProps.upsertKeyColumn.setValue("ExternalID__c");
    sfTestLookupProps.ceaseForError.setValue(true);
    // setup relation table
    sfTestLookupProps.upsertRelationTable.columnName.setValue(Arrays.asList("LookupModuleExternalId"));
    sfTestLookupProps.upsertRelationTable.lookupFieldName.setValue(Arrays.asList("TestLookupModule__c"));
    sfTestLookupProps.upsertRelationTable.lookupRelationshipFieldName.setValue(Arrays.asList("TestLookupModule__r"));
    sfTestLookupProps.upsertRelationTable.lookupFieldModuleName.setValue(Arrays.asList("TestLookupModule__c"));
    sfTestLookupProps.upsertRelationTable.lookupFieldExternalIdName.setValue(Arrays.asList("ExternalID__c"));
    records = new ArrayList<>();
    r1 = new GenericData.Record(CUSTOM_TEST_MODULE_SCHEMA);
    r1.put(0, "EXTERNAL_ID_" + UNIQUE_ID);
    r1.put(1, "EXTERNAL_ID_" + UNIQUE_ID);
    r1.put(2, UNIQUE_NAME + "_" + UNIQUE_ID);
    records.add(r1);
    salesforceSink = new SalesforceSink();
    salesforceSink.initialize(adaptor, sfTestLookupProps);
    salesforceSink.validate(adaptor);
    batchWriter = salesforceSink.createWriteOperation().createWriter(adaptor);
    writeRows(batchWriter, records);
    assertEquals(1, ((SalesforceWriter) batchWriter).getSuccessfulWrites().size());
    TSalesforceInputProperties sfInputProps = getSalesforceInputProperties();
    sfInputProps.copyValuesFrom(sfTestLookupProps);
    // "LookupModuleExternalId" is not the column of module. So "CUSTOM_LOOKUP_MODULE_SCHEMA" for query
    sfInputProps.module.main.schema.setValue(CUSTOM_LOOKUP_MODULE_SCHEMA);
    sfInputProps.condition.setValue("ExternalID__c = 'EXTERNAL_ID_" + UNIQUE_ID + "'");
    List<IndexedRecord> inpuRecords = readRows(sfInputProps);
    assertEquals(1, inpuRecords.size());
    LOGGER.debug("Upsert operation insert a record in module \"TestModule__c\" with ID: " + inpuRecords.get(0).get(2));
}

Example 15 with ComponentDefinition

use of org.talend.components.api.component.ComponentDefinition in project components by Talend.

the class SalesforceWriterTestIT method testSinkWorkflow_insert.

/**
 * Basic test that shows how the {@link SalesforceSink} is meant to be used to write data.
 */
@Test
public void testSinkWorkflow_insert() throws Exception {
    // Component framework objects.
    ComponentDefinition sfDef = new TSalesforceOutputDefinition();
    TSalesforceOutputProperties sfProps = (TSalesforceOutputProperties) sfDef.createRuntimeProperties();
    SalesforceTestBase.setupProps(sfProps.connection, false);
    sfProps.module.setValue("moduleName", "Account");
    sfProps.module.main.schema.setValue(SCHEMA_INSERT_ACCOUNT);
    sfProps.ceaseForError.setValue(false);
    // Automatically generate the out schemas.
    sfProps.module.schemaListener.afterSchema();
    DefaultComponentRuntimeContainerImpl container = new DefaultComponentRuntimeContainerImpl();
    // Initialize the Sink, WriteOperation and Writer
    SalesforceSink sfSink = new SalesforceSink();
    sfSink.initialize(container, sfProps);
    sfSink.validate(container);
    SalesforceWriteOperation sfWriteOp = sfSink.createWriteOperation();
    sfWriteOp.initialize(container);
    SalesforceWriter sfWriter = sfSink.createWriteOperation().createWriter(container);
    sfWriter.open("uid1");
    // Write one record.
    IndexedRecord r = new GenericData.Record(SCHEMA_INSERT_ACCOUNT);
    r.put(0, UNIQUE_NAME + "_" + UNIQUE_ID);
    r.put(1, "deleteme");
    r.put(2, "deleteme");
    r.put(3, "deleteme");
    sfWriter.write(r);
    sfWriter.close();
    assertThat(sfWriter.getRejectedWrites(), empty());
    assertThat(sfWriter.getSuccessfulWrites(), hasSize(1));
    assertThat(sfWriter.getSuccessfulWrites().get(0), is(r));
    sfWriter.cleanWrites();
    // Rejected and successful writes are reset on the next record.
    r = new GenericData.Record(SCHEMA_INSERT_ACCOUNT);
    r.put(0, UNIQUE_NAME + "_" + UNIQUE_ID);
    r.put(1, "deleteme2");
    r.put(2, "deleteme2");
    r.put(3, "deleteme2");
    sfWriter.write(r);
    sfWriter.close();
    assertThat(sfWriter.getRejectedWrites(), empty());
    assertThat(sfWriter.getSuccessfulWrites(), hasSize(1));
    assertThat(sfWriter.getSuccessfulWrites().get(0), is(r));
    // Finish the Writer, WriteOperation and Sink.
    Result wr1 = sfWriter.close();
    sfWriteOp.finalize(Arrays.asList(wr1), container);
}