use of org.talend.components.salesforce.tsalesforceoutput.TSalesforceOutputProperties in project components by Talend.
the class SalesforceWriterTestIT method testSinkWorkflow_insertAndRetrieveId.
/**
* Test for a Sink that has an output flow containing the salesforce id.
*/
@Test
public void testSinkWorkflow_insertAndRetrieveId() throws Exception {
// Component framework objects.
ComponentDefinition sfDef = new TSalesforceOutputDefinition();
TSalesforceOutputProperties sfProps = (TSalesforceOutputProperties) sfDef.createRuntimeProperties();
SalesforceTestBase.setupProps(sfProps.connection, false);
sfProps.module.setValue("moduleName", "Account");
sfProps.extendInsert.setValue(false);
sfProps.ceaseForError.setValue(false);
sfProps.retrieveInsertId.setValue(true);
sfProps.module.main.schema.setValue(SCHEMA_INSERT_ACCOUNT);
// Automatically generate the out schemas.
sfProps.module.schemaListener.afterSchema();
DefaultComponentRuntimeContainerImpl container = new DefaultComponentRuntimeContainerImpl();
// Initialize the Sink, WriteOperation and Writer
SalesforceSink sfSink = new SalesforceSink();
sfSink.initialize(container, sfProps);
sfSink.validate(container);
SalesforceWriteOperation sfWriteOp = sfSink.createWriteOperation();
sfWriteOp.initialize(container);
SalesforceWriter sfWriter = sfSink.createWriteOperation().createWriter(container);
sfWriter.open("uid1");
// Write one record.
IndexedRecord r = new GenericData.Record(SCHEMA_INSERT_ACCOUNT);
r.put(0, UNIQUE_NAME + "_" + UNIQUE_ID);
r.put(1, "deleteme");
r.put(2, "deleteme");
r.put(3, "deleteme");
sfWriter.write(r);
assertThat(sfWriter.getRejectedWrites(), empty());
assertThat(sfWriter.getSuccessfulWrites(), hasSize(1));
// Check the successful record (main output)
IndexedRecord main = sfWriter.getSuccessfulWrites().get(0);
assertThat(main.getSchema().getFields(), hasSize(5));
// Check the values copied from the incoming record.
for (int i = 0; i < r.getSchema().getFields().size(); i++) {
assertThat(main.getSchema().getFields().get(i), is(r.getSchema().getFields().get(i)));
assertThat(main.get(i), is(r.get(i)));
}
// The enriched fields.
assertThat(main.getSchema().getFields().get(4).name(), is("salesforce_id"));
assertThat(main.get(4), not(nullValue()));
// Finish the Writer, WriteOperation and Sink.
Result wr1 = sfWriter.close();
sfWriteOp.finalize(Arrays.asList(wr1), container);
}
use of org.talend.components.salesforce.tsalesforceoutput.TSalesforceOutputProperties in project components by Talend.
the class SalesforceWriterTestIT method testSourceIncludedId.
/**
* This test about: 1) Insert record which "Id" is passed from input data 2) Upsert with Id as a upsert key column
*/
@Test
public void testSourceIncludedId() throws Throwable {
// Generate log file path
String logFilePath = tempFolder.getRoot().getAbsolutePath() + "/salesforce_error_" + UNIQUE_ID + ".log";
File file = new File(logFilePath);
assertFalse(file.exists());
// Prepare the input properties for check record in server side
TSalesforceInputProperties inputProperties = getSalesforceInputProperties();
List<IndexedRecord> inputRecords = null;
// Component framework objects.
ComponentDefinition sfDef = new TSalesforceOutputDefinition();
TSalesforceOutputProperties sfProps = (TSalesforceOutputProperties) sfDef.createRuntimeProperties();
SalesforceTestBase.setupProps(sfProps.connection, false);
sfProps.module.setValue("moduleName", "Account");
sfProps.module.main.schema.setValue(SCHEMA_UPDATE_ACCOUNT);
sfProps.extendInsert.setValue(false);
sfProps.ceaseForError.setValue(false);
// Setup log file path
LOGGER.debug("Error log path: " + logFilePath);
sfProps.logFileName.setValue(logFilePath);
// ///////////////////////////////////////////////////////////////////////////////////////////////
// ///////////////////////// 1. Insert the record and get the record Id //////////////////////////
// ///////////////////////////////////////////////////////////////////////////////////////////////
// Automatically generate the out schemas.
sfProps.retrieveInsertId.setValue(true);
sfProps.module.schemaListener.afterSchema();
Schema flowSchema = sfProps.schemaFlow.schema.getValue();
Schema.Field field = flowSchema.getField(FIELD_SALESFORCE_ID);
assertEquals(6, flowSchema.getFields().size());
assertNotNull(field);
assertEquals(5, field.pos());
// Initialize the Writer
LOGGER.debug("Try to insert the record");
SalesforceWriter sfWriterInsert = (SalesforceWriter) createSalesforceOutputWriter(sfProps);
sfWriterInsert.open("uid_insert");
// Insert one record with Id column. The "Id" column should be ignore and insert successfully
IndexedRecord insertRecord_1 = new GenericData.Record(SCHEMA_UPDATE_ACCOUNT);
insertRecord_1.put(0, "bad id");
insertRecord_1.put(1, UNIQUE_NAME + "_" + UNIQUE_ID + "_insert");
insertRecord_1.put(2, "deleteme_insert");
insertRecord_1.put(3, "deleteme_insert");
insertRecord_1.put(4, "deleteme_insert");
IndexedRecord insertRecord_2 = new GenericData.Record(SCHEMA_UPDATE_ACCOUNT);
insertRecord_2.put(0, "bad id");
insertRecord_2.put(2, "deleteme_insert");
insertRecord_2.put(3, "deleteme_insert");
insertRecord_2.put(4, "deleteme_insert");
// Test wrong record
sfWriterInsert.write(insertRecord_2);
assertThat(sfWriterInsert.getSuccessfulWrites(), empty());
assertThat(sfWriterInsert.getRejectedWrites(), hasSize(1));
LOGGER.debug("1 record is reject by insert action.");
sfWriterInsert.cleanWrites();
sfWriterInsert.write(insertRecord_1);
assertThat(sfWriterInsert.getSuccessfulWrites(), hasSize(1));
assertThat(sfWriterInsert.getRejectedWrites(), empty());
// Check the rejected record.
IndexedRecord successRecord = sfWriterInsert.getSuccessfulWrites().get(0);
sfWriterInsert.cleanWrites();
assertThat(successRecord.getSchema().getFields(), hasSize(6));
assertEquals(FIELD_SALESFORCE_ID, successRecord.getSchema().getFields().get(5).name());
// The enriched fields.
String recordID = String.valueOf(successRecord.get(5));
LOGGER.debug("1 record insert successfully and get record Id: " + recordID);
// Finish the Writer, WriteOperation and Sink for insert action
Result wr1 = sfWriterInsert.close();
inputProperties.copyValuesFrom(sfProps);
inputProperties.condition.setValue("Name='" + UNIQUE_NAME + "_" + UNIQUE_ID + "_insert'");
inputRecords = readRows(inputProperties);
assertEquals(1, inputRecords.size());
// Check record in server side
successRecord = inputRecords.get(0);
assertThat(successRecord.get(1), is((Object) (UNIQUE_NAME + "_" + UNIQUE_ID + "_insert")));
assertThat(successRecord.get(2), is((Object) "deleteme_insert"));
assertThat(successRecord.get(3), is((Object) "deleteme_insert"));
assertThat(successRecord.get(4), is((Object) "deleteme_insert"));
// Check error log
assertTrue(file.exists());
assertNotEquals(0, file.length());
// removed this test caus the message is i18n
// runtimeTestUtil.compareFileContent(sfProps.logFileName.getValue(),
// new String[] { "\tStatus Code: REQUIRED_FIELD_MISSING", "", "\tRowKey/RowNo: 1", "\tFields: Name", "",
// "\tMessage: Required fields are missing: [Name]",
// "\t--------------------------------------------------------------------------------", "" });
// //////////////////////////////////////////////////////////////////////////////////////////////////////
// /////////////////////////////////// 2.Update the inserted record /////////////////////////////////////
// //////////////////////////////////////////////////////////////////////////////////////////////////////
// Flow schema change back to same with main schema
sfProps.extendInsert.setValue(true);
sfProps.outputAction.setValue(OutputAction.UPDATE);
sfProps.module.schemaListener.afterSchema();
flowSchema = sfProps.schemaFlow.schema.getValue();
assertEquals(5, flowSchema.getFields().size());
// Initialize the Writer
LOGGER.debug("Try to update the record which Id is: " + recordID);
SalesforceWriter sfWriter_Update = (SalesforceWriter) createSalesforceOutputWriter(sfProps);
sfWriter_Update.open("uid_update");
IndexedRecord updateRecord_1 = new GenericData.Record(SCHEMA_UPDATE_ACCOUNT);
updateRecord_1.put(0, "0019000001n3Kasss");
updateRecord_1.put(1, UNIQUE_NAME + "_" + UNIQUE_ID + "_update");
updateRecord_1.put(2, "deleteme_update");
updateRecord_1.put(3, "deleteme_update");
updateRecord_1.put(4, "deleteme_update");
IndexedRecord updateRecord_2 = new GenericData.Record(SCHEMA_UPDATE_ACCOUNT);
updateRecord_2.put(0, recordID);
updateRecord_2.put(1, UNIQUE_NAME + "_" + UNIQUE_ID + "_update");
updateRecord_2.put(2, "deleteme_update");
updateRecord_2.put(3, "deleteme_update");
updateRecord_2.put(4, "deleteme_update");
IndexedRecord updateRecord_3 = new GenericData.Record(SCHEMA_UPDATE_ACCOUNT);
updateRecord_3.put(0, "0019000001n3Kabbb");
updateRecord_3.put(1, UNIQUE_NAME + "_" + UNIQUE_ID + "_update");
updateRecord_3.put(2, "deleteme_update");
updateRecord_3.put(3, "deleteme_update");
updateRecord_3.put(4, "deleteme_update");
sfWriter_Update.write(updateRecord_1);
sfWriter_Update.write(updateRecord_2);
sfWriter_Update.write(updateRecord_3);
// Finish the Writer, WriteOperation and Sink for insert action
Result wr2 = sfWriter_Update.close();
assertEquals(1, wr2.getSuccessCount());
assertEquals(2, wr2.getRejectCount());
// Check record in server side
inputProperties.copyValuesFrom(sfProps);
inputProperties.condition.setValue("Name='" + UNIQUE_NAME + "_" + UNIQUE_ID + "_update'");
inputRecords = readRows(inputProperties);
assertEquals(1, inputRecords.size());
successRecord = inputRecords.get(0);
assertThat(successRecord.get(1), is((Object) (UNIQUE_NAME + "_" + UNIQUE_ID + "_update")));
assertThat(successRecord.get(2), is((Object) "deleteme_update"));
assertThat(successRecord.get(3), is((Object) "deleteme_update"));
assertThat(successRecord.get(4), is((Object) "deleteme_update"));
LOGGER.debug("1 record update successfully.");
LOGGER.debug("2 record is reject by update action");
// Check error log
assertTrue(file.exists());
assertNotEquals(0, file.length());
// removed the check on value cause it is i18n
// runtimeTestUtil.compareFileContent(sfProps.logFileName.getValue(),
// new String[] { "\tStatus Code: MALFORMED_ID", "", "\tRowKey/RowNo: 0019000001n3Kasss", "\tFields: Id", "",
// "\tMessage: Account ID: id value of incorrect type: 0019000001n3Kasss",
// "\t--------------------------------------------------------------------------------", "",
// "\tStatus Code: MALFORMED_ID", "", "\tRowKey/RowNo: 0019000001n3Kabbb", "\tFields: Id", "",
// "\tMessage: Account ID: id value of incorrect type: 0019000001n3Kabbb",
// "\t--------------------------------------------------------------------------------", "" });
// /////////////////////////////////////////////////////////////////////////////////////////////////////
// ////////////////////////// 3.Upsert the record with Id as upsertKeyColumn ///////////////////////////
// /////////////////////////////////////////////////////////////////////////////////////////////////////
sfProps.outputAction.setValue(OutputAction.UPSERT);
sfProps.module.schemaListener.afterSchema();
// Test upsertkey column is "Id"
sfProps.upsertKeyColumn.setValue("Id");
// Initialize the Writer
LOGGER.debug("Try to upsert the record which Id is: " + recordID);
SalesforceWriter sfWriter_Upsert = (SalesforceWriter) createSalesforceOutputWriter(sfProps);
sfWriter_Upsert.open("uid_upsert");
IndexedRecord upsertRecord_1 = new GenericData.Record(SCHEMA_UPDATE_ACCOUNT);
upsertRecord_1.put(0, "0019000001n3Kasss");
upsertRecord_1.put(1, UNIQUE_NAME + "_" + UNIQUE_ID + "_upsert");
upsertRecord_1.put(2, "deleteme_upsert");
upsertRecord_1.put(3, "deleteme_upsert");
upsertRecord_1.put(4, "deleteme_upsert");
IndexedRecord upsertRecord_2 = new GenericData.Record(SCHEMA_UPDATE_ACCOUNT);
upsertRecord_2.put(0, recordID);
upsertRecord_2.put(1, UNIQUE_NAME + "_" + UNIQUE_ID + "_upsert");
upsertRecord_2.put(2, "deleteme_upsert");
upsertRecord_2.put(3, "deleteme_upsert");
upsertRecord_2.put(4, "deleteme_upsert");
sfWriter_Upsert.write(upsertRecord_1);
sfWriter_Upsert.write(upsertRecord_2);
// Finish the Writer, WriteOperation and Sink for insert action
Result wr3 = sfWriter_Upsert.close();
assertEquals(1, wr3.getSuccessCount());
assertEquals(1, wr3.getRejectCount());
// Check record in server side
inputProperties.copyValuesFrom(sfProps);
inputProperties.condition.setValue("Name='" + UNIQUE_NAME + "_" + UNIQUE_ID + "_upsert'");
inputRecords = readRows(inputProperties);
assertEquals(1, inputRecords.size());
successRecord = inputRecords.get(0);
assertThat(successRecord.get(1), is((Object) (UNIQUE_NAME + "_" + UNIQUE_ID + "_upsert")));
assertThat(successRecord.get(2), is((Object) "deleteme_upsert"));
assertThat(successRecord.get(3), is((Object) "deleteme_upsert"));
assertThat(successRecord.get(4), is((Object) "deleteme_upsert"));
LOGGER.debug("1 record upsert successfully.");
LOGGER.debug("1 record is reject by upsert action.");
// Check error log
assertTrue(file.exists());
assertNotEquals(0, file.length());
runtimeTestUtil.compareFileContent(sfProps.logFileName.getValue(), new String[] { "\tStatus Code: MALFORMED_ID", "", "\tRowKey/RowNo: 0019000001n3Kasss", "\tFields: ", "", "\tMessage: Id in upsert is not valid", "\t--------------------------------------------------------------------------------", "" });
// Test wrong module name value
LOGGER.debug("Try to upsert the record which Id is: " + recordID);
sfWriter_Upsert = (SalesforceWriter) createSalesforceOutputWriter(sfProps);
sfWriter_Upsert.open("uid_upsert");
upsertRecord_1 = new GenericData.Record(SCHEMA_UPDATE_ACCOUNT);
upsertRecord_1.put(0, recordID);
upsertRecord_1.put(1, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n" + "\n");
upsertRecord_1.put(2, "deleteme_upsert");
upsertRecord_1.put(3, "deleteme_upsert");
upsertRecord_2.put(4, "deleteme_upsert");
sfWriter_Upsert.write(upsertRecord_1);
// Finish the Writer, WriteOperation and Sink for insert action
wr3 = sfWriter_Upsert.close();
assertEquals(0, wr3.getSuccessCount());
assertEquals(1, wr3.getRejectCount());
// Check error log
assertTrue(file.exists());
assertNotEquals(0, file.length());
// removed the check on value cause it is i18n
// runtimeTestUtil.compareFileContent(sfProps.logFileName.getValue(),
// new String[] { "\tStatus Code: STRING_TOO_LONG", "", "\tRowKey/RowNo: " + recordID, "\tFields: Name", "",
// "\tMessage: Account Name: data value too large: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
// + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
// + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
// + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa (max length=255)",
// "\t--------------------------------------------------------------------------------", "", });
// /////////////////////////////////////////////////////////////////////////////////////////////////////
// /////////////////////////////////// 4.Delete the record with Id /////////////////////////////////////
// /////////////////////////////////////////////////////////////////////////////////////////////////////
sfProps.outputAction.setValue(OutputAction.DELETE);
// Initialize the Writer
LOGGER.debug("Try to delete the record which Id is: " + recordID);
SalesforceWriter sfWriter_Delete = (SalesforceWriter) createSalesforceOutputWriter(sfProps);
sfWriter_Delete.open("uid_delete");
IndexedRecord deleteRecord_1 = new GenericData.Record(SCHEMA_UPDATE_ACCOUNT);
deleteRecord_1.put(0, recordID);
deleteRecord_1.put(1, UNIQUE_NAME + "_" + UNIQUE_ID + "_delete");
IndexedRecord deleteRecord_2 = new GenericData.Record(SCHEMA_UPDATE_ACCOUNT);
// Id not exist
deleteRecord_2.put(0, "0019000001n3Kabbb");
deleteRecord_2.put(1, UNIQUE_NAME + "_" + UNIQUE_ID + "_delete");
sfWriter_Delete.write(deleteRecord_1);
sfWriter_Delete.write(deleteRecord_2);
// Finish the Writer, WriteOperation and Sink for insert action
Result wr4 = sfWriter_Delete.close();
assertEquals(1, wr4.getSuccessCount());
assertEquals(1, wr4.getRejectCount());
// Check record in server side
inputProperties.copyValuesFrom(sfProps);
inputProperties.condition.setValue("Name='" + UNIQUE_NAME + "_" + UNIQUE_ID + "_upsert'");
inputRecords = readRows(inputProperties);
assertEquals(0, inputRecords.size());
LOGGER.debug("1 record delete successfully.");
LOGGER.debug("1 record is reject by delete action.");
// Check error log
assertTrue(file.exists());
assertNotEquals(0, file.length());
// removed the check on value cause it is i18n
// runtimeTestUtil.compareFileContent(sfProps.logFileName.getValue(),
// new String[] { "\tStatus Code: MALFORMED_ID", "", "\tRowKey/RowNo: 0019000001n3Kabbb", "\tFields: ", "",
// "\tMessage: bad id 0019000001n3Kabbb",
// "\t--------------------------------------------------------------------------------", "", });
}
use of org.talend.components.salesforce.tsalesforceoutput.TSalesforceOutputProperties in project components by Talend.
the class SalesforceWriterTestIT method testUpdateError.
// This is for reject and caseForError not real test for update
protected void testUpdateError(boolean ceaseForError) throws Exception {
// Generate log file path
String logFilePath = tempFolder.getRoot().getAbsolutePath() + "/salesforce_error_" + (ceaseForError ? 0 : 1) + ".log";
File file = new File(logFilePath);
assertFalse(file.exists());
// Component framework objects.
ComponentDefinition sfDef = new TSalesforceOutputDefinition();
TSalesforceOutputProperties sfProps = (TSalesforceOutputProperties) sfDef.createRuntimeProperties();
SalesforceTestBase.setupProps(sfProps.connection, false);
sfProps.module.setValue("moduleName", "Account");
sfProps.module.main.schema.setValue(SCHEMA_UPDATE_ACCOUNT);
sfProps.outputAction.setValue(OutputAction.UPDATE);
sfProps.extendInsert.setValue(false);
sfProps.ceaseForError.setValue(ceaseForError);
// Setup log file path
LOGGER.debug("Error log path: " + logFilePath);
sfProps.logFileName.setValue(logFilePath);
// Automatically generate the out schemas.
sfProps.module.schemaListener.afterSchema();
DefaultComponentRuntimeContainerImpl container = new DefaultComponentRuntimeContainerImpl();
// Initialize the Sink, WriteOperation and Writer
SalesforceSink sfSink = new SalesforceSink();
sfSink.initialize(container, sfProps);
sfSink.validate(container);
SalesforceWriteOperation sfWriteOp = sfSink.createWriteOperation();
sfWriteOp.initialize(container);
try {
SalesforceWriter sfWriter = sfSink.createWriteOperation().createWriter(container);
sfWriter.open("uid1");
// Write one record, which should fail for the bad ID
IndexedRecord r = new GenericData.Record(SCHEMA_UPDATE_ACCOUNT);
r.put(0, "bad id");
r.put(1, UNIQUE_NAME + "_" + UNIQUE_ID);
r.put(2, "deleteme");
r.put(3, "deleteme");
r.put(4, "deleteme");
if (!ceaseForError) {
sfWriter.write(r);
assertThat(sfWriter.getSuccessfulWrites(), empty());
assertThat(sfWriter.getRejectedWrites(), hasSize(1));
// Check the rejected record.
IndexedRecord rejected = sfWriter.getRejectedWrites().get(0);
assertThat(rejected.getSchema().getFields(), hasSize(8));
// Check the values copied from the incoming record.
for (int i = 0; i < r.getSchema().getFields().size(); i++) {
assertThat(rejected.getSchema().getFields().get(i), is(r.getSchema().getFields().get(i)));
assertThat(rejected.get(0), is(r.get(0)));
}
// The enriched fields.
assertThat(rejected.getSchema().getFields().get(5).name(), is("errorCode"));
assertThat(rejected.getSchema().getFields().get(6).name(), is("errorFields"));
assertThat(rejected.getSchema().getFields().get(7).name(), is("errorMessage"));
assertThat(rejected.get(5), is((Object) "MALFORMED_ID"));
assertThat(rejected.get(6), is((Object) "Id"));
// removed the check on value cause it is i18n
assertThat(rejected.get(7), instanceOf(String.class));
// Finish the Writer, WriteOperation and Sink.
Result wr1 = sfWriter.close();
sfWriteOp.finalize(Arrays.asList(wr1), container);
} else {
try {
sfWriter.write(r);
sfWriter.close();
fail("It should get error when insert data!");
} catch (IOException e) {
// assertThat(e.getMessage(), is((Object) "Account ID: id value of incorrect type: bad id\n"));
throw e;
}
}
} finally {
assertTrue(file.exists());
assertNotEquals(0, file.length());
}
}
use of org.talend.components.salesforce.tsalesforceoutput.TSalesforceOutputProperties in project components by Talend.
the class SalesforceTestBase method writeRows.
// Returns the rows written (having been re-read so they have their Ids)
protected List<IndexedRecord> writeRows(String random, SalesforceConnectionModuleProperties props, List<IndexedRecord> outputRows) throws Exception {
// $NON-NLS-1$
TSalesforceOutputProperties outputProps = new TSalesforceOutputProperties("output");
outputProps.copyValuesFrom(props);
outputProps.outputAction.setValue(TSalesforceOutputProperties.OutputAction.INSERT);
doWriteRows(outputProps, outputRows);
return readAndCheckRows(random, props, outputRows.size());
}
use of org.talend.components.salesforce.tsalesforceoutput.TSalesforceOutputProperties in project components by Talend.
the class SalesforceTestBase method writeRows.
protected static void writeRows(List<IndexedRecord> outputRows) throws Exception {
// $NON-NLS-1$
TSalesforceOutputProperties outputProps = createSalesforceoutputProperties(EXISTING_MODULE_NAME);
outputProps.outputAction.setValue(TSalesforceOutputProperties.OutputAction.INSERT);
doWriteRows(outputProps, outputRows);
}
Aggregations