Examples with Reader org.talend.components.api.component.runtime.Reader used on opensource projects

Example 26 with Reader

use of org.talend.components.api.component.runtime.Reader in project components by Talend.

the class JDBCInputTestIT method testReaderAllTypesString.

/**
 * Checks {@link JDBCInputReader} outputs {@link IndexedRecord} which contains nullable String type data for every SQL/JDBC
 * type
 */
@Test
public void testReaderAllTypesString() throws IOException {
    TJDBCInputDefinition definition = new TJDBCInputDefinition();
    TJDBCInputProperties properties = DBTestUtils.createCommonJDBCInputProperties(allSetting, definition);
    properties.main.schema.setValue(DBTestUtils.createAllTypesSchema(tablename_all_type));
    properties.tableSelection.tablename.setValue(tablename_all_type);
    properties.sql.setValue(DBTestUtils.getSQL(tablename_all_type));
    Reader reader = DBTestUtils.createCommonJDBCInputReader(properties);
    reader.start();
    IndexedRecord record = (IndexedRecord) reader.getCurrent();
    Short col0 = (Short) record.get(0);
    Integer col1 = (Integer) record.get(1);
    Long col2 = (Long) record.get(2);
    Float col3 = (Float) record.get(3);
    Double col4 = (Double) record.get(4);
    BigDecimal col5 = (BigDecimal) record.get(5);
    String col6 = (String) record.get(6);
    String col7 = (String) record.get(7);
    String col8 = (String) record.get(8);
    String col9 = (String) record.get(9);
    Long col10 = (Long) record.get(10);
    Long col11 = (Long) record.get(11);
    Long col12 = (Long) record.get(12);
    Boolean col13 = (Boolean) record.get(13);
    assertEquals(32767, col0.shortValue());
    assertEquals(2147483647, col1.intValue());
    assertEquals(9223372036854775807l, col2.longValue());
    assertTrue(col3 > 1);
    assertTrue(col4 > 2);
    assertEquals(new BigDecimal("1234567890.1234567890"), col5);
    assertEquals("abcd", col6);
    assertEquals("abcdefg", col7);
    assertEquals("00010203040506070809", col8);
    assertEquals("abcdefg", col9);
    assertEquals("2016-12-28", new SimpleDateFormat("yyyy-MM-dd").format(new Date(col10)));
    assertEquals("14:30:33", new SimpleDateFormat("HH:mm:ss").format(new Date(col11)));
    assertEquals("2016-12-28 14:31:56.123", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").format(new Date(col12)));
    assertEquals(true, col13);
    Schema actualSchema = record.getSchema();
    List<Field> actualFields = actualSchema.getFields();
    assertEquals(14, actualFields.size());
    reader.close();
}

Example 27 with Reader

use of org.talend.components.api.component.runtime.Reader in project components by Talend.

the class JDBCRowTestIT method test_basic_as_input.

@SuppressWarnings("rawtypes")
@Test
public void test_basic_as_input() throws Exception {
    TJDBCRowDefinition definition = new TJDBCRowDefinition();
    TJDBCRowProperties properties = DBTestUtils.createCommonJDBCRowProperties(allSetting, definition);
    Schema schema = DBTestUtils.createTestSchema4(tablename);
    properties.main.schema.setValue(schema);
    properties.updateOutputSchemas();
    properties.tableSelection.tablename.setValue(tablename);
    properties.sql.setValue("select id, name from " + tablename);
    properties.dieOnError.setValue(true);
    randomCommit(properties);
    // the field is the unique reason to use the component as a input
    properties.propagateQueryResultSet.setValue(true);
    // component
    properties.beforeUseColumn();
    properties.useColumn.setValue(properties.useColumn.getPossibleValues().get(0).toString());
    JDBCRowSource source = new JDBCRowSource();
    source.initialize(null, properties);
    ValidationResult result = source.validate(null);
    Assert.assertTrue(result.getStatus() == ValidationResult.Result.OK);
    Reader reader = source.createReader(null);
    try {
        reader.start();
        IndexedRecord row = (IndexedRecord) reader.getCurrent();
        ResultSet resultSet = (ResultSet) row.get(0);
        resultSet.next();
        Assert.assertEquals(1, resultSet.getInt(1));
        Assert.assertEquals("wangwei", resultSet.getString(2));
        resultSet.next();
        Assert.assertEquals(2, resultSet.getInt(1));
        Assert.assertEquals("gaoyan", resultSet.getString(2));
        resultSet.next();
        Assert.assertEquals(3, resultSet.getInt(1));
        Assert.assertEquals("dabao", resultSet.getString(2));
        resultSet.close();
        // only output one row when it works as a input component
        Assert.assertFalse(reader.advance());
        reader.close();
    } finally {
        reader.close();
    }
}

Example 28 with Reader

use of org.talend.components.api.component.runtime.Reader in project components by Talend.

the class SalesforceWriterTestIT method cleanupAllRecords.

@AfterClass
public static void cleanupAllRecords() throws NoSuchElementException, IOException {
    List<IndexedRecord> recordsToClean = new ArrayList<>();
    String prefixToDelete = UNIQUE_NAME + "_" + UNIQUE_ID;
    // Get the list of records that match the prefix to delete.
    {
        TSalesforceInputProperties sfProps = getSalesforceInputProperties();
        SalesforceTestBase.setupProps(sfProps.connection, false);
        sfProps.module.setValue("moduleName", "Account");
        sfProps.module.main.schema.setValue(SCHEMA_UPDATE_ACCOUNT);
        DefaultComponentRuntimeContainerImpl container = new DefaultComponentRuntimeContainerImpl();
        // Initialize the Source and Reader
        SalesforceSource sfSource = new SalesforceSource();
        sfSource.initialize(container, sfProps);
        sfSource.validate(container);
        int nameIndex = -1;
        @SuppressWarnings("unchecked") Reader<IndexedRecord> sfReader = sfSource.createReader(container);
        if (sfReader.start()) {
            do {
                IndexedRecord r = sfReader.getCurrent();
                if (nameIndex == -1) {
                    nameIndex = r.getSchema().getField("Name").pos();
                }
                if (String.valueOf(r.get(nameIndex)).startsWith(prefixToDelete)) {
                    recordsToClean.add(r);
                }
            } while (sfReader.advance());
        }
    }
    // Delete those records.
    {
        ComponentDefinition sfDef = new TSalesforceOutputDefinition();
        TSalesforceOutputProperties sfProps = (TSalesforceOutputProperties) sfDef.createRuntimeProperties();
        SalesforceTestBase.setupProps(sfProps.connection, false);
        sfProps.outputAction.setValue(OutputAction.DELETE);
        sfProps.module.setValue("moduleName", "Account");
        sfProps.module.main.schema.setValue(SCHEMA_UPDATE_ACCOUNT);
        DefaultComponentRuntimeContainerImpl container = new DefaultComponentRuntimeContainerImpl();
        // Initialize the Sink, WriteOperation and Writer
        SalesforceSink sfSink = new SalesforceSink();
        sfSink.initialize(container, sfProps);
        sfSink.validate(container);
        SalesforceWriteOperation sfWriteOp = sfSink.createWriteOperation();
        sfWriteOp.initialize(container);
        Writer<Result> sfWriter = sfSink.createWriteOperation().createWriter(container);
        sfWriter.open("uid1");
        // Write one record.
        for (IndexedRecord r : recordsToClean) {
            sfWriter.write(r);
        }
        // Finish the Writer, WriteOperation and Sink.
        Result wr1 = sfWriter.close();
        sfWriteOp.finalize(Arrays.asList(wr1), container);
    }
}

Example 29 with Reader

use of org.talend.components.api.component.runtime.Reader in project components by Talend.

the class SalesforceBulkLoadTestIT method testUpdate.

@Test
public void testUpdate() throws Throwable {
    List<String> ids = util.createTestData();
    String id = ids.get(0);
    final List<Map<String, String>> testData = new ArrayList<Map<String, String>>();
    Map<String, String> datarow = new HashMap<String, String>();
    datarow.put("Id", id);
    datarow.put("FirstName", "Wei");
    datarow.put("LastName", "Wang");
    // update the field
    datarow.put("Phone", "010-89492686");
    testData.add(datarow);
    datarow = new HashMap<String, String>();
    // should reject
    datarow.put("Id", "not_exist");
    datarow.put("FirstName", "Who");
    datarow.put("LastName", "Who");
    datarow.put("Phone", "010-89492686");
    testData.add(datarow);
    String data_file = tempFolder.newFile("data.txt").getAbsolutePath();
    // outputbulk part
    TSalesforceOutputBulkDefinition definition = (TSalesforceOutputBulkDefinition) getComponentService().getComponentDefinition(TSalesforceOutputBulkDefinition.COMPONENT_NAME);
    TSalesforceOutputBulkProperties modelProperties = util.simulateUserBasicAction(definition, data_file, util.getTestSchema4());
    util.simulateRuntimeCaller(definition, modelProperties, util.getTestSchema4(), testData);
    // bulkexec part
    TSalesforceBulkExecDefinition defin = (TSalesforceBulkExecDefinition) getComponentService().getComponentDefinition(TSalesforceBulkExecDefinition.COMPONENT_NAME);
    TSalesforceBulkExecProperties modelProps = (TSalesforceBulkExecProperties) defin.createRuntimeProperties();
    Reader reader = util.initReader(defin, data_file, modelProps, util.getTestSchema4(), util.getTestSchema4());
    modelProps.outputAction.setValue(TSalesforceBulkExecProperties.OutputAction.UPDATE);
    try {
        IndexedRecordConverter<Object, ? extends IndexedRecord> factory = null;
        for (boolean available = reader.start(); available; available = reader.advance()) {
            try {
                Object data = reader.getCurrent();
                factory = initCurrentData(factory, data);
                IndexedRecord record = factory.convertToAvro(data);
                String resultid = (String) record.get(0);
                String phone = (String) record.get(3);
                Assert.assertEquals(id, resultid);
                Assert.assertEquals("010-89492686", phone);
            } catch (DataRejectException e) {
                java.util.Map<String, Object> info = e.getRejectInfo();
                Object data = info.get("talend_record");
                String err = (String) info.get("error");
                factory = initCurrentData(factory, data);
                IndexedRecord record = factory.convertToAvro(data);
                String resultid = (String) record.get(0);
                String firstname = (String) record.get(1);
                String lastname = (String) record.get(2);
                String phone = (String) record.get(3);
                Assert.assertNull(resultid);
                Assert.assertEquals("Who", firstname);
                Assert.assertEquals("Who", lastname);
                Assert.assertEquals("010-89492686", phone);
                Assert.assertTrue(err != null);
            }
        }
    } finally {
        try {
            reader.close();
        } finally {
            util.deleteTestData(ids);
        }
    }
}

Example 30 with Reader

use of org.talend.components.api.component.runtime.Reader in project components by Talend.

the class SalesforceBulkLoadTestIT method testUpsert.

@Test
public void testUpsert() throws Throwable {
    List<String> ids = util.createTestData();
    String id = ids.get(0);
    final List<Map<String, String>> testData = new ArrayList<Map<String, String>>();
    Map<String, String> datarow = new HashMap<String, String>();
    // should update
    datarow.put("Id", id);
    datarow.put("FirstName", "Wei");
    datarow.put("LastName", "Wang");
    // update the field
    datarow.put("Phone", "010-89492686");
    testData.add(datarow);
    datarow = new HashMap<String, String>();
    // should insert
    datarow.put("Id", null);
    datarow.put("FirstName", "Who");
    datarow.put("LastName", "Who");
    datarow.put("Phone", "010-89492686");
    testData.add(datarow);
    String data_file = tempFolder.newFile("data.txt").getAbsolutePath();
    // outputbulk part
    TSalesforceOutputBulkDefinition definition = (TSalesforceOutputBulkDefinition) getComponentService().getComponentDefinition(TSalesforceOutputBulkDefinition.COMPONENT_NAME);
    TSalesforceOutputBulkProperties modelProperties = util.simulateUserBasicAction(definition, data_file, util.getTestSchema4());
    util.simulateRuntimeCaller(definition, modelProperties, util.getTestSchema4(), testData);
    // bulkexec part
    TSalesforceBulkExecDefinition defin = (TSalesforceBulkExecDefinition) getComponentService().getComponentDefinition(TSalesforceBulkExecDefinition.COMPONENT_NAME);
    TSalesforceBulkExecProperties modelProps = (TSalesforceBulkExecProperties) defin.createRuntimeProperties();
    Reader reader = util.initReader(defin, data_file, modelProps, util.getTestSchema4(), util.getTestSchema4());
    modelProps.outputAction.setValue(TSalesforceBulkExecProperties.OutputAction.UPSERT);
    modelProps.upsertKeyColumn.setValue("Id");
    try {
        IndexedRecordConverter<Object, ? extends IndexedRecord> factory = null;
        int index = -1;
        for (boolean available = reader.start(); available; available = reader.advance()) {
            try {
                Object data = reader.getCurrent();
                factory = initCurrentData(factory, data);
                IndexedRecord record = factory.convertToAvro(data);
                String resultid = (String) record.get(0);
                String phone = (String) record.get(3);
                index++;
                if (index == 0) {
                    Assert.assertEquals(id, resultid);
                    Assert.assertEquals("010-89492686", phone);
                } else if (index == 1) {
                    Assert.assertTrue(resultid != null);
                    Assert.assertEquals("010-89492686", phone);
                }
            } catch (DataRejectException e) {
                Assert.fail(e.getMessage());
            }
        }
    } finally {
        try {
            reader.close();
        } finally {
            util.deleteTestData(ids);
        }
    }
}