Examples with TFileInputDelimitedProperties org.talend.components.filedelimited.tfileinputdelimited.TFileInputDelimitedProperties used on opensource projects

Example 6 with TFileInputDelimitedProperties

use of org.talend.components.filedelimited.tfileinputdelimited.TFileInputDelimitedProperties in project components by Talend.

the class FileDelimitedReaderTestIT method testInputCompressFile.

protected void testInputCompressFile(boolean isCsvMode) throws Throwable {
    String resources = getClass().getResource("/runtime/input").toURI().getPath();
    String inputFile = resources + "/test_input_compress_delimited.zip";
    if (isCsvMode) {
        inputFile = resources + "/test_input_compress_csv.zip";
    }
    LOGGER.debug("Test file path: " + inputFile);
    TFileInputDelimitedProperties properties = createInputProperties(inputFile, isCsvMode);
    properties.uncompress.setValue(true);
    // Records count (15 + 35 + 8)=58
    testInputCompress(properties, 58);
    // Records count (13 + 33 + 6)=52
    properties.header.setValue(3);
    testInputCompress(properties, 52);
    properties.limit.setValue(10);
    // For read compressed file, the limit is for every single entry, not fore all records.
    // So here the return revords count is (10 + 10 +6)=26
    testInputCompress(properties, 26);
}

Example 7 with TFileInputDelimitedProperties

use of org.talend.components.filedelimited.tfileinputdelimited.TFileInputDelimitedProperties in project components by Talend.

the class FileDelimitedReaderTestIT method testInputDynamic.

protected void testInputDynamic(boolean isCsvMode) throws Throwable {
    String resources = getClass().getResource("/runtime/input").toURI().getPath();
    String inputFile = resources + "/input_delimited_dynamic.csv";
    if (isCsvMode) {
        inputFile = resources + "/input_csv_dynamic.csv";
    }
    LOGGER.debug("Test file path: " + inputFile);
    TFileInputDelimitedProperties properties = createInputProperties(inputFile, isCsvMode);
    properties.main.schema.setValue(BASIC_DYNAMIC_SCHEMA);
    // row1 & row2 in the file is empty
    // But "removeEmptyRow" default value is true. So would skip the empty row in the header part
    testInputDynamic(properties, 20);
    // Set header as not empty row
    properties.header.setValue(3);
    testInputDynamic(properties, 20);
    properties.limit.setValue(10);
    testInputDynamic(properties, 9);
    // when specified header line include empty field value like ";;;;"
    // Avro can be set field name empty. But we can do that before
    properties.removeEmptyRow.setValue(false);
// properties.header.setValue(1);
// testInputDynamic(properties, 9);
}

Example 8 with TFileInputDelimitedProperties

use of org.talend.components.filedelimited.tfileinputdelimited.TFileInputDelimitedProperties in project components by Talend.

the class FileDelimitedReaderTestIT method testInputDieOnErrorDelimitedMode.

// Test FileInputDelimited component read with delimited mode and die on error
@Test(expected = NumberFormatException.class)
public void testInputDieOnErrorDelimitedMode() throws Throwable {
    String resources = getClass().getResource("/runtime/input").toURI().getPath();
    String inputFile = resources + "/test_input_delimited_reject.csv";
    LOGGER.debug("Test file path: " + inputFile);
    TFileInputDelimitedProperties properties = createInputProperties(inputFile, false);
    properties.dieOnError.setValue(true);
    try {
        testInputReject(properties);
    } catch (Exception e) {
        LOGGER.debug("Expect exception: " + e.getMessage());
        throw e;
    }
}

Example 9 with TFileInputDelimitedProperties

use of org.talend.components.filedelimited.tfileinputdelimited.TFileInputDelimitedProperties in project components by Talend.

the class FileDelimitedReaderTestIT method testInputTransformNumberString.

@Test
public void testInputTransformNumberString() throws Throwable {
    String resources = getClass().getResource("/runtime/input").toURI().getPath();
    String inputFile = resources + "/test_input_transform_number.csv";
    LOGGER.debug("Test file path: " + inputFile);
    TFileInputDelimitedProperties properties = createInputProperties(inputFile, false);
    properties.main.schema.setValue(NUMBER_DECODE_SCHEMA);
    properties.dieOnError.setValue(true);
    properties.enableDecode.setValue(true);
    java.util.List<String> columnsName = new java.util.ArrayList<String>();
    columnsName.add("TestBoolean");
    columnsName.add("TestByte");
    columnsName.add("TestShort");
    columnsName.add("TestInteger");
    columnsName.add("TestLong");
    properties.decodeTable.setValue("columnName", columnsName);
    java.util.List<Boolean> decodes = new java.util.ArrayList<Boolean>();
    decodes.add(false);
    decodes.add(true);
    decodes.add(true);
    decodes.add(true);
    decodes.add(true);
    properties.decodeTable.setValue("decode", decodes);
    // Default thousandsSeparator and decimalSeparator setting is disabled
    try {
        List<IndexedRecord> records = readRows(properties);
        assertEquals(4, records.size());
        printLogRecords(records);
        fail("Expect get NumberFormatException !");
    } catch (Exception e) {
        // "TestInteger" parse value "01,000" fails
        e.printStackTrace();
        LOGGER.debug(e.getMessage());
        assertEquals(NumberFormatException.class, e.getClass());
    }
    // Enable thousandsSeparator and decimalSeparator setting
    properties.advancedSeparator.setValue(true);
    properties.thousandsSeparator.setValue(",");
    properties.decimalSeparator.setValue(".");
    List<IndexedRecord> records = readRows(properties);
    assertEquals(4, records.size());
    List<IndexedRecord> successRecords = printLogRecords(records);
    assertEquals(4, successRecords.size());
    assertEquals(true, records.get(3).get(0));
    // Decode HexDigits "0X18"
    assertEquals(24, records.get(3).get(1));
    // Decode HexDigits and transform 0X1,888 for Short type
    assertEquals(6280, records.get(3).get(2));
    // Decode HexDigits and transform "0X18,888" for Integer type
    assertEquals(100488, records.get(3).get(3));
    // Decode HexDigits and transform "0X188,888" for Long type
    assertEquals(1607816L, records.get(3).get(4));
}

Example 10 with TFileInputDelimitedProperties

use of org.talend.components.filedelimited.tfileinputdelimited.TFileInputDelimitedProperties in project components by Talend.

the class FileDelimitedReaderTestIT method testInputCheckDate.

// Test FileInputDelimited component read anc check date
@Test(expected = RuntimeException.class)
public void testInputCheckDate() throws Throwable {
    String resources = getClass().getResource("/runtime/input").toURI().getPath();
    String inputFile = resources + "/test_input_check_date.csv";
    LOGGER.debug("Test file path: " + inputFile);
    TFileInputDelimitedProperties properties = createInputProperties(inputFile, false);
    properties.dieOnError.setValue(true);
    // "Check date" not check. This means "2016-18-06T15:31:07" with wrong month number would be parsed lenient
    List<IndexedRecord> successRecords = printLogRecords(readRows(properties));
    assertEquals(20, successRecords.size());
    // "Check date" check. This means "2016-18-06T15:31:07" with wrong month number would throw exception
    properties.checkDate.setValue(true);
    try {
        printLogRecords(readRows(properties));
    } catch (Exception e) {
        LOGGER.debug("Expect exception: " + e.getMessage());
        throw e;
    }
}