Examples with TFileInputDelimitedProperties org.talend.components.filedelimited.tfileinputdelimited.TFileInputDelimitedProperties used on opensource projects

Example 1 with TFileInputDelimitedProperties

use of org.talend.components.filedelimited.tfileinputdelimited.TFileInputDelimitedProperties in project components by Talend.

the class FileDelimitedTestIT method testInputProperties.

protected void testInputProperties() throws Throwable {
    TFileInputDelimitedProperties input = (TFileInputDelimitedProperties) new TFileInputDelimitedDefinition().createProperties();
    Form inputMainForm = input.getForm(Form.MAIN);
    ComponentTestUtils.checkSerialize(input, errorCollector);
    LOGGER.debug(inputMainForm.toString());
    assertEquals(Form.MAIN, inputMainForm.getName());
    // Default properties
    assertFalse(input.csvOptions.getValue());
    assertFalse(inputMainForm.getWidget(input.rowSeparator.getName()).isHidden());
    assertEquals("\\n", input.rowSeparator.getValue());
    assertFalse(inputMainForm.getWidget(input.fieldSeparator.getName()).isHidden());
    assertEquals(";", input.fieldSeparator.getValue());
    assertTrue(inputMainForm.getWidget(input.escapeChar.getName()).isHidden());
    assertTrue(inputMainForm.getWidget(input.textEnclosure.getName()).isHidden());
    assertFalse(inputMainForm.getWidget(input.header.getName()).isHidden());
    assertEquals(0, (int) input.header.getValue());
    assertFalse(inputMainForm.getWidget(input.footer.getName()).isHidden());
    assertEquals(0, (int) input.footer.getValue());
    assertFalse(inputMainForm.getWidget(input.limit.getName()).isHidden());
    assertNull(input.limit.getValue());
    assertFalse(inputMainForm.getWidget(input.removeEmptyRow.getName()).isHidden());
    assertTrue(input.removeEmptyRow.getValue());
    assertFalse(inputMainForm.getWidget(input.dieOnError.getName()).isHidden());
    assertFalse(input.dieOnError.getValue());
    Form inputAdvancedForm = input.getForm(Form.ADVANCED);
    assertFalse(inputAdvancedForm.getWidget(input.advancedSeparator.getName()).isHidden());
    assertFalse(input.advancedSeparator.getValue());
    assertTrue(inputAdvancedForm.getWidget(input.thousandsSeparator.getName()).isHidden());
    assertTrue(inputAdvancedForm.getWidget(input.decimalSeparator.getName()).isHidden());
    assertFalse(inputAdvancedForm.getWidget(input.random.getName()).isHidden());
    assertFalse(input.random.getValue());
    assertTrue(inputAdvancedForm.getWidget(input.nbRandom.getName()).isHidden());
    Form trimForm = inputAdvancedForm.getChildForm(input.trimColumns.getName());
    assertFalse(trimForm.getWidget(input.trimColumns.trimAll.getName()).isHidden());
    assertFalse(input.trimColumns.trimAll.getValue());
    assertFalse(trimForm.getWidget(input.trimColumns.trimTable.getName()).isHidden());
    assertNull(input.trimColumns.trimTable.trim.getValue());
    assertFalse(inputAdvancedForm.getWidget(input.checkFieldsNum.getName()).isHidden());
    assertFalse(input.checkFieldsNum.getValue());
    assertFalse(inputAdvancedForm.getWidget(input.checkDate.getName()).isHidden());
    assertFalse(input.checkDate.getValue());
    Form encodingForm = inputAdvancedForm.getChildForm(input.encoding.getName());
    assertFalse(encodingForm.getWidget(input.encoding.encodingType.getName()).isHidden());
    assertTrue(encodingForm.getWidget(input.encoding.customEncoding.getName()).isHidden());
    assertEquals(EncodingTypeProperties.ENCODING_TYPE_ISO_8859_15, input.encoding.encodingType.getValue());
    assertFalse(inputAdvancedForm.getWidget(input.splitRecord.getName()).isHidden());
    assertFalse(input.splitRecord.getValue());
    assertFalse(inputAdvancedForm.getWidget(input.enableDecode.getName()).isHidden());
    assertFalse(input.enableDecode.getValue());
    assertTrue(inputAdvancedForm.getWidget(input.decodeTable.getName()).isHidden());
    // Use uncompress
    input.uncompress.setValue(true);
    assertTrue(inputMainForm.getWidget(input.uncompress.getName()).isCallAfter());
    getComponentService().afterProperty(input.uncompress.getName(), input);
    assertTrue(inputMainForm.getWidget(input.footer.getName()).isHidden());
    assertTrue(inputAdvancedForm.getWidget(input.random.getName()).isHidden());
    assertTrue(inputAdvancedForm.getWidget(input.nbRandom.getName()).isHidden());
    input.uncompress.setValue(false);
    assertTrue(inputMainForm.getWidget(input.uncompress.getName()).isCallAfter());
    getComponentService().afterProperty(input.uncompress.getName(), input);
    // Use random
    assertFalse(inputAdvancedForm.getWidget(input.random.getName()).isHidden());
    input.random.setValue(true);
    assertTrue(inputAdvancedForm.getWidget(input.random.getName()).isCallAfter());
    getComponentService().afterProperty(input.random.getName(), input);
    assertFalse(inputAdvancedForm.getWidget(input.nbRandom.getName()).isHidden());
    assertEquals(10, (int) input.nbRandom.getValue());
    // Change to CSV mode
    input.csvOptions.setValue(true);
    assertTrue(inputMainForm.getWidget(input.csvOptions.getName()).isCallAfter());
    getComponentService().afterProperty(input.csvOptions.getName(), input);
    assertFalse(inputMainForm.getWidget(input.rowSeparator.getName()).isHidden());
    assertFalse(inputMainForm.getWidget(input.escapeChar.getName()).isHidden());
    assertFalse(inputMainForm.getWidget(input.textEnclosure.getName()).isHidden());
    assertTrue(inputAdvancedForm.getWidget(input.random.getName()).isHidden());
    assertTrue(inputAdvancedForm.getWidget(input.splitRecord.getName()).isHidden());
    // Change to advanced separator
    input.advancedSeparator.setValue(true);
    assertTrue(inputAdvancedForm.getWidget(input.advancedSeparator.getName()).isCallAfter());
    getComponentService().afterProperty(input.advancedSeparator.getName(), input);
    assertFalse(inputAdvancedForm.getWidget(input.thousandsSeparator.getName()).isHidden());
    assertEquals(",", input.thousandsSeparator.getValue());
    assertFalse(inputAdvancedForm.getWidget(input.decimalSeparator.getName()).isHidden());
    assertEquals(".", input.decimalSeparator.getValue());
    // Schema change
    input.main.schema.setValue(BASIC_SCHEMA);
    input.schemaListener.afterSchema();
    Form schemaForm = inputMainForm.getChildForm(input.main.getName());
    assertTrue(schemaForm.getWidget(input.main.schema.getName()).isCallAfter());
    getComponentService().afterProperty(input.main.schema.getName(), input.main);
    // Trim table
    input.trimColumns.trimAll.setValue(true);
    assertTrue(trimForm.getWidget(input.trimColumns.trimAll.getName()).isCallAfter());
    getComponentService().afterProperty(input.trimColumns.trimAll.getName(), input.trimColumns);
    assertNotNull(input.trimColumns.trimTable.columnName.getValue());
    assertEquals(Arrays.asList("Id", "Name", "Age"), input.trimColumns.trimTable.columnName.getValue());
    // Decode table
    input.enableDecode.setValue(true);
    assertTrue(inputAdvancedForm.getWidget(input.enableDecode.getName()).isCallAfter());
    getComponentService().afterProperty(input.enableDecode.getName(), input);
    assertNotNull(input.decodeTable.columnName.getValue());
    assertEquals(Arrays.asList("Id", "Name", "Age"), input.decodeTable.columnName.getValue());
}

Example 2 with TFileInputDelimitedProperties

use of org.talend.components.filedelimited.tfileinputdelimited.TFileInputDelimitedProperties in project components by Talend.

the class FileDelimitedReaderTestIT method testInputTrim.

protected void testInputTrim(boolean isCsvMode) throws Throwable {
    String resources = getClass().getResource("/runtime/input").toURI().getPath();
    String inputFile = resources + "/test_input_trim.csv";
    LOGGER.debug("Test file path: " + inputFile);
    TFileInputDelimitedProperties properties = createInputProperties(inputFile, isCsvMode);
    List<IndexedRecord> records = readRows(properties);
    assertEquals(20, records.size());
    List<IndexedRecord> successRecords = printLogRecords(records);
    assertEquals(14, successRecords.size());
    // "true " for TestBoolean, would parse to false
    assertEquals(false, records.get(0).get(0));
    // " " for TestBytes
    assertEquals("   ", new String((byte[]) records.get(2).get(2)));
    // " I" for TestChar
    assertEquals(" ", records.get(3).get(3));
    // " " for TestDate, Date parser trim date string automatically
    assertNull(records.get(4).get(4));
    // " " for TestObject
    assertEquals("   ", new String((byte[]) records.get(10).get(10)));
    properties.dieOnError.setValue(true);
    try {
        printLogRecords(readRows(properties));
        fail("Expect get NumberFormatException !");
    } catch (Exception e) {
        LOGGER.debug("Expect exception: " + e.getMessage());
    }
    properties.trimColumns.trimAll.setValue(true);
    records = readRows(properties);
    assertEquals(20, records.size());
    successRecords = printLogRecords(records);
    assertEquals(20, successRecords.size());
    // "true " for TestBoolean. Would parse to true after trim
    assertEquals(true, records.get(0).get(0));
    // " " for TestByte
    assertNull(records.get(1).get(1));
    // " " for TestBytes
    assertNull(records.get(2).get(2));
    // " I" for TestChar
    assertEquals("I", records.get(3).get(3));
    // " " for TestDate, Date parser trim date string automatically
    assertNull(records.get(4).get(4));
    // " " for TestDouble
    assertNull(records.get(5).get(5));
    // " " for TestFloat
    assertNull(records.get(6).get(6));
    // " " for TestBigDecimal
    assertNull(records.get(7).get(7));
    // " " for TestInteger
    assertNull(records.get(8).get(8));
    // " " for TestLong
    assertNull(records.get(9).get(9));
    // " " for TestObject
    assertNull(records.get(10).get(10));
}

Example 3 with TFileInputDelimitedProperties

use of org.talend.components.filedelimited.tfileinputdelimited.TFileInputDelimitedProperties in project components by Talend.

the class FileDelimitedReaderTestIT method testInputNBLine.

// Test FileInputDelimited component nb line count
@Test
public void testInputNBLine() throws Throwable {
    String resources = getClass().getResource("/runtime/input").toURI().getPath();
    // CSV mode
    String inputFile = resources + "/test_input_csv_reject.csv";
    TFileInputDelimitedProperties properties = createInputProperties(inputFile, true);
    // For tFileInputDelimited, number of line is total line (means included reject lines)
    readAndCheck(properties, 20, false);
    // Delimited mode
    inputFile = resources + "/test_input_delimited_reject.csv";
    properties = createInputProperties(inputFile, false);
    readAndCheck(properties, 20, false);
}

Example 4 with TFileInputDelimitedProperties

use of org.talend.components.filedelimited.tfileinputdelimited.TFileInputDelimitedProperties in project components by Talend.

the class FileDelimitedTestBasic method createInputProperties.

protected TFileInputDelimitedProperties createInputProperties(Object file, boolean isCsvMode) {
    TFileInputDelimitedProperties properties = (TFileInputDelimitedProperties) new TFileInputDelimitedDefinition().createProperties().init();
    properties.fileName.setValue(file);
    properties.rowSeparator.setValue("\n");
    if (isCsvMode) {
        properties.csvOptions.setValue(true);
        properties.escapeChar.setValue("\"");
        properties.textEnclosure.setValue("\"");
    }
    properties.header.setValue(1);
    properties.main.schema.setValue(BASIC_SCHEMA);
    ComponentTestUtils.checkSerialize(properties, errorCollector);
    return properties;
}

Example 5 with TFileInputDelimitedProperties

use of org.talend.components.filedelimited.tfileinputdelimited.TFileInputDelimitedProperties in project components by Talend.

the class FileDelimitedReaderTestIT method testInputDecodeNumber.

@Test
public void testInputDecodeNumber() throws Throwable {
    String resources = getClass().getResource("/runtime/input").toURI().getPath();
    String inputFile = resources + "/test_input_decode.csv";
    LOGGER.debug("Test file path: " + inputFile);
    TFileInputDelimitedProperties properties = createInputProperties(inputFile, false);
    properties.main.schema.setValue(NUMBER_DECODE_SCHEMA);
    properties.dieOnError.setValue(true);
    // 1. Test enable decode
    properties.enableDecode.setValue(true);
    java.util.List<String> columnsName = new java.util.ArrayList<String>();
    columnsName.add("TestBoolean");
    columnsName.add("TestByte");
    columnsName.add("TestShort");
    columnsName.add("TestInteger");
    columnsName.add("TestLong");
    properties.decodeTable.setValue("columnName", columnsName);
    java.util.List<Boolean> decodes = new java.util.ArrayList<Boolean>();
    decodes.add(false);
    decodes.add(true);
    decodes.add(true);
    decodes.add(true);
    decodes.add(true);
    properties.decodeTable.setValue("decode", decodes);
    List<IndexedRecord> records = readRows(properties);
    assertEquals(4, records.size());
    List<IndexedRecord> successRecords = printLogRecords(records);
    assertEquals(4, successRecords.size());
    assertEquals(false, records.get(0).get(0));
    // Decode OctalDigits "010" for Byte type
    assertEquals(8, records.get(0).get(1));
    // Decode OctalDigits "0100" for Short type
    assertEquals(64, records.get(0).get(2));
    // Decode OctalDigits "01000" for Integer type
    assertEquals(512, records.get(0).get(3));
    // Decode OctalDigits "010000" for Long type
    assertEquals(4096L, records.get(0).get(4));
    assertEquals(true, records.get(3).get(0));
    // Decode HexDigits "0X18" for Byte type
    assertEquals(24, records.get(3).get(1));
    // Decode HexDigits "0X188" for Short type
    assertEquals(392, records.get(3).get(2));
    // Decode HexDigits "0X1888" for Integer type
    assertEquals(6280, records.get(3).get(3));
    // Decode HexDigits "0X18888" for Long type
    assertEquals(100488L, records.get(3).get(4));
    // 2. Test disable decode and disable "die on error"
    properties.dieOnError.setValue(false);
    properties.enableDecode.setValue(false);
    records = readRows(properties);
    assertEquals(4, records.size());
    successRecords = printLogRecords(records);
    assertEquals(1, successRecords.size());
    try {
        // 3. Test disable decode and enable "die on error"
        properties.dieOnError.setValue(true);
        records = readRows(properties);
        assertEquals(4, records.size());
        printLogRecords(records);
        fail("Expect get NumberFormatException !");
    } catch (Exception e) {
        // "TestByte" parse value "0X100" fails
        LOGGER.debug(e.getMessage());
        assertEquals(NumberFormatException.class, e.getClass());
    }
    try {
        // 3. Test enable decode, enable "die on error" and disable "TestInteger" decode
        properties.enableDecode.setValue(true);
        decodes.add(3, false);
        records = readRows(properties);
        assertEquals(4, records.size());
        printLogRecords(records);
        fail("Expect get NumberFormatException !");
    } catch (Exception e) {
        // "TestInteger" parse value "0X1000" fails
        LOGGER.debug(e.getMessage());
        assertEquals(NumberFormatException.class, e.getClass());
    }
}