Examples with SimpleFileIODatasetProperties org.talend.components.simplefileio.SimpleFileIODatasetProperties used on opensource projects

Example 6 with SimpleFileIODatasetProperties

use of org.talend.components.simplefileio.SimpleFileIODatasetProperties in project components by Talend.

the class SimpleFileIODatasetRuntimeTest method testBasic.

@Test
public void testBasic() throws Exception {
    File input = folder.newFile("stuff.csv");
    try (FileWriter fw = new FileWriter(input)) {
        fw.write("1;one");
    }
    SimpleFileIODatasetProperties props = createDatasetProperties();
    props.path.setValue(input.toURI().toString());
    final List<IndexedRecord> consumed = new ArrayList<>();
    RuntimeInfo ri = def.getRuntimeInfo(props);
    try (SandboxedInstance si = RuntimeUtil.createRuntimeClass(ri, getClass().getClassLoader())) {
        DatasetRuntime runtime = (DatasetRuntime) si.getInstance();
        runtime.initialize(null, props);
        assertThat(runtime, not(nullValue()));
        Schema s = runtime.getSchema();
        assertThat(s, not(nullValue()));
        runtime.getSample(100, new Consumer<IndexedRecord>() {

            @Override
            public void accept(IndexedRecord ir) {
                consumed.add(ir);
            }
        });
    }
    assertThat(consumed, hasSize(1));
    assertThat(consumed.get(0).get(0), is((Object) "1"));
    assertThat(consumed.get(0).get(1), is((Object) "one"));
}

Example 7 with SimpleFileIODatasetProperties

use of org.talend.components.simplefileio.SimpleFileIODatasetProperties in project components by Talend.

the class SimpleFileIOOutputPropertiesTest method setup.

@Before
public void setup() {
    properties = new SimpleFileIOOutputProperties("test");
    SimpleFileIODatastoreProperties datastoreProperties = new SimpleFileIODatastoreProperties("test");
    datastoreProperties.init();
    SimpleFileIODatasetProperties datasetProperties = new SimpleFileIODatasetProperties("test");
    datasetProperties.init();
    datasetProperties.setDatastoreProperties(datastoreProperties);
    properties.setDatasetProperties(datasetProperties);
    properties.init();
}

Example 8 with SimpleFileIODatasetProperties

use of org.talend.components.simplefileio.SimpleFileIODatasetProperties in project components by Talend.

the class SimpleFileIODatasetRuntimeTest method testGetSampleCsv_multipleSources.

@Test
public void testGetSampleCsv_multipleSources() throws Exception {
    RecordSet rs1 = getSimpleTestData(0);
    writeRandomCsvFile(mini.getFs(), "/user/test/input/part-00000", rs1);
    RecordSet rs2 = getSimpleTestData(100);
    writeRandomCsvFile(mini.getFs(), "/user/test/input/part-00001", rs2);
    RecordSet rs3 = getSimpleTestData(100);
    writeRandomCsvFile(mini.getFs(), "/user/test/input/part-00002", rs3);
    String fileSpec = mini.getFs().getUri().resolve("/user/test/input/").toString();
    // Configure the component.
    SimpleFileIODatasetProperties props = createDatasetProperties();
    props.format.setValue(SimpleFileIOFormat.CSV);
    props.path.setValue(fileSpec);
    // Create the runtime.
    SimpleFileIODatasetRuntime runtime = new SimpleFileIODatasetRuntime();
    runtime.initialize(null, props);
    // Attempt to get a sample using the runtime methods.
    {
        final List<IndexedRecord> actual = new ArrayList<>();
        runtime.getSample(15, new Consumer<IndexedRecord>() {

            @Override
            public void accept(IndexedRecord ir) {
                actual.add(ir);
            }
        });
        // Check that the result was limited (15 out of 30 records)
        assertThat(actual, hasSize(15));
    }
    // Run it again to verify that the static state is not retained.
    {
        final List<IndexedRecord> actual = new ArrayList<>();
        runtime.getSample(15, new Consumer<IndexedRecord>() {

            @Override
            public void accept(IndexedRecord ir) {
                actual.add(ir);
            }
        });
        assertThat(actual, hasSize(15));
    }
}

Example 9 with SimpleFileIODatasetProperties

use of org.talend.components.simplefileio.SimpleFileIODatasetProperties in project components by Talend.

the class SimpleFileIODatasetRuntimeTest method testGetSampleCsv.

@Test
public void testGetSampleCsv() throws Exception {
    RecordSet rs = getSimpleTestData(0);
    writeRandomCsvFile(mini.getFs(), "/user/test/input.csv", rs);
    String fileSpec = mini.getFs().getUri().resolve("/user/test/input.csv").toString();
    // Configure the component.
    SimpleFileIODatasetProperties props = createDatasetProperties();
    props.format.setValue(SimpleFileIOFormat.CSV);
    props.path.setValue(fileSpec);
    // Create the runtime.
    SimpleFileIODatasetRuntime runtime = new SimpleFileIODatasetRuntime();
    runtime.initialize(null, props);
    // Attempt to get a sample using the runtime methods.
    final List<IndexedRecord> actual = new ArrayList<>();
    runtime.getSample(100, new Consumer<IndexedRecord>() {

        @Override
        public void accept(IndexedRecord ir) {
            actual.add(ir);
        }
    });
    // Check the expected values match.
    assertThat(actual, hasSize(10));
// assertThat(actual, (Matcher) equalTo(rs.getAllData()));
}

Example 10 with SimpleFileIODatasetProperties

use of org.talend.components.simplefileio.SimpleFileIODatasetProperties in project components by Talend.

the class SimpleFileIODatasetRuntimeTest method testGetSampleAvro.

@Test
public void testGetSampleAvro() throws Exception {
    RecordSet rs = getSimpleTestData(0);
    writeRandomAvroFile(mini.getFs(), "/user/test/input.avro", rs);
    String fileSpec = mini.getFs().getUri().resolve("/user/test/input.avro").toString();
    // Configure the component.
    SimpleFileIODatasetProperties props = createDatasetProperties();
    props.format.setValue(SimpleFileIOFormat.AVRO);
    props.path.setValue(fileSpec);
    // Create the runtime.
    SimpleFileIODatasetRuntime runtime = new SimpleFileIODatasetRuntime();
    runtime.initialize(null, props);
    // Attempt to get a sample using the runtime methods.
    final List<IndexedRecord> actual = new ArrayList<>();
    runtime.getSample(100, new Consumer<IndexedRecord>() {

        @Override
        public void accept(IndexedRecord ir) {
            actual.add(ir);
        }
    });
    // Check the expected values.
    assertThat(actual, (Matcher) equalTo(rs.getAllData()));
}