Examples with CSVFileSource com.airhacks.enhydrator.in.CSVFileSource used on opensource projects

Example 1 with CSVFileSource

use of com.airhacks.enhydrator.in.CSVFileSource in project enhydrator by AdamBien.

the class ParentTest method copy.

/**
     * Name;Size;Folder
     */
@Test
public void copy() {
    Source source = new CSVFileSource("./src/test/files/files.csv", ";", "UTF-8", true);
    VirtualSinkSource vss = new VirtualSinkSource();
    Pump pump = new Pump.Engine().from(source).startWith(new SkipFirstRow()).startWith(new NonRecursiveTree("Name", "Folder")).to(vss).to(new LogSink()).build();
    pump.start();
    System.out.println(vss.getRows());
    int numberOfRows = vss.getNumberOfRows();
    assertThat(numberOfRows, is(2));
    Row parentWithChildren = vss.getRow(0);
    assertNotNull(parentWithChildren);
    assertThat(parentWithChildren.getNumberOfColumns(), is(3));
    List<Row> children = parentWithChildren.getChildren();
    assertThat(children.size(), is(2));
    children.forEach(e -> assertThat(e.getNumberOfColumns(), is(2)));
}

Example 2 with CSVFileSource

use of com.airhacks.enhydrator.in.CSVFileSource in project enhydrator by AdamBien.

the class PipelineTest method getCSVPipeline.

public static Pipeline getCSVPipeline() {
    DestinationMapper targetMapper = new DestinationMapper();
    targetMapper.addMapping(0, new TargetMapping("*", "*"));
    DatatypeIndexMapper datatypeMapper = new DatatypeIndexMapper();
    datatypeMapper.addMapping(0, Datatype.DOUBLE);
    Source source = new CSVFileSource("./src/test/files/pyramid.csv", ";", "UTF-8", true);
    NamedSink logSink = new LogSink();
    NamedSink jdbcSink = new VirtualSinkSource();
    ColumnTransformation e1 = new ColumnTransformation("name", "convert", true);
    ColumnTransformation e2 = new ColumnTransformation(42, "compress", true);
    Pipeline origin = new Pipeline("csv", "src/test/scripts", "select * from Coffee where name like ? and strength = ?", source);
    origin.addSink(logSink);
    origin.addSink(jdbcSink);
    origin.addQueryParam("arabica");
    origin.addQueryParam(2);
    origin.addEntryTransformation(e1);
    origin.addEntryTransformation(e2);
    origin.addPreRowTransformation(targetMapper);
    origin.addPreRowTransformation(datatypeMapper);
    origin.addEntryTransformation(new ColumnTransformation("1", "function execute(i){return 42;}", false));
    origin.addPreRowTransformation(new NashornRowTransformer("src/test/scripts", "encrypt"));
    origin.addPostRowTransformation(new NashornRowTransformer("src/test/scripts", "compress"));
    origin.addFilter("true");
    origin.addExpression("print($ROW); $ROW");
    return origin;
}

Example 3 with CSVFileSource

use of com.airhacks.enhydrator.in.CSVFileSource in project enhydrator by AdamBien.

the class CSVFileSinkTest method serializeLineWithHeaders.

@Test
public void serializeLineWithHeaders() {
    cut.init();
    final Row entries = getEntries();
    cut.processRow(entries);
    cut.close();
    CSVFileSource source = new CSVFileSource(FILE_NAME, DELIMITER, "utf-8", USE_HEADERS);
    Iterable<Row> result = source.query(null, null);
    Row read = result.iterator().next();
    assertNotNull(read);
    System.out.println(entries.getColumnNames() + " " + read.getColumnNames());
    System.out.println(entries.getColumnValues().values() + " " + read.getColumnValues().values());
    if (USE_HEADERS) {
        read.getColumnNames().stream().forEach(t -> assertTrue(entries.getColumnNames().contains(t)));
        // ensure the order of the columns
        assertEquals("Column One", read.getColumnByIndex(0).getName());
        assertEquals("Column Two", read.getColumnByIndex(1).getName());
        assertEquals("Column Three (empty)", read.getColumnByIndex(2).getName());
        assertEquals("Column Four", read.getColumnByIndex(3).getName());
        assertEquals("Column Five (empty)", read.getColumnByIndex(4).getName());
    } else {
        read.getColumnValues().values().stream().forEach(t -> assertTrue(entries.getColumnValues().values().contains(t)));
        // ensure the order of the columns
        assertEquals("java", read.getColumnByIndex(0).getValue());
        assertEquals("tengah", read.getColumnByIndex(1).getValue());
        assertNull(read.getColumnByIndex(2).getValue());
        assertEquals("groovy", read.getColumnByIndex(3).getValue());
        assertNull(read.getColumnByIndex(4).getValue());
    }
}

Example 4 with CSVFileSource

use of com.airhacks.enhydrator.in.CSVFileSource in project enhydrator by AdamBien.

the class ReadJDKVersionsTest method readFromCSV.

@Test
public void readFromCSV() {
    Source input = new CSVFileSource("./src/test/files/jdk-dates.csv", ";", "UTF-8", true);
    VirtualSinkSource output = new VirtualSinkSource();
    Pump pump = new Pump.Engine().from(input).startWith(new SkipFirstRow()).startWith(new DatatypeNameMapper().addMapping("Year", Datatype.INTEGER)).filter("$ROW.getColumnValue('Year') > 2000").to(new LogSink()).to(output).build();
    Memory memory = pump.start();
    Assert.assertFalse(memory.areErrorsOccured());
    List<Row> rows = output.getRows();
    assertFalse(rows.isEmpty());
    Row first = rows.get(1);
    Object year = first.getColumnValue("Year");
    System.out.println("The year is: " + year);
}

Example 5 with CSVFileSource

use of com.airhacks.enhydrator.in.CSVFileSource in project enhydrator by AdamBien.

the class CSVImportTest method getSource.

VirtualSinkSource getSource(final String fileName) {
    Source source = new CSVFileSource(fileName, ";", "UTF-8", true);
    VirtualSinkSource vss = new VirtualSinkSource();
    Pump pump = new Pump.Engine().from(source).to(vss).to(new LogSink()).build();
    pump.start();
    return vss;
}