Examples with LazySchema org.hillview.table.LazySchema used on opensource projects

Example 1 with LazySchema

use of org.hillview.table.LazySchema in project hillview by vmware.

the class LoadCsvColumnsSketch method create.

@Nullable
@Override
public ControlMessage.StatusList create(@Nullable ITable data) {
    HillviewLogger.instance.info("Loading CSV columns for table", "Columns are {0}", this.schema.toString());
    Converters.checkNull(data);
    CsvFileLoader.Config config = new CsvFileLoader.Config();
    config.hasHeaderRow = true;
    CsvFileLoader loader = new CsvFileLoader(// initially to load the table.
    Converters.checkNull(data.getSourceFile()), config, new LazySchema(this.schema));
    ITable loaded = loader.load();
    Converters.checkNull(loaded);
    for (String c : this.schema.getColumnNames()) {
        IColumn ld = loaded.getLoadedColumn(c);
        LazyColumn lc = data.getColumn(c).as(LazyColumn.class);
        Converters.checkNull(lc);
        if (lc.sizeInRows() != ld.sizeInRows())
            throw new RuntimeException("Loaded column has different size from original column:" + " file=" + data.getSourceFile() + " loaded=" + ld.toString() + " size=" + ld.sizeInRows() + " original=" + lc.toString() + " size=" + lc.sizeInRows());
        Converters.checkNull(lc).setData(ld);
    }
    return new ControlMessage.StatusList(new ControlMessage.Status("OK"));
}

Example 2 with LazySchema

use of org.hillview.table.LazySchema in project hillview by vmware.

the class OrcFileTest method writeSmallFileTest.

@Test
public void writeSmallFileTest() {
    String orcFile = orcFolder + "tmp.orc";
    File file = new File(orcFile);
    if (file.exists()) {
        boolean success = file.delete();
        Assert.assertTrue(success);
    }
    Table t = TestTables.testTable();
    OrcFileWriter ofw = new OrcFileWriter(orcFile);
    ofw.writeTable(t);
    OrcFileLoader loader = new OrcFileLoader(orcFile, new LazySchema(), false);
    ITable table = loader.load();
    Assert.assertNotNull(table);
    Assert.assertEquals(t.toLongString(20), table.toLongString(20));
    deleteOrcFile(orcFolder, "tmp.orc");
}

Example 3 with LazySchema

use of org.hillview.table.LazySchema in project hillview by vmware.

the class OrcFileTest method writeNullTest.

@Test
public void writeNullTest() {
    String orcFile = orcFolder + "tmp.orc";
    File file = new File(orcFile);
    if (file.exists()) {
        boolean success = file.delete();
        Assert.assertTrue(success);
    }
    ColumnDescription str = new ColumnDescription("Str", ContentsKind.String);
    StringArrayColumn strCol = new StringArrayColumn(str, 2);
    strCol.set(0, "Something");
    strCol.set(1, null);
    List<IColumn> cols = new ArrayList<IColumn>();
    cols.add(strCol);
    Table tbl = new Table(cols, null, null);
    OrcFileWriter ofw = new OrcFileWriter(orcFile);
    ofw.writeTable(tbl);
    OrcFileLoader loader = new OrcFileLoader(orcFile, new LazySchema(), false);
    ITable table = loader.load();
    Assert.assertNotNull(table);
    Assert.assertEquals(tbl.toLongString(20), table.toLongString(20));
    deleteOrcFile(orcFolder, "tmp.orc");
}

Example 4 with LazySchema

use of org.hillview.table.LazySchema in project hillview by vmware.

the class OrcFileTest method readOrcFileLazyTest.

@Test
public void readOrcFileLazyTest() {
    String orcFile = orcFolder + orcOutFile;
    OrcFileLoader loader = new OrcFileLoader(orcFile, new LazySchema(), true);
    ITable table = loader.load();
    Table ref = TestTables.testRepTable();
    Assert.assertNotNull(table);
    Assert.assertEquals(ref.toLongString(20), table.toLongString(20));
}

Example 5 with LazySchema

use of org.hillview.table.LazySchema in project hillview by vmware.

the class OrcFileTest method readOrcFileTestWithSchema.

@Test
public void readOrcFileTestWithSchema() {
    String orcFile = orcFolder + orcOutFile;
    Table ref = TestTables.testRepTable();
    Schema schema = ref.getSchema();
    String tmpSchema = "tmpOrcSchema";
    schema.writeToJsonFile(Paths.get(tmpSchema));
    OrcFileLoader loader = new OrcFileLoader(orcFile, new LazySchema(tmpSchema), false);
    ITable table = loader.load();
    Assert.assertNotNull(table);
    Assert.assertEquals(ref.toLongString(20), table.toLongString(20));
    Assert.assertEquals(ref.getSchema().toString(), table.getSchema().toString());
    File file = new File(tmpSchema);
    if (file.exists()) {
        boolean success = file.delete();
        Assert.assertTrue(success);
    }
}