Examples with TableRow alluxio.job.plan.transform.format.TableRow used on opensource projects

Example 1 with TableRow

use of alluxio.job.plan.transform.format.TableRow in project alluxio by Alluxio.

the class OrcReaderTest method testOrcFile.

@Test
public void testOrcFile() throws IOException {
    String resourceName = "TestOrcFile.columnProjection.orc";
    final int expectedRows = 21000;
    ClassLoader classLoader = getClass().getClassLoader();
    File file = new File(classLoader.getResource(resourceName).getFile());
    String absolutePath = file.getAbsolutePath();
    final OrcReader orcReader = OrcReader.create(new AlluxioURI("file://" + absolutePath));
    final TableRow row0 = orcReader.read();
    assertEquals(-1155869325L, row0.getColumn("int1"));
    assertEquals("bb2c72394b1ab9f8", new String((byte[]) row0.getColumn("string1")));
    final TableRow row1 = orcReader.read();
    assertEquals(431529176L, row1.getColumn("int1"));
    assertEquals("e6c5459001105f17", new String((byte[]) row1.getColumn("string1")));
    for (int i = 0; i < expectedRows - 2; i++) {
        assertNotNull(orcReader.read());
    }
    assertNull(orcReader.read());
}

Example 2 with TableRow

use of alluxio.job.plan.transform.format.TableRow in project alluxio by Alluxio.

the class CompactDefinition method runTask.

@Override
public SerializableVoid runTask(CompactConfig config, ArrayList<CompactTask> tasks, RunTaskContext context) throws Exception {
    for (CompactTask task : tasks) {
        ArrayList<String> inputs = task.getInputs();
        if (inputs.isEmpty()) {
            continue;
        }
        AlluxioURI output = new AlluxioURI(task.getOutput());
        TableSchema schema;
        try (TableReader reader = TableReader.create(new AlluxioURI(inputs.get(0)), config.getInputPartitionInfo())) {
            schema = reader.getSchema();
        }
        try (TableWriter writer = TableWriter.create(schema, output, config.getOutputPartitionInfo())) {
            for (String input : inputs) {
                try (TableReader reader = TableReader.create(new AlluxioURI(input), config.getInputPartitionInfo())) {
                    for (TableRow row = reader.read(); row != null; row = reader.read()) {
                        writer.write(row);
                    }
                }
            }
        } catch (Throwable e) {
            try {
                // outputUri is the output file
                context.getFileSystem().delete(output);
            } catch (Throwable t) {
                e.addSuppressed(t);
            }
            throw e;
        }
    }
    return null;
}