Examples with TableWriter alluxio.job.plan.transform.format.TableWriter used on opensource projects

Example 1 with TableWriter

use of alluxio.job.plan.transform.format.TableWriter in project alluxio by Alluxio.

the class CompactDefinition method runTask.

@Override
public SerializableVoid runTask(CompactConfig config, ArrayList<CompactTask> tasks, RunTaskContext context) throws Exception {
    for (CompactTask task : tasks) {
        ArrayList<String> inputs = task.getInputs();
        if (inputs.isEmpty()) {
            continue;
        }
        AlluxioURI output = new AlluxioURI(task.getOutput());
        TableSchema schema;
        try (TableReader reader = TableReader.create(new AlluxioURI(inputs.get(0)), config.getInputPartitionInfo())) {
            schema = reader.getSchema();
        }
        try (TableWriter writer = TableWriter.create(schema, output, config.getOutputPartitionInfo())) {
            for (String input : inputs) {
                try (TableReader reader = TableReader.create(new AlluxioURI(input), config.getInputPartitionInfo())) {
                    for (TableRow row = reader.read(); row != null; row = reader.read()) {
                        writer.write(row);
                    }
                }
            }
        } catch (Throwable e) {
            try {
                // outputUri is the output file
                context.getFileSystem().delete(output);
            } catch (Throwable t) {
                e.addSuppressed(t);
            }
            throw e;
        }
    }
    return null;
}