Examples with FileWriteInfo com.facebook.presto.hive.PartitionUpdate.FileWriteInfo used on opensource projects

Example 6 with FileWriteInfo

use of com.facebook.presto.hive.PartitionUpdate.FileWriteInfo in project presto by prestodb.

the class TestPartitionUpdate method testRoundTrip.

@Test
public void testRoundTrip() {
    PartitionUpdate expected = new PartitionUpdate("test", UpdateMode.APPEND, "/writePath", "/targetPath", ImmutableList.of(new PartitionUpdate.FileWriteInfo(".file1", "file1", Optional.empty()), new FileWriteInfo(".file3", "file3", Optional.empty())), 123, 456, 789, false);
    PartitionUpdate actual = CODEC.fromJson(CODEC.toJson(expected));
    assertEquals(actual.getName(), "test");
    assertEquals(actual.getUpdateMode(), UpdateMode.APPEND);
    assertEquals(actual.getWritePath(), new Path("/writePath"));
    assertEquals(actual.getTargetPath(), new Path("/targetPath"));
    assertEquals(actual.getFileWriteInfos(), ImmutableList.of(new FileWriteInfo(".file1", "file1", Optional.empty()), new FileWriteInfo(".file3", "file3", Optional.empty())));
    assertEquals(actual.getRowCount(), 123);
    assertEquals(actual.getInMemoryDataSizeInBytes(), 456);
    assertEquals(actual.getOnDiskDataSizeInBytes(), 789);
}

Example 7 with FileWriteInfo

use of com.facebook.presto.hive.PartitionUpdate.FileWriteInfo in project presto by prestodb.

the class HiveStagingFileCommitter method commitFiles.

@Override
public ListenableFuture<Void> commitFiles(ConnectorSession session, String schemaName, String tableName, String tablePath, boolean isCreateTable, List<PartitionUpdate> partitionUpdates) {
    HdfsContext context = new HdfsContext(session, schemaName, tableName, tablePath, isCreateTable);
    List<ListenableFuture<Void>> commitFutures = new ArrayList<>();
    for (PartitionUpdate partitionUpdate : partitionUpdates) {
        Path path = partitionUpdate.getWritePath();
        FileSystem fileSystem = getFileSystem(hdfsEnvironment, context, path);
        for (FileWriteInfo fileWriteInfo : partitionUpdate.getFileWriteInfos()) {
            checkState(!fileWriteInfo.getWriteFileName().equals(fileWriteInfo.getTargetFileName()));
            Path source = new Path(path, fileWriteInfo.getWriteFileName());
            Path target = new Path(path, fileWriteInfo.getTargetFileName());
            commitFutures.add(fileRenameExecutor.submit(() -> {
                renameFile(fileSystem, source, target);
                return null;
            }));
        }
    }
    ListenableFuture<Void> result = whenAllSucceed(commitFutures).call(() -> null, directExecutor());
    return catching(result, RuntimeException.class, e -> {
        checkState(e != null, "Null exception is caught during commitFiles");
        result.cancel(true);
        throw e;
    }, directExecutor());
}