use of com.facebook.presto.raptor.filesystem.LocalOrcDataEnvironment in project presto by prestodb.
the class TestOrcStorageManager method testWriter.
@Test
public void testWriter() throws Exception {
OrcStorageManager manager = createOrcStorageManager();
List<Long> columnIds = ImmutableList.of(3L, 7L);
List<Type> columnTypes = ImmutableList.of(BIGINT, createVarcharType(10));
StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes);
List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello").row(456L, "bye").build();
sink.appendPages(pages);
// shard is not recorded until flush
assertEquals(shardRecorder.getShards().size(), 0);
sink.flush();
// shard is recorded after flush
List<RecordedShard> recordedShards = shardRecorder.getShards();
assertEquals(recordedShards.size(), 1);
List<ShardInfo> shards = getFutureValue(sink.commit());
assertEquals(shards.size(), 1);
ShardInfo shardInfo = Iterables.getOnlyElement(shards);
UUID shardUuid = shardInfo.getShardUuid();
File file = new File(storageService.getStorageFile(shardUuid).toString());
File backupFile = fileBackupStore.getBackupFile(shardUuid);
assertEquals(recordedShards.get(0).getTransactionId(), TRANSACTION_ID);
assertEquals(recordedShards.get(0).getShardUuid(), shardUuid);
assertEquals(shardInfo.getRowCount(), 2);
assertEquals(shardInfo.getCompressedSize(), file.length());
assertEquals(shardInfo.getXxhash64(), xxhash64(new RaptorLocalFileSystem(new Configuration()), new Path(file.toURI())));
// verify primary and backup shard exist
assertFile(file, "primary shard");
assertFile(backupFile, "backup shard");
assertFileEquals(file, backupFile);
// remove primary shard to force recovery from backup
assertTrue(file.delete());
assertTrue(file.getParentFile().delete());
assertFalse(file.exists());
recoveryManager.restoreFromBackup(shardUuid, shardInfo.getCompressedSize(), OptionalLong.of(shardInfo.getXxhash64()));
FileSystem fileSystem = new LocalOrcDataEnvironment().getFileSystem(DEFAULT_RAPTOR_CONTEXT);
try (OrcDataSource dataSource = manager.openShard(fileSystem, shardUuid, READER_ATTRIBUTES)) {
OrcBatchRecordReader reader = createReader(dataSource, columnIds, columnTypes);
assertEquals(reader.nextBatch(), 2);
Block column0 = reader.readBlock(0);
assertEquals(column0.isNull(0), false);
assertEquals(column0.isNull(1), false);
assertEquals(BIGINT.getLong(column0, 0), 123L);
assertEquals(BIGINT.getLong(column0, 1), 456L);
Block column1 = reader.readBlock(1);
assertEquals(createVarcharType(10).getSlice(column1, 0), utf8Slice("hello"));
assertEquals(createVarcharType(10).getSlice(column1, 1), utf8Slice("bye"));
assertEquals(reader.nextBatch(), -1);
}
}
use of com.facebook.presto.raptor.filesystem.LocalOrcDataEnvironment in project presto by prestodb.
the class TestOrcStorageManager method setup.
@BeforeMethod
public void setup() {
temporary = createTempDir();
URI directory = new File(temporary, "data").toURI();
storageService = new LocalFileStorageService(new LocalOrcDataEnvironment(), directory);
storageService.start();
File backupDirectory = new File(temporary, "backup");
fileBackupStore = new FileBackupStore(backupDirectory);
fileBackupStore.start();
backupStore = Optional.of(fileBackupStore);
IDBI dbi = new DBI("jdbc:h2:mem:test" + System.nanoTime() + "_" + ThreadLocalRandom.current().nextInt());
dummyHandle = dbi.open();
createTablesWithRetry(dbi);
ShardManager shardManager = createShardManager(dbi);
Duration discoveryInterval = new Duration(5, TimeUnit.MINUTES);
recoveryManager = new ShardRecoveryManager(storageService, backupStore, new LocalOrcDataEnvironment(), nodeManager, shardManager, discoveryInterval, 10);
shardRecorder = new InMemoryShardRecorder();
}
use of com.facebook.presto.raptor.filesystem.LocalOrcDataEnvironment in project presto by prestodb.
the class TestShardRecovery method setup.
@BeforeMethod
public void setup() {
temporary = createTempDir();
File directory = new File(temporary, "data");
File backupDirectory = new File(temporary, "backup");
backupStore = new FileBackupStore(backupDirectory);
backupStore.start();
storageService = new LocalFileStorageService(new LocalOrcDataEnvironment(), directory.toURI());
storageService.start();
IDBI dbi = new DBI("jdbc:h2:mem:test" + System.nanoTime() + "_" + ThreadLocalRandom.current().nextInt());
dummyHandle = dbi.open();
createTablesWithRetry(dbi);
ShardManager shardManager = createShardManager(dbi);
recoveryManager = createShardRecoveryManager(storageService, Optional.of(backupStore), shardManager);
}
use of com.facebook.presto.raptor.filesystem.LocalOrcDataEnvironment in project presto by prestodb.
the class TestOrcFileRewriter method testRewriterDropThenAddSameColumns.
/**
* The following test drop and add the same columns; the legacy ORC rewriter will fail due to unchanged schema.
* However, if we enforce the newly added column to always have the largest ID, this won't happen.
*/
@Test
public void testRewriterDropThenAddSameColumns() throws Exception {
FunctionAndTypeManager functionAndTypeManager = createTestFunctionAndTypeManager();
DBI dbi = new DBI("jdbc:h2:mem:test" + System.nanoTime() + "_" + ThreadLocalRandom.current().nextInt());
dbi.registerMapper(new TableColumn.Mapper(functionAndTypeManager));
Handle dummyHandle = dbi.open();
File dataDir = Files.createTempDir();
StorageManager storageManager = createOrcStorageManager(dbi, dataDir);
List<Long> columnIds = ImmutableList.of(3L, 7L);
List<Type> columnTypes = ImmutableList.of(BIGINT, createVarcharType(20));
File file = new File(temporary, randomUUID().toString());
try (FileWriter writer = createFileWriter(columnIds, columnTypes, file, false)) {
List<Page> pages = rowPagesBuilder(columnTypes).row(2L, "2").build();
writer.appendPages(pages);
}
// Add a column
File newFile1 = new File(temporary, randomUUID().toString());
FileSystem fileSystem = new LocalOrcDataEnvironment().getFileSystem(DEFAULT_RAPTOR_CONTEXT);
OrcFileInfo info = createFileRewriter().rewrite(fileSystem, getColumnTypes(ImmutableList.of(3L, 7L, 10L), ImmutableList.of(BIGINT, createVarcharType(20), DOUBLE)), path(file), path(newFile1), new BitSet(5));
assertEquals(info.getRowCount(), 1);
// Drop a column
File newFile2 = new File(temporary, randomUUID().toString());
info = createFileRewriter().rewrite(fileSystem, getColumnTypes(ImmutableList.of(7L, 10L), ImmutableList.of(createVarcharType(20), DOUBLE)), path(newFile1), path(newFile2), new BitSet(5));
assertEquals(info.getRowCount(), 1);
// Add a column with the same ID but different type
File newFile3 = new File(temporary, randomUUID().toString());
info = createFileRewriter().rewrite(fileSystem, getColumnTypes(ImmutableList.of(7L, 10L, 3L), ImmutableList.of(createVarcharType(20), DOUBLE, createVarcharType(5))), path(newFile2), path(newFile3), new BitSet(5));
assertEquals(info.getRowCount(), 1);
// Get prepared for the final file; make sure it is accessible from storage manager
UUID uuid = randomUUID();
File newFile4 = getFileSystemPath(new File(dataDir, "data/storage"), uuid);
// Optimized ORC writer does not create the file itself
newFile4.getParentFile().mkdirs();
newFile4.createNewFile();
// Drop a column and add a column
info = createFileRewriter().rewrite(fileSystem, getColumnTypes(ImmutableList.of(7L, 3L, 8L), ImmutableList.of(createVarcharType(20), createVarcharType(5), INTEGER)), path(newFile3), path(newFile4), new BitSet(5));
assertEquals(info.getRowCount(), 1);
ConnectorPageSource source = storageManager.getPageSource(DEFAULT_RAPTOR_CONTEXT, DEFAULT_HIVE_FILE_CONTEXT, uuid, Optional.empty(), false, OptionalInt.empty(), ImmutableList.of(3L, 7L, 8L), ImmutableList.of(createVarcharType(5), createVarcharType(20), INTEGER), TupleDomain.all(), READER_ATTRIBUTES);
Page page = null;
while (page == null) {
page = source.getNextPage();
}
assertEquals(page.getPositionCount(), 1);
try {
// Column 3L
Block column0 = page.getBlock(0);
assertTrue(column0.isNull(0));
// Column 7L
Block column1 = page.getBlock(1);
assertEquals(createVarcharType(20).getSlice(column1, 0), utf8Slice("2"));
// Column 8L
Block column2 = page.getBlock(2);
assertTrue(column2.isNull(0));
dummyHandle.close();
deleteRecursively(dataDir.toPath(), ALLOW_INSECURE);
} catch (UnsupportedOperationException e) {
// Optimized ORC rewriter will respect the schema
fail();
}
}
use of com.facebook.presto.raptor.filesystem.LocalOrcDataEnvironment in project presto by prestodb.
the class TestBackupManager method setup.
@BeforeMethod
public void setup() {
temporary = createTempDir();
FileBackupStore fileStore = new FileBackupStore(new File(temporary, "backup"));
fileStore.start();
backupStore = new TestingBackupStore(fileStore);
storageService = new LocalFileStorageService(new LocalOrcDataEnvironment(), new File(temporary, "data").toURI());
storageService.start();
backupManager = new BackupManager(Optional.of(backupStore), storageService, new LocalOrcDataEnvironment(), 5);
}
Aggregations