Examples with RaptorLocalFileSystem com.facebook.presto.raptor.filesystem.RaptorLocalFileSystem used on opensource projects

Example 1 with RaptorLocalFileSystem

use of com.facebook.presto.raptor.filesystem.RaptorLocalFileSystem in project presto by prestodb.

the class TestShardRecovery method testShardRecoveryExistingFileChecksumMismatch.

@Test
public void testShardRecoveryExistingFileChecksumMismatch() throws Exception {
    UUID shardUuid = UUID.randomUUID();
    // write data and backup
    File tempFile = createTempFile("tmp", null, temporary);
    Files.write("test data", tempFile, UTF_8);
    backupStore.backupShard(shardUuid, tempFile);
    assertTrue(backupStore.shardExists(shardUuid));
    File backupFile = backupStore.getBackupFile(shardUuid);
    assertTrue(Files.equal(tempFile, backupFile));
    // write corrupt storage file with wrong data
    File storageFile = new File(storageService.getStorageFile(shardUuid).toString());
    storageService.createParents(new Path(storageFile.toURI()));
    Files.write("test xata", storageFile, UTF_8);
    assertTrue(storageFile.exists());
    assertEquals(storageFile.length(), tempFile.length());
    assertFalse(Files.equal(storageFile, tempFile));
    // restore from backup and verify
    recoveryManager.restoreFromBackup(shardUuid, tempFile.length(), OptionalLong.of(xxhash64(new RaptorLocalFileSystem(new Configuration()), new Path(tempFile.toURI()))));
    assertTrue(storageFile.exists());
    assertTrue(Files.equal(storageFile, tempFile));
    // verify quarantine exists
    List<String> quarantined = listFiles(new File(storageService.getQuarantineFile(shardUuid).getParent().toString()));
    assertEquals(quarantined.size(), 1);
    assertTrue(getOnlyElement(quarantined).startsWith(shardUuid + ".orc.corrupt"));
}

Example 2 with RaptorLocalFileSystem

use of com.facebook.presto.raptor.filesystem.RaptorLocalFileSystem in project presto by prestodb.

the class TestShardRecovery method testShardRecoveryBackupChecksumMismatch.

@Test
public void testShardRecoveryBackupChecksumMismatch() throws Exception {
    UUID shardUuid = UUID.randomUUID();
    // write storage file
    File storageFile = new File(storageService.getStorageFile(shardUuid).toString());
    storageService.createParents(new Path(storageFile.toURI()));
    Files.write("test data", storageFile, UTF_8);
    long size = storageFile.length();
    long xxhash64 = xxhash64(new RaptorLocalFileSystem(new Configuration()), new Path(storageFile.toURI()));
    // backup and verify
    backupStore.backupShard(shardUuid, storageFile);
    assertTrue(backupStore.shardExists(shardUuid));
    File backupFile = backupStore.getBackupFile(shardUuid);
    assertTrue(Files.equal(storageFile, backupFile));
    // corrupt backup file
    Files.write("test xata", backupFile, UTF_8);
    assertTrue(backupFile.exists());
    assertEquals(storageFile.length(), backupFile.length());
    assertFalse(Files.equal(storageFile, backupFile));
    // delete local file to force restore
    assertTrue(storageFile.delete());
    assertFalse(storageFile.exists());
    // restore should fail
    try {
        recoveryManager.restoreFromBackup(shardUuid, size, OptionalLong.of(xxhash64));
        fail("expected exception");
    } catch (PrestoException e) {
        assertEquals(e.getErrorCode(), RAPTOR_BACKUP_CORRUPTION.toErrorCode());
        assertEquals(e.getMessage(), "Backup is corrupt after read: " + shardUuid);
    }
    // verify quarantine exists
    List<String> quarantined = listFiles(new File(storageService.getQuarantineFile(shardUuid).getParent().toString()));
    assertEquals(quarantined.size(), 1);
    assertTrue(getOnlyElement(quarantined).startsWith(shardUuid + ".orc.corrupt"));
}

Example 3 with RaptorLocalFileSystem

use of com.facebook.presto.raptor.filesystem.RaptorLocalFileSystem in project presto by prestodb.

the class TestOrcStorageManager method testWriter.

@Test
public void testWriter() throws Exception {
    OrcStorageManager manager = createOrcStorageManager();
    List<Long> columnIds = ImmutableList.of(3L, 7L);
    List<Type> columnTypes = ImmutableList.of(BIGINT, createVarcharType(10));
    StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes);
    List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello").row(456L, "bye").build();
    sink.appendPages(pages);
    // shard is not recorded until flush
    assertEquals(shardRecorder.getShards().size(), 0);
    sink.flush();
    // shard is recorded after flush
    List<RecordedShard> recordedShards = shardRecorder.getShards();
    assertEquals(recordedShards.size(), 1);
    List<ShardInfo> shards = getFutureValue(sink.commit());
    assertEquals(shards.size(), 1);
    ShardInfo shardInfo = Iterables.getOnlyElement(shards);
    UUID shardUuid = shardInfo.getShardUuid();
    File file = new File(storageService.getStorageFile(shardUuid).toString());
    File backupFile = fileBackupStore.getBackupFile(shardUuid);
    assertEquals(recordedShards.get(0).getTransactionId(), TRANSACTION_ID);
    assertEquals(recordedShards.get(0).getShardUuid(), shardUuid);
    assertEquals(shardInfo.getRowCount(), 2);
    assertEquals(shardInfo.getCompressedSize(), file.length());
    assertEquals(shardInfo.getXxhash64(), xxhash64(new RaptorLocalFileSystem(new Configuration()), new Path(file.toURI())));
    // verify primary and backup shard exist
    assertFile(file, "primary shard");
    assertFile(backupFile, "backup shard");
    assertFileEquals(file, backupFile);
    // remove primary shard to force recovery from backup
    assertTrue(file.delete());
    assertTrue(file.getParentFile().delete());
    assertFalse(file.exists());
    recoveryManager.restoreFromBackup(shardUuid, shardInfo.getCompressedSize(), OptionalLong.of(shardInfo.getXxhash64()));
    FileSystem fileSystem = new LocalOrcDataEnvironment().getFileSystem(DEFAULT_RAPTOR_CONTEXT);
    try (OrcDataSource dataSource = manager.openShard(fileSystem, shardUuid, READER_ATTRIBUTES)) {
        OrcBatchRecordReader reader = createReader(dataSource, columnIds, columnTypes);
        assertEquals(reader.nextBatch(), 2);
        Block column0 = reader.readBlock(0);
        assertEquals(column0.isNull(0), false);
        assertEquals(column0.isNull(1), false);
        assertEquals(BIGINT.getLong(column0, 0), 123L);
        assertEquals(BIGINT.getLong(column0, 1), 456L);
        Block column1 = reader.readBlock(1);
        assertEquals(createVarcharType(10).getSlice(column1, 0), utf8Slice("hello"));
        assertEquals(createVarcharType(10).getSlice(column1, 1), utf8Slice("bye"));
        assertEquals(reader.nextBatch(), -1);
    }
}