Examples with ShardManager com.facebook.presto.raptor.metadata.ShardManager used on opensource projects

Example 1 with ShardManager

use of com.facebook.presto.raptor.metadata.ShardManager in project presto by prestodb.

the class TestShardMetadataRecordCursor method testSimple.

@Test
public void testSimple() {
    ShardManager shardManager = createShardManager(dbi);
    // Add shards to the table
    long tableId = 1;
    OptionalInt bucketNumber = OptionalInt.empty();
    UUID uuid1 = UUID.randomUUID();
    UUID uuid2 = UUID.randomUUID();
    UUID uuid3 = UUID.randomUUID();
    ShardInfo shardInfo1 = new ShardInfo(uuid1, bucketNumber, ImmutableSet.of("node1"), ImmutableList.of(), 1, 10, 100, 0x1234);
    ShardInfo shardInfo2 = new ShardInfo(uuid2, bucketNumber, ImmutableSet.of("node2"), ImmutableList.of(), 2, 20, 200, 0xCAFEBABEDEADBEEFL);
    ShardInfo shardInfo3 = new ShardInfo(uuid3, bucketNumber, ImmutableSet.of("node3"), ImmutableList.of(), 3, 30, 300, 0xFEDCBA0987654321L);
    List<ShardInfo> shards = ImmutableList.of(shardInfo1, shardInfo2, shardInfo3);
    long transactionId = shardManager.beginTransaction();
    shardManager.commitShards(transactionId, tableId, ImmutableList.of(new ColumnInfo(1, BIGINT), new ColumnInfo(2, DATE)), shards, Optional.empty(), 0);
    Slice schema = utf8Slice(DEFAULT_TEST_ORDERS.getSchemaName());
    Slice table = utf8Slice(DEFAULT_TEST_ORDERS.getTableName());
    DateTime date1 = DateTime.parse("2015-01-01T00:00");
    DateTime date2 = DateTime.parse("2015-01-02T00:00");
    TupleDomain<Integer> tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.<Integer, Domain>builder().put(0, Domain.singleValue(createVarcharType(10), schema)).put(1, Domain.create(ValueSet.ofRanges(lessThanOrEqual(createVarcharType(10), table)), true)).put(8, Domain.create(ValueSet.ofRanges(lessThanOrEqual(BIGINT, date1.getMillis()), greaterThan(BIGINT, date2.getMillis())), true)).put(9, Domain.create(ValueSet.ofRanges(lessThanOrEqual(BIGINT, date1.getMillis()), greaterThan(BIGINT, date2.getMillis())), true)).build());
    List<MaterializedRow> actual;
    try (RecordCursor cursor = new ShardMetadataSystemTable(dbi).cursor(null, SESSION, tupleDomain)) {
        actual = getMaterializedResults(cursor, SHARD_METADATA.getColumns());
    }
    assertEquals(actual.size(), 3);
    List<MaterializedRow> expected = ImmutableList.of(new MaterializedRow(DEFAULT_PRECISION, schema, table, utf8Slice(uuid1.toString()), null, 100L, 10L, 1L, utf8Slice("0000000000001234"), null, null, null, null), new MaterializedRow(DEFAULT_PRECISION, schema, table, utf8Slice(uuid2.toString()), null, 200L, 20L, 2L, utf8Slice("cafebabedeadbeef"), null, null, null, null), new MaterializedRow(DEFAULT_PRECISION, schema, table, utf8Slice(uuid3.toString()), null, 300L, 30L, 3L, utf8Slice("fedcba0987654321"), null, null, null, null));
    assertEquals(actual, expected);
}

Example 2 with ShardManager

use of com.facebook.presto.raptor.metadata.ShardManager in project presto by prestodb.

the class TestOrcStorageManager method createOrcStorageManager.

public static OrcStorageManager createOrcStorageManager(IDBI dbi, File temporary, int maxShardRows) {
    URI directory = new File(temporary, "data").toURI();
    StorageService storageService = new LocalFileStorageService(new LocalOrcDataEnvironment(), directory);
    storageService.start();
    File backupDirectory = new File(temporary, "backup");
    FileBackupStore fileBackupStore = new FileBackupStore(backupDirectory);
    fileBackupStore.start();
    Optional<BackupStore> backupStore = Optional.of(fileBackupStore);
    ShardManager shardManager = createShardManager(dbi);
    ShardRecoveryManager recoveryManager = new ShardRecoveryManager(storageService, backupStore, new LocalOrcDataEnvironment(), new TestingNodeManager(), shardManager, MISSING_SHARD_DISCOVERY, 10);
    return createOrcStorageManager(storageService, backupStore, recoveryManager, new InMemoryShardRecorder(), maxShardRows, MAX_FILE_SIZE);
}

Example 3 with ShardManager

use of com.facebook.presto.raptor.metadata.ShardManager in project presto by prestodb.

the class TestOrcStorageManager method setup.

@BeforeMethod
public void setup() {
    temporary = createTempDir();
    URI directory = new File(temporary, "data").toURI();
    storageService = new LocalFileStorageService(new LocalOrcDataEnvironment(), directory);
    storageService.start();
    File backupDirectory = new File(temporary, "backup");
    fileBackupStore = new FileBackupStore(backupDirectory);
    fileBackupStore.start();
    backupStore = Optional.of(fileBackupStore);
    IDBI dbi = new DBI("jdbc:h2:mem:test" + System.nanoTime() + "_" + ThreadLocalRandom.current().nextInt());
    dummyHandle = dbi.open();
    createTablesWithRetry(dbi);
    ShardManager shardManager = createShardManager(dbi);
    Duration discoveryInterval = new Duration(5, TimeUnit.MINUTES);
    recoveryManager = new ShardRecoveryManager(storageService, backupStore, new LocalOrcDataEnvironment(), nodeManager, shardManager, discoveryInterval, 10);
    shardRecorder = new InMemoryShardRecorder();
}

Example 4 with ShardManager

use of com.facebook.presto.raptor.metadata.ShardManager in project presto by prestodb.

the class TestShardRecovery method setup.

@BeforeMethod
public void setup() {
    temporary = createTempDir();
    File directory = new File(temporary, "data");
    File backupDirectory = new File(temporary, "backup");
    backupStore = new FileBackupStore(backupDirectory);
    backupStore.start();
    storageService = new LocalFileStorageService(new LocalOrcDataEnvironment(), directory.toURI());
    storageService.start();
    IDBI dbi = new DBI("jdbc:h2:mem:test" + System.nanoTime() + "_" + ThreadLocalRandom.current().nextInt());
    dummyHandle = dbi.open();
    createTablesWithRetry(dbi);
    ShardManager shardManager = createShardManager(dbi);
    recoveryManager = createShardRecoveryManager(storageService, Optional.of(backupStore), shardManager);
}

Example 5 with ShardManager

use of com.facebook.presto.raptor.metadata.ShardManager in project presto by prestodb.

the class TestRaptorConnector method setup.

@BeforeMethod
public void setup() throws Exception {
    FunctionAndTypeManager functionAndTypeManager = createTestFunctionAndTypeManager();
    DBI dbi = new DBI("jdbc:h2:mem:test" + System.nanoTime() + "_" + ThreadLocalRandom.current().nextInt());
    dbi.registerMapper(new TableColumn.Mapper(functionAndTypeManager));
    dummyHandle = dbi.open();
    metadataDao = dbi.onDemand(MetadataDao.class);
    createTablesWithRetry(dbi);
    dataDir = Files.createTempDir();
    RaptorConnectorId connectorId = new RaptorConnectorId("test");
    NodeManager nodeManager = new TestingNodeManager();
    NodeSupplier nodeSupplier = nodeManager::getWorkerNodes;
    ShardManager shardManager = createShardManager(dbi);
    StorageManager storageManager = createOrcStorageManager(dbi, dataDir);
    StorageManagerConfig config = new StorageManagerConfig();
    connector = new RaptorConnector(new LifeCycleManager(ImmutableList.of(), null), new TestingNodeManager(), new RaptorMetadataFactory(connectorId, dbi, shardManager, functionAndTypeManager), new RaptorSplitManager(connectorId, nodeSupplier, shardManager, false), new RaptorPageSourceProvider(storageManager), new RaptorPageSinkProvider(storageManager, new PagesIndexPageSorter(new PagesIndex.TestingFactory(false)), new TemporalFunction(DateTimeZone.forID("America/Los_Angeles")), config), new RaptorNodePartitioningProvider(nodeSupplier), new RaptorSessionProperties(config), new RaptorTableProperties(functionAndTypeManager), ImmutableSet.of(), new AllowAllAccessControl(), dbi, ImmutableSet.of());
}