Examples with NodeManager com.facebook.presto.spi.NodeManager used on opensource projects

Example 1 with NodeManager

use of com.facebook.presto.spi.NodeManager in project presto by prestodb.

the class RaptorConnectorFactory method create.

@Override
public Connector create(String catalogName, Map<String, String> config, ConnectorContext context) {
    NodeManager nodeManager = context.getNodeManager();
    try {
        Bootstrap app = new Bootstrap(new JsonModule(), new MBeanModule(), binder -> {
            MBeanServer mbeanServer = new RebindSafeMBeanServer(getPlatformMBeanServer());
            binder.bind(MBeanServer.class).toInstance(mbeanServer);
            binder.bind(NodeManager.class).toInstance(nodeManager);
            binder.bind(PageSorter.class).toInstance(context.getPageSorter());
            binder.bind(TypeManager.class).toInstance(context.getTypeManager());
        }, metadataModule, new FileSystemModule(fileSystemProviders), new BackupModule(backupProviders), new StorageModule(catalogName), new RaptorModule(catalogName), new RaptorSecurityModule(), new RaptorProcedureModule());
        Injector injector = app.doNotInitializeLogging().setRequiredConfigurationProperties(config).initialize();
        return injector.getInstance(RaptorConnector.class);
    } catch (Exception e) {
        throwIfUnchecked(e);
        throw new RuntimeException(e);
    }
}

Example 2 with NodeManager

use of com.facebook.presto.spi.NodeManager in project presto by prestodb.

the class IndexedTpchConnectorFactory method create.

@Override
public Connector create(String catalogName, Map<String, String> properties, ConnectorContext context) {
    int splitsPerNode = getSplitsPerNode(properties);
    TpchIndexedData indexedData = new TpchIndexedData(catalogName, indexSpec);
    NodeManager nodeManager = context.getNodeManager();
    return new Connector() {

        @Override
        public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly) {
            return TpchTransactionHandle.INSTANCE;
        }

        @Override
        public ConnectorMetadata getMetadata(ConnectorTransactionHandle transactionHandle) {
            return new TpchIndexMetadata(catalogName, indexedData);
        }

        @Override
        public ConnectorSplitManager getSplitManager() {
            return new TpchSplitManager(nodeManager, splitsPerNode);
        }

        @Override
        public ConnectorRecordSetProvider getRecordSetProvider() {
            return new TpchRecordSetProvider();
        }

        @Override
        public ConnectorIndexProvider getIndexProvider() {
            return new TpchIndexProvider(indexedData);
        }

        @Override
        public Set<SystemTable> getSystemTables() {
            return ImmutableSet.of(new ExampleSystemTable());
        }

        @Override
        public ConnectorNodePartitioningProvider getNodePartitioningProvider() {
            return new TpchNodePartitioningProvider(nodeManager, splitsPerNode);
        }
    };
}

Example 3 with NodeManager

use of com.facebook.presto.spi.NodeManager in project presto by prestodb.

the class TpchConnectorFactory method create.

@Override
public Connector create(String catalogName, Map<String, String> properties, ConnectorContext context) {
    int splitsPerNode = getSplitsPerNode(properties);
    ColumnNaming columnNaming = ColumnNaming.valueOf(properties.getOrDefault(TPCH_COLUMN_NAMING_PROPERTY, ColumnNaming.SIMPLIFIED.name()).toUpperCase());
    NodeManager nodeManager = context.getNodeManager();
    return new Connector() {

        @Override
        public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly) {
            return TpchTransactionHandle.INSTANCE;
        }

        @Override
        public ConnectorMetadata getMetadata(ConnectorTransactionHandle transaction) {
            return new TpchMetadata(catalogName, columnNaming, predicatePushdownEnabled, isPartitioningEnabled(properties));
        }

        @Override
        public ConnectorSplitManager getSplitManager() {
            return new TpchSplitManager(nodeManager, splitsPerNode);
        }

        @Override
        public ConnectorRecordSetProvider getRecordSetProvider() {
            return new TpchRecordSetProvider();
        }

        @Override
        public ConnectorNodePartitioningProvider getNodePartitioningProvider() {
            return new TpchNodePartitioningProvider(nodeManager, splitsPerNode);
        }
    };
}

Example 4 with NodeManager

use of com.facebook.presto.spi.NodeManager in project presto by prestodb.

the class TpcdsConnectorFactory method create.

@Override
public Connector create(String catalogName, Map<String, String> config, ConnectorContext context) {
    int splitsPerNode = getSplitsPerNode(config);
    NodeManager nodeManager = context.getNodeManager();
    return new Connector() {

        @Override
        public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly) {
            return TpcdsTransactionHandle.INSTANCE;
        }

        @Override
        public ConnectorMetadata getMetadata(ConnectorTransactionHandle transactionHandle) {
            return new TpcdsMetadata();
        }

        @Override
        public ConnectorSplitManager getSplitManager() {
            return new TpcdsSplitManager(nodeManager, splitsPerNode, isWithNoSexism(config));
        }

        @Override
        public ConnectorRecordSetProvider getRecordSetProvider() {
            return new TpcdsRecordSetProvider();
        }

        @Override
        public ConnectorNodePartitioningProvider getNodePartitioningProvider() {
            return new TpcdsNodePartitioningProvider(nodeManager, splitsPerNode);
        }
    };
}

Example 5 with NodeManager

use of com.facebook.presto.spi.NodeManager in project presto by prestodb.

the class TestShardEjector method testEjector.

@Test(invocationCount = 20)
public void testEjector() throws Exception {
    NodeManager nodeManager = createNodeManager("node1", "node2", "node3", "node4", "node5");
    ShardEjector ejector = new ShardEjector(nodeManager.getCurrentNode().getNodeIdentifier(), nodeManager::getWorkerNodes, shardManager, storageService, new Duration(1, HOURS), Optional.of(new TestingBackupStore()), new LocalOrcDataEnvironment(), "test");
    List<ShardInfo> shards = ImmutableList.<ShardInfo>builder().add(shardInfo("node1", 14)).add(shardInfo("node1", 13)).add(shardInfo("node1", 12)).add(shardInfo("node1", 11)).add(shardInfo("node1", 10)).add(shardInfo("node1", 10)).add(shardInfo("node1", 10)).add(shardInfo("node1", 10)).add(shardInfo("node2", 5)).add(shardInfo("node2", 5)).add(shardInfo("node3", 10)).add(shardInfo("node4", 10)).add(shardInfo("node5", 10)).add(shardInfo("node6", 200)).build();
    long tableId = createTable("test");
    List<ColumnInfo> columns = ImmutableList.of(new ColumnInfo(1, BIGINT));
    shardManager.createTable(tableId, columns, false, OptionalLong.empty(), false);
    long transactionId = shardManager.beginTransaction();
    shardManager.commitShards(transactionId, tableId, columns, shards, Optional.empty(), 0);
    for (ShardInfo shard : shards.subList(0, 8)) {
        File file = new File(storageService.getStorageFile(shard.getShardUuid()).toString());
        storageService.createParents(new Path(file.toURI()));
        assertTrue(file.createNewFile());
    }
    ejector.process();
    shardManager.getShardNodes(tableId, TupleDomain.all(), false);
    Set<UUID> ejectedShards = shards.subList(0, 4).stream().map(ShardInfo::getShardUuid).collect(toSet());
    Set<UUID> keptShards = shards.subList(4, 8).stream().map(ShardInfo::getShardUuid).collect(toSet());
    Set<UUID> remaining = uuids(shardManager.getNodeShardsAndDeltas("node1"));
    for (UUID uuid : ejectedShards) {
        assertFalse(remaining.contains(uuid));
        assertFalse(new File(storageService.getStorageFile(uuid).toString()).exists());
    }
    assertEquals(remaining, keptShards);
    for (UUID uuid : keptShards) {
        assertTrue(new File(storageService.getStorageFile(uuid).toString()).exists());
    }
    Set<UUID> others = ImmutableSet.<UUID>builder().addAll(uuids(shardManager.getNodeShardsAndDeltas("node2"))).addAll(uuids(shardManager.getNodeShardsAndDeltas("node3"))).addAll(uuids(shardManager.getNodeShardsAndDeltas("node4"))).addAll(uuids(shardManager.getNodeShardsAndDeltas("node5"))).build();
    assertTrue(others.containsAll(ejectedShards));
}