Examples with ConnectorId com.facebook.presto.spi.ConnectorId used on opensource projects

Example 91 with ConnectorId

use of com.facebook.presto.spi.ConnectorId in project presto by prestodb.

the class TestResourceManagerClusterStateProvider method testOutOfOrderUpdatesIgnored.

@Test(timeOut = 15_000)
public void testOutOfOrderUpdatesIgnored() throws Exception {
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node1", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node2", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    ResourceManagerClusterStateProvider provider = new ResourceManagerClusterStateProvider(nodeManager, new SessionPropertyManager(), 10, Duration.valueOf("4s"), Duration.valueOf("8s"), Duration.valueOf("5s"), Duration.valueOf("0s"), true, newSingleThreadScheduledExecutor());
    assertEquals(provider.getClusterQueries(), ImmutableList.of());
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", QUEUED), 1);
    provider.registerQueryHeartbeat("node1", createQueryInfo("2", FINISHED), 2);
    assertQueryInfos(provider.getClusterQueries(), 2, 1);
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", FINISHED), 0);
    provider.registerQueryHeartbeat("node1", createQueryInfo("2", RUNNING), 1);
    assertQueryInfos(provider.getClusterQueries(), 2, 1);
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", FINISHED), 2);
    assertQueryInfos(provider.getClusterQueries(), 2, 2);
}

Example 92 with ConnectorId

use of com.facebook.presto.spi.ConnectorId in project presto by prestodb.

the class TestResourceManagerClusterStateProvider method testResourceGroupsMerged.

@Test(timeOut = 15_000)
public void testResourceGroupsMerged() throws Exception {
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node1", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node2", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node3", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node4", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node5", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node6", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    ResourceManagerClusterStateProvider provider = new ResourceManagerClusterStateProvider(nodeManager, new SessionPropertyManager(), 10, Duration.valueOf("4s"), Duration.valueOf("8s"), Duration.valueOf("50s"), Duration.valueOf("0s"), true, newSingleThreadScheduledExecutor());
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("local"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node1"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node2"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node3"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node4"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node5"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node6"));
    assertEquals(provider.getClusterQueries(), ImmutableList.of());
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", QUEUED, "rg4", GENERAL_POOL), 0);
    assertTrue(provider.getClusterResourceGroups("node1").isEmpty());
    assertResourceGroup(provider, "node2", "rg4", 1, 0, DataSize.valueOf("1MB"));
    provider.registerQueryHeartbeat("node2", createQueryInfo("2", RUNNING, "rg4", GENERAL_POOL), 0);
    assertResourceGroup(provider, "node1", "rg4", 0, 1, DataSize.valueOf("1MB"));
    assertResourceGroup(provider, "node2", "rg4", 1, 0, DataSize.valueOf("1MB"));
    assertResourceGroup(provider, "node3", "rg4", 1, 1, DataSize.valueOf("2MB"));
    provider.registerQueryHeartbeat("node3", createQueryInfo("3", FINISHED, "rg4", GENERAL_POOL), 0);
    assertResourceGroup(provider, "node1", "rg4", 0, 1, DataSize.valueOf("1MB"));
    assertResourceGroup(provider, "node2", "rg4", 1, 0, DataSize.valueOf("1MB"));
    assertResourceGroup(provider, "node3", "rg4", 1, 1, DataSize.valueOf("2MB"));
    assertResourceGroup(provider, "node4", "rg4", 1, 1, DataSize.valueOf("2MB"));
    provider.registerQueryHeartbeat("node4", createQueryInfo("4", FAILED, "rg4", GENERAL_POOL), 0);
    assertResourceGroup(provider, "node1", "rg4", 0, 1, DataSize.valueOf("1MB"));
    assertResourceGroup(provider, "node2", "rg4", 1, 0, DataSize.valueOf("1MB"));
    assertResourceGroup(provider, "node3", "rg4", 1, 1, DataSize.valueOf("2MB"));
    assertResourceGroup(provider, "node4", "rg4", 1, 1, DataSize.valueOf("2MB"));
    assertResourceGroup(provider, "node5", "rg4", 1, 1, DataSize.valueOf("2MB"));
    // Add queries which are in non-terminal states other than RUNNING and QUEUED
    provider.registerQueryHeartbeat("node1", createQueryInfo("5", WAITING_FOR_RESOURCES, "rg4", GENERAL_POOL), 0);
    provider.registerQueryHeartbeat("node2", createQueryInfo("6", DISPATCHING, "rg4", GENERAL_POOL), 0);
    provider.registerQueryHeartbeat("node3", createQueryInfo("7", PLANNING, "rg4", GENERAL_POOL), 0);
    provider.registerQueryHeartbeat("node4", createQueryInfo("8", STARTING, "rg4", GENERAL_POOL), 0);
    provider.registerQueryHeartbeat("node5", createQueryInfo("9", FINISHING, "rg4", GENERAL_POOL), 0);
    assertResourceGroup(provider, "node1", "rg4", 0, 5, DataSize.valueOf("5MB"));
    assertResourceGroup(provider, "node2", "rg4", 1, 4, DataSize.valueOf("5MB"));
    assertResourceGroup(provider, "node3", "rg4", 1, 5, DataSize.valueOf("6MB"));
    assertResourceGroup(provider, "node4", "rg4", 1, 5, DataSize.valueOf("6MB"));
    assertResourceGroup(provider, "node5", "rg4", 1, 5, DataSize.valueOf("6MB"));
    assertResourceGroup(provider, "node6", "rg4", 1, 6, DataSize.valueOf("7MB"));
    // Expire running queries
    Thread.sleep(SECONDS.toMillis(5));
    assertTrue(provider.getClusterResourceGroups("node1").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node2").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node3").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node4").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node5").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node6").isEmpty());
}

Example 93 with ConnectorId

use of com.facebook.presto.spi.ConnectorId in project presto by prestodb.

the class TestResourceManagerClusterStateProvider method testQueryInfo.

@Test(timeOut = 15_000)
public void testQueryInfo() throws Exception {
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node1", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node2", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    ResourceManagerClusterStateProvider provider = new ResourceManagerClusterStateProvider(nodeManager, new SessionPropertyManager(), 10, Duration.valueOf("4s"), Duration.valueOf("8s"), Duration.valueOf("5s"), Duration.valueOf("0s"), true, newSingleThreadScheduledExecutor());
    assertEquals(provider.getClusterQueries(), ImmutableList.of());
    long query1Sequence = 0;
    long query2Sequence = 0;
    long query3Sequence = 0;
    long query4Sequence = 0;
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", QUEUED), query1Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("2", RUNNING), query2Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("3", FINISHED), query3Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("4", FAILED), query4Sequence++);
    assertQueryInfos(provider.getClusterQueries(), 4, 2);
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", RUNNING), query1Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("2", FINISHING), query2Sequence++);
    assertQueryInfos(provider.getClusterQueries(), 4, 2);
    // Update query 2 to FINISHED to verify this is now completed in the resource manager
    provider.registerQueryHeartbeat("node1", createQueryInfo("2", FINISHED), query2Sequence++);
    assertQueryInfos(provider.getClusterQueries(), 4, 3);
    // Mix in queries from another coordinator
    provider.registerQueryHeartbeat("node2", createQueryInfo("1", QUEUED), query1Sequence++);
    provider.registerQueryHeartbeat("node2", createQueryInfo("2", RUNNING), query2Sequence++);
    provider.registerQueryHeartbeat("node2", createQueryInfo("3", FINISHED), query3Sequence++);
    provider.registerQueryHeartbeat("node2", createQueryInfo("4", FAILED), query4Sequence++);
    assertQueryInfos(provider.getClusterQueries(), 8, 5);
    // Expire completed queries
    Thread.sleep(SECONDS.toMillis(5));
    assertQueryInfos(provider.getClusterQueries(), 8, 5);
    // Expire all queries
    Thread.sleep(SECONDS.toMillis(5));
    assertQueryInfos(provider.getClusterQueries(), 0, 0);
}

Example 94 with ConnectorId

use of com.facebook.presto.spi.ConnectorId in project presto by prestodb.

the class TestTransactionManager method registerConnector.

private static void registerConnector(CatalogManager catalogManager, TransactionManager transactionManager, String catalogName, ConnectorId connectorId, Connector connector) {
    ConnectorId systemId = createSystemTablesConnectorId(connectorId);
    InternalNodeManager nodeManager = new InMemoryNodeManager();
    MetadataManager metadata = MetadataManager.createTestMetadataManager(catalogManager);
    catalogManager.registerCatalog(new Catalog(catalogName, connectorId, connector, createInformationSchemaConnectorId(connectorId), new InformationSchemaConnector(catalogName, nodeManager, metadata, new AllowAllAccessControl(), ImmutableList.of()), systemId, new SystemConnector(systemId, nodeManager, connector.getSystemTables(), transactionId -> transactionManager.getConnectorTransaction(transactionId, connectorId))));
}

Example 95 with ConnectorId

use of com.facebook.presto.spi.ConnectorId in project presto by prestodb.

the class ThriftMetastoreModule method setup.

@Override
protected void setup(Binder binder) {
    binder.bind(HiveMetastoreClientFactory.class).in(Scopes.SINGLETON);
    binder.bind(HiveCluster.class).to(StaticHiveCluster.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(StaticMetastoreConfig.class);
    binder.bind(HiveMetastore.class).to(ThriftHiveMetastore.class).in(Scopes.SINGLETON);
    binder.bind(ConnectorId.class).toInstance(new ConnectorId(connectorId));
    if (buildConfigObject(MetastoreClientConfig.class).getRecordingPath() != null) {
        binder.bind(ExtendedHiveMetastore.class).annotatedWith(ForRecordingHiveMetastore.class).to(BridgingHiveMetastore.class).in(Scopes.SINGLETON);
        binder.bind(ExtendedHiveMetastore.class).annotatedWith(ForCachingHiveMetastore.class).to(RecordingHiveMetastore.class).in(Scopes.SINGLETON);
        binder.bind(RecordingHiveMetastore.class).in(Scopes.SINGLETON);
        newExporter(binder).export(RecordingHiveMetastore.class).as(generatedNameOf(RecordingHiveMetastore.class, connectorId));
    } else {
        binder.bind(ExtendedHiveMetastore.class).annotatedWith(ForCachingHiveMetastore.class).to(BridgingHiveMetastore.class).in(Scopes.SINGLETON);
    }
    binder.bind(ExtendedHiveMetastore.class).to(CachingHiveMetastore.class).in(Scopes.SINGLETON);
    newExporter(binder).export(HiveMetastore.class).as(generatedNameOf(ThriftHiveMetastore.class, connectorId));
    newExporter(binder).export(ExtendedHiveMetastore.class).as(generatedNameOf(CachingHiveMetastore.class, connectorId));
}