Examples with ConnectorId com.facebook.presto.spi.ConnectorId used on opensource projects

Example 76 with ConnectorId

use of com.facebook.presto.spi.ConnectorId in project presto by prestodb.

the class TestAccessControlManager method registerBogusConnector.

private static ConnectorId registerBogusConnector(CatalogManager catalogManager, TransactionManager transactionManager, AccessControl accessControl, String catalogName) {
    ConnectorId connectorId = new ConnectorId(catalogName);
    Connector connector = new TpchConnectorFactory().create(catalogName, ImmutableMap.of(), new TestingConnectorContext());
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    MetadataManager metadata = MetadataManager.createTestMetadataManager(catalogManager);
    ConnectorId systemId = createSystemTablesConnectorId(connectorId);
    catalogManager.registerCatalog(new Catalog(catalogName, connectorId, connector, createInformationSchemaConnectorId(connectorId), new InformationSchemaConnector(catalogName, nodeManager, metadata, accessControl, ImmutableList.of()), systemId, new SystemConnector(systemId, nodeManager, connector.getSystemTables(), transactionId -> transactionManager.getConnectorTransaction(transactionId, connectorId))));
    return connectorId;
}

Example 77 with ConnectorId

use of com.facebook.presto.spi.ConnectorId in project presto by prestodb.

the class TestAccessControlManager method testDenyCatalogAccessControl.

@Test(expectedExceptions = PrestoException.class, expectedExceptionsMessageRegExp = "Access Denied: Cannot select from columns \\[column\\] in table or view schema.table")
public void testDenyCatalogAccessControl() {
    CatalogManager catalogManager = new CatalogManager();
    TransactionManager transactionManager = createTestTransactionManager(catalogManager);
    AccessControlManager accessControlManager = new AccessControlManager(transactionManager);
    TestSystemAccessControlFactory accessControlFactory = new TestSystemAccessControlFactory("test");
    accessControlManager.addSystemAccessControlFactory(accessControlFactory);
    accessControlManager.setSystemAccessControl("test", ImmutableMap.of());
    ConnectorId connectorId = registerBogusConnector(catalogManager, transactionManager, accessControlManager, "catalog");
    accessControlManager.addCatalogAccessControl(connectorId, new DenyConnectorAccessControl());
    transaction(transactionManager, accessControlManager).execute(transactionId -> {
        accessControlManager.checkCanSelectFromColumns(transactionId, new Identity(USER_NAME, Optional.of(PRINCIPAL)), new AccessControlContext(new QueryId(QUERY_ID), Optional.empty(), Optional.empty()), new QualifiedObjectName("catalog", "schema", "table"), ImmutableSet.of("column"));
    });
}

Example 78 with ConnectorId

use of com.facebook.presto.spi.ConnectorId in project presto by prestodb.

the class TestAccessControlManager method testDenySystemAccessControl.

@Test(expectedExceptions = PrestoException.class, expectedExceptionsMessageRegExp = "Access Denied: Cannot select from table secured_catalog.schema.table")
public void testDenySystemAccessControl() {
    CatalogManager catalogManager = new CatalogManager();
    TransactionManager transactionManager = createTestTransactionManager(catalogManager);
    AccessControlManager accessControlManager = new AccessControlManager(transactionManager);
    TestSystemAccessControlFactory accessControlFactory = new TestSystemAccessControlFactory("test");
    accessControlManager.addSystemAccessControlFactory(accessControlFactory);
    accessControlManager.setSystemAccessControl("test", ImmutableMap.of());
    registerBogusConnector(catalogManager, transactionManager, accessControlManager, "connector");
    accessControlManager.addCatalogAccessControl(new ConnectorId("connector"), new DenyConnectorAccessControl());
    transaction(transactionManager, accessControlManager).execute(transactionId -> {
        accessControlManager.checkCanSelectFromColumns(transactionId, new Identity(USER_NAME, Optional.of(PRINCIPAL)), new AccessControlContext(new QueryId(QUERY_ID), Optional.empty(), Optional.empty()), new QualifiedObjectName("secured_catalog", "schema", "table"), ImmutableSet.of("column"));
    });
}

Example 79 with ConnectorId

use of com.facebook.presto.spi.ConnectorId in project presto by prestodb.

the class TestResourceManagerClusterStateProvider method testNonLeafResourceGroupsMerged.

@Test
public void testNonLeafResourceGroupsMerged() throws Exception {
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node1", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node2", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node3", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node4", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node5", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node6", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    ResourceManagerClusterStateProvider provider = new ResourceManagerClusterStateProvider(nodeManager, new SessionPropertyManager(), 10, Duration.valueOf("4s"), Duration.valueOf("8s"), Duration.valueOf("50s"), Duration.valueOf("0s"), true, newSingleThreadScheduledExecutor());
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("local"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node1"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node2"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node3"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node4"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node5"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node6"));
    long query1Sequence = 0;
    long query2Sequence = 0;
    long query3Sequence = 0;
    long query4Sequence = 0;
    long query5Sequence = 0;
    long query6Sequence = 0;
    long query7Sequence = 0;
    long query8Sequence = 0;
    long query9Sequence = 0;
    assertEquals(provider.getClusterQueries(), ImmutableList.of());
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", QUEUED, "root.rg4", GENERAL_POOL), query1Sequence++);
    assertTrue(provider.getClusterResourceGroups("node1").isEmpty());
    assertResourceGroup(provider, "node2", "root.rg4", 1, 0, DataSize.valueOf("1MB"));
    assertNonLeafResourceGroup(provider, "node2", "root", 0, 0, 1, 0);
    provider.registerQueryHeartbeat("node2", createQueryInfo("2", RUNNING, "root.rg4", GENERAL_POOL), query2Sequence++);
    assertResourceGroup(provider, "node1", "root.rg4", 0, 1, DataSize.valueOf("1MB"));
    assertNonLeafResourceGroup(provider, "node1", "root", 0, 0, 0, 1);
    assertResourceGroup(provider, "node2", "root.rg4", 1, 0, DataSize.valueOf("1MB"));
    assertNonLeafResourceGroup(provider, "node1", "root", 0, 0, 0, 1);
    assertResourceGroup(provider, "node3", "root.rg4", 1, 1, DataSize.valueOf("2MB"));
    provider.registerQueryHeartbeat("node3", createQueryInfo("3", FINISHED, "root.rg4", GENERAL_POOL), query3Sequence++);
    assertResourceGroup(provider, "node1", "root.rg4", 0, 1, DataSize.valueOf("1MB"));
    assertNonLeafResourceGroup(provider, "node1", "root", 0, 0, 0, 1);
    assertResourceGroup(provider, "node2", "root.rg4", 1, 0, DataSize.valueOf("1MB"));
    assertNonLeafResourceGroup(provider, "node2", "root", 0, 0, 1, 0);
    assertResourceGroup(provider, "node3", "root.rg4", 1, 1, DataSize.valueOf("2MB"));
    assertNonLeafResourceGroup(provider, "node3", "root", 0, 0, 1, 1);
    assertResourceGroup(provider, "node4", "root.rg4", 1, 1, DataSize.valueOf("2MB"));
    assertNonLeafResourceGroup(provider, "node4", "root", 0, 0, 1, 1);
    provider.registerQueryHeartbeat("node4", createQueryInfo("4", FAILED, "root.rg4", GENERAL_POOL), query4Sequence++);
    assertResourceGroup(provider, "node1", "root.rg4", 0, 1, DataSize.valueOf("1MB"));
    assertNonLeafResourceGroup(provider, "node1", "root", 0, 0, 0, 1);
    assertResourceGroup(provider, "node2", "root.rg4", 1, 0, DataSize.valueOf("1MB"));
    assertNonLeafResourceGroup(provider, "node2", "root", 0, 0, 1, 0);
    assertResourceGroup(provider, "node3", "root.rg4", 1, 1, DataSize.valueOf("2MB"));
    assertNonLeafResourceGroup(provider, "node3", "root", 0, 0, 1, 1);
    assertResourceGroup(provider, "node4", "root.rg4", 1, 1, DataSize.valueOf("2MB"));
    assertNonLeafResourceGroup(provider, "node4", "root", 0, 0, 1, 1);
    assertResourceGroup(provider, "node5", "root.rg4", 1, 1, DataSize.valueOf("2MB"));
    assertNonLeafResourceGroup(provider, "node5", "root", 0, 0, 1, 1);
    // Add queries which are in non-terminal states other than RUNNING and QUEUED
    provider.registerQueryHeartbeat("node1", createQueryInfo("5", WAITING_FOR_RESOURCES, "root.rg4", GENERAL_POOL), query5Sequence++);
    provider.registerQueryHeartbeat("node2", createQueryInfo("6", DISPATCHING, "root.rg4", GENERAL_POOL), query6Sequence++);
    provider.registerQueryHeartbeat("node3", createQueryInfo("7", PLANNING, "root.rg4", GENERAL_POOL), query7Sequence++);
    provider.registerQueryHeartbeat("node4", createQueryInfo("8", STARTING, "root.rg4", GENERAL_POOL), query8Sequence++);
    provider.registerQueryHeartbeat("node5", createQueryInfo("9", FINISHING, "root.rg4", GENERAL_POOL), query9Sequence++);
    assertResourceGroup(provider, "node1", "root.rg4", 0, 5, DataSize.valueOf("5MB"));
    assertNonLeafResourceGroup(provider, "node1", "root", 0, 0, 0, 5);
    assertResourceGroup(provider, "node2", "root.rg4", 1, 4, DataSize.valueOf("5MB"));
    assertNonLeafResourceGroup(provider, "node2", "root", 0, 0, 1, 4);
    assertResourceGroup(provider, "node3", "root.rg4", 1, 5, DataSize.valueOf("6MB"));
    assertNonLeafResourceGroup(provider, "node3", "root", 0, 0, 1, 5);
    assertResourceGroup(provider, "node4", "root.rg4", 1, 5, DataSize.valueOf("6MB"));
    assertNonLeafResourceGroup(provider, "node4", "root", 0, 0, 1, 5);
    assertResourceGroup(provider, "node5", "root.rg4", 1, 5, DataSize.valueOf("6MB"));
    assertNonLeafResourceGroup(provider, "node5", "root", 0, 0, 1, 5);
    assertResourceGroup(provider, "node6", "root.rg4", 1, 6, DataSize.valueOf("7MB"));
    assertNonLeafResourceGroup(provider, "node6", "root", 0, 0, 1, 6);
    // Expire running queries
    Thread.sleep(SECONDS.toMillis(5));
    nodeManager.refreshNodes();
    assertTrue(provider.getClusterResourceGroups("node1").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node2").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node3").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node4").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node5").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node6").isEmpty());
}

Example 80 with ConnectorId

use of com.facebook.presto.spi.ConnectorId in project presto by prestodb.

the class TestResourceManagerClusterStateProvider method testClusterMemoryPoolInfo.

@Test(timeOut = 15_000)
public void testClusterMemoryPoolInfo() throws Exception {
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("nodeId1", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("nodeId2", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("nodeId3", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("nodeId4", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    long query1Sequence = 0;
    long query2Sequence = 0;
    long query3Sequence = 0;
    ResourceManagerClusterStateProvider provider = new ResourceManagerClusterStateProvider(nodeManager, new SessionPropertyManager(), 10, Duration.valueOf("4s"), Duration.valueOf("8s"), Duration.valueOf("4s"), Duration.valueOf("0s"), true, newSingleThreadScheduledExecutor());
    // Memory pool starts off empty
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 0, 0, 0, 0, 0, Optional.empty());
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 0, 0, 0, 0, 0, Optional.empty());
    // Create a node and heartbeat to the resource manager
    provider.registerNodeHeartbeat(createNodeStatus("nodeId", GENERAL_POOL, createMemoryPoolInfo(100, 2, 1)));
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 0, 0, 100, 2, 1, Optional.empty());
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 0, 0, 0, 0, 0, Optional.empty());
    // Register a query and heartbeat that to the resource manager
    provider.registerQueryHeartbeat("nodeId1", createQueryInfo("1", QUEUED, "rg4", GENERAL_POOL), query1Sequence++);
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 1, 0, 100, 2, 1, Optional.of("1"));
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 0, 0, 0, 0, 0, Optional.empty());
    // Create another node and heartbeat to the resource manager
    provider.registerNodeHeartbeat(createNodeStatus("nodeId2", GENERAL_POOL, createMemoryPoolInfo(1000, 20, 10)));
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 1, 0, 1100, 22, 11, Optional.of("1"));
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 0, 0, 0, 0, 0, Optional.empty());
    // Create a blocked node and heartbeat to the resource manager
    provider.registerNodeHeartbeat(createNodeStatus("nodeId3", GENERAL_POOL, createMemoryPoolInfo(1, 2, 3)));
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 1, 1, 1101, 24, 14, Optional.of("1"));
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 0, 0, 0, 0, 0, Optional.empty());
    // Create a node that has only reserved pool allocations
    provider.registerNodeHeartbeat(createNodeStatus("nodeId4", RESERVED_POOL, createMemoryPoolInfo(5, 3, 2)));
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 1, 1, 1101, 24, 14, Optional.of("1"));
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 0, 0, 5, 3, 2, Optional.empty());
    // Add a larger query and verify that the largest query is updated
    provider.registerQueryHeartbeat("nodeId2", createQueryInfo("2", RUNNING, "rg4", GENERAL_POOL, DataSize.valueOf("25GB")), query2Sequence++);
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 2, 1, 1101, 24, 14, Optional.of("2"));
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 0, 0, 5, 3, 2, Optional.empty());
    // Adding a larger reserved pool query does not affect largest query in general pool
    provider.registerQueryHeartbeat("nodeId1", createQueryInfo("3", RUNNING, "rg4", RESERVED_POOL, DataSize.valueOf("50GB")), query3Sequence++);
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 2, 1, 1101, 24, 14, Optional.of("2"));
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 1, 0, 5, 3, 2, Optional.empty());
    // Expire nodes
    Thread.sleep(SECONDS.toMillis(5));
    // All nodes expired, memory pools emptied
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 0, 0, 0, 0, 0, Optional.empty());
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 0, 0, 0, 0, 0, Optional.empty());
}