Examples with SessionPropertyManager com.facebook.presto.metadata.SessionPropertyManager used on opensource projects

Example 11 with SessionPropertyManager

use of com.facebook.presto.metadata.SessionPropertyManager in project presto by prestodb.

the class TestResourceManagerClusterStateProvider method testShuttingDownCoordinatorHeartbeat.

@Test(timeOut = 15_000)
public void testShuttingDownCoordinatorHeartbeat() {
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    nodeManager.addShuttingDownNode(new InternalNode("node1", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    ResourceManagerClusterStateProvider provider = new ResourceManagerClusterStateProvider(nodeManager, new SessionPropertyManager(), 10, Duration.valueOf("4s"), Duration.valueOf("8s"), Duration.valueOf("5s"), Duration.valueOf("0s"), true, newSingleThreadScheduledExecutor());
    assertEquals(provider.getClusterQueries(), ImmutableList.of());
    long query1Sequence = 0;
    long query2Sequence = 0;
    long query3Sequence = 0;
    long query4Sequence = 0;
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", QUEUED), query1Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("2", RUNNING), query2Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("3", FINISHED), query3Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("4", FAILED), query4Sequence++);
    assertQueryInfos(provider.getClusterQueries(), 4, 2);
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", RUNNING), query1Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("2", FINISHING), query2Sequence++);
    assertQueryInfos(provider.getClusterQueries(), 4, 2);
    provider.registerQueryHeartbeat("node1", createQueryInfo("2", FINISHED), query2Sequence++);
    assertQueryInfos(provider.getClusterQueries(), 4, 3);
}

Example 12 with SessionPropertyManager

use of com.facebook.presto.metadata.SessionPropertyManager in project presto by prestodb.

the class TestResourceManagerClusterStateProvider method testClusterMemoryPoolInfo.

@Test(timeOut = 15_000)
public void testClusterMemoryPoolInfo() throws Exception {
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("nodeId1", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("nodeId2", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("nodeId3", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("nodeId4", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    long query1Sequence = 0;
    long query2Sequence = 0;
    long query3Sequence = 0;
    ResourceManagerClusterStateProvider provider = new ResourceManagerClusterStateProvider(nodeManager, new SessionPropertyManager(), 10, Duration.valueOf("4s"), Duration.valueOf("8s"), Duration.valueOf("4s"), Duration.valueOf("0s"), true, newSingleThreadScheduledExecutor());
    // Memory pool starts off empty
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 0, 0, 0, 0, 0, Optional.empty());
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 0, 0, 0, 0, 0, Optional.empty());
    // Create a node and heartbeat to the resource manager
    provider.registerNodeHeartbeat(createNodeStatus("nodeId", GENERAL_POOL, createMemoryPoolInfo(100, 2, 1)));
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 0, 0, 100, 2, 1, Optional.empty());
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 0, 0, 0, 0, 0, Optional.empty());
    // Register a query and heartbeat that to the resource manager
    provider.registerQueryHeartbeat("nodeId1", createQueryInfo("1", QUEUED, "rg4", GENERAL_POOL), query1Sequence++);
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 1, 0, 100, 2, 1, Optional.of("1"));
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 0, 0, 0, 0, 0, Optional.empty());
    // Create another node and heartbeat to the resource manager
    provider.registerNodeHeartbeat(createNodeStatus("nodeId2", GENERAL_POOL, createMemoryPoolInfo(1000, 20, 10)));
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 1, 0, 1100, 22, 11, Optional.of("1"));
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 0, 0, 0, 0, 0, Optional.empty());
    // Create a blocked node and heartbeat to the resource manager
    provider.registerNodeHeartbeat(createNodeStatus("nodeId3", GENERAL_POOL, createMemoryPoolInfo(1, 2, 3)));
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 1, 1, 1101, 24, 14, Optional.of("1"));
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 0, 0, 0, 0, 0, Optional.empty());
    // Create a node that has only reserved pool allocations
    provider.registerNodeHeartbeat(createNodeStatus("nodeId4", RESERVED_POOL, createMemoryPoolInfo(5, 3, 2)));
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 1, 1, 1101, 24, 14, Optional.of("1"));
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 0, 0, 5, 3, 2, Optional.empty());
    // Add a larger query and verify that the largest query is updated
    provider.registerQueryHeartbeat("nodeId2", createQueryInfo("2", RUNNING, "rg4", GENERAL_POOL, DataSize.valueOf("25GB")), query2Sequence++);
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 2, 1, 1101, 24, 14, Optional.of("2"));
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 0, 0, 5, 3, 2, Optional.empty());
    // Adding a larger reserved pool query does not affect largest query in general pool
    provider.registerQueryHeartbeat("nodeId1", createQueryInfo("3", RUNNING, "rg4", RESERVED_POOL, DataSize.valueOf("50GB")), query3Sequence++);
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 2, 1, 1101, 24, 14, Optional.of("2"));
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 1, 0, 5, 3, 2, Optional.empty());
    // Expire nodes
    Thread.sleep(SECONDS.toMillis(5));
    // All nodes expired, memory pools emptied
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 0, 0, 0, 0, 0, Optional.empty());
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 0, 0, 0, 0, 0, Optional.empty());
}

Example 13 with SessionPropertyManager

use of com.facebook.presto.metadata.SessionPropertyManager in project presto by prestodb.

the class TestResourceManagerClusterStateProvider method testResourceGroups.

@Test(timeOut = 15_000)
public void testResourceGroups() throws Exception {
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node1", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node2", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node3", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    ResourceManagerClusterStateProvider provider = new ResourceManagerClusterStateProvider(nodeManager, new SessionPropertyManager(), 10, Duration.valueOf("4s"), Duration.valueOf("8s"), Duration.valueOf("50s"), Duration.valueOf("0s"), true, newSingleThreadScheduledExecutor());
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("local"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node1"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node2"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node3"));
    assertEquals(provider.getClusterQueries(), ImmutableList.of());
    long query1Sequence = 0;
    long query2Sequence = 0;
    long query3Sequence = 0;
    long query4Sequence = 0;
    long query5Sequence = 0;
    long query6Sequence = 0;
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", QUEUED, "rg1", GENERAL_POOL), query1Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("2", RUNNING, "rg2", GENERAL_POOL), query2Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("3", FINISHING, "rg3", GENERAL_POOL), query3Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("4", FINISHED, "rg4", GENERAL_POOL), query4Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("5", FAILED, "rg5", GENERAL_POOL), query5Sequence++);
    assertResourceGroups(provider, "node1", 0);
    assertResourceGroups(provider, "node2", 3);
    // Add an existing leaf node from another node
    provider.registerQueryHeartbeat("node3", createQueryInfo("6", QUEUED, "rg6", GENERAL_POOL), query6Sequence++);
    assertResourceGroups(provider, "node1", 1);
    assertResourceGroups(provider, "node2", 4);
    assertResourceGroups(provider, "node3", 3);
    // Expire running queries
    Thread.sleep(SECONDS.toMillis(5));
    assertResourceGroups(provider, "node1", 0);
    assertResourceGroups(provider, "node2", 0);
    assertResourceGroups(provider, "node3", 0);
}

Example 14 with SessionPropertyManager

use of com.facebook.presto.metadata.SessionPropertyManager in project presto by prestodb.

the class TestResourceManagerClusterStateProvider method testWorkerMemoryInfo.

@Test(timeOut = 15_000)
public void testWorkerMemoryInfo() throws Exception {
    ResourceManagerClusterStateProvider provider = new ResourceManagerClusterStateProvider(new InMemoryNodeManager(), new SessionPropertyManager(), 10, Duration.valueOf("4s"), Duration.valueOf("8s"), Duration.valueOf("4s"), Duration.valueOf("0s"), true, newSingleThreadScheduledExecutor());
    assertWorkerMemoryInfo(provider, 0);
    provider.registerNodeHeartbeat(createNodeStatus("nodeId", GENERAL_POOL, createMemoryPoolInfo(100, 2, 1)));
    assertWorkerMemoryInfo(provider, 1);
    provider.registerNodeHeartbeat(createNodeStatus("nodeId2", GENERAL_POOL, createMemoryPoolInfo(200, 20, 10)));
    assertWorkerMemoryInfo(provider, 2);
    // Expire nodes
    Thread.sleep(SECONDS.toMillis(5));
    assertWorkerMemoryInfo(provider, 0);
}

Example 15 with SessionPropertyManager

use of com.facebook.presto.metadata.SessionPropertyManager in project presto by prestodb.

the class SystemConnectorSessionUtil method toSession.

// this does not preserve any connector properties (for the system connector)
public static Session toSession(ConnectorTransactionHandle transactionHandle, ConnectorSession session) {
    TransactionId transactionId = ((GlobalSystemTransactionHandle) transactionHandle).getTransactionId();
    ConnectorIdentity connectorIdentity = session.getIdentity();
    Identity identity = new Identity(connectorIdentity.getUser(), connectorIdentity.getPrincipal());
    return Session.builder(new SessionPropertyManager(SYSTEM_SESSION_PROPERTIES)).setQueryId(new QueryId(session.getQueryId())).setTransactionId(transactionId).setCatalog("catalog").setSchema("schema").setIdentity(identity).setTimeZoneKey(session.getSqlFunctionProperties().getTimeZoneKey()).setLocale(session.getLocale()).setStartTime(session.getStartTime()).build();
}