Examples with SessionPropertyManager com.facebook.presto.metadata.SessionPropertyManager used on opensource projects

Example 21 with SessionPropertyManager

use of com.facebook.presto.metadata.SessionPropertyManager in project presto by prestodb.

the class TestResourceManagerClusterStateProvider method testOutOfOrderUpdatesIgnored.

@Test(timeOut = 15_000)
public void testOutOfOrderUpdatesIgnored() throws Exception {
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node1", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node2", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    ResourceManagerClusterStateProvider provider = new ResourceManagerClusterStateProvider(nodeManager, new SessionPropertyManager(), 10, Duration.valueOf("4s"), Duration.valueOf("8s"), Duration.valueOf("5s"), Duration.valueOf("0s"), true, newSingleThreadScheduledExecutor());
    assertEquals(provider.getClusterQueries(), ImmutableList.of());
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", QUEUED), 1);
    provider.registerQueryHeartbeat("node1", createQueryInfo("2", FINISHED), 2);
    assertQueryInfos(provider.getClusterQueries(), 2, 1);
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", FINISHED), 0);
    provider.registerQueryHeartbeat("node1", createQueryInfo("2", RUNNING), 1);
    assertQueryInfos(provider.getClusterQueries(), 2, 1);
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", FINISHED), 2);
    assertQueryInfos(provider.getClusterQueries(), 2, 2);
}

Example 22 with SessionPropertyManager

use of com.facebook.presto.metadata.SessionPropertyManager in project presto by prestodb.

the class TestResourceManagerClusterStateProvider method testResourceGroupsMerged.

@Test(timeOut = 15_000)
public void testResourceGroupsMerged() throws Exception {
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node1", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node2", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node3", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node4", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node5", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node6", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    ResourceManagerClusterStateProvider provider = new ResourceManagerClusterStateProvider(nodeManager, new SessionPropertyManager(), 10, Duration.valueOf("4s"), Duration.valueOf("8s"), Duration.valueOf("50s"), Duration.valueOf("0s"), true, newSingleThreadScheduledExecutor());
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("local"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node1"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node2"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node3"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node4"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node5"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node6"));
    assertEquals(provider.getClusterQueries(), ImmutableList.of());
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", QUEUED, "rg4", GENERAL_POOL), 0);
    assertTrue(provider.getClusterResourceGroups("node1").isEmpty());
    assertResourceGroup(provider, "node2", "rg4", 1, 0, DataSize.valueOf("1MB"));
    provider.registerQueryHeartbeat("node2", createQueryInfo("2", RUNNING, "rg4", GENERAL_POOL), 0);
    assertResourceGroup(provider, "node1", "rg4", 0, 1, DataSize.valueOf("1MB"));
    assertResourceGroup(provider, "node2", "rg4", 1, 0, DataSize.valueOf("1MB"));
    assertResourceGroup(provider, "node3", "rg4", 1, 1, DataSize.valueOf("2MB"));
    provider.registerQueryHeartbeat("node3", createQueryInfo("3", FINISHED, "rg4", GENERAL_POOL), 0);
    assertResourceGroup(provider, "node1", "rg4", 0, 1, DataSize.valueOf("1MB"));
    assertResourceGroup(provider, "node2", "rg4", 1, 0, DataSize.valueOf("1MB"));
    assertResourceGroup(provider, "node3", "rg4", 1, 1, DataSize.valueOf("2MB"));
    assertResourceGroup(provider, "node4", "rg4", 1, 1, DataSize.valueOf("2MB"));
    provider.registerQueryHeartbeat("node4", createQueryInfo("4", FAILED, "rg4", GENERAL_POOL), 0);
    assertResourceGroup(provider, "node1", "rg4", 0, 1, DataSize.valueOf("1MB"));
    assertResourceGroup(provider, "node2", "rg4", 1, 0, DataSize.valueOf("1MB"));
    assertResourceGroup(provider, "node3", "rg4", 1, 1, DataSize.valueOf("2MB"));
    assertResourceGroup(provider, "node4", "rg4", 1, 1, DataSize.valueOf("2MB"));
    assertResourceGroup(provider, "node5", "rg4", 1, 1, DataSize.valueOf("2MB"));
    // Add queries which are in non-terminal states other than RUNNING and QUEUED
    provider.registerQueryHeartbeat("node1", createQueryInfo("5", WAITING_FOR_RESOURCES, "rg4", GENERAL_POOL), 0);
    provider.registerQueryHeartbeat("node2", createQueryInfo("6", DISPATCHING, "rg4", GENERAL_POOL), 0);
    provider.registerQueryHeartbeat("node3", createQueryInfo("7", PLANNING, "rg4", GENERAL_POOL), 0);
    provider.registerQueryHeartbeat("node4", createQueryInfo("8", STARTING, "rg4", GENERAL_POOL), 0);
    provider.registerQueryHeartbeat("node5", createQueryInfo("9", FINISHING, "rg4", GENERAL_POOL), 0);
    assertResourceGroup(provider, "node1", "rg4", 0, 5, DataSize.valueOf("5MB"));
    assertResourceGroup(provider, "node2", "rg4", 1, 4, DataSize.valueOf("5MB"));
    assertResourceGroup(provider, "node3", "rg4", 1, 5, DataSize.valueOf("6MB"));
    assertResourceGroup(provider, "node4", "rg4", 1, 5, DataSize.valueOf("6MB"));
    assertResourceGroup(provider, "node5", "rg4", 1, 5, DataSize.valueOf("6MB"));
    assertResourceGroup(provider, "node6", "rg4", 1, 6, DataSize.valueOf("7MB"));
    // Expire running queries
    Thread.sleep(SECONDS.toMillis(5));
    assertTrue(provider.getClusterResourceGroups("node1").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node2").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node3").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node4").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node5").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node6").isEmpty());
}

Example 23 with SessionPropertyManager

use of com.facebook.presto.metadata.SessionPropertyManager in project presto by prestodb.

the class TestResourceManagerClusterStateProvider method testQueryInfo.

@Test(timeOut = 15_000)
public void testQueryInfo() throws Exception {
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node1", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node2", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    ResourceManagerClusterStateProvider provider = new ResourceManagerClusterStateProvider(nodeManager, new SessionPropertyManager(), 10, Duration.valueOf("4s"), Duration.valueOf("8s"), Duration.valueOf("5s"), Duration.valueOf("0s"), true, newSingleThreadScheduledExecutor());
    assertEquals(provider.getClusterQueries(), ImmutableList.of());
    long query1Sequence = 0;
    long query2Sequence = 0;
    long query3Sequence = 0;
    long query4Sequence = 0;
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", QUEUED), query1Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("2", RUNNING), query2Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("3", FINISHED), query3Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("4", FAILED), query4Sequence++);
    assertQueryInfos(provider.getClusterQueries(), 4, 2);
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", RUNNING), query1Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("2", FINISHING), query2Sequence++);
    assertQueryInfos(provider.getClusterQueries(), 4, 2);
    // Update query 2 to FINISHED to verify this is now completed in the resource manager
    provider.registerQueryHeartbeat("node1", createQueryInfo("2", FINISHED), query2Sequence++);
    assertQueryInfos(provider.getClusterQueries(), 4, 3);
    // Mix in queries from another coordinator
    provider.registerQueryHeartbeat("node2", createQueryInfo("1", QUEUED), query1Sequence++);
    provider.registerQueryHeartbeat("node2", createQueryInfo("2", RUNNING), query2Sequence++);
    provider.registerQueryHeartbeat("node2", createQueryInfo("3", FINISHED), query3Sequence++);
    provider.registerQueryHeartbeat("node2", createQueryInfo("4", FAILED), query4Sequence++);
    assertQueryInfos(provider.getClusterQueries(), 8, 5);
    // Expire completed queries
    Thread.sleep(SECONDS.toMillis(5));
    assertQueryInfos(provider.getClusterQueries(), 8, 5);
    // Expire all queries
    Thread.sleep(SECONDS.toMillis(5));
    assertQueryInfos(provider.getClusterQueries(), 0, 0);
}

Example 24 with SessionPropertyManager

use of com.facebook.presto.metadata.SessionPropertyManager in project presto by prestodb.

the class DispatchManager method createQueryInternal.

/**
 *  Creates and registers a dispatch query with the query tracker.  This method will never fail to register a query with the query
 *  tracker.  If an error occurs while, creating a dispatch query a failed dispatch will be created and registered.
 */
private <C> void createQueryInternal(QueryId queryId, String slug, int retryCount, SessionContext sessionContext, String query, ResourceGroupManager<C> resourceGroupManager) {
    Session session = null;
    PreparedQuery preparedQuery;
    try {
        if (query.length() > maxQueryLength) {
            int queryLength = query.length();
            query = query.substring(0, maxQueryLength);
            throw new PrestoException(QUERY_TEXT_TOO_LARGE, format("Query text length (%s) exceeds the maximum length (%s)", queryLength, maxQueryLength));
        }
        // decode session
        session = sessionSupplier.createSession(queryId, sessionContext);
        // prepare query
        WarningCollector warningCollector = warningCollectorFactory.create(getWarningHandlingLevel(session));
        preparedQuery = queryPreparer.prepareQuery(session, query, warningCollector);
        query = preparedQuery.getFormattedQuery().orElse(query);
        // select resource group
        Optional<QueryType> queryType = getQueryType(preparedQuery.getStatement().getClass());
        SelectionContext<C> selectionContext = resourceGroupManager.selectGroup(new SelectionCriteria(sessionContext.getIdentity().getPrincipal().isPresent(), sessionContext.getIdentity().getUser(), Optional.ofNullable(sessionContext.getSource()), sessionContext.getClientTags(), sessionContext.getResourceEstimates(), queryType.map(Enum::name)));
        // apply system default session properties (does not override user set properties)
        session = sessionPropertyDefaults.newSessionWithDefaultProperties(session, queryType.map(Enum::name), Optional.of(selectionContext.getResourceGroupId()));
        // mark existing transaction as active
        transactionManager.activateTransaction(session, isTransactionControlStatement(preparedQuery.getStatement()), accessControl);
        DispatchQuery dispatchQuery = dispatchQueryFactory.createDispatchQuery(session, query, preparedQuery, slug, retryCount, selectionContext.getResourceGroupId(), queryType, warningCollector, (dq) -> resourceGroupManager.submit(preparedQuery.getStatement(), dq, selectionContext, queryExecutor));
        boolean queryAdded = queryCreated(dispatchQuery);
        if (queryAdded && !dispatchQuery.isDone()) {
            try {
                clusterStatusSender.registerQuery(dispatchQuery);
                dispatchQuery.startWaitingForPrerequisites();
            } catch (Throwable e) {
                // dispatch query has already been registered, so just fail it directly
                dispatchQuery.fail(e);
            }
        }
    } catch (Throwable throwable) {
        // creation must never fail, so register a failed query in this case
        if (session == null) {
            session = Session.builder(new SessionPropertyManager()).setQueryId(queryId).setIdentity(sessionContext.getIdentity()).setSource(sessionContext.getSource()).build();
        }
        DispatchQuery failedDispatchQuery = failedDispatchQueryFactory.createFailedDispatchQuery(session, query, Optional.empty(), throwable);
        queryCreated(failedDispatchQuery);
    }
}

Example 25 with SessionPropertyManager

use of com.facebook.presto.metadata.SessionPropertyManager in project presto by prestodb.

the class TestQuerySessionSupplier method testCreateSession.

@Test
public void testCreateSession() {
    HttpRequestSessionContext context = new HttpRequestSessionContext(TEST_REQUEST, new SqlParserOptions());
    QuerySessionSupplier sessionSupplier = new QuerySessionSupplier(createTestTransactionManager(), new AllowAllAccessControl(), new SessionPropertyManager(), new SqlEnvironmentConfig());
    Session session = sessionSupplier.createSession(new QueryId("test_query_id"), context);
    assertEquals(session.getQueryId(), new QueryId("test_query_id"));
    assertEquals(session.getUser(), "testUser");
    assertEquals(session.getSource().get(), "testSource");
    assertEquals(session.getCatalog().get(), "testCatalog");
    assertEquals(session.getSchema().get(), "testSchema");
    assertEquals(session.getLocale(), Locale.TAIWAN);
    assertEquals(session.getTimeZoneKey(), getTimeZoneKey("Asia/Taipei"));
    assertEquals(session.getRemoteUserAddress().get(), "testRemote");
    assertEquals(session.getClientInfo().get(), "client-info");
    assertEquals(session.getClientTags(), ImmutableSet.of("tag1", "tag2", "tag3"));
    assertEquals(session.getSystemProperties(), ImmutableMap.<String, String>builder().put(QUERY_MAX_MEMORY, "1GB").put(JOIN_DISTRIBUTION_TYPE, "partitioned").put(HASH_PARTITION_COUNT, "43").build());
    assertEquals(session.getPreparedStatements(), ImmutableMap.<String, String>builder().put("query1", "select * from foo").put("query2", "select * from bar").build());
    assertEquals(session.getSessionFunctions(), ImmutableMap.of(SQL_FUNCTION_ID_ADD, SQL_FUNCTION_ADD));
}