Examples with ResourceGroupRuntimeInfo com.facebook.presto.execution.resourceGroups.ResourceGroupRuntimeInfo used on opensource projects

Example 1 with ResourceGroupRuntimeInfo

use of com.facebook.presto.execution.resourceGroups.ResourceGroupRuntimeInfo in project presto by prestodb.

the class TestDistributedQueuesDb method testDistributedQueue.

@Test(timeOut = 60_000)
public void testDistributedQueue() throws Exception {
    QueryId firstAdhocQuery = createQuery(queryRunner, 1, adhocSession(), LONG_LASTING_QUERY);
    QueryId secondAdhocQuery = createQuery(queryRunner, 1, adhocSession(), LONG_LASTING_QUERY);
    QueryId thirdAdhocQuery = createQuery(queryRunner, 0, adhocSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, 1, firstAdhocQuery, RUNNING);
    waitForQueryState(queryRunner, 1, secondAdhocQuery, RUNNING);
    waitForQueryState(queryRunner, 0, thirdAdhocQuery, RUNNING);
    Map<ResourceGroupId, ResourceGroupRuntimeInfo> resourceGroupRuntimeInfoSnapshot;
    int globalRunningQueries = 0;
    do {
        MILLISECONDS.sleep(100);
        globalRunningQueries = 0;
        for (int coordinator = 0; coordinator < 2; coordinator++) {
            resourceGroupRuntimeInfoSnapshot = queryRunner.getCoordinator(coordinator).getResourceGroupManager().get().getResourceGroupRuntimeInfosSnapshot();
            ResourceGroupRuntimeInfo resourceGroupRuntimeInfo = resourceGroupRuntimeInfoSnapshot.get(new ResourceGroupId("global"));
            if (resourceGroupRuntimeInfo != null) {
                globalRunningQueries += resourceGroupRuntimeInfo.getDescendantRunningQueries();
            }
        }
    } while (globalRunningQueries != 3);
    QueryId firstDashboardQuery = createQuery(queryRunner, 0, dashboardSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, 0, firstDashboardQuery, QUEUED);
    cancelQuery(queryRunner, 0, thirdAdhocQuery);
    waitForQueryState(queryRunner, 0, firstDashboardQuery, RUNNING);
}

Example 2 with ResourceGroupRuntimeInfo

use of com.facebook.presto.execution.resourceGroups.ResourceGroupRuntimeInfo in project presto by prestodb.

the class TestDistributedQueuesDb method testMultiResourceGroupConcurrencyThreshold.

@Test(timeOut = 60_000)
public void testMultiResourceGroupConcurrencyThreshold() throws Exception {
    QueryId firstAdhocQuery = createQuery(queryRunner, 1, adhocSession(), LONG_LASTING_QUERY);
    QueryId secondAdhocQuery = createQuery(queryRunner, 1, adhocSession(), LONG_LASTING_QUERY);
    QueryId thirdAdhocQuery = createQuery(queryRunner, 1, adhocSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, 1, firstAdhocQuery, RUNNING);
    waitForQueryState(queryRunner, 1, secondAdhocQuery, RUNNING);
    waitForQueryState(queryRunner, 1, thirdAdhocQuery, RUNNING);
    Map<ResourceGroupId, ResourceGroupRuntimeInfo> resourceGroupRuntimeInfoSnapshot;
    int globalRunningQueries = 0;
    do {
        MILLISECONDS.sleep(100);
        resourceGroupRuntimeInfoSnapshot = queryRunner.getCoordinator(0).getResourceGroupManager().get().getResourceGroupRuntimeInfosSnapshot();
        ResourceGroupRuntimeInfo resourceGroupRuntimeInfo = resourceGroupRuntimeInfoSnapshot.get(new ResourceGroupId("global"));
        if (resourceGroupRuntimeInfo != null) {
            globalRunningQueries = resourceGroupRuntimeInfo.getDescendantRunningQueries();
        }
    } while (globalRunningQueries != 3);
    QueryId firstDashboardQuery = createQuery(queryRunner, 0, dashboardSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, 0, firstDashboardQuery, QUEUED);
    cancelQuery(queryRunner, 1, firstAdhocQuery);
    waitForQueryState(queryRunner, 0, firstDashboardQuery, RUNNING);
}

Example 3 with ResourceGroupRuntimeInfo

use of com.facebook.presto.execution.resourceGroups.ResourceGroupRuntimeInfo in project presto by prestodb.

the class TestDistributedQueuesDb method testResourceGroupConcurrencyThreshold.

@Test(timeOut = 60_000)
public void testResourceGroupConcurrencyThreshold() throws Exception {
    QueryId firstAdhocQuery = createQuery(queryRunner, 1, adhocSession(), LONG_LASTING_QUERY);
    QueryId secondAdhocQuery = createQuery(queryRunner, 1, adhocSession(), LONG_LASTING_QUERY);
    QueryId thirdAdhocQuery = createQuery(queryRunner, 1, adhocSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, 1, firstAdhocQuery, RUNNING);
    waitForQueryState(queryRunner, 1, secondAdhocQuery, RUNNING);
    waitForQueryState(queryRunner, 1, thirdAdhocQuery, RUNNING);
    Map<ResourceGroupId, ResourceGroupRuntimeInfo> resourceGroupRuntimeInfoSnapshot;
    int globalRunningQueries = 0;
    do {
        MILLISECONDS.sleep(100);
        resourceGroupRuntimeInfoSnapshot = queryRunner.getCoordinator(0).getResourceGroupManager().get().getResourceGroupRuntimeInfosSnapshot();
        ResourceGroupRuntimeInfo resourceGroupRuntimeInfo = resourceGroupRuntimeInfoSnapshot.get(new ResourceGroupId("global"));
        if (resourceGroupRuntimeInfo != null) {
            globalRunningQueries = resourceGroupRuntimeInfo.getDescendantRunningQueries();
        }
    } while (globalRunningQueries != 3);
    QueryId fourthAdhocQuery = createQuery(queryRunner, 0, adhocSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, 0, fourthAdhocQuery, QUEUED);
    cancelQuery(queryRunner, 1, firstAdhocQuery);
    waitForQueryState(queryRunner, 0, fourthAdhocQuery, RUNNING);
}

Example 4 with ResourceGroupRuntimeInfo

use of com.facebook.presto.execution.resourceGroups.ResourceGroupRuntimeInfo in project presto by prestodb.

the class ResourceManagerClusterStateProvider method getClusterResourceGroups.

public List<ResourceGroupRuntimeInfo> getClusterResourceGroups(String excludingNode) throws ResourceManagerInconsistentException {
    requireNonNull(excludingNode, "excludingNode is null");
    validateCoordinatorConsistency();
    Map<ResourceGroupId, ResourceGroupRuntimeInfo.Builder> resourceGroupBuilders = new HashMap<>();
    nodeQueryStates.values().stream().filter(state -> !state.getNodeId().equals(excludingNode)).map(CoordinatorQueriesState::getActiveQueries).flatMap(Collection::stream).map(Query::getBasicQueryInfo).filter(info -> info.getResourceGroupId().isPresent()).forEach(info -> {
        ResourceGroupId resourceGroupId = info.getResourceGroupId().get();
        ResourceGroupRuntimeInfo.Builder builder = resourceGroupBuilders.computeIfAbsent(resourceGroupId, ResourceGroupRuntimeInfo::builder);
        if (info.getState() == QUEUED) {
            builder.addQueuedQueries(1);
        } else if (!info.getState().isDone()) {
            builder.addRunningQueries(1);
        }
        builder.addUserMemoryReservationBytes(info.getQueryStats().getUserMemoryReservation().toBytes());
        while (resourceGroupId.getParent().isPresent()) {
            resourceGroupId = resourceGroupId.getParent().get();
            ResourceGroupRuntimeInfo.Builder parentBuilder = resourceGroupBuilders.computeIfAbsent(resourceGroupId, ResourceGroupRuntimeInfo::builder);
            if (info.getState() == QUEUED) {
                parentBuilder.addDescendantQueuedQueries(1);
            } else if (!info.getState().isDone()) {
                parentBuilder.addDescendantRunningQueries(1);
            }
        }
    });
    return resourceGroupBuilders.values().stream().map(ResourceGroupRuntimeInfo.Builder::build).collect(toImmutableList());
}

Example 5 with ResourceGroupRuntimeInfo

use of com.facebook.presto.execution.resourceGroups.ResourceGroupRuntimeInfo in project presto by prestodb.

the class TestResourceManagerClusterStateProvider method assertResourceGroup.

private void assertResourceGroup(ResourceManagerClusterStateProvider provider, String excludingNode, String resourceGroupId, int queuedQueries, int runningQueries, DataSize userMemoryReservation) throws ResourceManagerInconsistentException {
    ResourceGroupId currResourceGroupId = new ResourceGroupId(Arrays.asList(resourceGroupId.split("\\.")));
    List<ResourceGroupRuntimeInfo> list = provider.getClusterResourceGroups(excludingNode);
    Optional<ResourceGroupRuntimeInfo> resourceGroupRuntimeInfo = list.stream().filter(resourceGroupInfo -> currResourceGroupId.equals(resourceGroupInfo.getResourceGroupId())).findFirst();
    assertTrue(resourceGroupRuntimeInfo.isPresent(), "Resource group " + resourceGroupId + " not found");
    ResourceGroupRuntimeInfo info = resourceGroupRuntimeInfo.get();
    ResourceGroupId rg = new ResourceGroupId(Arrays.asList(resourceGroupId.split("\\.")));
    assertEquals(info.getQueuedQueries(), queuedQueries, format("Expected %s queued queries, found %s", queuedQueries, info.getQueuedQueries()));
    assertEquals(info.getRunningQueries(), runningQueries, format("Expected %s running queries, found %s", runningQueries, info.getRunningQueries()));
    assertEquals(info.getResourceGroupId(), rg, format("Expected resource group id %s, found %s", resourceGroupId, info.getResourceGroupId()));
    assertEquals(info.getMemoryUsageBytes(), userMemoryReservation.toBytes(), format("Expected %s user memory reservation found %s", userMemoryReservation, DataSize.succinctBytes(info.getMemoryUsageBytes())));
}