Examples with ResourceGroupInfo com.facebook.presto.server.ResourceGroupInfo used on opensource projects

Example 6 with ResourceGroupInfo

use of com.facebook.presto.server.ResourceGroupInfo in project presto by prestodb.

the class DistributedResourceGroupInfoResource method getResourceGroupInfos.

@GET
@Produces(MediaType.APPLICATION_JSON)
@Encoded
@Path("{resourceGroupId: .+}")
public void getResourceGroupInfos(@PathParam("resourceGroupId") String resourceGroupIdString, @Context UriInfo uriInfo, @Context HttpServletRequest servletRequest, @Suspended AsyncResponse asyncResponse) {
    if (isNullOrEmpty(resourceGroupIdString)) {
        asyncResponse.resume(Response.status(NOT_FOUND).build());
    }
    try {
        ImmutableList.Builder<ListenableFuture<ResourceGroupInfo>> resourceGroupInfoFutureBuilder = ImmutableList.builder();
        for (InternalNode coordinator : internalNodeManager.getCoordinators()) {
            resourceGroupInfoFutureBuilder.add(getResourceGroupInfoFromCoordinator(uriInfo, coordinator));
        }
        List<ListenableFuture<ResourceGroupInfo>> resourceGroupInfoFutureList = resourceGroupInfoFutureBuilder.build();
        Futures.whenAllComplete(resourceGroupInfoFutureList).call(() -> {
            try {
                ResourceGroupInfo aggregatedResourceGroupInfo = aggregateResourceGroupInfo(resourceGroupInfoFutureList);
                if (aggregatedResourceGroupInfo == null) {
                    return asyncResponse.resume(Response.status(NOT_FOUND).build());
                }
                return asyncResponse.resume(Response.ok(aggregatedResourceGroupInfo).build());
            } catch (Exception ex) {
                log.error(ex, "Error in getting resource group info from one of the coordinators");
                return asyncResponse.resume(Response.serverError().entity(ex.getMessage()).build());
            }
        }, executor);
    } catch (IOException ex) {
        log.error(ex, "Error in getting resource group info");
        asyncResponse.resume(Response.serverError().entity(ex.getMessage()).build());
    }
}

Example 7 with ResourceGroupInfo

use of com.facebook.presto.server.ResourceGroupInfo in project presto by prestodb.

the class TestThriftResourceGroupInfo method testRoundTripSerializeTFacebookCompactProtocol.

@Test(dataProvider = "codecCombinations")
public void testRoundTripSerializeTFacebookCompactProtocol(ThriftCodec<ResourceGroupInfo> readCodec, ThriftCodec<ResourceGroupInfo> writeCodec) throws Exception {
    ResourceGroupInfo groupInfo = getRoundTripSerialize(readCodec, writeCodec, TFacebookCompactProtocol::new);
    assertSerde(groupInfo);
}

Example 8 with ResourceGroupInfo

use of com.facebook.presto.server.ResourceGroupInfo in project presto by prestodb.

the class TestThriftResourceGroupInfo method testRoundTripSerializeBinaryProtocol.

@Test(dataProvider = "codecCombinations")
public void testRoundTripSerializeBinaryProtocol(ThriftCodec<ResourceGroupInfo> readCodec, ThriftCodec<ResourceGroupInfo> writeCodec) throws Exception {
    ResourceGroupInfo groupInfo = getRoundTripSerialize(readCodec, writeCodec, TBinaryProtocol::new);
    assertSerde(groupInfo);
}

Example 9 with ResourceGroupInfo

use of com.facebook.presto.server.ResourceGroupInfo in project presto by prestodb.

the class TestResourceGroups method testGetResourceGroupStateInfo.

@Test
public void testGetResourceGroupStateInfo() {
    RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {
    }, directExecutor(), ignored -> Optional.empty(), rg -> false);
    root.setSoftMemoryLimit(new DataSize(1, GIGABYTE));
    root.setMaxQueuedQueries(40);
    root.setHardConcurrencyLimit(10);
    root.setSchedulingPolicy(WEIGHTED);
    InternalResourceGroup rootA = root.getOrCreateSubGroup("a", true);
    rootA.setSoftMemoryLimit(new DataSize(10, MEGABYTE));
    rootA.setMaxQueuedQueries(20);
    rootA.setHardConcurrencyLimit(0);
    InternalResourceGroup rootB = root.getOrCreateSubGroup("b", true);
    rootB.setSoftMemoryLimit(new DataSize(5, MEGABYTE));
    rootB.setMaxQueuedQueries(20);
    rootB.setHardConcurrencyLimit(1);
    rootB.setSchedulingWeight(2);
    rootB.setSchedulingPolicy(QUERY_PRIORITY);
    InternalResourceGroup rootAX = rootA.getOrCreateSubGroup("x", true);
    rootAX.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    rootAX.setMaxQueuedQueries(10);
    rootAX.setHardConcurrencyLimit(10);
    InternalResourceGroup rootAY = rootA.getOrCreateSubGroup("y", true);
    rootAY.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    rootAY.setMaxQueuedQueries(10);
    rootAY.setHardConcurrencyLimit(10);
    Set<MockManagedQueryExecution> queries = fillGroupTo(rootAX, ImmutableSet.of(), 5, false);
    queries.addAll(fillGroupTo(rootAY, ImmutableSet.of(), 5, false));
    queries.addAll(fillGroupTo(rootB, ImmutableSet.of(), 10, true));
    ResourceGroupInfo rootInfo = root.getResourceGroupInfo(true, true, false);
    assertEquals(rootInfo.getId(), root.getId());
    assertEquals(rootInfo.getState(), CAN_RUN);
    assertEquals(rootInfo.getSoftMemoryLimit(), root.getSoftMemoryLimit());
    assertEquals(rootInfo.getMemoryUsage(), new DataSize(0, BYTE));
    assertEquals(rootInfo.getSubGroups().size(), 2);
    assertGroupInfoEquals(rootInfo.getSubGroups().get(0), rootA.getInfo());
    assertEquals(rootInfo.getSubGroups().get(0).getId(), rootA.getId());
    assertEquals(rootInfo.getSubGroups().get(0).getState(), CAN_QUEUE);
    assertEquals(rootInfo.getSubGroups().get(0).getSoftMemoryLimit(), rootA.getSoftMemoryLimit());
    assertEquals(rootInfo.getSubGroups().get(0).getHardConcurrencyLimit(), rootA.getHardConcurrencyLimit());
    assertEquals(rootInfo.getSubGroups().get(0).getMaxQueuedQueries(), rootA.getMaxQueuedQueries());
    assertEquals(rootInfo.getSubGroups().get(0).getNumEligibleSubGroups(), 2);
    assertEquals(rootInfo.getSubGroups().get(0).getNumRunningQueries(), 0);
    assertEquals(rootInfo.getSubGroups().get(0).getNumQueuedQueries(), 10);
    assertGroupInfoEquals(rootInfo.getSubGroups().get(1), rootB.getInfo());
    assertEquals(rootInfo.getSubGroups().get(1).getId(), rootB.getId());
    assertEquals(rootInfo.getSubGroups().get(1).getState(), CAN_QUEUE);
    assertEquals(rootInfo.getSubGroups().get(1).getSoftMemoryLimit(), rootB.getSoftMemoryLimit());
    assertEquals(rootInfo.getSubGroups().get(1).getHardConcurrencyLimit(), rootB.getHardConcurrencyLimit());
    assertEquals(rootInfo.getSubGroups().get(1).getMaxQueuedQueries(), rootB.getMaxQueuedQueries());
    assertEquals(rootInfo.getSubGroups().get(1).getNumEligibleSubGroups(), 0);
    assertEquals(rootInfo.getSubGroups().get(1).getNumRunningQueries(), 1);
    assertEquals(rootInfo.getSubGroups().get(1).getNumQueuedQueries(), 9);
    assertEquals(rootInfo.getSoftConcurrencyLimit(), root.getSoftConcurrencyLimit());
    assertEquals(rootInfo.getHardConcurrencyLimit(), root.getHardConcurrencyLimit());
    assertEquals(rootInfo.getMaxQueuedQueries(), root.getMaxQueuedQueries());
    assertEquals(rootInfo.getNumQueuedQueries(), 19);
    assertEquals(rootInfo.getRunningQueries().size(), 1);
    QueryStateInfo queryInfo = rootInfo.getRunningQueries().get(0);
    assertEquals(queryInfo.getResourceGroupId(), Optional.of(rootB.getId()));
}

Example 10 with ResourceGroupInfo

use of com.facebook.presto.server.ResourceGroupInfo in project presto by prestodb.

the class TestResourceGroups method testGetStaticResourceGroupInfo.

@Test
public void testGetStaticResourceGroupInfo() {
    RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {
    }, directExecutor(), ignored -> Optional.empty(), rg -> false);
    root.setSoftMemoryLimit(new DataSize(1, GIGABYTE));
    root.setMaxQueuedQueries(100);
    root.setHardConcurrencyLimit(10);
    root.setSchedulingPolicy(WEIGHTED);
    InternalResourceGroup rootA = root.getOrCreateSubGroup("a", true);
    rootA.setSoftMemoryLimit(new DataSize(10, MEGABYTE));
    rootA.setMaxQueuedQueries(100);
    rootA.setHardConcurrencyLimit(0);
    InternalResourceGroup rootB = root.getOrCreateSubGroup("b", true);
    rootB.setSoftMemoryLimit(new DataSize(5, MEGABYTE));
    rootB.setMaxQueuedQueries(100);
    rootB.setHardConcurrencyLimit(1);
    rootB.setSchedulingWeight(2);
    rootB.setSchedulingPolicy(QUERY_PRIORITY);
    // x is a dynamic resource group
    InternalResourceGroup rootAX = rootA.getOrCreateSubGroup("x", false);
    rootAX.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    rootAX.setMaxQueuedQueries(10);
    rootAX.setHardConcurrencyLimit(10);
    InternalResourceGroup rootAY = rootA.getOrCreateSubGroup("y", true);
    rootAY.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    rootAY.setMaxQueuedQueries(10);
    rootAY.setHardConcurrencyLimit(10);
    for (int i = 0; i < 10; i++) {
        InternalResourceGroup subGroup = rootAX.getOrCreateSubGroup("ax" + i, false);
        subGroup.setSoftMemoryLimit(new DataSize(i, MEGABYTE));
        subGroup.setMaxQueuedQueries(10);
        subGroup.setHardConcurrencyLimit(10);
    }
    for (int i = 0; i < 10; i++) {
        fillGroupTo(rootAX.getOrCreateSubGroup("ax" + i, true), ImmutableSet.of(), 1, false);
    }
    fillGroupTo(rootAY, ImmutableSet.of(), 5, false);
    fillGroupTo(rootB, ImmutableSet.of(), 10, true);
    ResourceGroupInfo rootInfo = root.getResourceGroupInfo(false, false, true);
    assertEquals(rootInfo.getId(), root.getId());
    assertNotNull(rootInfo.getSubGroups());
    assertEquals(rootInfo.getSubGroups().size(), 2);
    Optional<ResourceGroupInfo> rootAInfo = getResourceGroupInfoForId(rootA, rootInfo);
    assertTrue(rootAInfo.isPresent());
    assertNotNull(rootAInfo.get().getSubGroups());
    assertEquals(rootAInfo.get().getSubGroups().size(), 1);
    Optional<ResourceGroupInfo> rootAXInfo = getResourceGroupInfoForId(rootAX, rootAInfo.get());
    // dynamic resource groups should not be returned.
    assertFalse(rootAXInfo.isPresent());
    Optional<ResourceGroupInfo> rootAYInfo = getResourceGroupInfoForId(rootAY, rootAInfo.get());
    assertTrue(rootAYInfo.isPresent());
    assertNotNull(rootAYInfo.get().getSubGroups());
    assertEquals(rootAYInfo.get().getSubGroups().size(), 0);
    Optional<ResourceGroupInfo> rootBInfo = getResourceGroupInfoForId(rootB, rootInfo);
    assertTrue(rootBInfo.isPresent());
    assertNotNull(rootBInfo.get().getSubGroups());
    assertEquals(rootBInfo.get().getSubGroups().size(), 0);
}