Examples with ResourcePoolTree org.apache.drill.exec.resourcemgr.config.ResourcePoolTree used on opensource projects

Example 1 with ResourcePoolTree

use of org.apache.drill.exec.resourcemgr.config.ResourcePoolTree in project drill by apache.

the class TestResourcePoolTree method testSelectionPolicyLowerCase.

@Test
public void testSelectionPolicyLowerCase() throws Exception {
    // leaf pool with queue
    poolTreeConfig.put(ResourcePoolImpl.POOL_NAME_KEY, "drill");
    poolTreeConfig.put(ResourcePoolImpl.POOL_QUEUE_KEY, queue1);
    poolTreeConfig.put(ResourcePoolImpl.POOL_SELECTOR_KEY, tagSelectorConfig1);
    poolTreeConfig.put(ResourcePoolTreeImpl.ROOT_POOL_QUEUE_SELECTION_POLICY_KEY, "bestfit");
    Config rmConfig = ConfigFactory.empty().withValue("drill.exec.rm", ConfigValueFactory.fromMap(poolTreeConfig));
    ResourcePoolTree poolTree = new ResourcePoolTreeImpl(rmConfig, 10000, 10, 2);
    assertTrue("Root pool is not a leaf pool", poolTree.getRootPool().isLeafPool());
    assertEquals("Root pool name is not drill", "drill", poolTree.getRootPool().getPoolName());
    assertTrue("Root pool is not the only leaf pool", poolTree.getAllLeafQueues().size() == 1);
    assertTrue("Root pool name is not same as leaf pool name", poolTree.getAllLeafQueues().containsKey("drill"));
    assertFalse("Root pool should not be a default pool", poolTree.getRootPool().isDefaultPool());
    assertTrue("Selection policy is not bestfit", poolTree.getSelectionPolicyInUse().getSelectionPolicy().toString().equals("bestfit"));
}

Example 2 with ResourcePoolTree

use of org.apache.drill.exec.resourcemgr.config.ResourcePoolTree in project drill by apache.

the class TestResourcePoolTree method testTreeWith2LeafPool.

@Test
public void testTreeWith2LeafPool() throws Exception {
    // pool with tag selector
    pool1.put(ResourcePoolImpl.POOL_QUEUE_KEY, queue1);
    pool1.put(ResourcePoolImpl.POOL_SELECTOR_KEY, tagSelectorConfig1);
    // pool with default selector
    pool2.put(ResourcePoolImpl.POOL_QUEUE_KEY, queue1);
    childResourcePools.add(pool1);
    childResourcePools.add(pool2);
    ResourcePoolTree configTree = getPoolTreeConfig();
    // get all leaf queues names
    Set<String> expectedLeafQueue = new HashSet<>();
    expectedLeafQueue.add((String) pool1.get("pool_name"));
    expectedLeafQueue.add((String) pool2.get("pool_name"));
    assertEquals("Root pool is different than expected", "drill", configTree.getRootPool().getPoolName());
    assertEquals("Expected and actual leaf queue names are different", expectedLeafQueue, configTree.getAllLeafQueues().keySet());
    assertEquals("Unexpected Selection policy is in use", ROOT_POOL_DEFAULT_QUEUE_SELECTION_POLICY, configTree.getSelectionPolicyInUse().getSelectionPolicy());
}

Example 3 with ResourcePoolTree

use of org.apache.drill.exec.resourcemgr.config.ResourcePoolTree in project drill by apache.

the class TestResourcePoolTree method testRootPoolAsLeaf.

@Test
public void testRootPoolAsLeaf() throws Exception {
    // leaf pool with queue
    poolTreeConfig.put(ResourcePoolImpl.POOL_NAME_KEY, "drill");
    poolTreeConfig.put(ResourcePoolImpl.POOL_QUEUE_KEY, queue1);
    poolTreeConfig.put(ResourcePoolImpl.POOL_SELECTOR_KEY, tagSelectorConfig1);
    Config rmConfig = ConfigFactory.empty().withValue("drill.exec.rm", ConfigValueFactory.fromMap(poolTreeConfig));
    ResourcePoolTree poolTree = new ResourcePoolTreeImpl(rmConfig, 10000, 10, 2);
    assertTrue("Root pool is not a leaf pool", poolTree.getRootPool().isLeafPool());
    assertEquals("Root pool name is not drill", "drill", poolTree.getRootPool().getPoolName());
    assertTrue("Root pool is not the only leaf pool", poolTree.getAllLeafQueues().size() == 1);
    assertTrue("Root pool name is not same as leaf pool name", poolTree.getAllLeafQueues().containsKey("drill"));
    assertFalse("Root pool should not be a default pool", poolTree.getRootPool().isDefaultPool());
}

Example 4 with ResourcePoolTree

use of org.apache.drill.exec.resourcemgr.config.ResourcePoolTree in project drill by apache.

the class TestRMConfigLoad method testDefaultRMConfig.

@Test
public void testDefaultRMConfig() throws Exception {
    ClusterFixtureBuilder fixtureBuilder = ClusterFixture.builder(dirTestWatcher).configProperty(ExecConstants.RM_ENABLED, true).configProperty(ExecConstants.DRILL_PORT_HUNT, true).withLocalZk();
    try (ClusterFixture cluster = fixtureBuilder.build()) {
        ResourceManager resourceManager = cluster.drillbit().getContext().getResourceManager();
        assertTrue(resourceManager instanceof DistributedResourceManager);
        ResourcePoolTree poolTree = ((DistributedResourceManager) resourceManager).getRmPoolTree();
        assertTrue("In drill-rm-default root pool is not leaf pool", poolTree.getRootPool().isLeafPool());
        assertTrue("selector in drill-rm-default is not acl selector", poolTree.getRootPool().getSelector() instanceof AclSelector);
        assertEquals("max_query_memory_per_node in drill-rm-default is not configured with expected default value", 8 * 1024L, poolTree.getRootPool().getMaxQueryMemoryPerNode());
        assertEquals("queue_selection_policy in drill-rm-default is not configured with expected default value", RMCommonDefaults.ROOT_POOL_DEFAULT_QUEUE_SELECTION_POLICY, poolTree.getSelectionPolicyInUse().getSelectionPolicy());
        assertEquals("memory share of root pool in drill-rm-default is not configured with expected default value", RMCommonDefaults.ROOT_POOL_DEFAULT_MEMORY_PERCENT, poolTree.getResourceShare(), 0);
        final QueryQueueConfig defaultQueue = poolTree.getRootPool().getQueryQueue();
        assertEquals("max_admissible in drill-rm-default is not configured with expected default value", RMCommonDefaults.MAX_ADMISSIBLE_QUERY_COUNT, defaultQueue.getMaxAdmissibleQueries());
        assertEquals("max_waiting in drill-rm-default is not configured with expected default value", RMCommonDefaults.MAX_WAITING_QUERY_COUNT, defaultQueue.getMaxWaitingQueries());
        assertEquals("max_wait_timeout in drill-rm-default is not configured with expected default value", RMCommonDefaults.MAX_WAIT_TIMEOUT_IN_MS, defaultQueue.getWaitTimeoutInMs());
        assertEquals("wait_for_preferred_nodes in drill-rm-default is not configured with expected default value", RMCommonDefaults.WAIT_FOR_PREFERRED_NODES, defaultQueue.waitForPreferredNodes());
    }
}

Example 5 with ResourcePoolTree

use of org.apache.drill.exec.resourcemgr.config.ResourcePoolTree in project drill by apache.

the class TestResourcePoolTree method testTreeWithLeafAndIntermediatePool.

@Test
public void testTreeWithLeafAndIntermediatePool() throws Exception {
    // left leaf pool1 with tag selector
    pool1.put(ResourcePoolImpl.POOL_QUEUE_KEY, queue1);
    pool1.put(ResourcePoolImpl.POOL_SELECTOR_KEY, tagSelectorConfig1);
    // left leaf pool2 with default selector
    pool2.put(ResourcePoolImpl.POOL_QUEUE_KEY, queue1);
    // intermediate left pool1 with 2 leaf pools (pool1, pool2)
    Map<String, Object> interPool1 = new HashMap<>();
    List<Object> childPools1 = new ArrayList<>();
    childPools1.add(pool1);
    childPools1.add(pool2);
    interPool1.put(ResourcePoolImpl.POOL_NAME_KEY, "eng");
    interPool1.put(ResourcePoolImpl.POOL_MEMORY_SHARE_KEY, 0.9);
    interPool1.put(ResourcePoolImpl.POOL_CHILDREN_POOLS_KEY, childPools1);
    // right leaf pool
    Map<String, Object> rightLeafPool = new HashMap<>();
    rightLeafPool.put(ResourcePoolImpl.POOL_NAME_KEY, "marketing");
    rightLeafPool.put(ResourcePoolImpl.POOL_MEMORY_SHARE_KEY, 0.1);
    rightLeafPool.put(ResourcePoolImpl.POOL_QUEUE_KEY, queue1);
    rightLeafPool.put(ResourcePoolImpl.POOL_SELECTOR_KEY, tagSelectorConfig2);
    childResourcePools.add(interPool1);
    childResourcePools.add(rightLeafPool);
    ResourcePoolTree configTree = getPoolTreeConfig();
    // Test successful selection of all leaf pools
    OptionValue testOption = OptionValue.create(OptionValue.AccessibleScopes.SESSION_AND_QUERY, ExecConstants.RM_QUERY_TAGS_KEY, "small,large", OptionValue.OptionScope.SESSION);
    when(mockContext.getOption(ExecConstants.RM_QUERY_TAGS_KEY)).thenReturn(testOption);
    QueueAssignmentResult assignmentResult = configTree.selectAllQueues(mockContext);
    List<ResourcePool> selectedPools = assignmentResult.getSelectedLeafPools();
    List<String> expectedPools = new ArrayList<>();
    expectedPools.add("dev");
    expectedPools.add("qa");
    expectedPools.add("marketing");
    assertTrue("All leaf pools are not selected", selectedPools.size() == 3);
    assertTrue("Selected leaf pools and expected pools are different", checkExpectedVsActualPools(selectedPools, expectedPools));
    // Test successful selection of multiple leaf pools
    expectedPools.clear();
    testOption = OptionValue.create(OptionValue.AccessibleScopes.SESSION_AND_QUERY, ExecConstants.RM_QUERY_TAGS_KEY, "small", OptionValue.OptionScope.SESSION);
    when(mockContext.getOption(ExecConstants.RM_QUERY_TAGS_KEY)).thenReturn(testOption);
    assignmentResult = configTree.selectAllQueues(mockContext);
    selectedPools = assignmentResult.getSelectedLeafPools();
    expectedPools.add("qa");
    expectedPools.add("dev");
    assertTrue("Expected 2 pools to be selected", selectedPools.size() == 2);
    assertTrue("Selected leaf pools and expected pools are different", checkExpectedVsActualPools(selectedPools, expectedPools));
    // Test successful selection of only left default pool
    expectedPools.clear();
    testOption = OptionValue.create(OptionValue.AccessibleScopes.SESSION_AND_QUERY, ExecConstants.RM_QUERY_TAGS_KEY, "medium", OptionValue.OptionScope.SESSION);
    when(mockContext.getOption(ExecConstants.RM_QUERY_TAGS_KEY)).thenReturn(testOption);
    assignmentResult = configTree.selectAllQueues(mockContext);
    selectedPools = assignmentResult.getSelectedLeafPools();
    expectedPools.add("qa");
    assertTrue("More than one leaf pool is selected", selectedPools.size() == 1);
    assertTrue("Selected leaf pools and expected pools are different", checkExpectedVsActualPools(selectedPools, expectedPools));
    // cleanup
    interPool1.clear();
    rightLeafPool.clear();
    expectedPools.clear();
}