Examples with OptionManager org.apache.drill.exec.server.options.OptionManager used on opensource projects

Example 16 with OptionManager

use of org.apache.drill.exec.server.options.OptionManager in project drill by apache.

the class HiveStoragePlugin method getPhysicalOptimizerRules.

@Override
public Set<StoragePluginOptimizerRule> getPhysicalOptimizerRules(OptimizerRulesContext optimizerRulesContext) {
    ImmutableSet.Builder<StoragePluginOptimizerRule> ruleBuilder = ImmutableSet.builder();
    OptionManager options = optimizerRulesContext.getPlannerSettings().getOptions();
    // once "store.parquet.reader.int96_as_timestamp" will be true by default
    if (options.getBoolean(ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS) || options.getBoolean(ExecConstants.HIVE_OPTIMIZE_PARQUET_SCAN_WITH_NATIVE_READER)) {
        ruleBuilder.add(ConvertHiveParquetScanToDrillParquetScan.INSTANCE);
    }
    if (options.getBoolean(ExecConstants.HIVE_OPTIMIZE_MAPRDB_JSON_SCAN_WITH_NATIVE_READER)) {
        try {
            Class<?> hiveToDrillMapRDBJsonRuleClass = Class.forName("org.apache.drill.exec.planner.sql.logical.ConvertHiveMapRDBJsonScanToDrillMapRDBJsonScan");
            ruleBuilder.add((StoragePluginOptimizerRule) hiveToDrillMapRDBJsonRuleClass.getField("INSTANCE").get(null));
        } catch (ReflectiveOperationException e) {
            logger.warn("Current Drill build is not designed for working with Hive MapR-DB tables. " + "Please disable {} option", ExecConstants.HIVE_OPTIMIZE_MAPRDB_JSON_SCAN_WITH_NATIVE_READER);
        }
    }
    return ruleBuilder.build();
}

Example 17 with OptionManager

use of org.apache.drill.exec.server.options.OptionManager in project drill by apache.

the class MemoryAllocationUtilities method setupBufferedOpsMemoryAllocations.

/**
 * Helper method to setup Memory Allocations
 * <p>
 * Plan the memory for buffered operators (the only ones that can spill in this release)
 * based on assumptions. These assumptions are the amount of memory per node to give
 * to each query and the number of sort operators per node.
 * <p>
 * The reason the total
 * memory is an assumption is that we have know knowledge of the number of queries
 * that can run, so we need the user to tell use that information by configuring the
 * amount of memory to be assumed available to each query.
 * <p>
 * The number of sorts per node could be calculated, but we instead simply take
 * the worst case: the maximum per-query, per-node parallization and assume that
 * all sorts appear in all fragments &mdash; a gross oversimplification, but one
 * that Drill has long made.
 * <p>
 * since this method can be used in multiple places adding it in this class
 * rather than keeping it in Foreman
 * @param planHasMemory defines the memory planning needs to be done or not.
 *                             generally skipped when the plan contains memory allocation.
 * @param bufferedOperators list of buffered operators in the plan.
 * @param queryContext context of the query.
 */
public static void setupBufferedOpsMemoryAllocations(boolean planHasMemory, List<PhysicalOperator> bufferedOperators, final QueryContext queryContext) {
    if (planHasMemory || bufferedOperators.isEmpty()) {
        return;
    }
    // Setup options, etc.
    final OptionManager optionManager = queryContext.getOptions();
    final long directMemory = DrillConfig.getMaxDirectMemory();
    // Compute per-node, per-query memory.
    final long maxAllocPerNode = computeQueryMemory(queryContext.getConfig(), optionManager, directMemory);
    logger.debug("Memory per query per node: {}", maxAllocPerNode);
    // Now divide up the memory by slices and operators.
    final long opMinMem = computeOperatorMemory(optionManager, maxAllocPerNode, bufferedOperators.size());
    for (final PhysicalOperator op : bufferedOperators) {
        final long alloc = Math.max(opMinMem, op.getInitialAllocation());
        op.setMaxAllocation(alloc);
    }
}

Example 18 with OptionManager

use of org.apache.drill.exec.server.options.OptionManager in project drill by apache.

the class TestQueryMemoryAlloc method testDefaultOptions.

@Test
public void testDefaultOptions() throws Exception {
    OperatorFixture.Builder builder = OperatorFixture.builder(dirTestWatcher);
    builder.systemOption(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.05);
    builder.systemOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 2 * ONE_GB);
    try (OperatorFixture fixture = builder.build()) {
        final OptionManager optionManager = fixture.getOptionManager();
        optionManager.setLocalOption(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.05);
        optionManager.setLocalOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 2 * ONE_GB);
        // Out-of-box memory, use query memory per node as floor.
        long mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), optionManager, 8 * ONE_GB);
        assertEquals(2 * ONE_GB, mem);
        // Up to 40 GB, query memory dominates.
        mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), optionManager, 40 * ONE_GB);
        assertEquals(2 * ONE_GB, mem);
        // After 40 GB, the percent dominates
        mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), optionManager, 100 * ONE_GB);
        assertEquals(5 * ONE_GB, mem);
    }
}

Example 19 with OptionManager

use of org.apache.drill.exec.server.options.OptionManager in project drill by apache.

the class TestQueryMemoryAlloc method testCustomPercent.

@Test
public void testCustomPercent() throws Exception {
    OperatorFixture.Builder builder = OperatorFixture.builder(dirTestWatcher);
    builder.systemOption(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.10);
    builder.systemOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 2 * ONE_GB);
    try (OperatorFixture fixture = builder.build()) {
        final OptionManager optionManager = fixture.getOptionManager();
        optionManager.setLocalOption(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.10);
        optionManager.setLocalOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 2 * ONE_GB);
        // Out-of-box memory, use query memory per node as floor.
        long mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), optionManager, 8 * ONE_GB);
        assertEquals(2 * ONE_GB, mem);
        // Up to 20 GB, query memory dominates.
        mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), optionManager, 20 * ONE_GB);
        assertEquals(2 * ONE_GB, mem);
        // After 20 GB, the percent dominates
        mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), optionManager, 30 * ONE_GB);
        assertEquals(3 * ONE_GB, mem);
    }
}

Example 20 with OptionManager

use of org.apache.drill.exec.server.options.OptionManager in project drill by apache.

the class TestQueryMemoryAlloc method testCustomFloor.

@Test
public void testCustomFloor() throws Exception {
    OperatorFixture.Builder builder = OperatorFixture.builder(dirTestWatcher);
    builder.systemOption(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.05);
    builder.systemOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 2 * ONE_GB);
    try (OperatorFixture fixture = builder.build()) {
        final OptionManager optionManager = fixture.getOptionManager();
        optionManager.setLocalOption(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.05);
        optionManager.setLocalOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 2 * ONE_GB);
        // Out-of-box memory, use query memory per node as floor.
        long mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), optionManager, 8 * ONE_GB);
        assertEquals(2 * ONE_GB, mem);
        // Up to 60 GB, query memory dominates.
        mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), optionManager, 60 * ONE_GB);
        assertEquals(3 * ONE_GB, mem);
        // After 60 GB, the percent dominates
        mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), optionManager, 100 * ONE_GB);
        assertEquals(5 * ONE_GB, mem);
    }
}