Examples with ExternalSort org.apache.drill.exec.physical.config.ExternalSort used on opensource projects

Example 1 with ExternalSort

use of org.apache.drill.exec.physical.config.ExternalSort in project drill by apache.

the class MemoryAllocationUtilities method setupSortMemoryAllocations.

/**
   * Helper method to setup SortMemoryAllocations
   * since this method can be used in multiple places adding it in this class
   * rather than keeping it in Foreman
   * @param plan
   * @param queryContext
   */
public static void setupSortMemoryAllocations(final PhysicalPlan plan, final QueryContext queryContext) {
    if (plan.getProperties().hasResourcePlan) {
        return;
    }
    // look for external sorts
    final List<ExternalSort> sortList = new LinkedList<>();
    for (final PhysicalOperator op : plan.getSortedOperators()) {
        if (op instanceof ExternalSort) {
            sortList.add((ExternalSort) op);
        }
    }
    // if there are any sorts, compute the maximum allocation, and set it on them
    if (sortList.size() > 0) {
        final OptionManager optionManager = queryContext.getOptions();
        final long maxWidthPerNode = optionManager.getOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY).num_val;
        long maxAllocPerNode = Math.min(DrillConfig.getMaxDirectMemory(), queryContext.getConfig().getLong(RootAllocatorFactory.TOP_LEVEL_MAX_ALLOC));
        maxAllocPerNode = Math.min(maxAllocPerNode, optionManager.getOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY).num_val);
        final long maxSortAlloc = maxAllocPerNode / (sortList.size() * maxWidthPerNode);
        logger.debug("Max sort alloc: {}", maxSortAlloc);
        for (final ExternalSort externalSort : sortList) {
            // Ensure that the sort receives the minimum memory needed to make progress.
            // Without this, the math might work out to allocate too little memory.
            long alloc = Math.max(maxSortAlloc, externalSort.getInitialAllocation());
            externalSort.setMaxAllocation(alloc);
        }
    }
    plan.getProperties().hasResourcePlan = true;
}

Example 2 with ExternalSort

use of org.apache.drill.exec.physical.config.ExternalSort in project drill by axbaretto.

the class BasicPhysicalOpUnitTest method testExternalSort.

@Test
public void testExternalSort() {
    ExternalSort sortConf = new ExternalSort(null, Lists.newArrayList(ordering("b", RelFieldCollation.Direction.ASCENDING, RelFieldCollation.NullDirection.FIRST)), false);
    List<String> inputJsonBatches = Lists.newArrayList("[{\"a\": 5, \"b\" : 1 }]", "[{\"a\": 5, \"b\" : 5},{\"a\": 3, \"b\" : 8}]", "[{\"a\": 40, \"b\" : 3},{\"a\": 13, \"b\" : 100}]");
    opTestBuilder().physicalOperator(sortConf).maxAllocation(15_000_000L).inputDataStreamJson(inputJsonBatches).baselineColumns("a", "b").baselineValues(5l, 1l).baselineValues(40l, 3l).baselineValues(5l, 5l).baselineValues(3l, 8l).baselineValues(13l, 100l).go();
}

Example 3 with ExternalSort

use of org.apache.drill.exec.physical.config.ExternalSort in project drill by axbaretto.

the class BasicPhysicalOpUnitTest method externalSortLowMemoryHelper.

private void externalSortLowMemoryHelper(int batchSize, int numberOfBatches, long initReservation, long maxAllocation) {
    ExternalSort sortConf = new ExternalSort(null, Lists.newArrayList(ordering("b", RelFieldCollation.Direction.ASCENDING, RelFieldCollation.NullDirection.FIRST)), false);
    List<String> inputJsonBatches = Lists.newArrayList();
    StringBuilder batchString = new StringBuilder();
    for (int j = 0; j < numberOfBatches; j++) {
        batchString.append("[");
        for (int i = 0; i < batchSize; i++) {
            batchString.append("{\"a\": 5, \"b\" : 5},{\"a\": 3, \"b\" : 8},");
        }
        batchString.append("{\"a\": 5, \"b\" : 1 }");
        batchString.append("]");
        inputJsonBatches.add(batchString.toString());
    }
    OperatorTestBuilder opTestBuilder = opTestBuilder().initReservation(initReservation).maxAllocation(maxAllocation).physicalOperator(sortConf).inputDataStreamJson(inputJsonBatches).baselineColumns("a", "b");
    for (int i = 0; i < numberOfBatches; i++) {
        opTestBuilder.baselineValues(5l, 1l);
    }
    for (int i = 0; i < batchSize * numberOfBatches; i++) {
        opTestBuilder.baselineValues(5l, 5l);
    }
    for (int i = 0; i < batchSize * numberOfBatches; i++) {
        opTestBuilder.baselineValues(3l, 8l);
    }
    opTestBuilder.go();
}

Example 4 with ExternalSort

use of org.apache.drill.exec.physical.config.ExternalSort in project drill by apache.

the class TestExternalSortExec method testSortSpec.

@Test
public void testSortSpec() {
    FieldReference expr = FieldReference.getWithQuotedRef("foo");
    Ordering ordering = new Ordering(Ordering.ORDER_ASC, expr, Ordering.NULLS_FIRST);
    // Basics
    ExternalSort popConfig = new ExternalSort(null, Lists.newArrayList(ordering), false);
    assertSame(ordering, popConfig.getOrderings().get(0));
    assertFalse(popConfig.getReverse());
    assertEquals(SelectionVectorMode.FOUR_BYTE, popConfig.getSVMode());
    assertEquals(ExternalSort.OPERATOR_TYPE, popConfig.getOperatorType());
    assertEquals(ExternalSort.DEFAULT_SORT_ALLOCATION, popConfig.getInitialAllocation());
    assertEquals(AbstractBase.MAX_ALLOCATION, popConfig.getMaxAllocation());
    assertTrue(popConfig.isExecutable());
    // Non-default settings
    popConfig = new ExternalSort(null, Lists.newArrayList(ordering), true);
    assertTrue(popConfig.getReverse());
    long maxAlloc = 50_000_000;
    popConfig.setMaxAllocation(maxAlloc);
    assertEquals(ExternalSort.DEFAULT_SORT_ALLOCATION, popConfig.getInitialAllocation());
    assertEquals(maxAlloc, popConfig.getMaxAllocation());
}

Example 5 with ExternalSort

use of org.apache.drill.exec.physical.config.ExternalSort in project drill by apache.

the class TestNullInputMiniPlan method testSortEmpty.

@Test
public void testSortEmpty() throws Exception {
    final PhysicalOperator sort = new ExternalSort(null, Lists.newArrayList(ordering("b", RelFieldCollation.Direction.ASCENDING, RelFieldCollation.NullDirection.FIRST)), false);
    testSingleInputNullBatchHandling(sort);
}