Examples with PhysicalOperator org.apache.drill.exec.physical.base.PhysicalOperator used on opensource projects

Example 21 with PhysicalOperator

use of org.apache.drill.exec.physical.base.PhysicalOperator in project drill by apache.

the class MemoryAllocationUtilities method setupSortMemoryAllocations.

/**
   * Helper method to setup SortMemoryAllocations
   * since this method can be used in multiple places adding it in this class
   * rather than keeping it in Foreman
   * @param plan
   * @param queryContext
   */
public static void setupSortMemoryAllocations(final PhysicalPlan plan, final QueryContext queryContext) {
    if (plan.getProperties().hasResourcePlan) {
        return;
    }
    // look for external sorts
    final List<ExternalSort> sortList = new LinkedList<>();
    for (final PhysicalOperator op : plan.getSortedOperators()) {
        if (op instanceof ExternalSort) {
            sortList.add((ExternalSort) op);
        }
    }
    // if there are any sorts, compute the maximum allocation, and set it on them
    if (sortList.size() > 0) {
        final OptionManager optionManager = queryContext.getOptions();
        final long maxWidthPerNode = optionManager.getOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY).num_val;
        long maxAllocPerNode = Math.min(DrillConfig.getMaxDirectMemory(), queryContext.getConfig().getLong(RootAllocatorFactory.TOP_LEVEL_MAX_ALLOC));
        maxAllocPerNode = Math.min(maxAllocPerNode, optionManager.getOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY).num_val);
        final long maxSortAlloc = maxAllocPerNode / (sortList.size() * maxWidthPerNode);
        logger.debug("Max sort alloc: {}", maxSortAlloc);
        for (final ExternalSort externalSort : sortList) {
            // Ensure that the sort receives the minimum memory needed to make progress.
            // Without this, the math might work out to allocate too little memory.
            long alloc = Math.max(maxSortAlloc, externalSort.getInitialAllocation());
            externalSort.setMaxAllocation(alloc);
        }
    }
    plan.getProperties().hasResourcePlan = true;
}

Example 22 with PhysicalOperator

use of org.apache.drill.exec.physical.base.PhysicalOperator in project drill by apache.

the class Foreman method getQueryWorkUnit.

private QueryWorkUnit getQueryWorkUnit(final PhysicalPlan plan) throws ExecutionSetupException {
    final PhysicalOperator rootOperator = plan.getSortedOperators(false).iterator().next();
    final Fragment rootFragment = rootOperator.accept(MakeFragmentsVisitor.INSTANCE, null);
    final SimpleParallelizer parallelizer = new SimpleParallelizer(queryContext);
    final QueryWorkUnit queryWorkUnit = parallelizer.getFragments(queryContext.getOptions().getOptionList(), queryContext.getCurrentEndpoint(), queryId, queryContext.getActiveEndpoints(), drillbitContext.getPlanReader(), rootFragment, initiatingClient.getSession(), queryContext.getQueryContextInfo());
    if (logger.isTraceEnabled()) {
        final StringBuilder sb = new StringBuilder();
        sb.append("PlanFragments for query ");
        sb.append(queryId);
        sb.append('\n');
        final List<PlanFragment> planFragments = queryWorkUnit.getFragments();
        final int fragmentCount = planFragments.size();
        int fragmentIndex = 0;
        for (final PlanFragment planFragment : planFragments) {
            final FragmentHandle fragmentHandle = planFragment.getHandle();
            sb.append("PlanFragment(");
            sb.append(++fragmentIndex);
            sb.append('/');
            sb.append(fragmentCount);
            sb.append(") major_fragment_id ");
            sb.append(fragmentHandle.getMajorFragmentId());
            sb.append(" minor_fragment_id ");
            sb.append(fragmentHandle.getMinorFragmentId());
            sb.append('\n');
            final DrillbitEndpoint endpointAssignment = planFragment.getAssignment();
            sb.append("  DrillbitEndpoint address ");
            sb.append(endpointAssignment.getAddress());
            sb.append('\n');
            String jsonString = "<<malformed JSON>>";
            sb.append("  fragment_json: ");
            final ObjectMapper objectMapper = new ObjectMapper();
            try {
                final Object json = objectMapper.readValue(planFragment.getFragmentJson(), Object.class);
                jsonString = objectMapper.defaultPrettyPrintingWriter().writeValueAsString(json);
            } catch (final Exception e) {
            // we've already set jsonString to a fallback value
            }
            sb.append(jsonString);
            logger.trace(sb.toString());
        }
    }
    return queryWorkUnit;
}

Example 23 with PhysicalOperator

use of org.apache.drill.exec.physical.base.PhysicalOperator in project drill by apache.

the class TestAllocators method testAllocators.

@Test
public void testAllocators() throws Exception {
    // Setup a drillbit (initializes a root allocator)
    final DrillConfig config = DrillConfig.create(TEST_CONFIGURATIONS);
    try (final RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
        final Drillbit bit = new Drillbit(config, serviceSet)) {
        ;
        bit.run();
        final DrillbitContext bitContext = bit.getContext();
        FunctionImplementationRegistry functionRegistry = bitContext.getFunctionImplementationRegistry();
        StoragePluginRegistry storageRegistry = new StoragePluginRegistryImpl(bitContext);
        // Create a few Fragment Contexts
        BitControl.PlanFragment.Builder pfBuilder1 = BitControl.PlanFragment.newBuilder();
        pfBuilder1.setMemInitial(1500000);
        BitControl.PlanFragment pf1 = pfBuilder1.build();
        BitControl.PlanFragment.Builder pfBuilder2 = BitControl.PlanFragment.newBuilder();
        pfBuilder2.setMemInitial(500000);
        BitControl.PlanFragment pf2 = pfBuilder1.build();
        FragmentContext fragmentContext1 = new FragmentContext(bitContext, pf1, null, functionRegistry);
        FragmentContext fragmentContext2 = new FragmentContext(bitContext, pf2, null, functionRegistry);
        // Get a few physical operators. Easiest way is to read a physical plan.
        PhysicalPlanReader planReader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(bitContext, storageRegistry);
        PhysicalPlan plan = planReader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile(planFile), Charsets.UTF_8));
        List<PhysicalOperator> physicalOperators = plan.getSortedOperators();
        Iterator<PhysicalOperator> physicalOperatorIterator = physicalOperators.iterator();
        PhysicalOperator physicalOperator1 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator2 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator3 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator4 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator5 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator6 = physicalOperatorIterator.next();
        // Create some bogus Operator profile defs and stats to create operator contexts
        OpProfileDef def;
        OperatorStats stats;
        // Use some bogus operator type to create a new operator context.
        def = new OpProfileDef(physicalOperator1.getOperatorId(), UserBitShared.CoreOperatorType.MOCK_SUB_SCAN_VALUE, OperatorUtilities.getChildCount(physicalOperator1));
        stats = fragmentContext1.getStats().newOperatorStats(def, fragmentContext1.getAllocator());
        // Add a couple of Operator Contexts
        // Initial allocation = 1000000 bytes for all operators
        OperatorContext oContext11 = fragmentContext1.newOperatorContext(physicalOperator1);
        DrillBuf b11 = oContext11.getAllocator().buffer(1000000);
        OperatorContext oContext12 = fragmentContext1.newOperatorContext(physicalOperator2, stats);
        DrillBuf b12 = oContext12.getAllocator().buffer(500000);
        OperatorContext oContext21 = fragmentContext1.newOperatorContext(physicalOperator3);
        def = new OpProfileDef(physicalOperator4.getOperatorId(), UserBitShared.CoreOperatorType.TEXT_WRITER_VALUE, OperatorUtilities.getChildCount(physicalOperator4));
        stats = fragmentContext2.getStats().newOperatorStats(def, fragmentContext2.getAllocator());
        OperatorContext oContext22 = fragmentContext2.newOperatorContext(physicalOperator4, stats);
        DrillBuf b22 = oContext22.getAllocator().buffer(2000000);
        // New Fragment begins
        BitControl.PlanFragment.Builder pfBuilder3 = BitControl.PlanFragment.newBuilder();
        pfBuilder3.setMemInitial(1000000);
        BitControl.PlanFragment pf3 = pfBuilder3.build();
        FragmentContext fragmentContext3 = new FragmentContext(bitContext, pf3, null, functionRegistry);
        // New fragment starts an operator that allocates an amount within the limit
        def = new OpProfileDef(physicalOperator5.getOperatorId(), UserBitShared.CoreOperatorType.UNION_VALUE, OperatorUtilities.getChildCount(physicalOperator5));
        stats = fragmentContext3.getStats().newOperatorStats(def, fragmentContext3.getAllocator());
        OperatorContext oContext31 = fragmentContext3.newOperatorContext(physicalOperator5, stats);
        DrillBuf b31a = oContext31.getAllocator().buffer(200000);
        // Previously running operator completes
        b22.release();
        ((AutoCloseable) oContext22).close();
        // Fragment 3 asks for more and fails
        boolean outOfMem = false;
        try {
            oContext31.getAllocator().buffer(44000000);
            fail("Fragment 3 should fail to allocate buffer");
        } catch (OutOfMemoryException e) {
            // Expected.
            outOfMem = true;
        }
        assertTrue(outOfMem);
        // Operator is Exempt from Fragment limits. Fragment 3 asks for more and succeeds
        OperatorContext oContext32 = fragmentContext3.newOperatorContext(physicalOperator6);
        try {
            DrillBuf b32 = oContext32.getAllocator().buffer(4400000);
            b32.release();
        } catch (OutOfMemoryException e) {
            fail("Fragment 3 failed to allocate buffer");
        } finally {
            closeOp(oContext32);
        }
        b11.release();
        closeOp(oContext11);
        b12.release();
        closeOp(oContext12);
        closeOp(oContext21);
        b31a.release();
        closeOp(oContext31);
        fragmentContext1.close();
        fragmentContext2.close();
        fragmentContext3.close();
    }
}

Example 24 with PhysicalOperator

use of org.apache.drill.exec.physical.base.PhysicalOperator in project drill by apache.

the class PlanSplitter method getFragments.

private List<PlanFragment> getFragments(final DrillbitContext dContext, final GetQueryPlanFragments req, final QueryContext queryContext, final QueryId queryId) throws Exception {
    final PhysicalPlan plan;
    final String query = req.getQuery();
    switch(req.getType()) {
        case SQL:
            final Pointer<String> textPlan = new Pointer<>();
            plan = DrillSqlWorker.getPlan(queryContext, query, textPlan);
            break;
        case PHYSICAL:
            plan = dContext.getPlanReader().readPhysicalPlan(query);
            break;
        default:
            throw new IllegalStateException("Planning fragments supports only SQL or PHYSICAL QueryType");
    }
    MemoryAllocationUtilities.setupSortMemoryAllocations(plan, queryContext);
    final PhysicalOperator rootOperator = plan.getSortedOperators(false).iterator().next();
    final Fragment rootFragment = rootOperator.accept(MakeFragmentsVisitor.INSTANCE, null);
    final SimpleParallelizer parallelizer = new SplittingParallelizer(queryContext);
    List<PlanFragment> fragments = Lists.newArrayList();
    if (req.getSplitPlan()) {
        final List<QueryWorkUnit> queryWorkUnits = parallelizer.getSplitFragments(queryContext.getOptions().getOptionList(), queryContext.getCurrentEndpoint(), queryId, queryContext.getActiveEndpoints(), dContext.getPlanReader(), rootFragment, queryContext.getSession(), queryContext.getQueryContextInfo());
        for (QueryWorkUnit queryWorkUnit : queryWorkUnits) {
            fragments.add(queryWorkUnit.getRootFragment());
            List<PlanFragment> childFragments = queryWorkUnit.getFragments();
            if (!childFragments.isEmpty()) {
                throw new IllegalStateException("Split plans can not have more then one fragment");
            }
        }
    } else {
        final QueryWorkUnit queryWorkUnit = parallelizer.getFragments(queryContext.getOptions().getOptionList(), queryContext.getCurrentEndpoint(), queryId, queryContext.getActiveEndpoints(), dContext.getPlanReader(), rootFragment, queryContext.getSession(), queryContext.getQueryContextInfo());
        fragments.add(queryWorkUnit.getRootFragment());
        fragments.addAll(queryWorkUnit.getFragments());
    }
    return fragments;
}

Example 25 with PhysicalOperator

use of org.apache.drill.exec.physical.base.PhysicalOperator in project drill by apache.

the class UnorderedDeMuxExchangePrel method getPhysicalOperator.

@Override
public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException {
    Prel child = (Prel) this.getInput();
    PhysicalOperator childPOP = child.getPhysicalOperator(creator);
    UnorderedDeMuxExchange p = new UnorderedDeMuxExchange(childPOP, HashPrelUtil.getHashExpression(this.fields, getInput().getRowType()));
    return creator.addMetadata(this, p);
}