Search in sources :

Example 6 with PhysicalOperator

use of org.apache.drill.exec.physical.base.PhysicalOperator in project drill by apache.

the class ExchangeRemoverMaterializer method visitStore.

@Override
public PhysicalOperator visitStore(Store store, IndexedFragmentNode iNode) throws ExecutionSetupException {
    PhysicalOperator child = store.getChild().accept(this, iNode);
    iNode.addAllocation(store);
    try {
        PhysicalOperator o = store.getSpecificStore(child, iNode.getMinorFragmentId());
        return o;
    } catch (PhysicalOperatorSetupException e) {
        throw new FragmentSetupException("Failure while generating a specific Store materialization.", e);
    }
}
Also used : PhysicalOperator(org.apache.drill.exec.physical.base.PhysicalOperator) PhysicalOperatorSetupException(org.apache.drill.exec.physical.PhysicalOperatorSetupException) FragmentSetupException(org.apache.drill.exec.exception.FragmentSetupException)

Example 7 with PhysicalOperator

use of org.apache.drill.exec.physical.base.PhysicalOperator in project drill by apache.

the class ExchangeRemoverMaterializer method visitOp.

@Override
public PhysicalOperator visitOp(PhysicalOperator op, IndexedFragmentNode iNode) throws ExecutionSetupException {
    iNode.addAllocation(op);
    List<PhysicalOperator> children = Lists.newArrayList();
    for (PhysicalOperator child : op) {
        children.add(child.accept(this, iNode));
    }
    PhysicalOperator newOp = op.getNewWithChildren(children);
    newOp.setCost(op.getCost());
    return newOp;
}
Also used : PhysicalOperator(org.apache.drill.exec.physical.base.PhysicalOperator)

Example 8 with PhysicalOperator

use of org.apache.drill.exec.physical.base.PhysicalOperator in project drill by apache.

the class ExchangeRemoverMaterializer method visitExchange.

@Override
public PhysicalOperator visitExchange(Exchange exchange, IndexedFragmentNode iNode) throws ExecutionSetupException {
    iNode.addAllocation(exchange);
    PhysicalOperator childEx = exchange.getChild().accept(this, iNode);
    return childEx;
}
Also used : PhysicalOperator(org.apache.drill.exec.physical.base.PhysicalOperator)

Example 9 with PhysicalOperator

use of org.apache.drill.exec.physical.base.PhysicalOperator in project drill by apache.

the class MemoryAllocationUtilities method setupSortMemoryAllocations.

/**
   * Helper method to setup SortMemoryAllocations
   * since this method can be used in multiple places adding it in this class
   * rather than keeping it in Foreman
   * @param plan
   * @param queryContext
   */
public static void setupSortMemoryAllocations(final PhysicalPlan plan, final QueryContext queryContext) {
    if (plan.getProperties().hasResourcePlan) {
        return;
    }
    // look for external sorts
    final List<ExternalSort> sortList = new LinkedList<>();
    for (final PhysicalOperator op : plan.getSortedOperators()) {
        if (op instanceof ExternalSort) {
            sortList.add((ExternalSort) op);
        }
    }
    // if there are any sorts, compute the maximum allocation, and set it on them
    if (sortList.size() > 0) {
        final OptionManager optionManager = queryContext.getOptions();
        final long maxWidthPerNode = optionManager.getOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY).num_val;
        long maxAllocPerNode = Math.min(DrillConfig.getMaxDirectMemory(), queryContext.getConfig().getLong(RootAllocatorFactory.TOP_LEVEL_MAX_ALLOC));
        maxAllocPerNode = Math.min(maxAllocPerNode, optionManager.getOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY).num_val);
        final long maxSortAlloc = maxAllocPerNode / (sortList.size() * maxWidthPerNode);
        logger.debug("Max sort alloc: {}", maxSortAlloc);
        for (final ExternalSort externalSort : sortList) {
            // Ensure that the sort receives the minimum memory needed to make progress.
            // Without this, the math might work out to allocate too little memory.
            long alloc = Math.max(maxSortAlloc, externalSort.getInitialAllocation());
            externalSort.setMaxAllocation(alloc);
        }
    }
    plan.getProperties().hasResourcePlan = true;
}
Also used : ExternalSort(org.apache.drill.exec.physical.config.ExternalSort) PhysicalOperator(org.apache.drill.exec.physical.base.PhysicalOperator) LinkedList(java.util.LinkedList) OptionManager(org.apache.drill.exec.server.options.OptionManager)

Example 10 with PhysicalOperator

use of org.apache.drill.exec.physical.base.PhysicalOperator in project drill by apache.

the class Foreman method getQueryWorkUnit.

private QueryWorkUnit getQueryWorkUnit(final PhysicalPlan plan) throws ExecutionSetupException {
    final PhysicalOperator rootOperator = plan.getSortedOperators(false).iterator().next();
    final Fragment rootFragment = rootOperator.accept(MakeFragmentsVisitor.INSTANCE, null);
    final SimpleParallelizer parallelizer = new SimpleParallelizer(queryContext);
    final QueryWorkUnit queryWorkUnit = parallelizer.getFragments(queryContext.getOptions().getOptionList(), queryContext.getCurrentEndpoint(), queryId, queryContext.getActiveEndpoints(), drillbitContext.getPlanReader(), rootFragment, initiatingClient.getSession(), queryContext.getQueryContextInfo());
    if (logger.isTraceEnabled()) {
        final StringBuilder sb = new StringBuilder();
        sb.append("PlanFragments for query ");
        sb.append(queryId);
        sb.append('\n');
        final List<PlanFragment> planFragments = queryWorkUnit.getFragments();
        final int fragmentCount = planFragments.size();
        int fragmentIndex = 0;
        for (final PlanFragment planFragment : planFragments) {
            final FragmentHandle fragmentHandle = planFragment.getHandle();
            sb.append("PlanFragment(");
            sb.append(++fragmentIndex);
            sb.append('/');
            sb.append(fragmentCount);
            sb.append(") major_fragment_id ");
            sb.append(fragmentHandle.getMajorFragmentId());
            sb.append(" minor_fragment_id ");
            sb.append(fragmentHandle.getMinorFragmentId());
            sb.append('\n');
            final DrillbitEndpoint endpointAssignment = planFragment.getAssignment();
            sb.append("  DrillbitEndpoint address ");
            sb.append(endpointAssignment.getAddress());
            sb.append('\n');
            String jsonString = "<<malformed JSON>>";
            sb.append("  fragment_json: ");
            final ObjectMapper objectMapper = new ObjectMapper();
            try {
                final Object json = objectMapper.readValue(planFragment.getFragmentJson(), Object.class);
                jsonString = objectMapper.defaultPrettyPrintingWriter().writeValueAsString(json);
            } catch (final Exception e) {
            // we've already set jsonString to a fallback value
            }
            sb.append(jsonString);
            logger.trace(sb.toString());
        }
    }
    return queryWorkUnit;
}
Also used : QueryWorkUnit(org.apache.drill.exec.work.QueryWorkUnit) SimpleParallelizer(org.apache.drill.exec.planner.fragment.SimpleParallelizer) FragmentHandle(org.apache.drill.exec.proto.ExecProtos.FragmentHandle) PlanFragment(org.apache.drill.exec.proto.BitControl.PlanFragment) Fragment(org.apache.drill.exec.planner.fragment.Fragment) PlanFragment(org.apache.drill.exec.proto.BitControl.PlanFragment) DrillbitEndpoint(org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint) UserException(org.apache.drill.common.exceptions.UserException) RpcException(org.apache.drill.exec.rpc.RpcException) InvalidProtocolBufferException(com.google.protobuf.InvalidProtocolBufferException) OptimizerException(org.apache.drill.exec.exception.OptimizerException) OutOfMemoryException(org.apache.drill.exec.exception.OutOfMemoryException) ExecutionSetupException(org.apache.drill.common.exceptions.ExecutionSetupException) IOException(java.io.IOException) DrillbitEndpoint(org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint) PhysicalOperator(org.apache.drill.exec.physical.base.PhysicalOperator) ObjectMapper(org.codehaus.jackson.map.ObjectMapper)

Aggregations

PhysicalOperator (org.apache.drill.exec.physical.base.PhysicalOperator)57 PhysicalPlan (org.apache.drill.exec.physical.PhysicalPlan)13 FragmentRoot (org.apache.drill.exec.physical.base.FragmentRoot)6 PhysicalPlanReader (org.apache.drill.exec.planner.PhysicalPlanReader)6 Test (org.junit.Test)6 FunctionImplementationRegistry (org.apache.drill.exec.expr.fn.FunctionImplementationRegistry)4 Fragment (org.apache.drill.exec.planner.fragment.Fragment)4 DrillParseContext (org.apache.drill.exec.planner.logical.DrillParseContext)4 PlanFragment (org.apache.drill.exec.proto.BitControl.PlanFragment)4 IOException (java.io.IOException)3 RelNode (org.apache.calcite.rel.RelNode)3 RelDataType (org.apache.calcite.rel.type.RelDataType)3 DrillConfig (org.apache.drill.common.config.DrillConfig)3 FragmentContext (org.apache.drill.exec.ops.FragmentContext)3 OpProfileDef (org.apache.drill.exec.ops.OpProfileDef)3 OperatorStats (org.apache.drill.exec.ops.OperatorStats)3 QueryWorkUnit (org.apache.drill.exec.work.QueryWorkUnit)3 JsonProcessingException (com.fasterxml.jackson.core.JsonProcessingException)2 JoinRelType (org.apache.calcite.rel.core.JoinRelType)2 ExecutionSetupException (org.apache.drill.common.exceptions.ExecutionSetupException)2