Examples with PhysicalOperator org.apache.drill.exec.physical.base.PhysicalOperator used on opensource projects

Example 11 with PhysicalOperator

use of org.apache.drill.exec.physical.base.PhysicalOperator in project drill by apache.

the class TestAllocators method testAllocators.

@Test
public void testAllocators() throws Exception {
    // Setup a drillbit (initializes a root allocator)
    final DrillConfig config = DrillConfig.create(TEST_CONFIGURATIONS);
    try (final RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
        final Drillbit bit = new Drillbit(config, serviceSet)) {
        ;
        bit.run();
        final DrillbitContext bitContext = bit.getContext();
        FunctionImplementationRegistry functionRegistry = bitContext.getFunctionImplementationRegistry();
        StoragePluginRegistry storageRegistry = new StoragePluginRegistryImpl(bitContext);
        // Create a few Fragment Contexts
        BitControl.PlanFragment.Builder pfBuilder1 = BitControl.PlanFragment.newBuilder();
        pfBuilder1.setMemInitial(1500000);
        BitControl.PlanFragment pf1 = pfBuilder1.build();
        BitControl.PlanFragment.Builder pfBuilder2 = BitControl.PlanFragment.newBuilder();
        pfBuilder2.setMemInitial(500000);
        BitControl.PlanFragment pf2 = pfBuilder1.build();
        FragmentContext fragmentContext1 = new FragmentContext(bitContext, pf1, null, functionRegistry);
        FragmentContext fragmentContext2 = new FragmentContext(bitContext, pf2, null, functionRegistry);
        // Get a few physical operators. Easiest way is to read a physical plan.
        PhysicalPlanReader planReader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(bitContext, storageRegistry);
        PhysicalPlan plan = planReader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile(planFile), Charsets.UTF_8));
        List<PhysicalOperator> physicalOperators = plan.getSortedOperators();
        Iterator<PhysicalOperator> physicalOperatorIterator = physicalOperators.iterator();
        PhysicalOperator physicalOperator1 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator2 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator3 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator4 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator5 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator6 = physicalOperatorIterator.next();
        // Create some bogus Operator profile defs and stats to create operator contexts
        OpProfileDef def;
        OperatorStats stats;
        // Use some bogus operator type to create a new operator context.
        def = new OpProfileDef(physicalOperator1.getOperatorId(), UserBitShared.CoreOperatorType.MOCK_SUB_SCAN_VALUE, OperatorUtilities.getChildCount(physicalOperator1));
        stats = fragmentContext1.getStats().newOperatorStats(def, fragmentContext1.getAllocator());
        // Add a couple of Operator Contexts
        // Initial allocation = 1000000 bytes for all operators
        OperatorContext oContext11 = fragmentContext1.newOperatorContext(physicalOperator1);
        DrillBuf b11 = oContext11.getAllocator().buffer(1000000);
        OperatorContext oContext12 = fragmentContext1.newOperatorContext(physicalOperator2, stats);
        DrillBuf b12 = oContext12.getAllocator().buffer(500000);
        OperatorContext oContext21 = fragmentContext1.newOperatorContext(physicalOperator3);
        def = new OpProfileDef(physicalOperator4.getOperatorId(), UserBitShared.CoreOperatorType.TEXT_WRITER_VALUE, OperatorUtilities.getChildCount(physicalOperator4));
        stats = fragmentContext2.getStats().newOperatorStats(def, fragmentContext2.getAllocator());
        OperatorContext oContext22 = fragmentContext2.newOperatorContext(physicalOperator4, stats);
        DrillBuf b22 = oContext22.getAllocator().buffer(2000000);
        // New Fragment begins
        BitControl.PlanFragment.Builder pfBuilder3 = BitControl.PlanFragment.newBuilder();
        pfBuilder3.setMemInitial(1000000);
        BitControl.PlanFragment pf3 = pfBuilder3.build();
        FragmentContext fragmentContext3 = new FragmentContext(bitContext, pf3, null, functionRegistry);
        // New fragment starts an operator that allocates an amount within the limit
        def = new OpProfileDef(physicalOperator5.getOperatorId(), UserBitShared.CoreOperatorType.UNION_VALUE, OperatorUtilities.getChildCount(physicalOperator5));
        stats = fragmentContext3.getStats().newOperatorStats(def, fragmentContext3.getAllocator());
        OperatorContext oContext31 = fragmentContext3.newOperatorContext(physicalOperator5, stats);
        DrillBuf b31a = oContext31.getAllocator().buffer(200000);
        // Previously running operator completes
        b22.release();
        ((AutoCloseable) oContext22).close();
        // Fragment 3 asks for more and fails
        boolean outOfMem = false;
        try {
            oContext31.getAllocator().buffer(44000000);
            fail("Fragment 3 should fail to allocate buffer");
        } catch (OutOfMemoryException e) {
            // Expected.
            outOfMem = true;
        }
        assertTrue(outOfMem);
        // Operator is Exempt from Fragment limits. Fragment 3 asks for more and succeeds
        OperatorContext oContext32 = fragmentContext3.newOperatorContext(physicalOperator6);
        try {
            DrillBuf b32 = oContext32.getAllocator().buffer(4400000);
            b32.release();
        } catch (OutOfMemoryException e) {
            fail("Fragment 3 failed to allocate buffer");
        } finally {
            closeOp(oContext32);
        }
        b11.release();
        closeOp(oContext11);
        b12.release();
        closeOp(oContext12);
        closeOp(oContext21);
        b31a.release();
        closeOp(oContext31);
        fragmentContext1.close();
        fragmentContext2.close();
        fragmentContext3.close();
    }
}

Example 12 with PhysicalOperator

use of org.apache.drill.exec.physical.base.PhysicalOperator in project drill by apache.

the class BasicOptimizer method optimize.

@Override
public PhysicalPlan optimize(final OptimizationContext context, final LogicalPlan plan) throws OptimizerException {
    final Object obj = new Object();
    final Collection<SinkOperator> roots = plan.getGraph().getRoots();
    final List<PhysicalOperator> physOps = new ArrayList<>(roots.size());
    final LogicalConverter converter = new LogicalConverter(plan);
    for (SinkOperator op : roots) {
        final PhysicalOperator pop = op.accept(converter, obj);
        physOps.add(pop);
    }
    final PlanProperties logicalProperties = plan.getProperties();
    final PlanProperties props = PlanProperties.builder().type(PlanProperties.PlanType.APACHE_DRILL_PHYSICAL).version(logicalProperties.version).generator(logicalProperties.generator).options(new JSONOptions(context.getOptions().getOptionList())).build();
    final PhysicalPlan p = new PhysicalPlan(props, physOps);
    return p;
}

Example 13 with PhysicalOperator

use of org.apache.drill.exec.physical.base.PhysicalOperator in project drill by apache.

the class TraceInjector method getExec.

public static PhysicalOperator getExec(FragmentContext context, FragmentRoot root) throws ExecutionSetupException {
    TraceInjector tI = new TraceInjector();
    PhysicalOperator newOp = root.accept(tI, context);
    return newOp;
}

Example 14 with PhysicalOperator

use of org.apache.drill.exec.physical.base.PhysicalOperator in project drill by apache.

the class TraceInjector method visitOp.

/**
     * Traverse the physical plan and inject the trace operator after
     * every operator.
     * @param op Physical operator under which the trace operator will be injected
     * @param context Fragment context
     * @return same physical operator as passed in, but its child will be a trace operator
     *         whose child will be the original child of this operator
     * @throws ExecutionSetupException
     */
@Override
public PhysicalOperator visitOp(PhysicalOperator op, FragmentContext context) throws ExecutionSetupException {
    List<PhysicalOperator> newChildren = Lists.newArrayList();
    List<PhysicalOperator> list = null;
    PhysicalOperator newOp = op;
    /* Get the list of child operators */
    for (PhysicalOperator child : op) {
        newChildren.add(child.accept(this, context));
    }
    list = Lists.newArrayList();
    /* For every child operator create a trace operator as its parent */
    for (int i = 0; i < newChildren.size(); i++) {
        String traceTag = newChildren.get(i).toString() + Integer.toString(traceTagCount++);
        /* Trace operator */
        Trace traceOp = new Trace(newChildren.get(i), traceTag);
        list.add(traceOp);
    }
    /* Inject trace operator */
    if (list.size() > 0) {
        newOp = op.getNewWithChildren(list);
    }
    newOp.setOperatorId(op.getOperatorId());
    return newOp;
}

Example 15 with PhysicalOperator

use of org.apache.drill.exec.physical.base.PhysicalOperator in project drill by apache.

the class HashJoinPrel method getHashJoinPop.

private PhysicalOperator getHashJoinPop(PhysicalPlanCreator creator, RelNode left, RelNode right, List<Integer> leftKeys, List<Integer> rightKeys) throws IOException {
    final List<String> fields = getRowType().getFieldNames();
    assert isUnique(fields);
    final List<String> leftFields = left.getRowType().getFieldNames();
    final List<String> rightFields = right.getRowType().getFieldNames();
    PhysicalOperator leftPop = ((Prel) left).getPhysicalOperator(creator);
    PhysicalOperator rightPop = ((Prel) right).getPhysicalOperator(creator);
    JoinRelType jtype = this.getJoinType();
    List<JoinCondition> conditions = Lists.newArrayList();
    buildJoinConditions(conditions, leftFields, rightFields, leftKeys, rightKeys);
    HashJoinPOP hjoin = new HashJoinPOP(leftPop, rightPop, conditions, jtype);
    return creator.addMetadata(this, hjoin);
}