Examples with PhysicalPlanReader org.apache.drill.exec.planner.PhysicalPlanReader used on opensource projects

Example 16 with PhysicalPlanReader

use of org.apache.drill.exec.planner.PhysicalPlanReader in project drill by apache.

the class TestMergeJoin method orderedEqualityInnerJoin.

@Test
@Ignore
public void orderedEqualityInnerJoin(@Injectable final DrillbitContext bitContext, @Injectable UserClientConnection connection) throws Throwable {
    mockDrillbitContext(bitContext);
    final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c, new StoragePluginRegistryImpl(bitContext));
    final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/join/merge_inner_single_batch.json"), Charsets.UTF_8).replace("#{LEFT_FILE}", FileUtils.getResourceAsFile("/join/merge_single_batch.left.json").toURI().toString()).replace("#{RIGHT_FILE}", FileUtils.getResourceAsFile("/join/merge_single_batch.right.json").toURI().toString()));
    final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
    final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
    final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
    int totalRecordCount = 0;
    while (exec.next()) {
        totalRecordCount += exec.getRecordCount();
        System.out.println("got next with record count: " + exec.getRecordCount() + " (total: " + totalRecordCount + "):");
        System.out.println("       t1                 t2");
        for (int valueIdx = 0; valueIdx < exec.getRecordCount(); valueIdx++) {
            final List<Object> row = Lists.newArrayList();
            for (final ValueVector v : exec) {
                row.add(v.getField().getPath() + ":" + v.getAccessor().getObject(valueIdx));
            }
            for (final Object cell : row) {
                if (cell == null) {
                    System.out.print("<null>    ");
                    continue;
                }
                final int len = cell.toString().length();
                System.out.print(cell + " ");
                for (int i = 0; i < (10 - len); ++i) {
                    System.out.print(" ");
                }
            }
            System.out.println();
        }
    }
    System.out.println("Total Record Count: " + totalRecordCount);
    assertEquals(23, totalRecordCount);
    if (context.getFailureCause() != null) {
        throw context.getFailureCause();
    }
    assertTrue(!context.isFailed());
}

Example 17 with PhysicalPlanReader

use of org.apache.drill.exec.planner.PhysicalPlanReader in project drill by apache.

the class TestSimpleLimit method verifyLimitCount.

private void verifyLimitCount(DrillbitContext bitContext, UserClientConnection connection, String testPlan, int expectedCount) throws Throwable {
    final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
    final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/limit/" + testPlan), Charsets.UTF_8));
    final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
    final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
    final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
    int recordCount = 0;
    while (exec.next()) {
        recordCount += exec.getRecordCount();
    }
    assertEquals(expectedCount, recordCount);
    if (context.getFailureCause() != null) {
        throw context.getFailureCause();
    }
    assertTrue(!context.isFailed());
}

Example 18 with PhysicalPlanReader

use of org.apache.drill.exec.planner.PhysicalPlanReader in project drill by apache.

the class TestSimpleLimit method verifySum.

private void verifySum(DrillbitContext bitContext, UserClientConnection connection, String testPlan, int expectedCount, long expectedSum) throws Throwable {
    final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
    final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/limit/" + testPlan), Charsets.UTF_8));
    final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
    final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
    final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
    int recordCount = 0;
    long sum = 0;
    while (exec.next()) {
        recordCount += exec.getRecordCount();
        final BigIntVector v = (BigIntVector) exec.iterator().next();
        for (int i = 0; i < v.getAccessor().getValueCount(); i++) {
            sum += v.getAccessor().get(i);
        }
    }
    assertEquals(expectedCount, recordCount);
    assertEquals(expectedSum, sum);
    if (context.getFailureCause() != null) {
        throw context.getFailureCause();
    }
    assertTrue(!context.isFailed());
}

Example 19 with PhysicalPlanReader

use of org.apache.drill.exec.planner.PhysicalPlanReader in project drill by apache.

the class TestOptiqPlans method doLogicalTest.

private SimpleRootExec doLogicalTest(final BootStrapContext context, UserClientConnection connection, String file, ClusterCoordinator coord, DataConnectionCreator com, Controller controller, WorkEventBus workBus) throws Exception {
    new NonStrictExpectations() {

        {
            context.getMetrics();
            result = new MetricRegistry();
            context.getAllocator();
            result = RootAllocatorFactory.newRoot(config);
            context.getConfig();
            result = config;
        }
    };
    final RemoteServiceSet lss = RemoteServiceSet.getLocalServiceSet();
    final DrillbitContext bitContext = new DrillbitContext(DrillbitEndpoint.getDefaultInstance(), context, coord, controller, com, workBus, new LocalPersistentStoreProvider(config));
    final QueryContext qc = new QueryContext(UserSession.Builder.newBuilder().setSupportComplexTypes(true).build(), bitContext, QueryId.getDefaultInstance());
    final PhysicalPlanReader reader = bitContext.getPlanReader();
    final LogicalPlan plan = reader.readLogicalPlan(Files.toString(FileUtils.getResourceAsFile(file), Charsets.UTF_8));
    final PhysicalPlan pp = new BasicOptimizer(qc, connection).optimize(new BasicOptimizer.BasicOptimizationContext(qc), plan);
    final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(config);
    final FragmentContext fctxt = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
    final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(fctxt, (FragmentRoot) pp.getSortedOperators(false).iterator().next()));
    return exec;
}

Example 20 with PhysicalPlanReader

use of org.apache.drill.exec.planner.PhysicalPlanReader in project drill by apache.

the class TestRecordIterator method testMarkResetIterator.

@Test
public void testMarkResetIterator(@Injectable final DrillbitContext bitContext, @Injectable UserClientConnection connection) throws Throwable {
    mockDrillbitContext(bitContext);
    final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
    final String planStr = Files.toString(FileUtils.getResourceAsFile("/record/test_recorditerator.json"), Charsets.UTF_8);
    final PhysicalPlan plan = reader.readPhysicalPlan(planStr);
    final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
    final FragmentContext context = new FragmentContext(bitContext, BitControl.PlanFragment.getDefaultInstance(), connection, registry);
    final List<PhysicalOperator> operatorList = plan.getSortedOperators(false);
    SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) operatorList.iterator().next()));
    RecordBatch singleBatch = exec.getIncoming();
    PhysicalOperator dummyPop = operatorList.iterator().next();
    OpProfileDef def = new OpProfileDef(dummyPop.getOperatorId(), UserBitShared.CoreOperatorType.MOCK_SUB_SCAN_VALUE, OperatorUtilities.getChildCount(dummyPop));
    OperatorStats stats = exec.getContext().getStats().newOperatorStats(def, exec.getContext().getAllocator());
    RecordIterator iter = new RecordIterator(singleBatch, null, exec.getContext().newOperatorContext(dummyPop, stats), 0);
    List<ValueVector> vectors = null;
    // batche sizes
    // 1, 100, 10, 10000, 1, 1000
    // total = 11112
    // BATCH 1 : 1, starting outerposition: 0
    iter.next();
    assertFalse(iter.finished());
    assertEquals(1, iter.getTotalRecordCount());
    assertEquals(0, iter.getCurrentPosition());
    assertEquals(0, iter.getOuterPosition());
    assertEquals(1, iter.cachedBatches().size());
    vectors = Lists.newArrayList();
    for (VectorWrapper vw : iter) {
        vectors.add(vw.getValueVector());
    }
    // mark at position 0
    iter.mark();
    checkValues(vectors, 0);
    // BATCH 2: 100, starting outerposition: 1
    iter.next();
    assertFalse(iter.finished());
    assertEquals(101, iter.getTotalRecordCount(), 101);
    assertEquals(0, iter.getCurrentPosition());
    assertEquals(100, iter.getInnerRecordCount());
    assertEquals(1, iter.getOuterPosition());
    assertEquals(2, iter.cachedBatches().size());
    for (int i = 0; i < 100; i++) {
        checkValues(vectors, i);
        iter.next();
    }
    // BATCH 3 :10, starting outerposition: 101
    assertFalse(iter.finished());
    assertEquals(111, iter.getTotalRecordCount());
    assertEquals(0, iter.getCurrentPosition());
    assertEquals(10, iter.getInnerRecordCount());
    assertEquals(101, iter.getOuterPosition());
    assertEquals(3, iter.cachedBatches().size());
    for (int i = 0; i < 10; i++) {
        checkValues(vectors, i);
        iter.next();
    }
    // BATCH 4 : 10000, starting outerposition: 111
    assertFalse(iter.finished());
    assertEquals(10111, iter.getTotalRecordCount());
    assertEquals(0, iter.getCurrentPosition(), 0);
    assertEquals(10000, iter.getInnerRecordCount());
    assertEquals(111, iter.getOuterPosition());
    assertEquals(4, iter.cachedBatches().size());
    for (int i = 0; i < 10000; i++) {
        checkValues(vectors, i);
        iter.next();
    }
    // BATCH 5 : 1, starting outerposition: 10111
    assertFalse(iter.finished());
    assertEquals(10112, iter.getTotalRecordCount());
    assertEquals(0, iter.getCurrentPosition());
    assertEquals(1, iter.getInnerRecordCount());
    assertEquals(10111, iter.getOuterPosition());
    assertEquals(5, iter.cachedBatches().size());
    checkValues(vectors, 0);
    iter.next();
    // BATCH 6 : 1000, starting outerposition: 10112
    assertFalse(iter.finished());
    assertEquals(11112, iter.getTotalRecordCount());
    assertEquals(0, iter.getCurrentPosition());
    assertEquals(1000, iter.getInnerRecordCount());
    assertEquals(10112, iter.getOuterPosition());
    assertEquals(6, iter.cachedBatches().size());
    for (int i = 0; i < 1000; i++) {
        checkValues(vectors, i);
        iter.next();
    }
    assertTrue(iter.finished());
    assertEquals(6, iter.cachedBatches().size());
    // back to batch 1
    iter.reset();
    assertFalse(iter.finished());
    assertEquals(iter.getTotalRecordCount(), 11112);
    assertEquals(6, iter.cachedBatches().size());
    assertEquals(iter.getCurrentPosition(), 0);
    assertEquals(1, iter.getInnerRecordCount());
    checkValues(vectors, 0);
    iter.next();
    // mark start of batch 2
    iter.mark();
    assertFalse(iter.finished());
    assertEquals(iter.getTotalRecordCount(), 11112);
    assertEquals(5, iter.cachedBatches().size());
    assertEquals(iter.getCurrentPosition(), 0);
    assertEquals(100, iter.getInnerRecordCount());
    for (int i = 0; i < 100; i++) {
        iter.next();
    }
    // mark start of batch 3
    iter.mark();
    assertFalse(iter.finished());
    assertEquals(iter.getTotalRecordCount(), 11112);
    assertEquals(4, iter.cachedBatches().size());
    assertEquals(iter.getCurrentPosition(), 0);
    assertEquals(10, iter.getInnerRecordCount());
    for (int i = 0; i < 10; i++) {
        iter.next();
    }
    // jump into middle of largest batch #4.
    for (int i = 0; i < 5000; i++) {
        iter.next();
    }
    assertEquals(4, iter.cachedBatches().size());
    iter.mark();
    assertEquals(3, iter.cachedBatches().size());
    for (int i = 0; i < 5000; i++) {
        iter.next();
    }
    // mark start of batch 5
    iter.mark();
    assertFalse(iter.finished());
    assertEquals(iter.getTotalRecordCount(), 11112);
    assertEquals(2, iter.cachedBatches().size());
    assertEquals(iter.getCurrentPosition(), 0);
    assertEquals(1, iter.getInnerRecordCount());
    // move to last batch
    iter.next();
    // skip to the middle of last batch
    for (int i = 0; i < 500; i++) {
        iter.next();
    }
    checkValues(vectors, 499);
    checkValues(vectors, 500);
    iter.reset();
    checkValues(vectors, 0);
    assertFalse(iter.finished());
    assertEquals(iter.getTotalRecordCount(), 11112);
    assertEquals(2, iter.cachedBatches().size());
    assertEquals(iter.getCurrentPosition(), 0);
    assertEquals(1, iter.getInnerRecordCount());
    // move to last batch
    iter.next();
    assertEquals(0, iter.getCurrentPosition());
    for (int i = 0; i < 500; i++) {
        iter.next();
    }
    // This should free 5th batch.
    iter.mark();
    assertFalse(iter.finished());
    assertEquals(iter.getTotalRecordCount(), 11112);
    assertEquals(1, iter.cachedBatches().size());
    assertEquals(500, iter.getCurrentPosition());
    assertEquals(1000, iter.getInnerRecordCount());
    // go to the end of iterator
    for (int i = 0; i < 500; i++) {
        iter.next();
    }
    assertTrue(iter.finished());
    iter.reset();
    assertFalse(iter.finished());
    assertEquals(iter.getTotalRecordCount(), 11112);
    assertEquals(1, iter.cachedBatches().size());
    assertEquals(500, iter.getCurrentPosition());
    assertEquals(1000, iter.getInnerRecordCount());
    iter.close();
    assertEquals(0, iter.cachedBatches().size());
}