Examples with OpProfileDef org.apache.drill.exec.ops.OpProfileDef used on opensource projects

Example 6 with OpProfileDef

use of org.apache.drill.exec.ops.OpProfileDef in project drill by apache.

the class TestRecordIterator method testMarkResetIterator.

@Test
public void testMarkResetIterator(@Injectable final DrillbitContext bitContext, @Injectable UserClientConnection connection) throws Throwable {
    mockDrillbitContext(bitContext);
    final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
    final String planStr = Files.toString(FileUtils.getResourceAsFile("/record/test_recorditerator.json"), Charsets.UTF_8);
    final PhysicalPlan plan = reader.readPhysicalPlan(planStr);
    final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
    final FragmentContext context = new FragmentContext(bitContext, BitControl.PlanFragment.getDefaultInstance(), connection, registry);
    final List<PhysicalOperator> operatorList = plan.getSortedOperators(false);
    SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) operatorList.iterator().next()));
    RecordBatch singleBatch = exec.getIncoming();
    PhysicalOperator dummyPop = operatorList.iterator().next();
    OpProfileDef def = new OpProfileDef(dummyPop.getOperatorId(), UserBitShared.CoreOperatorType.MOCK_SUB_SCAN_VALUE, OperatorUtilities.getChildCount(dummyPop));
    OperatorStats stats = exec.getContext().getStats().newOperatorStats(def, exec.getContext().getAllocator());
    RecordIterator iter = new RecordIterator(singleBatch, null, exec.getContext().newOperatorContext(dummyPop, stats), 0);
    List<ValueVector> vectors = null;
    // batche sizes
    // 1, 100, 10, 10000, 1, 1000
    // total = 11112
    // BATCH 1 : 1, starting outerposition: 0
    iter.next();
    assertFalse(iter.finished());
    assertEquals(1, iter.getTotalRecordCount());
    assertEquals(0, iter.getCurrentPosition());
    assertEquals(0, iter.getOuterPosition());
    assertEquals(1, iter.cachedBatches().size());
    vectors = Lists.newArrayList();
    for (VectorWrapper vw : iter) {
        vectors.add(vw.getValueVector());
    }
    // mark at position 0
    iter.mark();
    checkValues(vectors, 0);
    // BATCH 2: 100, starting outerposition: 1
    iter.next();
    assertFalse(iter.finished());
    assertEquals(101, iter.getTotalRecordCount(), 101);
    assertEquals(0, iter.getCurrentPosition());
    assertEquals(100, iter.getInnerRecordCount());
    assertEquals(1, iter.getOuterPosition());
    assertEquals(2, iter.cachedBatches().size());
    for (int i = 0; i < 100; i++) {
        checkValues(vectors, i);
        iter.next();
    }
    // BATCH 3 :10, starting outerposition: 101
    assertFalse(iter.finished());
    assertEquals(111, iter.getTotalRecordCount());
    assertEquals(0, iter.getCurrentPosition());
    assertEquals(10, iter.getInnerRecordCount());
    assertEquals(101, iter.getOuterPosition());
    assertEquals(3, iter.cachedBatches().size());
    for (int i = 0; i < 10; i++) {
        checkValues(vectors, i);
        iter.next();
    }
    // BATCH 4 : 10000, starting outerposition: 111
    assertFalse(iter.finished());
    assertEquals(10111, iter.getTotalRecordCount());
    assertEquals(0, iter.getCurrentPosition(), 0);
    assertEquals(10000, iter.getInnerRecordCount());
    assertEquals(111, iter.getOuterPosition());
    assertEquals(4, iter.cachedBatches().size());
    for (int i = 0; i < 10000; i++) {
        checkValues(vectors, i);
        iter.next();
    }
    // BATCH 5 : 1, starting outerposition: 10111
    assertFalse(iter.finished());
    assertEquals(10112, iter.getTotalRecordCount());
    assertEquals(0, iter.getCurrentPosition());
    assertEquals(1, iter.getInnerRecordCount());
    assertEquals(10111, iter.getOuterPosition());
    assertEquals(5, iter.cachedBatches().size());
    checkValues(vectors, 0);
    iter.next();
    // BATCH 6 : 1000, starting outerposition: 10112
    assertFalse(iter.finished());
    assertEquals(11112, iter.getTotalRecordCount());
    assertEquals(0, iter.getCurrentPosition());
    assertEquals(1000, iter.getInnerRecordCount());
    assertEquals(10112, iter.getOuterPosition());
    assertEquals(6, iter.cachedBatches().size());
    for (int i = 0; i < 1000; i++) {
        checkValues(vectors, i);
        iter.next();
    }
    assertTrue(iter.finished());
    assertEquals(6, iter.cachedBatches().size());
    // back to batch 1
    iter.reset();
    assertFalse(iter.finished());
    assertEquals(iter.getTotalRecordCount(), 11112);
    assertEquals(6, iter.cachedBatches().size());
    assertEquals(iter.getCurrentPosition(), 0);
    assertEquals(1, iter.getInnerRecordCount());
    checkValues(vectors, 0);
    iter.next();
    // mark start of batch 2
    iter.mark();
    assertFalse(iter.finished());
    assertEquals(iter.getTotalRecordCount(), 11112);
    assertEquals(5, iter.cachedBatches().size());
    assertEquals(iter.getCurrentPosition(), 0);
    assertEquals(100, iter.getInnerRecordCount());
    for (int i = 0; i < 100; i++) {
        iter.next();
    }
    // mark start of batch 3
    iter.mark();
    assertFalse(iter.finished());
    assertEquals(iter.getTotalRecordCount(), 11112);
    assertEquals(4, iter.cachedBatches().size());
    assertEquals(iter.getCurrentPosition(), 0);
    assertEquals(10, iter.getInnerRecordCount());
    for (int i = 0; i < 10; i++) {
        iter.next();
    }
    // jump into middle of largest batch #4.
    for (int i = 0; i < 5000; i++) {
        iter.next();
    }
    assertEquals(4, iter.cachedBatches().size());
    iter.mark();
    assertEquals(3, iter.cachedBatches().size());
    for (int i = 0; i < 5000; i++) {
        iter.next();
    }
    // mark start of batch 5
    iter.mark();
    assertFalse(iter.finished());
    assertEquals(iter.getTotalRecordCount(), 11112);
    assertEquals(2, iter.cachedBatches().size());
    assertEquals(iter.getCurrentPosition(), 0);
    assertEquals(1, iter.getInnerRecordCount());
    // move to last batch
    iter.next();
    // skip to the middle of last batch
    for (int i = 0; i < 500; i++) {
        iter.next();
    }
    checkValues(vectors, 499);
    checkValues(vectors, 500);
    iter.reset();
    checkValues(vectors, 0);
    assertFalse(iter.finished());
    assertEquals(iter.getTotalRecordCount(), 11112);
    assertEquals(2, iter.cachedBatches().size());
    assertEquals(iter.getCurrentPosition(), 0);
    assertEquals(1, iter.getInnerRecordCount());
    // move to last batch
    iter.next();
    assertEquals(0, iter.getCurrentPosition());
    for (int i = 0; i < 500; i++) {
        iter.next();
    }
    // This should free 5th batch.
    iter.mark();
    assertFalse(iter.finished());
    assertEquals(iter.getTotalRecordCount(), 11112);
    assertEquals(1, iter.cachedBatches().size());
    assertEquals(500, iter.getCurrentPosition());
    assertEquals(1000, iter.getInnerRecordCount());
    // go to the end of iterator
    for (int i = 0; i < 500; i++) {
        iter.next();
    }
    assertTrue(iter.finished());
    iter.reset();
    assertFalse(iter.finished());
    assertEquals(iter.getTotalRecordCount(), 11112);
    assertEquals(1, iter.cachedBatches().size());
    assertEquals(500, iter.getCurrentPosition());
    assertEquals(1000, iter.getInnerRecordCount());
    iter.close();
    assertEquals(0, iter.cachedBatches().size());
}

Example 7 with OpProfileDef

use of org.apache.drill.exec.ops.OpProfileDef in project drill by axbaretto.

the class TestAllocators method testAllocators.

@Test
public void testAllocators() throws Exception {
    // Setup a drillbit (initializes a root allocator)
    final DrillConfig config = DrillConfig.create(TEST_CONFIGURATIONS);
    try (final RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
        final Drillbit bit = new Drillbit(config, serviceSet)) {
        ;
        bit.run();
        final DrillbitContext bitContext = bit.getContext();
        FunctionImplementationRegistry functionRegistry = bitContext.getFunctionImplementationRegistry();
        StoragePluginRegistry storageRegistry = new StoragePluginRegistryImpl(bitContext);
        // Create a few Fragment Contexts
        BitControl.PlanFragment.Builder pfBuilder1 = BitControl.PlanFragment.newBuilder();
        pfBuilder1.setMemInitial(1500000);
        BitControl.PlanFragment pf1 = pfBuilder1.build();
        BitControl.PlanFragment.Builder pfBuilder2 = BitControl.PlanFragment.newBuilder();
        pfBuilder2.setMemInitial(500000);
        BitControl.PlanFragment pf2 = pfBuilder1.build();
        FragmentContextImpl fragmentContext1 = new FragmentContextImpl(bitContext, pf1, null, functionRegistry);
        FragmentContextImpl fragmentContext2 = new FragmentContextImpl(bitContext, pf2, null, functionRegistry);
        // Get a few physical operators. Easiest way is to read a physical plan.
        PhysicalPlanReader planReader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(bitContext, storageRegistry);
        PhysicalPlan plan = planReader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile(planFile), Charsets.UTF_8));
        List<PhysicalOperator> physicalOperators = plan.getSortedOperators();
        Iterator<PhysicalOperator> physicalOperatorIterator = physicalOperators.iterator();
        PhysicalOperator physicalOperator1 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator2 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator3 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator4 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator5 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator6 = physicalOperatorIterator.next();
        // Create some bogus Operator profile defs and stats to create operator contexts
        OpProfileDef def;
        OperatorStats stats;
        // Use some bogus operator type to create a new operator context.
        def = new OpProfileDef(physicalOperator1.getOperatorId(), UserBitShared.CoreOperatorType.MOCK_SUB_SCAN_VALUE, OperatorUtilities.getChildCount(physicalOperator1));
        stats = fragmentContext1.getStats().newOperatorStats(def, fragmentContext1.getAllocator());
        // Add a couple of Operator Contexts
        // Initial allocation = 1000000 bytes for all operators
        OperatorContext oContext11 = fragmentContext1.newOperatorContext(physicalOperator1);
        DrillBuf b11 = oContext11.getAllocator().buffer(1000000);
        OperatorContext oContext12 = fragmentContext1.newOperatorContext(physicalOperator2, stats);
        DrillBuf b12 = oContext12.getAllocator().buffer(500000);
        OperatorContext oContext21 = fragmentContext1.newOperatorContext(physicalOperator3);
        def = new OpProfileDef(physicalOperator4.getOperatorId(), UserBitShared.CoreOperatorType.TEXT_WRITER_VALUE, OperatorUtilities.getChildCount(physicalOperator4));
        stats = fragmentContext2.getStats().newOperatorStats(def, fragmentContext2.getAllocator());
        OperatorContext oContext22 = fragmentContext2.newOperatorContext(physicalOperator4, stats);
        DrillBuf b22 = oContext22.getAllocator().buffer(2000000);
        // New Fragment begins
        BitControl.PlanFragment.Builder pfBuilder3 = BitControl.PlanFragment.newBuilder();
        pfBuilder3.setMemInitial(1000000);
        BitControl.PlanFragment pf3 = pfBuilder3.build();
        FragmentContextImpl fragmentContext3 = new FragmentContextImpl(bitContext, pf3, null, functionRegistry);
        // New fragment starts an operator that allocates an amount within the limit
        def = new OpProfileDef(physicalOperator5.getOperatorId(), UserBitShared.CoreOperatorType.UNION_VALUE, OperatorUtilities.getChildCount(physicalOperator5));
        stats = fragmentContext3.getStats().newOperatorStats(def, fragmentContext3.getAllocator());
        OperatorContext oContext31 = fragmentContext3.newOperatorContext(physicalOperator5, stats);
        DrillBuf b31a = oContext31.getAllocator().buffer(200000);
        // Previously running operator completes
        b22.release();
        ((AutoCloseable) oContext22).close();
        // Fragment 3 asks for more and fails
        boolean outOfMem = false;
        try {
            oContext31.getAllocator().buffer(44000000);
            fail("Fragment 3 should fail to allocate buffer");
        } catch (OutOfMemoryException e) {
            // Expected.
            outOfMem = true;
        }
        assertTrue(outOfMem);
        // Operator is Exempt from Fragment limits. Fragment 3 asks for more and succeeds
        OperatorContext oContext32 = fragmentContext3.newOperatorContext(physicalOperator6);
        try {
            DrillBuf b32 = oContext32.getAllocator().buffer(4400000);
            b32.release();
        } catch (OutOfMemoryException e) {
            fail("Fragment 3 failed to allocate buffer");
        } finally {
            closeOp(oContext32);
        }
        b11.release();
        closeOp(oContext11);
        b12.release();
        closeOp(oContext12);
        closeOp(oContext21);
        b31a.release();
        closeOp(oContext31);
        fragmentContext1.close();
        fragmentContext2.close();
        fragmentContext3.close();
    }
}

Example 8 with OpProfileDef

use of org.apache.drill.exec.ops.OpProfileDef in project drill by axbaretto.

the class TestRecordIterator method testMarkResetIterator.

@Test
public void testMarkResetIterator() throws Throwable {
    final DrillbitContext bitContext = mockDrillbitContext();
    final UserClientConnection connection = Mockito.mock(UserClientConnection.class);
    final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
    final String planStr = Files.toString(DrillFileUtils.getResourceAsFile("/record/test_recorditerator.json"), Charsets.UTF_8);
    final PhysicalPlan plan = reader.readPhysicalPlan(planStr);
    final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
    final FragmentContextImpl context = new FragmentContextImpl(bitContext, BitControl.PlanFragment.getDefaultInstance(), connection, registry);
    final List<PhysicalOperator> operatorList = plan.getSortedOperators(false);
    SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) operatorList.iterator().next()));
    RecordBatch singleBatch = exec.getIncoming();
    PhysicalOperator dummyPop = operatorList.iterator().next();
    OpProfileDef def = new OpProfileDef(dummyPop.getOperatorId(), UserBitShared.CoreOperatorType.MOCK_SUB_SCAN_VALUE, OperatorUtilities.getChildCount(dummyPop));
    OperatorStats stats = exec.getContext().getStats().newOperatorStats(def, exec.getContext().getAllocator());
    RecordIterator iter = new RecordIterator(singleBatch, null, exec.getContext().newOperatorContext(dummyPop, stats), 0, null);
    List<ValueVector> vectors = null;
    // batche sizes
    // 1, 100, 10, 10000, 1, 1000
    // total = 11112
    // BATCH 1 : 1, starting outerposition: 0
    iter.next();
    assertFalse(iter.finished());
    assertEquals(1, iter.getTotalRecordCount());
    assertEquals(0, iter.getCurrentPosition());
    assertEquals(0, iter.getOuterPosition());
    assertEquals(1, iter.cachedBatches().size());
    vectors = Lists.newArrayList();
    for (VectorWrapper vw : iter) {
        vectors.add(vw.getValueVector());
    }
    // mark at position 0
    iter.mark();
    checkValues(vectors, 0);
    // BATCH 2: 100, starting outerposition: 1
    iter.next();
    assertFalse(iter.finished());
    assertEquals(101, iter.getTotalRecordCount(), 101);
    assertEquals(0, iter.getCurrentPosition());
    assertEquals(100, iter.getInnerRecordCount());
    assertEquals(1, iter.getOuterPosition());
    assertEquals(2, iter.cachedBatches().size());
    for (int i = 0; i < 100; i++) {
        checkValues(vectors, i);
        iter.next();
    }
    // BATCH 3 :10, starting outerposition: 101
    assertFalse(iter.finished());
    assertEquals(111, iter.getTotalRecordCount());
    assertEquals(0, iter.getCurrentPosition());
    assertEquals(10, iter.getInnerRecordCount());
    assertEquals(101, iter.getOuterPosition());
    assertEquals(3, iter.cachedBatches().size());
    for (int i = 0; i < 10; i++) {
        checkValues(vectors, i);
        iter.next();
    }
    // BATCH 4 : 10000, starting outerposition: 111
    assertFalse(iter.finished());
    assertEquals(10111, iter.getTotalRecordCount());
    assertEquals(0, iter.getCurrentPosition(), 0);
    assertEquals(10000, iter.getInnerRecordCount());
    assertEquals(111, iter.getOuterPosition());
    assertEquals(4, iter.cachedBatches().size());
    for (int i = 0; i < 10000; i++) {
        checkValues(vectors, i);
        iter.next();
    }
    // BATCH 5 : 1, starting outerposition: 10111
    assertFalse(iter.finished());
    assertEquals(10112, iter.getTotalRecordCount());
    assertEquals(0, iter.getCurrentPosition());
    assertEquals(1, iter.getInnerRecordCount());
    assertEquals(10111, iter.getOuterPosition());
    assertEquals(5, iter.cachedBatches().size());
    checkValues(vectors, 0);
    iter.next();
    // BATCH 6 : 1000, starting outerposition: 10112
    assertFalse(iter.finished());
    assertEquals(11112, iter.getTotalRecordCount());
    assertEquals(0, iter.getCurrentPosition());
    assertEquals(1000, iter.getInnerRecordCount());
    assertEquals(10112, iter.getOuterPosition());
    assertEquals(6, iter.cachedBatches().size());
    for (int i = 0; i < 1000; i++) {
        checkValues(vectors, i);
        iter.next();
    }
    assertTrue(iter.finished());
    assertEquals(6, iter.cachedBatches().size());
    // back to batch 1
    iter.reset();
    assertFalse(iter.finished());
    assertEquals(iter.getTotalRecordCount(), 11112);
    assertEquals(6, iter.cachedBatches().size());
    assertEquals(iter.getCurrentPosition(), 0);
    assertEquals(1, iter.getInnerRecordCount());
    checkValues(vectors, 0);
    iter.next();
    // mark start of batch 2
    iter.mark();
    assertFalse(iter.finished());
    assertEquals(iter.getTotalRecordCount(), 11112);
    assertEquals(5, iter.cachedBatches().size());
    assertEquals(iter.getCurrentPosition(), 0);
    assertEquals(100, iter.getInnerRecordCount());
    for (int i = 0; i < 100; i++) {
        iter.next();
    }
    // mark start of batch 3
    iter.mark();
    assertFalse(iter.finished());
    assertEquals(iter.getTotalRecordCount(), 11112);
    assertEquals(4, iter.cachedBatches().size());
    assertEquals(iter.getCurrentPosition(), 0);
    assertEquals(10, iter.getInnerRecordCount());
    for (int i = 0; i < 10; i++) {
        iter.next();
    }
    // jump into middle of largest batch #4.
    for (int i = 0; i < 5000; i++) {
        iter.next();
    }
    assertEquals(4, iter.cachedBatches().size());
    iter.mark();
    assertEquals(3, iter.cachedBatches().size());
    for (int i = 0; i < 5000; i++) {
        iter.next();
    }
    // mark start of batch 5
    iter.mark();
    assertFalse(iter.finished());
    assertEquals(iter.getTotalRecordCount(), 11112);
    assertEquals(2, iter.cachedBatches().size());
    assertEquals(iter.getCurrentPosition(), 0);
    assertEquals(1, iter.getInnerRecordCount());
    // move to last batch
    iter.next();
    // skip to the middle of last batch
    for (int i = 0; i < 500; i++) {
        iter.next();
    }
    checkValues(vectors, 499);
    checkValues(vectors, 500);
    iter.reset();
    checkValues(vectors, 0);
    assertFalse(iter.finished());
    assertEquals(iter.getTotalRecordCount(), 11112);
    assertEquals(2, iter.cachedBatches().size());
    assertEquals(iter.getCurrentPosition(), 0);
    assertEquals(1, iter.getInnerRecordCount());
    // move to last batch
    iter.next();
    assertEquals(0, iter.getCurrentPosition());
    for (int i = 0; i < 500; i++) {
        iter.next();
    }
    // This should free 5th batch.
    iter.mark();
    assertFalse(iter.finished());
    assertEquals(iter.getTotalRecordCount(), 11112);
    assertEquals(1, iter.cachedBatches().size());
    assertEquals(500, iter.getCurrentPosition());
    assertEquals(1000, iter.getInnerRecordCount());
    // go to the end of iterator
    for (int i = 0; i < 500; i++) {
        iter.next();
    }
    assertTrue(iter.finished());
    iter.reset();
    assertFalse(iter.finished());
    assertEquals(iter.getTotalRecordCount(), 11112);
    assertEquals(1, iter.cachedBatches().size());
    assertEquals(500, iter.getCurrentPosition());
    assertEquals(1000, iter.getInnerRecordCount());
    iter.close();
    assertEquals(0, iter.cachedBatches().size());
}

Example 9 with OpProfileDef

use of org.apache.drill.exec.ops.OpProfileDef in project drill by axbaretto.

the class TestDrillFileSystem method testIOStats.

@Test
public void testIOStats() throws Exception {
    DrillFileSystem dfs = null;
    InputStream is = null;
    Configuration conf = new Configuration();
    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, FileSystem.DEFAULT_FS);
    OpProfileDef profileDef = new OpProfileDef(0, /*operatorId*/
    0, /*operatorType*/
    0);
    OperatorStats stats = new OperatorStats(profileDef, null);
    // start wait time method in OperatorStats expects the OperatorStats state to be in "processing"
    stats.startProcessing();
    try {
        dfs = new DrillFileSystem(conf, stats);
        is = dfs.open(new Path(tempFilePath));
        byte[] buf = new byte[8000];
        while (is.read(buf, 0, buf.length) != -1) {
        }
    } finally {
        stats.stopProcessing();
        if (is != null) {
            is.close();
        }
        if (dfs != null) {
            dfs.close();
        }
    }
    OperatorProfile operatorProfile = stats.getProfile();
    assertTrue("Expected wait time is non-zero, but got zero wait time", operatorProfile.getWaitNanos() > 0);
}

Example 10 with OpProfileDef

use of org.apache.drill.exec.ops.OpProfileDef in project drill by apache.

the class TestDrillFileSystem method testIOStats.

@Test
public void testIOStats() throws Exception {
    DrillFileSystem dfs = null;
    InputStream is = null;
    Configuration conf = new Configuration();
    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, FileSystem.DEFAULT_FS);
    OpProfileDef profileDef = new OpProfileDef(0, /*operatorId*/
    "", /*operatorType*/
    0);
    OperatorStats stats = new OperatorStats(profileDef, null);
    // start wait time method in OperatorStats expects the OperatorStats state to be in "processing"
    stats.startProcessing();
    try {
        dfs = new DrillFileSystem(conf, stats);
        is = dfs.open(new Path(tempFilePath));
        byte[] buf = new byte[8000];
        while (is.read(buf, 0, buf.length) != -1) {
        }
    } finally {
        stats.stopProcessing();
        if (is != null) {
            is.close();
        }
        if (dfs != null) {
            dfs.close();
        }
    }
    OperatorProfile operatorProfile = stats.getProfile();
    assertTrue("Expected wait time is non-zero, but got zero wait time", operatorProfile.getWaitNanos() > 0);
}