Examples with VectorAccessibleSerializable org.apache.drill.exec.cache.VectorAccessibleSerializable used on opensource projects

Example 6 with VectorAccessibleSerializable

use of org.apache.drill.exec.cache.VectorAccessibleSerializable in project drill by axbaretto.

the class BatchGroup method getBatch.

private VectorContainer getBatch() throws IOException {
    assert fs != null;
    assert path != null;
    if (inputStream == null) {
        inputStream = fs.open(path);
    }
    VectorAccessibleSerializable vas = new VectorAccessibleSerializable(allocator);
    Stopwatch watch = Stopwatch.createStarted();
    vas.readFromStream(inputStream);
    VectorContainer c = vas.get();
    if (schema != null) {
        c = SchemaUtil.coerceContainer(c, schema, context);
    }
    logger.trace("Took {} us to read {} records", watch.elapsed(TimeUnit.MICROSECONDS), c.getRecordCount());
    spilledBatches--;
    currentContainer.zeroVectors();
    Iterator<VectorWrapper<?>> wrapperIterator = c.iterator();
    for (VectorWrapper<?> w : currentContainer) {
        TransferPair pair = wrapperIterator.next().getValueVector().makeTransferPair(w.getValueVector());
        pair.transfer();
    }
    currentContainer.setRecordCount(c.getRecordCount());
    c.zeroVectors();
    return c;
}

Example 7 with VectorAccessibleSerializable

use of org.apache.drill.exec.cache.VectorAccessibleSerializable in project drill by apache.

the class DumpCat method doBatch.

/**
 * Batch mode:
 * $drill-dumpcat --file=local:///tmp/drilltrace/[queryid]_[tag]_[majorid]_[minor]_[operator] --batch=123 --include-headers=true
 * Records: 1/1
 * Average Record Size: 8 bytes
 * Total Data Size: 8 bytes
 * Schema Information
 * name: col1, minor_type: int4, data_mode: nullable
 * name: col2, minor_type: int4, data_mode: non-nullable
 * @param targetBatchNum
 * @throws Exception
 */
protected void doBatch(FileInputStream input, int targetBatchNum, boolean showHeader) throws Exception {
    int batchNum = -1;
    VectorAccessibleSerializable vcSerializable = null;
    while (input.available() > 0 && batchNum++ < targetBatchNum) {
        vcSerializable = new VectorAccessibleSerializable(DumpCat.allocator);
        vcSerializable.readFromStream(input);
        if (batchNum != targetBatchNum) {
            final VectorContainer vectorContainer = vcSerializable.get();
            vectorContainer.zeroVectors();
        }
    }
    if (batchNum < targetBatchNum) {
        System.out.println(String.format("Wrong input of batch # ! Total # of batch in the file is %d. Please input a number 0..%d as batch #", batchNum + 1, batchNum));
        input.close();
        System.exit(-1);
    }
    if (vcSerializable != null) {
        showSingleBatch(vcSerializable, showHeader);
        final VectorContainer vectorContainer = vcSerializable.get();
        vectorContainer.zeroVectors();
    }
}

Example 8 with VectorAccessibleSerializable

use of org.apache.drill.exec.cache.VectorAccessibleSerializable in project drill by apache.

the class TraceRecordBatch method doWork.

/**
 * Invoked for every record batch and it simply dumps the buffers
 * associated with all the value vectors in this record batch to a log file.
 */
@Override
protected IterOutcome doWork() {
    boolean incomingHasSv2 = incoming.getSchema().getSelectionVectorMode() == SelectionVectorMode.TWO_BYTE;
    if (incomingHasSv2) {
        sv = incoming.getSelectionVector2();
    } else {
        sv = null;
    }
    WritableBatch batch = WritableBatch.getBatchNoHVWrap(incoming.getContainer().getRecordCount(), incoming, incomingHasSv2);
    VectorAccessibleSerializable wrap = new VectorAccessibleSerializable(batch, sv, oContext.getAllocator());
    try {
        wrap.writeToStreamAndRetain(fos);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    batch.reconstructContainer(localAllocator, container);
    if (incomingHasSv2) {
        sv = wrap.getSv2();
        container.setRecordCount(sv.getBatchActualRecordCount());
    } else {
        container.setRecordCount(batch.getDef().getRecordCount());
    }
    return getFinalOutcome(false);
}

Example 9 with VectorAccessibleSerializable

use of org.apache.drill.exec.cache.VectorAccessibleSerializable in project drill by apache.

the class TestTraceOutputDump method testFilter.

@Test
public void testFilter() throws Throwable {
    final DrillbitContext bitContext = mockDrillbitContext();
    final UserClientConnection connection = Mockito.mock(UserClientConnection.class);
    final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
    final PhysicalPlan plan = reader.readPhysicalPlan(Files.asCharSource(DrillFileUtils.getResourceAsFile("/trace/simple_trace.json"), Charsets.UTF_8).read());
    final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
    final FragmentContextImpl context = new FragmentContextImpl(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
    final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
    while (exec.next()) {
    }
    exec.close();
    if (context.getExecutorState().getFailureCause() != null) {
        throw context.getExecutorState().getFailureCause();
    }
    assertTrue(!context.getExecutorState().isFailed());
    final FragmentHandle handle = context.getHandle();
    /* Form the file name to which the trace output will dump the record batches */
    final String qid = QueryIdHelper.getQueryId(handle.getQueryId());
    final int majorFragmentId = handle.getMajorFragmentId();
    final int minorFragmentId = handle.getMinorFragmentId();
    final String logLocation = c.getString(ExecConstants.TRACE_DUMP_DIRECTORY);
    final String filename = String.format("%s//%s_%d_%d_mock-scan", logLocation, qid, majorFragmentId, minorFragmentId);
    final Configuration conf = new Configuration();
    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, c.getString(ExecConstants.TRACE_DUMP_FILESYSTEM));
    final FileSystem fs = FileSystem.get(conf);
    final Path path = new Path(filename);
    assertTrue("Trace file does not exist", fs.exists(path));
    final FSDataInputStream in = fs.open(path);
    final VectorAccessibleSerializable wrap = new VectorAccessibleSerializable(context.getAllocator());
    wrap.readFromStream(in);
    final VectorAccessible container = wrap.get();
    /* Assert there are no selection vectors */
    assertNull(wrap.getSv2());
    /* Assert there is only one record */
    assertEquals(1, container.getRecordCount());
    /* Read the Integer value and ASSERT its Integer.MIN_VALUE */
    final int value = (int) container.iterator().next().getValueVector().getAccessor().getObject(0);
    assertEquals(value, Integer.MIN_VALUE);
}

Example 10 with VectorAccessibleSerializable

use of org.apache.drill.exec.cache.VectorAccessibleSerializable in project drill by apache.

the class BatchGroup method getBatch.

private VectorContainer getBatch() throws IOException {
    assert fs != null;
    assert path != null;
    if (inputStream == null) {
        inputStream = fs.open(path);
    }
    VectorAccessibleSerializable vas = new VectorAccessibleSerializable(allocator);
    Stopwatch watch = Stopwatch.createStarted();
    vas.readFromStream(inputStream);
    VectorContainer c = vas.get();
    if (schema != null) {
        c = SchemaUtil.coerceContainer(c, schema, context);
    }
    logger.trace("Took {} us to read {} records", watch.elapsed(TimeUnit.MICROSECONDS), c.getRecordCount());
    spilledBatches--;
    currentContainer.zeroVectors();
    Iterator<VectorWrapper<?>> wrapperIterator = c.iterator();
    for (VectorWrapper w : currentContainer) {
        TransferPair pair = wrapperIterator.next().getValueVector().makeTransferPair(w.getValueVector());
        pair.transfer();
    }
    currentContainer.setRecordCount(c.getRecordCount());
    c.zeroVectors();
    return c;
}