Examples with VectorContainer org.apache.drill.exec.record.VectorContainer used on opensource projects

Example 96 with VectorContainer

use of org.apache.drill.exec.record.VectorContainer in project drill by axbaretto.

the class TestOperatorRecordBatch method testSchemaChange.

@Test
public void testSchemaChange() {
    BatchSchema schema = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.VARCHAR).build();
    SingleRowSet rs = fixture.rowSetBuilder(schema).addRow(10, "fred").addRow(20, "wilma").build();
    VectorContainer container = rs.container();
    MockOperatorExec opExec = new MockOperatorExec(container);
    int schemaVersion = opExec.batchAccessor().schemaVersion();
    // Be tidy: start at 1.
    assertEquals(1, schemaVersion);
    // Changing data does not trigger schema change
    container.zeroVectors();
    opExec.batchAccessor.setContainer(container);
    assertEquals(schemaVersion, opExec.batchAccessor().schemaVersion());
    // Different container, same vectors, does not trigger a change
    VectorContainer c2 = new VectorContainer(fixture.allocator());
    for (VectorWrapper<?> vw : container) {
        c2.add(vw.getValueVector());
    }
    c2.buildSchema(SelectionVectorMode.NONE);
    opExec.batchAccessor.setContainer(c2);
    assertEquals(schemaVersion, opExec.batchAccessor().schemaVersion());
    opExec.batchAccessor.setContainer(container);
    assertEquals(schemaVersion, opExec.batchAccessor().schemaVersion());
    // Replacing a vector with another of the same type does trigger
    // a change.
    VectorContainer c3 = new VectorContainer(fixture.allocator());
    c3.add(container.getValueVector(0).getValueVector());
    c3.add(TypeHelper.getNewVector(container.getValueVector(1).getValueVector().getField(), fixture.allocator(), null));
    c3.buildSchema(SelectionVectorMode.NONE);
    opExec.batchAccessor.setContainer(c3);
    assertEquals(schemaVersion + 1, opExec.batchAccessor().schemaVersion());
    schemaVersion = opExec.batchAccessor().schemaVersion();
    // No change if same schema again
    opExec.batchAccessor.setContainer(c3);
    assertEquals(schemaVersion, opExec.batchAccessor().schemaVersion());
    // Adding a vector triggers a change
    MaterializedField c = SchemaBuilder.columnSchema("c", MinorType.INT, DataMode.OPTIONAL);
    c3.add(TypeHelper.getNewVector(c, fixture.allocator(), null));
    c3.buildSchema(SelectionVectorMode.NONE);
    opExec.batchAccessor.setContainer(c3);
    assertEquals(schemaVersion + 1, opExec.batchAccessor().schemaVersion());
    schemaVersion = opExec.batchAccessor().schemaVersion();
    // No change if same schema again
    opExec.batchAccessor.setContainer(c3);
    assertEquals(schemaVersion, opExec.batchAccessor().schemaVersion());
    // Removing a vector triggers a change
    c3.remove(c3.getValueVector(2).getValueVector());
    c3.buildSchema(SelectionVectorMode.NONE);
    assertEquals(2, c3.getNumberOfColumns());
    opExec.batchAccessor.setContainer(c3);
    assertEquals(schemaVersion + 1, opExec.batchAccessor().schemaVersion());
    schemaVersion = opExec.batchAccessor().schemaVersion();
    // Clean up
    opExec.close();
    c2.clear();
    c3.clear();
}

Example 97 with VectorContainer

use of org.apache.drill.exec.record.VectorContainer in project drill by axbaretto.

the class TestSortImpl method runWideRowsTest.

/**
 * Run a test using wide rows. This stresses the "copier" portion of the sort
 * and allows us to test the original generated copier and the revised "generic"
 * copier.
 *
 * @param fixture operator test fixture
 * @param colCount number of data (non-key) columns
 * @param rowCount number of rows to generate
 */
public void runWideRowsTest(OperatorFixture fixture, int colCount, int rowCount) {
    SchemaBuilder builder = new SchemaBuilder().add("key", MinorType.INT);
    for (int i = 0; i < colCount; i++) {
        builder.add("col" + (i + 1), MinorType.INT);
    }
    BatchSchema schema = builder.build();
    ExtendableRowSet rowSet = fixture.rowSet(schema);
    RowSetWriter writer = rowSet.writer(rowCount);
    for (int i = 0; i < rowCount; i++) {
        writer.set(0, i);
        for (int j = 0; j < colCount; j++) {
            writer.set(j + 1, i * 100_000 + j);
        }
        writer.save();
    }
    writer.done();
    VectorContainer dest = new VectorContainer();
    SortImpl sort = makeSortImpl(fixture, Ordering.ORDER_ASC, Ordering.NULLS_UNSPECIFIED, dest);
    sort.setSchema(rowSet.container().getSchema());
    sort.addBatch(rowSet.vectorAccessible());
    SortResults results = sort.startMerge();
    if (results.getContainer() != dest) {
        dest.clear();
        dest = results.getContainer();
    }
    assertTrue(results.next());
    assertFalse(results.next());
    results.close();
    dest.clear();
    sort.close();
    sort.opContext().close();
}

Example 98 with VectorContainer

use of org.apache.drill.exec.record.VectorContainer in project drill by axbaretto.

the class AbstractGenericCopierTest method testAppendRecords.

@Test
public void testAppendRecords() throws SchemaChangeException {
    try (RootAllocator allocator = new RootAllocator(10_000_000)) {
        final BatchSchema batchSchema = createTestSchema(BatchSchema.SelectionVectorMode.NONE);
        final RowSet srcRowSet = createSrcRowSet(allocator);
        final RowSet destRowSet = new RowSetBuilder(allocator, batchSchema).build();
        final VectorContainer destContainer = destRowSet.container();
        final Copier copier = createCopier();
        final RowSet expectedRowSet = createExpectedRowset(allocator);
        copier.setup(new RowSetBatch(srcRowSet), destContainer);
        copier.appendRecord(0);
        copier.appendRecords(1, 2);
        try {
            new RowSetComparison(expectedRowSet).verify(destRowSet);
        } finally {
            srcRowSet.clear();
            if (srcRowSet instanceof RowSet.HyperRowSet) {
                ((RowSet.HyperRowSet) srcRowSet).getSv4().clear();
            }
            destRowSet.clear();
            expectedRowSet.clear();
        }
    }
}

Example 99 with VectorContainer

use of org.apache.drill.exec.record.VectorContainer in project drill by axbaretto.

the class GenericSV4CopierTest method createSrcRowSet.

@Override
public RowSet createSrcRowSet(RootAllocator allocator) throws SchemaChangeException {
    final BatchSchema batchSchema = createTestSchema(BatchSchema.SelectionVectorMode.NONE);
    final DrillBuf drillBuf = allocator.buffer(4 * 3);
    final SelectionVector4 sv4 = new SelectionVector4(drillBuf, 3, Character.MAX_VALUE);
    final VectorContainer batch1 = new RowSetBuilder(allocator, batchSchema).addRow(row1()).addRow(row4()).build().container();
    final VectorContainer batch2 = new RowSetBuilder(allocator, batchSchema).addRow(row2()).addRow(row5()).addRow(row3()).build().container();
    final ExpandableHyperContainer hyperContainer = new ExpandableHyperContainer(batch1);
    hyperContainer.addBatch(batch2);
    sv4.set(0, 0, 0);
    sv4.set(1, 1, 0);
    sv4.set(2, 1, 2);
    return new HyperRowSetImpl(hyperContainer, sv4);
}

Example 100 with VectorContainer

use of org.apache.drill.exec.record.VectorContainer in project drill by axbaretto.

the class DumpCat method doBatch.

/**
 * Batch mode:
 * $drill-dumpcat --file=local:///tmp/drilltrace/[queryid]_[tag]_[majorid]_[minor]_[operator] --batch=123 --include-headers=true
 * Records: 1/1
 * Average Record Size: 8 bytes
 * Total Data Size: 8 bytes
 * Schema Information
 * name: col1, minor_type: int4, data_mode: nullable
 * name: col2, minor_type: int4, data_mode: non-nullable
 * @param targetBatchNum
 * @throws Exception
 */
protected void doBatch(FileInputStream input, int targetBatchNum, boolean showHeader) throws Exception {
    int batchNum = -1;
    VectorAccessibleSerializable vcSerializable = null;
    while (input.available() > 0 && batchNum++ < targetBatchNum) {
        vcSerializable = new VectorAccessibleSerializable(DumpCat.allocator);
        vcSerializable.readFromStream(input);
        if (batchNum != targetBatchNum) {
            final VectorContainer vectorContainer = vcSerializable.get();
            vectorContainer.zeroVectors();
        }
    }
    if (batchNum < targetBatchNum) {
        System.out.println(String.format("Wrong input of batch # ! Total # of batch in the file is %d. Please input a number 0..%d as batch #", batchNum + 1, batchNum));
        input.close();
        System.exit(-1);
    }
    if (vcSerializable != null) {
        showSingleBatch(vcSerializable, showHeader);
        final VectorContainer vectorContainer = vcSerializable.get();
        vectorContainer.zeroVectors();
    }
}