Examples with MapVector org.apache.drill.exec.vector.complex.MapVector used on opensource projects

Example 26 with MapVector

use of org.apache.drill.exec.vector.complex.MapVector in project drill by axbaretto.

the class TestRecordBatchSizer method testSizerNestedMap.

@Test
public void testSizerNestedMap() {
    BatchSchema schema = new SchemaBuilder().addMap("map").add("key", MinorType.INT).add("value", MinorType.VARCHAR).addMap("childMap").add("childKey", MinorType.INT).add("childValue", MinorType.VARCHAR).resumeMap().resumeSchema().build();
    RowSetBuilder builder = fixture.rowSetBuilder(schema);
    for (int i = 0; i < 10; i++) {
        builder.addRow((Object) (new Object[] { 10, "a", new Object[] { 5, "b" } }));
    }
    RowSet rows = builder.build();
    // Run the record batch sizer on the resulting batch.
    RecordBatchSizer sizer = new RecordBatchSizer(rows.container());
    assertEquals(1, sizer.columns().size());
    /**
     * stdDataSize:(50+4)*2, stdNetSize:(50+4)*2+4+4, dataSizePerEntry:(4+1)*2,
     * netSizePerEntry: 4*2+1*2+4*2,
     * totalDataSize:5*2*10, totalNetSize:netSizePerEntry*2,
     * valueCount:10,
     * elementCount:10, estElementCountPerArray:1, isVariableWidth:true
     */
    verifyColumnValues(sizer.columns().get("map"), 108, 116, 10, 18, 100, 180, 10, 10, 1, false);
    SingleRowSet empty = fixture.rowSet(schema);
    VectorAccessible accessible = empty.vectorAccessible();
    UInt4Vector offsetVector;
    for (VectorWrapper<?> vw : accessible) {
        ValueVector v = vw.getValueVector();
        RecordBatchSizer.ColumnSize colSize = sizer.getColumn(v.getField().getName());
        // Allocates to nearest power of two
        colSize.allocateVector(v, testRowCount);
        MapVector mapVector = (MapVector) v;
        ValueVector keyVector = mapVector.getChild("key");
        ValueVector valueVector1 = mapVector.getChild("value");
        assertEquals((Integer.highestOneBit(testRowCount) << 1), keyVector.getValueCapacity());
        offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
        assertEquals((Integer.highestOneBit(testRowCount) << 1), offsetVector.getValueCapacity());
        assertEquals(Integer.highestOneBit(testRowCount << 1) - 1, valueVector1.getValueCapacity());
        MapVector childMapVector = (MapVector) mapVector.getChild("childMap");
        ValueVector childKeyVector = childMapVector.getChild("childKey");
        ValueVector childValueVector1 = childMapVector.getChild("childValue");
        assertEquals((Integer.highestOneBit(testRowCount) << 1), childKeyVector.getValueCapacity());
        offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
        assertEquals((Integer.highestOneBit(testRowCount) << 1), offsetVector.getValueCapacity());
        assertEquals(Integer.highestOneBit(testRowCount << 1) - 1, childValueVector1.getValueCapacity());
        // Allocates the same as value passed since it is already power of two.
        colSize.allocateVector(v, testRowCountPowerTwo - 1);
        mapVector = (MapVector) v;
        keyVector = mapVector.getChild("key");
        valueVector1 = mapVector.getChild("value");
        assertEquals(testRowCountPowerTwo, keyVector.getValueCapacity());
        offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
        assertEquals(testRowCountPowerTwo, offsetVector.getValueCapacity());
        assertEquals(testRowCountPowerTwo - 1, valueVector1.getValueCapacity());
        childMapVector = (MapVector) mapVector.getChild("childMap");
        childKeyVector = childMapVector.getChild("childKey");
        childValueVector1 = childMapVector.getChild("childValue");
        assertEquals(testRowCountPowerTwo, childKeyVector.getValueCapacity());
        offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
        assertEquals(testRowCountPowerTwo, offsetVector.getValueCapacity());
        assertEquals(testRowCountPowerTwo - 1, childValueVector1.getValueCapacity());
        // Allocate for max rows.
        colSize.allocateVector(v, ValueVector.MAX_ROW_COUNT - 1);
        mapVector = (MapVector) v;
        keyVector = mapVector.getChild("key");
        valueVector1 = mapVector.getChild("value");
        assertEquals(ValueVector.MAX_ROW_COUNT, keyVector.getValueCapacity());
        offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
        assertEquals(ValueVector.MAX_ROW_COUNT, offsetVector.getValueCapacity());
        assertEquals(ValueVector.MAX_ROW_COUNT - 1, valueVector1.getValueCapacity());
        childMapVector = (MapVector) mapVector.getChild("childMap");
        childKeyVector = childMapVector.getChild("childKey");
        childValueVector1 = childMapVector.getChild("childValue");
        assertEquals(ValueVector.MAX_ROW_COUNT, childKeyVector.getValueCapacity());
        offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
        assertEquals(ValueVector.MAX_ROW_COUNT, offsetVector.getValueCapacity());
        assertEquals(ValueVector.MAX_ROW_COUNT - 1, childValueVector1.getValueCapacity());
        // Allocate for 0 rows. should atleast do allocation for 1 row.
        colSize.allocateVector(v, 0);
        mapVector = (MapVector) v;
        keyVector = mapVector.getChild("key");
        valueVector1 = mapVector.getChild("value");
        assertEquals(ValueVector.MIN_ROW_COUNT, keyVector.getValueCapacity());
        offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
        assertEquals(ValueVector.MIN_ROW_COUNT + 1, offsetVector.getValueCapacity());
        assertEquals(ValueVector.MIN_ROW_COUNT, valueVector1.getValueCapacity());
        childMapVector = (MapVector) mapVector.getChild("childMap");
        childKeyVector = childMapVector.getChild("childKey");
        childValueVector1 = childMapVector.getChild("childValue");
        assertEquals(ValueVector.MIN_ROW_COUNT, childKeyVector.getValueCapacity());
        offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
        assertEquals(ValueVector.MIN_ROW_COUNT + 1, offsetVector.getValueCapacity());
        assertEquals(ValueVector.MIN_ROW_COUNT, childValueVector1.getValueCapacity());
        v.clear();
    }
    empty.clear();
    rows.clear();
}

Example 27 with MapVector

use of org.apache.drill.exec.vector.complex.MapVector in project drill by axbaretto.

the class ComplexWriterImpl method directMap.

public MapWriter directMap() {
    Preconditions.checkArgument(name == null);
    switch(mode) {
        case INIT:
            MapVector map = (MapVector) container;
            mapRoot = new SingleMapWriter(map, this, unionEnabled);
            mapRoot.setPosition(idx());
            mode = Mode.MAP;
            break;
        case MAP:
            break;
        default:
            check(Mode.INIT, Mode.MAP);
    }
    return mapRoot;
}

Example 28 with MapVector

use of org.apache.drill.exec.vector.complex.MapVector in project drill by axbaretto.

the class ComplexWriterImpl method rootAsMap.

@Override
public MapWriter rootAsMap() {
    switch(mode) {
        case INIT:
            MapVector map = container.addOrGet(name, Types.required(MinorType.MAP), MapVector.class);
            mapRoot = new SingleMapWriter(map, this, unionEnabled);
            mapRoot.setPosition(idx());
            mode = Mode.MAP;
            break;
        case MAP:
            break;
        default:
            check(Mode.INIT, Mode.MAP);
    }
    return mapRoot;
}

Example 29 with MapVector

use of org.apache.drill.exec.vector.complex.MapVector in project drill by apache.

the class TestUnnestCorrectness method testUnnest.

// test unnest for various input conditions optionally invoking kill. if the kill or killBatch
// parameter is greater than 0 then the record batch is sent a kill after that many batches have been processed
private <T> void testUnnest(TupleMetadata[] incomingSchemas, RecordBatch.IterOutcome[] iterOutcomes, // kill unnest after every 'unnestLimit' number of values in every record
int unnestLimit, // number of batches after which to kill the execution (!)
int execKill, T[][] data, T[][] baseline) throws Exception {
    // Get the incoming container with dummy data for LJ
    final List<VectorContainer> incomingContainer = new ArrayList<>(data.length);
    // Create data
    ArrayList<RowSet.SingleRowSet> rowSets = new ArrayList<>();
    int rowNumber = 0;
    int batchNum = 0;
    for (Object[] recordBatch : data) {
        RowSetBuilder rowSetBuilder = fixture.rowSetBuilder(incomingSchemas[batchNum]);
        for (Object rowData : recordBatch) {
            rowSetBuilder.addRow(++rowNumber, rowData);
        }
        RowSet.SingleRowSet rowSet = rowSetBuilder.build();
        rowSets.add(rowSet);
        incomingContainer.add(rowSet.container());
        batchNum++;
    }
    // Get the unnest POPConfig
    final UnnestPOP unnestPopConfig = new UnnestPOP(null, new SchemaPath(new PathSegment.NameSegment("unnestColumn")), DrillUnnestRelBase.IMPLICIT_COLUMN);
    // Get the IterOutcomes for LJ
    final List<RecordBatch.IterOutcome> outcomes = new ArrayList<>(iterOutcomes.length);
    for (RecordBatch.IterOutcome o : iterOutcomes) {
        outcomes.add(o);
    }
    // Create incoming MockRecordBatch
    final MockRecordBatch incomingMockBatch = new MockRecordBatch(fixture.getFragmentContext(), operatorContext, incomingContainer, outcomes, incomingContainer.get(0).getSchema());
    final MockLateralJoinBatch lateralJoinBatch = new MockLateralJoinBatch(fixture.getFragmentContext(), operatorContext, incomingMockBatch);
    // setup Unnest record batch
    final UnnestRecordBatch unnestBatch = new UnnestRecordBatch(unnestPopConfig, fixture.getFragmentContext());
    // set pointer to Lateral in unnest pop config
    unnestBatch.setIncoming((LateralContract) lateralJoinBatch);
    // set backpointer to lateral join in unnest
    lateralJoinBatch.setUnnest(unnestBatch);
    lateralJoinBatch.setUnnestLimit(unnestLimit);
    // Simulate the pipeline by calling next on the incoming
    List<ValueVector> results = null;
    int batchesProcessed = 0;
    try {
        while (!isTerminal(lateralJoinBatch.next())) {
            batchesProcessed++;
            if (batchesProcessed == execKill) {
                lateralJoinBatch.getContext().getExecutorState().fail(new DrillException("Testing failure of execution."));
                lateralJoinBatch.cancel();
            }
        // else nothing to do
        }
        // Check results against baseline
        results = lateralJoinBatch.getResultList();
        int i = 0;
        for (ValueVector vv : results) {
            int valueCount = vv.getAccessor().getValueCount();
            if (valueCount != baseline[i].length) {
                fail("Test failed in validating unnest output. Value count mismatch.");
            }
            for (int j = 0; j < valueCount; j++) {
                if (vv instanceof MapVector) {
                    if (!compareMapBaseline(baseline[i][j], vv.getAccessor().getObject(j))) {
                        fail("Test failed in validating unnest(Map) output. Value mismatch");
                    }
                } else if (vv instanceof VarCharVector) {
                    Object val = vv.getAccessor().getObject(j);
                    if (((String) baseline[i][j]).compareTo(val.toString()) != 0) {
                        fail("Test failed in validating unnest output. Value mismatch. Baseline value[]" + i + "][" + j + "]" + ": " + baseline[i][j] + "   VV.getObject(j): " + val);
                    }
                } else {
                    Object val = vv.getAccessor().getObject(j);
                    if (!baseline[i][j].equals(val)) {
                        fail("Test failed in validating unnest output. Value mismatch. Baseline value[" + i + "][" + j + "]" + ": " + baseline[i][j] + "   VV.getObject(j): " + val);
                    }
                }
            }
            i++;
        }
        assertTrue(lateralJoinBatch.isCompleted());
    } catch (UserException e) {
        // Valid exception
        throw e;
    } catch (Exception e) {
        fail("Test failed in validating unnest output. Exception : " + e.getMessage());
    } finally {
        // Close all the resources for this test case
        unnestBatch.close();
        lateralJoinBatch.close();
        incomingMockBatch.close();
        if (results != null) {
            for (ValueVector vv : results) {
                vv.clear();
            }
        }
        for (RowSet.SingleRowSet rowSet : rowSets) {
            rowSet.clear();
        }
    }
}

Example 30 with MapVector

use of org.apache.drill.exec.vector.complex.MapVector in project drill by apache.

the class TestMapAccessors method testBasics.

@Test
public void testBasics() {
    final TupleMetadata schema = new SchemaBuilder().add("a", MinorType.INT).addMap("m").add("c", MinorType.INT).add("d", MinorType.VARCHAR).resumeSchema().add("e", MinorType.VARCHAR).buildSchema();
    RowSetBuilder builder = fixture.rowSetBuilder(schema);
    RowSetWriter rootWriter = builder.writer();
    // Verify structure and schema
    final TupleMetadata actualSchema = rootWriter.tupleSchema();
    assertEquals(3, actualSchema.size());
    assertTrue(actualSchema.metadata(1).isMap());
    assertEquals(2, actualSchema.metadata("m").tupleSchema().size());
    assertEquals(2, actualSchema.column("m").getChildren().size());
    // Write a row the way that clients will do.
    final ScalarWriter aWriter = rootWriter.scalar("a");
    final TupleWriter mWriter = rootWriter.tuple("m");
    final ScalarWriter cWriter = mWriter.scalar("c");
    final ScalarWriter dWriter = mWriter.scalar("d");
    final ScalarWriter eWriter = rootWriter.scalar("e");
    aWriter.setInt(10);
    cWriter.setInt(110);
    dWriter.setString("fred");
    eWriter.setString("pebbles");
    rootWriter.save();
    // Write another using the test-time conveniences
    rootWriter.addRow(20, mapValue(210, "barney"), "bam-bam");
    RowSet result = builder.build();
    assertEquals(2, result.rowCount());
    // Validate internal structure.
    VectorContainer container = result.container();
    assertEquals(3, container.getNumberOfColumns());
    ValueVector v = container.getValueVector(1).getValueVector();
    assertTrue(v instanceof MapVector);
    MapVector mv = (MapVector) v;
    assertEquals(2, mv.getAccessor().getValueCount());
    // Validate data. Do so using the readers to avoid verifying
    // using the very mechanisms we want to test.
    RowSetReader rootReader = result.reader();
    final ScalarReader aReader = rootReader.scalar("a");
    final TupleReader mReader = rootReader.tuple("m");
    final ScalarReader cReader = mReader.scalar("c");
    final ScalarReader dReader = mReader.scalar("d");
    final ScalarReader eReader = rootReader.scalar("e");
    rootReader.next();
    assertEquals(10, aReader.getInt());
    assertEquals(110, cReader.getInt());
    assertEquals("fred", dReader.getString());
    assertEquals("pebbles", eReader.getString());
    rootReader.next();
    assertEquals(20, aReader.getInt());
    assertEquals(210, cReader.getInt());
    assertEquals("barney", dReader.getString());
    assertEquals("bam-bam", eReader.getString());
    // Verify using the convenience methods.
    final SingleRowSet expected = fixture.rowSetBuilder(schema).addRow(10, mapValue(110, "fred"), "pebbles").addRow(20, mapValue(210, "barney"), "bam-bam").build();
    new RowSetComparison(expected).verify(result);
    // Test that the row set rebuilds its internal structure from
    // a vector container.
    RowSet wrapped = fixture.wrap(result.container());
    RowSetUtilities.verify(expected, wrapped);
}