Examples with RepeatedMapVector org.apache.drill.exec.vector.complex.RepeatedMapVector used on opensource projects

Example 16 with RepeatedMapVector

use of org.apache.drill.exec.vector.complex.RepeatedMapVector in project drill by apache.

the class VectorTest method buildRepeatedMap.

private RepeatedMapVector buildRepeatedMap() {
    MaterializedField field = MaterializedField.create("repeated_map", Types.repeated(TypeProtos.MinorType.MAP));
    RepeatedMapVector v = new RepeatedMapVector(field, allocator, null);
    getInner(v);
    v.allocateNew(5, 10);
    return v;
}

Example 17 with RepeatedMapVector

use of org.apache.drill.exec.vector.complex.RepeatedMapVector in project drill by apache.

the class VectorTest method testRepeatedCopySafe.

@Test
public void testRepeatedCopySafe() throws Exception {
    try (RepeatedMapVector v = buildRepeatedMap();
        RepeatedMapVector f = buildFromMap()) {
        RepeatedMapVector.Mutator vm = v.getMutator();
        UInt4Vector ov = v.getOffsetVector();
        UInt4Vector.Accessor oa = ov.getAccessor();
        UInt4Vector.Mutator om = ov.getMutator();
        TransferPair tp = f.makeTransferPair(v);
        tp.copyValueSafe(0, 0);
        // CopyValue does not change the value count
        // assertEquals(1, va.getValueCount());
        // assertEquals(2, oa.getValueCount());
        // assertEquals(2, ia.getValueCount());
        assertEquals(0, oa.get(0));
        assertEquals(2, oa.get(1));
        tp.copyValueSafe(1, 1);
        assertEquals(0, oa.get(0));
        assertEquals(2, oa.get(1));
        assertEquals(5, oa.get(2));
        tp.copyValueSafe(2, 2);
        assertEquals(2, oa.get(1));
        assertEquals(5, oa.get(2));
        assertEquals(5, oa.get(3));
        vm.setValueCount(3);
        // v should now be the same as f
        validateFrom(v);
    }
}

Example 18 with RepeatedMapVector

use of org.apache.drill.exec.vector.complex.RepeatedMapVector in project drill by apache.

the class VectorTest method testRepeatedMapCount.

@Test
public void testRepeatedMapCount() throws Exception {
    try (RepeatedMapVector v = buildRepeatedMap()) {
        IntVector iv = getInner(v);
        RepeatedMapVector.Accessor va = v.getAccessor();
        RepeatedMapVector.Mutator vm = v.getMutator();
        IntVector.Accessor ia = iv.getAccessor();
        IntVector.Mutator im = iv.getMutator();
        UInt4Vector ov = v.getOffsetVector();
        UInt4Vector.Accessor oa = ov.getAccessor();
        UInt4Vector.Mutator om = ov.getMutator();
        // Zero fill so the following is deterministic
        ov.getBuffer().setZero(0, 6 * 4);
        // Initial state
        assertEquals(0, va.getValueCount());
        assertEquals(0, ia.getValueCount());
        assertEquals(0, oa.getValueCount());
        assertEquals(0, oa.get(0));
        // Record size = 0
        vm.setValueCount(0);
        assertEquals(0, va.getValueCount());
        assertEquals(0, ia.getValueCount());
        assertEquals(0, oa.getValueCount());
        assertEquals(0, oa.get(0));
        // Record size = 1, so, implicit record 1 (1-based) of []
        vm.setValueCount(1);
        assertEquals(1, va.getValueCount());
        assertEquals(0, ia.getValueCount());
        assertEquals(2, oa.getValueCount());
        assertEquals(0, oa.get(0));
        assertEquals(0, oa.get(1));
        // Record 2 (1-based) is [10, 20]
        im.set(0, 10);
        im.set(1, 20);
        im.setValueCount(2);
        om.set(2, 2);
        om.setValueCount(3);
        vm.setValueCount(2);
        assertEquals(2, va.getValueCount());
        assertEquals(2, ia.getValueCount());
        assertEquals(3, oa.getValueCount());
        assertEquals(0, oa.get(0));
        assertEquals(0, oa.get(1));
        assertEquals(2, oa.get(2));
        assertEquals(0, oa.get(3));
        // Batch size = 4, so implicit record 2, 4 of []
        vm.setValueCount(4);
        assertEquals(4, va.getValueCount());
        assertEquals(2, ia.getValueCount());
        assertEquals(5, oa.getValueCount());
        assertEquals(0, oa.get(0));
        assertEquals(0, oa.get(1));
        assertEquals(2, oa.get(2));
        assertEquals(2, oa.get(3));
        assertEquals(2, oa.get(4));
        assertEquals(0, oa.get(5));
    }
}

Example 19 with RepeatedMapVector

use of org.apache.drill.exec.vector.complex.RepeatedMapVector in project drill by apache.

the class TestRowSet method testRepeatedMapStructure.

@Test
public void testRepeatedMapStructure() {
    final TupleMetadata schema = new SchemaBuilder().add("a", MinorType.INT).addMapArray("m").add("b", MinorType.INT).add("c", MinorType.INT).resumeSchema().buildSchema();
    final ExtendableRowSet rowSet = fixture.rowSet(schema);
    final RowSetWriter writer = rowSet.writer();
    // Map and Int
    // Pick out components and lightly test. (Assumes structure
    // tested earlier is still valid, so no need to exhaustively
    // test again.)
    assertEquals(ObjectType.SCALAR, writer.column("a").type());
    assertEquals(ObjectType.ARRAY, writer.column("m").type());
    final ArrayWriter maWriter = writer.column(1).array();
    assertEquals(ObjectType.TUPLE, maWriter.entryType());
    final TupleWriter mapWriter = maWriter.tuple();
    assertEquals(ObjectType.SCALAR, mapWriter.column("b").type());
    assertEquals(ObjectType.SCALAR, mapWriter.column("c").type());
    final ScalarWriter aWriter = writer.column("a").scalar();
    final ScalarWriter bWriter = mapWriter.scalar("b");
    final ScalarWriter cWriter = mapWriter.scalar("c");
    assertEquals(ValueType.INTEGER, aWriter.valueType());
    assertEquals(ValueType.INTEGER, bWriter.valueType());
    assertEquals(ValueType.INTEGER, cWriter.valueType());
    // Write data
    aWriter.setInt(10);
    bWriter.setInt(101);
    cWriter.setInt(102);
    // Advance to next array position
    maWriter.save();
    bWriter.setInt(111);
    cWriter.setInt(112);
    maWriter.save();
    writer.save();
    aWriter.setInt(20);
    bWriter.setInt(201);
    cWriter.setInt(202);
    maWriter.save();
    bWriter.setInt(211);
    cWriter.setInt(212);
    maWriter.save();
    writer.save();
    aWriter.setInt(30);
    bWriter.setInt(301);
    cWriter.setInt(302);
    maWriter.save();
    bWriter.setInt(311);
    cWriter.setInt(312);
    maWriter.save();
    writer.save();
    // Finish the row set and get a reader.
    final SingleRowSet actual = writer.done();
    final RowSetReader reader = actual.reader();
    // Verify reader structure
    assertEquals(ObjectType.SCALAR, reader.column("a").type());
    assertEquals(ObjectType.ARRAY, reader.column("m").type());
    final ArrayReader maReader = reader.column(1).array();
    assertEquals(ObjectType.TUPLE, maReader.entryType());
    final TupleReader mapReader = maReader.tuple();
    assertEquals(ObjectType.SCALAR, mapReader.column("b").type());
    assertEquals(ObjectType.SCALAR, mapReader.column("c").type());
    final ScalarReader aReader = reader.column("a").scalar();
    final ScalarReader bReader = mapReader.scalar("b");
    final ScalarReader cReader = mapReader.scalar("c");
    assertEquals(ValueType.INTEGER, aReader.valueType());
    assertEquals(ValueType.INTEGER, bReader.valueType());
    assertEquals(ValueType.INTEGER, cReader.valueType());
    // Row 1: Use iterator-like accessors
    assertTrue(reader.next());
    assertEquals(10, aReader.getInt());
    // Array itself is not null
    assertFalse(maReader.isNull());
    assertTrue(maReader.next());
    // Tuple 0 is not null
    assertFalse(mapReader.isNull());
    assertEquals(101, mapReader.scalar(0).getInt());
    assertEquals(102, mapReader.scalar(1).getInt());
    assertTrue(maReader.next());
    assertEquals(111, mapReader.scalar(0).getInt());
    assertEquals(112, mapReader.scalar(1).getInt());
    // Row 2: use explicit positioning,
    // but access scalars through the map reader.
    assertTrue(reader.next());
    assertEquals(20, aReader.getInt());
    maReader.setPosn(0);
    assertEquals(201, mapReader.scalar(0).getInt());
    assertEquals(202, mapReader.scalar(1).getInt());
    maReader.setPosn(1);
    assertEquals(211, mapReader.scalar(0).getInt());
    assertEquals(212, mapReader.scalar(1).getInt());
    // Row 3: use scalar accessor
    assertTrue(reader.next());
    assertEquals(30, aReader.getInt());
    assertTrue(maReader.next());
    assertEquals(301, bReader.getInt());
    assertEquals(302, cReader.getInt());
    assertTrue(maReader.next());
    assertEquals(311, bReader.getInt());
    assertEquals(312, cReader.getInt());
    assertFalse(reader.next());
    // Verify that the map accessor's value count was set.
    final RepeatedMapVector mapVector = (RepeatedMapVector) actual.container().getValueVector(1).getValueVector();
    assertEquals(3, mapVector.getAccessor().getValueCount());
    // Verify the readers and writers again using the testing tools.
    final SingleRowSet expected = fixture.rowSetBuilder(schema).addRow(10, objArray(objArray(101, 102), objArray(111, 112))).addRow(20, objArray(objArray(201, 202), objArray(211, 212))).addRow(30, objArray(objArray(301, 302), objArray(311, 312))).build();
    RowSetUtilities.verify(expected, actual);
}

Example 20 with RepeatedMapVector

use of org.apache.drill.exec.vector.complex.RepeatedMapVector in project drill by apache.

the class TestRecordBatchSizer method testSizerRepeatedMap.

@Test
public void testSizerRepeatedMap() {
    TupleMetadata schema = new SchemaBuilder().addMapArray("map").add("key", MinorType.INT).add("value", MinorType.VARCHAR).resumeSchema().buildSchema();
    RowSetBuilder builder = fixture.rowSetBuilder(schema);
    for (int i = 0; i < 10; i++) {
        builder.addRow((Object) new Object[] { new Object[] { 110, "a" }, new Object[] { 120, "b" } });
    }
    RowSet rows = builder.build();
    // Run the record batch sizer on the resulting batch.
    RecordBatchSizer sizer = new RecordBatchSizer(rows.container());
    assertEquals(1, sizer.columns().size());
    /**
     * stdDataSize:50+4, stdNetSize:50+4+4+4, dataSizePerEntry:(4+1)*2,
     * netSizePerEntry: 4*2+1*2+4*2+4,
     * totalDataSize:5*2*10, totalNetSize:netSizePerEntry*2,
     * valueCount:10,
     * elementCount:20, cardinality:2, isVariableWidth:true
     */
    verifyColumnValues(sizer.columns().get("map"), 54, 62, 10, 22, 100, 220, 10, 20, 2, false);
    SingleRowSet empty = fixture.rowSet(schema);
    VectorAccessible accessible = empty.vectorAccessible();
    UInt4Vector offsetVector;
    for (VectorWrapper<?> vw : accessible) {
        ValueVector v = vw.getValueVector();
        RecordBatchSizer.ColumnSize colSize = sizer.getColumn(v.getField().getName());
        // Allocates to nearest power of two
        colSize.allocateVector(v, testRowCount);
        RepeatedMapVector mapVector = (RepeatedMapVector) v;
        offsetVector = ((RepeatedValueVector) mapVector).getOffsetVector();
        assertEquals((Integer.highestOneBit(testRowCount) << 1), offsetVector.getValueCapacity());
        ValueVector keyVector = mapVector.getChild("key");
        ValueVector valueVector1 = mapVector.getChild("value");
        assertEquals(((Integer.highestOneBit(testRowCount) << 1) * 2), keyVector.getValueCapacity());
        offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
        assertEquals((Integer.highestOneBit(testRowCount) << 1) * 2, offsetVector.getValueCapacity());
        assertEquals(Integer.highestOneBit(testRowCount << 1) * 2 - 1, valueVector1.getValueCapacity());
        // Allocates the same as value passed since it is already power of two.
        colSize.allocateVector(v, testRowCountPowerTwo - 1);
        mapVector = (RepeatedMapVector) v;
        offsetVector = ((RepeatedValueVector) mapVector).getOffsetVector();
        assertEquals(testRowCountPowerTwo, offsetVector.getValueCapacity());
        keyVector = mapVector.getChild("key");
        valueVector1 = mapVector.getChild("value");
        assertEquals(testRowCountPowerTwo * 2, keyVector.getValueCapacity());
        offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
        assertEquals(testRowCountPowerTwo * 2, offsetVector.getValueCapacity());
        assertEquals(testRowCountPowerTwo * 2 - 1, valueVector1.getValueCapacity());
        // Allocate for max rows.
        colSize.allocateVector(v, ValueVector.MAX_ROW_COUNT - 1);
        mapVector = (RepeatedMapVector) v;
        offsetVector = ((RepeatedValueVector) mapVector).getOffsetVector();
        assertEquals(ValueVector.MAX_ROW_COUNT, offsetVector.getValueCapacity());
        keyVector = mapVector.getChild("key");
        valueVector1 = mapVector.getChild("value");
        assertEquals(ValueVector.MAX_ROW_COUNT * 2, keyVector.getValueCapacity());
        offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
        assertEquals(ValueVector.MAX_ROW_COUNT * 2, offsetVector.getValueCapacity());
        assertEquals(ValueVector.MAX_ROW_COUNT * 2 - 1, valueVector1.getValueCapacity());
        // Allocate for 0 rows. should atleast do allocation for 1 row.
        colSize.allocateVector(v, 0);
        mapVector = (RepeatedMapVector) v;
        offsetVector = ((RepeatedValueVector) mapVector).getOffsetVector();
        assertEquals(ValueVector.MIN_ROW_COUNT, offsetVector.getValueCapacity());
        keyVector = mapVector.getChild("key");
        valueVector1 = mapVector.getChild("value");
        assertEquals(ValueVector.MIN_ROW_COUNT, keyVector.getValueCapacity());
        offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
        assertEquals(ValueVector.MIN_ROW_COUNT + 1, offsetVector.getValueCapacity());
        assertEquals(ValueVector.MIN_ROW_COUNT, valueVector1.getValueCapacity());
        v.clear();
    }
    empty.clear();
    rows.clear();
}