Examples with DictVector org.apache.drill.exec.vector.complex.DictVector used on opensource projects

Example 6 with DictVector

use of org.apache.drill.exec.vector.complex.DictVector in project drill by apache.

the class TestDictVector method testLoadBatchLoader.

@Test
public void testLoadBatchLoader() throws Exception {
    MaterializedField field = MaterializedField.create("map", DictVector.TYPE);
    try (DictVector mapVector = new DictVector(field, allocator, null)) {
        mapVector.allocateNew();
        List<Map<Object, Object>> maps = Arrays.asList(TestBuilder.mapOfObject(4f, 1L, 5.3f, 2L, 0.3f, 3L, -0.2f, 4L, 102.07f, 5L), TestBuilder.mapOfObject(45f, 6L, 9.2f, 7L), TestBuilder.mapOfObject(4.01f, 8L, 9.2f, 9L, -2.3f, 10L), TestBuilder.mapOfObject(), TestBuilder.mapOfObject(11f, 11L, 9.73f, 12L, 0.03f, 13L));
        BaseWriter.DictWriter mapWriter = new SingleDictWriter(mapVector, null);
        int index = 0;
        for (Map<Object, Object> map : maps) {
            mapWriter.setPosition(index++);
            mapWriter.start();
            for (Map.Entry<Object, Object> entry : map.entrySet()) {
                mapWriter.startKeyValuePair();
                mapWriter.float4(DictVector.FIELD_KEY_NAME).writeFloat4((float) entry.getKey());
                mapWriter.bigInt(DictVector.FIELD_VALUE_NAME).writeBigInt((long) entry.getValue());
                mapWriter.endKeyValuePair();
            }
            mapWriter.end();
        }
        WritableBatch writableBatch = WritableBatch.getBatchNoHV(maps.size(), Collections.singletonList(mapVector), false);
        // Serialize the vector
        DrillBuf byteBuf = TestLoad.serializeBatch(allocator, writableBatch);
        RecordBatchLoader batchLoader = new RecordBatchLoader(allocator);
        batchLoader.load(writableBatch.getDef(), byteBuf);
        byteBuf.release();
        assertEquals(maps.size(), batchLoader.getRecordCount());
        writableBatch.clear();
        batchLoader.clear();
    }
}

Example 7 with DictVector

use of org.apache.drill.exec.vector.complex.DictVector in project drill by apache.

the class TestDictVector method testVectorCreationListValue.

@SuppressWarnings("unchecked")
@Test
public void testVectorCreationListValue() {
    MaterializedField field = MaterializedField.create("map", DictVector.TYPE);
    try (DictVector mapVector = new DictVector(field, allocator, null)) {
        mapVector.allocateNew();
        List<Map<Object, Object>> maps = Arrays.asList(TestBuilder.mapOfObject(1, TestBuilder.listOf(1.0, 2.3, 3.1), 2, TestBuilder.listOf(4.9, -5.002)), TestBuilder.mapOfObject(3, TestBuilder.listOf(6.901), 4, TestBuilder.listOf(), 5, TestBuilder.listOf(7.03, -8.973)), TestBuilder.mapOfObject(), TestBuilder.mapOfObject(6, TestBuilder.listOf(9.0, 10.0, 11.0), 7, TestBuilder.listOf(12.07, 13.01, 14.58, 15.039), 8, TestBuilder.listOf(-16.0, -17.0, 18.0, 19.23, 20.1234)));
        BaseWriter.DictWriter mapWriter = new SingleDictWriter(mapVector, null);
        int index = 0;
        for (Map<Object, Object> map : maps) {
            mapWriter.setPosition(index++);
            mapWriter.start();
            for (Map.Entry<Object, Object> entry : map.entrySet()) {
                mapWriter.startKeyValuePair();
                mapWriter.integer(DictVector.FIELD_KEY_NAME).writeInt((int) entry.getKey());
                BaseWriter.ListWriter valueWriter = mapWriter.list(DictVector.FIELD_VALUE_NAME);
                valueWriter.startList();
                for (Object element : (List<Object>) entry.getValue()) {
                    valueWriter.float8().writeFloat8((double) element);
                }
                valueWriter.endList();
                mapWriter.endKeyValuePair();
            }
            mapWriter.end();
        }
        BaseReader.DictReader mapReader = mapVector.getReader();
        index = 0;
        for (Map<Object, Object> map : maps) {
            mapReader.setPosition(index++);
            for (Map.Entry<Object, Object> entry : map.entrySet()) {
                mapReader.next();
                Integer actualKey = mapReader.reader(DictVector.FIELD_KEY_NAME).readInteger();
                Object actualValue = mapReader.reader(DictVector.FIELD_VALUE_NAME).readObject();
                assertEquals(entry.getKey(), actualKey);
                assertEquals(entry.getValue(), actualValue);
            }
        }
    }
}

Example 8 with DictVector

use of org.apache.drill.exec.vector.complex.DictVector in project drill by apache.

the class TestDictVector method testSplitAndTransfer.

@Test
public void testSplitAndTransfer() {
    MaterializedField field = MaterializedField.create("map", DictVector.TYPE);
    try (DictVector mapVector = new DictVector(field, allocator, null)) {
        mapVector.allocateNew();
        List<Map<Object, Object>> maps = Arrays.asList(TestBuilder.mapOfObject(4f, 1L, 5.3f, 2L, 0.3f, 3L, -0.2f, 4L, 102.07f, 5L), TestBuilder.mapOfObject(45f, 6L, 9.2f, 7L), TestBuilder.mapOfObject(4.01f, 8L, 9.2f, 9L, -2.3f, 10L), TestBuilder.mapOfObject(), TestBuilder.mapOfObject(11f, 11L, 9.73f, 12L, 0.03f, 13L));
        BaseWriter.DictWriter mapWriter = new SingleDictWriter(mapVector, null);
        int index = 0;
        for (Map<Object, Object> map : maps) {
            mapWriter.setPosition(index++);
            mapWriter.start();
            for (Map.Entry<Object, Object> entry : map.entrySet()) {
                mapWriter.startKeyValuePair();
                mapWriter.float4(DictVector.FIELD_KEY_NAME).writeFloat4((float) entry.getKey());
                mapWriter.bigInt(DictVector.FIELD_VALUE_NAME).writeBigInt((long) entry.getValue());
                mapWriter.endKeyValuePair();
            }
            mapWriter.end();
        }
        int start = 1;
        int length = 2;
        DictVector newMapVector = new DictVector(field, allocator, null);
        TransferPair transferPair = mapVector.makeTransferPair(newMapVector);
        transferPair.splitAndTransfer(start, length);
        BaseReader.DictReader mapReader = newMapVector.getReader();
        index = 0;
        for (Map<Object, Object> map : maps.subList(start, start + length)) {
            mapReader.setPosition(index++);
            for (Map.Entry<Object, Object> entry : map.entrySet()) {
                mapReader.next();
                Float actualKey = mapReader.reader(DictVector.FIELD_KEY_NAME).readFloat();
                Long actualValue = mapReader.reader(DictVector.FIELD_VALUE_NAME).readLong();
                assertEquals(entry.getKey(), actualKey);
                assertEquals(entry.getValue(), actualValue);
            }
        }
        newMapVector.clear();
    }
}

Example 9 with DictVector

use of org.apache.drill.exec.vector.complex.DictVector in project drill by apache.

the class TestRowSet method testDictStructure.

@Test
public void testDictStructure() {
    final String dictName = "d";
    final TupleMetadata schema = new SchemaBuilder().add("id", MinorType.INT).addDict(dictName, MinorType.INT).value(// required int
    MinorType.VARCHAR).resumeSchema().buildSchema();
    final ExtendableRowSet rowSet = fixture.rowSet(schema);
    final RowSetWriter writer = rowSet.writer();
    // Dict
    // Pick out components and lightly test. (Assumes structure
    // tested earlier is still valid, so no need to exhaustively
    // test again.)
    assertEquals(ObjectType.ARRAY, writer.column(dictName).type());
    assertTrue(writer.column(dictName).schema().isDict());
    final ScalarWriter idWriter = writer.column(0).scalar();
    final DictWriter dictWriter = writer.column(1).dict();
    assertEquals(ValueType.INTEGER, dictWriter.keyType());
    assertEquals(ObjectType.SCALAR, dictWriter.valueType());
    final ScalarWriter keyWriter = dictWriter.keyWriter();
    final ScalarWriter valueWriter = dictWriter.valueWriter().scalar();
    assertEquals(ValueType.INTEGER, keyWriter.valueType());
    assertEquals(ValueType.STRING, valueWriter.valueType());
    // Write data
    idWriter.setInt(1);
    keyWriter.setInt(11);
    valueWriter.setString("a");
    // Advance to next entry position
    dictWriter.save();
    keyWriter.setInt(12);
    valueWriter.setString("b");
    dictWriter.save();
    writer.save();
    idWriter.setInt(2);
    keyWriter.setInt(21);
    valueWriter.setString("c");
    dictWriter.save();
    writer.save();
    idWriter.setInt(3);
    keyWriter.setInt(31);
    valueWriter.setString("d");
    dictWriter.save();
    keyWriter.setInt(32);
    valueWriter.setString("e");
    dictWriter.save();
    writer.save();
    // Finish the row set and get a reader.
    final SingleRowSet actual = writer.done();
    final RowSetReader reader = actual.reader();
    // Verify reader structure
    assertEquals(ObjectType.ARRAY, reader.column(dictName).type());
    final DictReader dictReader = reader.dict(1);
    assertEquals(ObjectType.ARRAY, dictReader.type());
    assertEquals(ValueType.INTEGER, dictReader.keyColumnType());
    assertEquals(ObjectType.SCALAR, dictReader.valueColumnType());
    // Row 1: get value reader with its position set to entry corresponding to a key
    assertTrue(reader.next());
    // dict itself is not null
    assertFalse(dictReader.isNull());
    dictReader.getAsString();
    final KeyAccessor keyAccessor = dictReader.keyAccessor();
    final ScalarReader valueReader = dictReader.valueReader().scalar();
    assertTrue(keyAccessor.find(12));
    assertEquals("b", valueReader.getString());
    assertTrue(keyAccessor.find(11));
    assertEquals("a", valueReader.getString());
    // compare entire dict
    Map<Object, Object> map = map(11, "a", 12, "b");
    assertEquals(map, dictReader.getObject());
    // Row 2
    assertTrue(reader.next());
    // the dict does not contain an entry with the key
    assertFalse(keyAccessor.find(22));
    assertTrue(keyAccessor.find(21));
    assertEquals("c", valueReader.getString());
    map = map(21, "c");
    assertEquals(map, dictReader.getObject());
    // Row 3
    assertTrue(reader.next());
    assertTrue(keyAccessor.find(31));
    assertEquals("d", valueReader.getString());
    assertFalse(keyAccessor.find(33));
    assertTrue(keyAccessor.find(32));
    assertEquals("e", valueReader.getString());
    map = map(31, "d", 32, "e");
    assertEquals(map, dictReader.getObject());
    assertFalse(reader.next());
    // Verify that the dict accessor's value count was set.
    final DictVector dictVector = (DictVector) actual.container().getValueVector(1).getValueVector();
    assertEquals(3, dictVector.getAccessor().getValueCount());
    final SingleRowSet expected = fixture.rowSetBuilder(schema).addRow(1, map(11, "a", 12, "b")).addRow(2, map(21, "c")).addRow(3, map(31, "d", 32, "e")).build();
    RowSetUtilities.verify(expected, actual);
}

Example 10 with DictVector

use of org.apache.drill.exec.vector.complex.DictVector in project drill by apache.

the class TestRowSet method testDictStructureMapValue.

@Test
public void testDictStructureMapValue() {
    final String dictName = "d";
    final int bScale = 1;
    final TupleMetadata schema = new SchemaBuilder().add("id", MinorType.INT).addDict(dictName, MinorType.INT).mapValue().add("a", MinorType.INT).add("b", MinorType.VARDECIMAL, 8, bScale).resumeDict().resumeSchema().buildSchema();
    final ExtendableRowSet rowSet = fixture.rowSet(schema);
    final RowSetWriter writer = rowSet.writer();
    // Dict with Map value
    assertEquals(ObjectType.ARRAY, writer.column(dictName).type());
    final ScalarWriter idWriter = writer.scalar(0);
    final DictWriter dictWriter = writer.column(1).dict();
    assertEquals(ValueType.INTEGER, dictWriter.keyType());
    assertEquals(ObjectType.TUPLE, dictWriter.valueType());
    final ScalarWriter keyWriter = dictWriter.keyWriter();
    final TupleWriter valueWriter = dictWriter.valueWriter().tuple();
    assertEquals(ValueType.INTEGER, keyWriter.valueType());
    ScalarWriter aWriter = valueWriter.scalar("a");
    ScalarWriter bWriter = valueWriter.scalar("b");
    assertEquals(ValueType.INTEGER, aWriter.valueType());
    assertEquals(ValueType.DECIMAL, bWriter.valueType());
    // Write data
    idWriter.setInt(1);
    keyWriter.setInt(11);
    aWriter.setInt(10);
    bWriter.setDecimal(BigDecimal.valueOf(1));
    // advance to next entry position
    dictWriter.save();
    keyWriter.setInt(12);
    aWriter.setInt(11);
    bWriter.setDecimal(BigDecimal.valueOf(2));
    dictWriter.save();
    writer.save();
    idWriter.setInt(2);
    keyWriter.setInt(21);
    aWriter.setInt(20);
    bWriter.setDecimal(BigDecimal.valueOf(3));
    dictWriter.save();
    writer.save();
    idWriter.setInt(3);
    keyWriter.setInt(31);
    aWriter.setInt(30);
    bWriter.setDecimal(BigDecimal.valueOf(4));
    dictWriter.save();
    keyWriter.setInt(32);
    aWriter.setInt(31);
    bWriter.setDecimal(BigDecimal.valueOf(5));
    dictWriter.save();
    keyWriter.setInt(33);
    aWriter.setInt(32);
    bWriter.setDecimal(BigDecimal.valueOf(6));
    dictWriter.save();
    writer.save();
    // Finish the row set and get a reader.
    final SingleRowSet actual = writer.done();
    final RowSetReader reader = actual.reader();
    // Verify reader structure
    assertEquals(ObjectType.ARRAY, reader.column(dictName).type());
    final DictReader dictReader = reader.dict(1);
    assertEquals(ObjectType.ARRAY, dictReader.type());
    assertEquals(ValueType.INTEGER, dictReader.keyColumnType());
    assertEquals(ObjectType.TUPLE, dictReader.valueColumnType());
    final KeyAccessor keyAccessor = dictReader.keyAccessor();
    final TupleReader valueReader = dictReader.valueReader().tuple();
    // Row 1: get value reader with its position set to entry corresponding to a key
    assertTrue(reader.next());
    // dict itself is not null
    assertFalse(dictReader.isNull());
    assertTrue(keyAccessor.find(12));
    assertEquals(11, valueReader.scalar("a").getInt());
    assertEquals(BigDecimal.valueOf(2.0), valueReader.scalar("b").getDecimal());
    // MapReader#getObject() returns a List containing values for each column
    // rather than mapping of column name to it's value, hence List is expected for Dict's value.
    Map<Object, Object> map = map(11, Arrays.asList(10, BigDecimal.valueOf(1.0)), 12, Arrays.asList(11, BigDecimal.valueOf(2.0)));
    assertEquals(map, dictReader.getObject());
    // Row 2
    assertTrue(reader.next());
    assertFalse(keyAccessor.find(222));
    assertTrue(keyAccessor.find(21));
    assertEquals(Arrays.asList(20, BigDecimal.valueOf(3.0)), valueReader.getObject());
    map = map(21, Arrays.asList(20, BigDecimal.valueOf(3.0)));
    assertEquals(map, dictReader.getObject());
    // Row 3
    assertTrue(reader.next());
    assertTrue(keyAccessor.find(32));
    assertFalse(valueReader.isNull());
    assertEquals(31, valueReader.scalar("a").getInt());
    assertEquals(BigDecimal.valueOf(5.0), valueReader.scalar("b").getDecimal());
    assertTrue(keyAccessor.find(31));
    assertEquals(30, valueReader.scalar("a").getInt());
    assertEquals(BigDecimal.valueOf(4.0), valueReader.scalar("b").getDecimal());
    assertFalse(keyAccessor.find(404));
    map = map(31, Arrays.asList(30, BigDecimal.valueOf(4.0)), 32, Arrays.asList(31, BigDecimal.valueOf(5.0)), 33, Arrays.asList(32, BigDecimal.valueOf(6.0)));
    assertEquals(map, dictReader.getObject());
    assertFalse(reader.next());
    // Verify that the dict accessor's value count was set.
    final DictVector dictVector = (DictVector) actual.container().getValueVector(1).getValueVector();
    assertEquals(3, dictVector.getAccessor().getValueCount());
    final SingleRowSet expected = fixture.rowSetBuilder(schema).addRow(1, map(11, objArray(10, BigDecimal.valueOf(1.0)), 12, objArray(11, BigDecimal.valueOf(2.0)))).addRow(2, map(21, objArray(20, BigDecimal.valueOf(3.0)))).addRow(3, map(31, objArray(30, BigDecimal.valueOf(4.0)), 32, objArray(31, BigDecimal.valueOf(5.0)), 33, objArray(32, BigDecimal.valueOf(6.0)))).build();
    RowSetUtilities.verify(expected, actual);
}