Examples with ScalarWriter org.apache.drill.exec.vector.accessor.ScalarWriter used on opensource projects

Example 41 with ScalarWriter

use of org.apache.drill.exec.vector.accessor.ScalarWriter in project drill by apache.

the class TestVariantAccessors method testScalarList.

/**
 * Test a scalar list. Should act just like a repeated type, with the
 * addition of allowing the list for a row to be null. But, a list
 * writer does not do auto-increment, so we must do that explicitly
 * after each write.
 */
@Test
public void testScalarList() {
    final TupleMetadata schema = new SchemaBuilder().addList("list").addType(MinorType.VARCHAR).resumeSchema().buildSchema();
    final ExtendableRowSet rowSet = fixture.rowSet(schema);
    final RowSetWriter writer = rowSet.writer();
    {
        final ObjectWriter listObj = writer.column(0);
        assertEquals(ObjectType.ARRAY, listObj.type());
        final ArrayWriter listArray = listObj.array();
        // The list contains only a scalar. But, because lists can,
        // in general, contain multiple contents, the list requires
        // an explicit save after each entry.
        final ObjectWriter itemObj = listArray.entry();
        assertEquals(ObjectType.SCALAR, itemObj.type());
        final ScalarWriter strWriter = itemObj.scalar();
        // First row: two strings and a null
        // Unlike a repeated type, a list can mark individual elements
        // as null.
        // List will automatically detect that data was written.
        strWriter.setString("fred");
        listArray.save();
        strWriter.setNull();
        listArray.save();
        strWriter.setString("wilma");
        listArray.save();
        writer.save();
        // Second row: null
        writer.save();
        // Third row: one string
        strWriter.setString("dino");
        listArray.save();
        writer.save();
        // Fourth row: empty array. Note that there is no trigger
        // to say that the column is not null, so we have to do it
        // explicitly.
        listArray.setNull(false);
        writer.save();
        // Last row: a null string and non-null
        strWriter.setNull();
        listArray.save();
        strWriter.setString("pebbles");
        listArray.save();
        writer.save();
    }
    final SingleRowSet result = writer.done();
    assertEquals(5, result.rowCount());
    {
        final RowSetReader reader = result.reader();
        final ObjectReader listObj = reader.column(0);
        assertEquals(ObjectType.ARRAY, listObj.type());
        final ArrayReader listArray = listObj.array();
        // The list is a repeated scalar
        assertEquals(ObjectType.SCALAR, listArray.entry().type());
        final ScalarReader strReader = listArray.scalar();
        // First row: two strings and a null
        assertTrue(reader.next());
        assertFalse(listArray.isNull());
        assertEquals(3, listArray.size());
        assertTrue(listArray.next());
        assertFalse(strReader.isNull());
        assertEquals("fred", strReader.getString());
        assertTrue(listArray.next());
        assertTrue(strReader.isNull());
        assertTrue(listArray.next());
        assertFalse(strReader.isNull());
        assertEquals("wilma", strReader.getString());
        assertFalse(listArray.next());
        // Second row: null
        assertTrue(reader.next());
        assertTrue(listArray.isNull());
        assertEquals(0, listArray.size());
        // Third row: one string
        assertTrue(reader.next());
        assertFalse(listArray.isNull());
        assertEquals(1, listArray.size());
        assertTrue(listArray.next());
        assertEquals("dino", strReader.getString());
        assertFalse(listArray.next());
        // Fourth row: empty array.
        assertTrue(reader.next());
        assertFalse(listArray.isNull());
        assertEquals(0, listArray.size());
        assertFalse(listArray.next());
        // Last row: a null string and non-null
        assertTrue(reader.next());
        assertFalse(listArray.isNull());
        assertEquals(2, listArray.size());
        assertTrue(listArray.next());
        assertTrue(strReader.isNull());
        assertTrue(listArray.next());
        assertFalse(strReader.isNull());
        assertEquals("pebbles", strReader.getString());
        assertFalse(listArray.next());
        assertFalse(reader.next());
    }
    result.clear();
}

Example 42 with ScalarWriter

use of org.apache.drill.exec.vector.accessor.ScalarWriter in project drill by apache.

the class TestIndirectReaders method testArray.

/**
 * More complex case with two levels of offset vector (one for the
 * array, another for the Varchar values.) Only the top level goes
 * through the indirection.
 */
@Test
public void testArray() {
    TupleMetadata schema = new SchemaBuilder().addArray("a", MinorType.VARCHAR).buildSchema();
    ExtendableRowSet rowSet = fixture.rowSet(schema);
    RowSetWriter writer = rowSet.writer();
    ArrayWriter aWriter = writer.array(0);
    ScalarWriter strWriter = aWriter.scalar();
    for (int i = 0; i < 10; i++) {
        for (int j = 0; j < 5; j++) {
            strWriter.setString("value" + i + "." + j);
        }
        writer.save();
    }
    SingleRowSet result = writer.done().toIndirect();
    SelectionVector2 sv2 = result.getSv2();
    for (int i = 0; i < 10; i++) {
        sv2.setIndex(i, 9 - i);
    }
    RowSetReader reader = result.reader();
    ArrayReader aReader = reader.array(0);
    ScalarReader strReader = aReader.scalar();
    for (int i = 9; i >= 0; i--) {
        assertTrue(reader.next());
        for (int j = 0; j < 5; j++) {
            assertTrue(aReader.next());
            assertEquals("value" + i + "." + j, strReader.getString());
        }
    }
    result.clear();
}

Example 43 with ScalarWriter

use of org.apache.drill.exec.vector.accessor.ScalarWriter in project drill by apache.

the class TestOffsetVectorWriter method testSizeLimit.

/**
 * Test resize monitoring. Add a listener to an offsets writer,
 * capture each resize, and refuse a resize when the number
 * of ints exceeds 8K values. This will trigger an overflow,
 * which will throw an exception which we then check for.
 */
@Test
public void testSizeLimit() {
    try (UInt4Vector vector = allocVector(1000)) {
        TestIndex index = new TestIndex();
        OffsetVectorWriterImpl writer = makeWriter(vector, index);
        writer.bindListener(new ColumnWriterListener() {

            int totalAlloc = 4096;

            @Override
            public void overflowed(ScalarWriter writer) {
                throw new IllegalStateException("overflow called");
            }

            @Override
            public boolean canExpand(ScalarWriter writer, int delta) {
                totalAlloc += delta;
                return totalAlloc < 16_384 * 4;
            }
        });
        writer.startWrite();
        try {
            for (int i = 0; ; i++) {
                index.index = i;
                writer.startRow();
                writer.setNextOffset(i);
                writer.saveRow();
            }
        } catch (IllegalStateException e) {
            assertTrue(e.getMessage().contains("overflow called"));
        }
        // Should have failed on 8191, which doubled vector
        // to 16K, which was rejected. Note the 8191 value,
        // because offsets are one ahead of the index.
        assertEquals(8191, index.index);
    }
}

Example 44 with ScalarWriter

use of org.apache.drill.exec.vector.accessor.ScalarWriter in project drill by apache.

the class TestRowSet method testDictStructure.

@Test
public void testDictStructure() {
    final String dictName = "d";
    final TupleMetadata schema = new SchemaBuilder().add("id", MinorType.INT).addDict(dictName, MinorType.INT).value(// required int
    MinorType.VARCHAR).resumeSchema().buildSchema();
    final ExtendableRowSet rowSet = fixture.rowSet(schema);
    final RowSetWriter writer = rowSet.writer();
    // Dict
    // Pick out components and lightly test. (Assumes structure
    // tested earlier is still valid, so no need to exhaustively
    // test again.)
    assertEquals(ObjectType.ARRAY, writer.column(dictName).type());
    assertTrue(writer.column(dictName).schema().isDict());
    final ScalarWriter idWriter = writer.column(0).scalar();
    final DictWriter dictWriter = writer.column(1).dict();
    assertEquals(ValueType.INTEGER, dictWriter.keyType());
    assertEquals(ObjectType.SCALAR, dictWriter.valueType());
    final ScalarWriter keyWriter = dictWriter.keyWriter();
    final ScalarWriter valueWriter = dictWriter.valueWriter().scalar();
    assertEquals(ValueType.INTEGER, keyWriter.valueType());
    assertEquals(ValueType.STRING, valueWriter.valueType());
    // Write data
    idWriter.setInt(1);
    keyWriter.setInt(11);
    valueWriter.setString("a");
    // Advance to next entry position
    dictWriter.save();
    keyWriter.setInt(12);
    valueWriter.setString("b");
    dictWriter.save();
    writer.save();
    idWriter.setInt(2);
    keyWriter.setInt(21);
    valueWriter.setString("c");
    dictWriter.save();
    writer.save();
    idWriter.setInt(3);
    keyWriter.setInt(31);
    valueWriter.setString("d");
    dictWriter.save();
    keyWriter.setInt(32);
    valueWriter.setString("e");
    dictWriter.save();
    writer.save();
    // Finish the row set and get a reader.
    final SingleRowSet actual = writer.done();
    final RowSetReader reader = actual.reader();
    // Verify reader structure
    assertEquals(ObjectType.ARRAY, reader.column(dictName).type());
    final DictReader dictReader = reader.dict(1);
    assertEquals(ObjectType.ARRAY, dictReader.type());
    assertEquals(ValueType.INTEGER, dictReader.keyColumnType());
    assertEquals(ObjectType.SCALAR, dictReader.valueColumnType());
    // Row 1: get value reader with its position set to entry corresponding to a key
    assertTrue(reader.next());
    // dict itself is not null
    assertFalse(dictReader.isNull());
    dictReader.getAsString();
    final KeyAccessor keyAccessor = dictReader.keyAccessor();
    final ScalarReader valueReader = dictReader.valueReader().scalar();
    assertTrue(keyAccessor.find(12));
    assertEquals("b", valueReader.getString());
    assertTrue(keyAccessor.find(11));
    assertEquals("a", valueReader.getString());
    // compare entire dict
    Map<Object, Object> map = map(11, "a", 12, "b");
    assertEquals(map, dictReader.getObject());
    // Row 2
    assertTrue(reader.next());
    // the dict does not contain an entry with the key
    assertFalse(keyAccessor.find(22));
    assertTrue(keyAccessor.find(21));
    assertEquals("c", valueReader.getString());
    map = map(21, "c");
    assertEquals(map, dictReader.getObject());
    // Row 3
    assertTrue(reader.next());
    assertTrue(keyAccessor.find(31));
    assertEquals("d", valueReader.getString());
    assertFalse(keyAccessor.find(33));
    assertTrue(keyAccessor.find(32));
    assertEquals("e", valueReader.getString());
    map = map(31, "d", 32, "e");
    assertEquals(map, dictReader.getObject());
    assertFalse(reader.next());
    // Verify that the dict accessor's value count was set.
    final DictVector dictVector = (DictVector) actual.container().getValueVector(1).getValueVector();
    assertEquals(3, dictVector.getAccessor().getValueCount());
    final SingleRowSet expected = fixture.rowSetBuilder(schema).addRow(1, map(11, "a", 12, "b")).addRow(2, map(21, "c")).addRow(3, map(31, "d", 32, "e")).build();
    RowSetUtilities.verify(expected, actual);
}

Example 45 with ScalarWriter

use of org.apache.drill.exec.vector.accessor.ScalarWriter in project drill by apache.

the class TestVariableWidthWriter method testSizeLimit.

/**
 * Test resize monitoring. Add a listener to an Varchar writer,
 * capture each resize, and refuse a resize when the s
 * of the vector exceeds 1 MB. This will trigger an overflow,
 * which will throw an exception which we then check for.
 */
@Test
public void testSizeLimit() {
    try (VarCharVector vector = allocVector(1000)) {
        TestIndex index = new TestIndex();
        VarCharColumnWriter writer = makeWriter(vector, index);
        writer.bindListener(new ColumnWriterListener() {

            // Because assumed array size is 10, so 10 * 1000 = 10,000
            // rounded to 16K
            int totalAlloc = 16384;

            @Override
            public void overflowed(ScalarWriter writer) {
                throw new IllegalStateException("overflow called");
            }

            @Override
            public boolean canExpand(ScalarWriter writer, int delta) {
                totalAlloc += delta;
                return totalAlloc < 1024 * 1024;
            }
        });
        writer.startWrite();
        byte[] value = new byte[423];
        Arrays.fill(value, (byte) 'X');
        try {
            for (int i = 0; ; i++) {
                index.index = i;
                writer.startRow();
                writer.setBytes(value, value.length);
                writer.saveRow();
            }
        } catch (IllegalStateException e) {
            assertTrue(e.getMessage().contains("overflow called"));
        }
    }
}