Examples with ScalarWriter org.apache.drill.exec.vector.accessor.ScalarWriter used on opensource projects

Example 66 with ScalarWriter

use of org.apache.drill.exec.vector.accessor.ScalarWriter in project drill by axbaretto.

the class TestFixedWidthWriter method testSizeLimit.

/**
 * Test resize monitoring. Add a listener to an int writer,
 * capture each resize, and refuse a resize when the number
 * of ints exceeds 8K values. This will trigger an overflow,
 * which will throw an exception which we then check for.
 */
@Test
public void testSizeLimit() {
    try (IntVector vector = allocVector(1000)) {
        TestIndex index = new TestIndex();
        IntColumnWriter writer = makeWriter(vector, index);
        writer.bindListener(new ColumnWriterListener() {

            int totalAlloc = 4096;

            @Override
            public void overflowed(ScalarWriter writer) {
                throw new IllegalStateException("overflow called");
            }

            @Override
            public boolean canExpand(ScalarWriter writer, int delta) {
                // System.out.println("Delta: " + delta);
                totalAlloc += delta;
                return totalAlloc < 16_384 * 4;
            }
        });
        writer.startWrite();
        try {
            for (int i = 0; ; i++) {
                index.index = i;
                writer.startRow();
                writer.setInt(i);
                writer.saveRow();
            }
        } catch (IllegalStateException e) {
            assertTrue(e.getMessage().contains("overflow called"));
        }
        // Should have failed on 8192, which doubled vector
        // to 16K, which was rejected.
        assertEquals(8192, index.index);
    }
}

Example 67 with ScalarWriter

use of org.apache.drill.exec.vector.accessor.ScalarWriter in project drill by axbaretto.

the class RowSetUtilities method setFromInt.

/**
 * Set a test data value from an int. Uses the type information of the
 * column to handle interval types. Else, uses the value type of the
 * accessor. The value set here is purely for testing; the mapping
 * from ints to intervals has no real meaning.
 *
 * @param rowWriter
 * @param index
 * @param value
 */
public static void setFromInt(RowSetWriter rowWriter, int index, int value) {
    ScalarWriter writer = rowWriter.scalar(index);
    MaterializedField field = rowWriter.schema().column(index);
    writer.setObject(testDataFromInt(writer.valueType(), field.getType(), value));
}

Example 68 with ScalarWriter

use of org.apache.drill.exec.vector.accessor.ScalarWriter in project drill by apache.

the class RowSetUtilities method setFromInt.

/**
 * Set a test data value from an int. Uses the type information of the
 * column to handle interval types. Else, uses the value type of the
 * accessor. The value set here is purely for testing; the mapping
 * from ints to intervals has no real meaning.
 *
 * @param rowWriter writer where value will be written to
 * @param index     target index
 * @param value     value to write
 */
public static void setFromInt(RowSetWriter rowWriter, int index, int value) {
    ScalarWriter writer = rowWriter.scalar(index);
    MaterializedField field = rowWriter.tupleSchema().column(index);
    writer.setObject(testDataFromInt(writer.valueType(), field.getType(), value));
}

Example 69 with ScalarWriter

use of org.apache.drill.exec.vector.accessor.ScalarWriter in project drill by apache.

the class TestResultSetLoaderRepeatedList method test2DLateSchemaIncremental.

@Test
public void test2DLateSchemaIncremental() {
    final TupleMetadata schema = new SchemaBuilder().add("id", MinorType.INT).addRepeatedList("list1").addArray(MinorType.VARCHAR).resumeSchema().addRepeatedList("list2").addArray(MinorType.VARCHAR).resumeSchema().buildSchema();
    final ResultSetLoaderImpl.ResultSetOptions options = new ResultSetOptionBuilder().build();
    final ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
    final RowSetLoader writer = rsLoader.writer();
    // Add columns dynamically
    writer.addColumn(schema.metadata(0));
    // Write a row without the array.
    rsLoader.startBatch();
    writer.addRow(1);
    // Add the repeated list, but without contents.
    writer.addColumn(schema.metadata(1).cloneEmpty());
    // Sanity check of writer structure
    assertEquals(2, writer.size());
    final ObjectWriter listObj = writer.column("list1");
    assertEquals(ObjectType.ARRAY, listObj.type());
    final ArrayWriter listWriter = listObj.array();
    // No child defined yet. A dummy child is inserted instead.
    assertEquals(MinorType.NULL, listWriter.entry().schema().type());
    assertEquals(ObjectType.ARRAY, listWriter.entryType());
    assertEquals(ObjectType.SCALAR, listWriter.array().entryType());
    assertEquals(ValueType.NULL, listWriter.array().scalar().valueType());
    // Although we don't know the type of the inner, we can still
    // create null (empty) elements in the outer array.
    writer.addRow(2, null).addRow(3, objArray()).addRow(4, objArray(objArray(), null));
    // Define the inner type.
    final RepeatedListWriter listWriterImpl = (RepeatedListWriter) listWriter;
    listWriterImpl.defineElement(MaterializedField.create("list1", Types.repeated(MinorType.VARCHAR)));
    // Sanity check of completed structure
    assertEquals(ObjectType.ARRAY, listWriter.entryType());
    final ArrayWriter innerWriter = listWriter.array();
    assertEquals(ObjectType.SCALAR, innerWriter.entryType());
    final ScalarWriter strWriter = innerWriter.scalar();
    assertEquals(ValueType.STRING, strWriter.valueType());
    // Write values
    writer.addRow(5, objArray(strArray("a1", "b1"), strArray("c1", "d1")));
    // Add the second list, with a complete type
    writer.addColumn(schema.metadata(2));
    // Sanity check of writer structure
    assertEquals(3, writer.size());
    final ObjectWriter list2Obj = writer.column("list2");
    assertEquals(ObjectType.ARRAY, list2Obj.type());
    final ArrayWriter list2Writer = list2Obj.array();
    assertEquals(ObjectType.ARRAY, list2Writer.entryType());
    final ArrayWriter inner2Writer = list2Writer.array();
    assertEquals(ObjectType.SCALAR, inner2Writer.entryType());
    final ScalarWriter str2Writer = inner2Writer.scalar();
    assertEquals(ValueType.STRING, str2Writer.valueType());
    // Write values
    writer.addRow(6, objArray(strArray("a2", "b2"), strArray("c2", "d2")), objArray(strArray("w2", "x2"), strArray("y2", "z2")));
    // Add the second list, with a complete type
    // Verify the values.
    // (Relies on the row set level repeated list tests having passed.)
    final RowSet expected = fixture.rowSetBuilder(schema).addRow(1, objArray(), objArray()).addRow(2, objArray(), objArray()).addRow(3, objArray(), objArray()).addRow(4, objArray(objArray(), null), objArray()).addRow(5, objArray(strArray("a1", "b1"), strArray("c1", "d1")), objArray()).addRow(6, objArray(strArray("a2", "b2"), strArray("c2", "d2")), objArray(strArray("w2", "x2"), strArray("y2", "z2"))).build();
    RowSetUtilities.verify(expected, fixture.wrap(rsLoader.harvest()));
    rsLoader.close();
}

Example 70 with ScalarWriter

use of org.apache.drill.exec.vector.accessor.ScalarWriter in project drill by apache.

the class TestResultSetLoaderDictArray method testKeyOverflow.

@Test
public void testKeyOverflow() {
    TupleMetadata schema = new SchemaBuilder().addDictArray("d", MinorType.VARCHAR).value(MinorType.INT).resumeSchema().buildSchema();
    ResultSetLoaderImpl.ResultSetOptions options = new ResultSetOptionBuilder().rowCountLimit(ValueVector.MAX_ROW_COUNT).readerSchema(schema).build();
    ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
    RowSetLoader rootWriter = rsLoader.writer();
    rsLoader.startBatch();
    byte[] key = new byte[523];
    Arrays.fill(key, (byte) 'X');
    // number of dicts in each row
    int arraySize = 3;
    // number of entries in each dict
    int dictSize = 1;
    // Number of rows should be driven by vector size.
    // Our row count should include the overflow row
    ArrayWriter arrayDictWriter = rootWriter.array(0);
    DictWriter dictWriter = arrayDictWriter.dict();
    ScalarWriter keyWriter = dictWriter.keyWriter();
    ScalarWriter valueWriter = dictWriter.valueWriter().scalar();
    int expectedCount = ValueVector.MAX_BUFFER_SIZE / (key.length * dictSize * arraySize);
    // System.out.println("expectedCoutn: " + expectedCount);
    {
        int count = 0;
        while (!rootWriter.isFull()) {
            rootWriter.start();
            for (int i = 0; i < arraySize; i++) {
                for (int j = 0; j < dictSize; j++) {
                    keyWriter.setBytes(key, key.length);
                    // acts as a placeholder, the actual value is not important
                    valueWriter.setInt(0);
                    // not necessary for scalars, just for completeness
                    dictWriter.save();
                }
                arrayDictWriter.save();
            }
            rootWriter.save();
            count++;
        }
        assertEquals(expectedCount + 1, count);
        // System.out.println("count: " + count);
        // Loader's row count should include only "visible" rows
        assertEquals(expectedCount, rootWriter.rowCount());
        // Total count should include invisible and look-ahead rows.
        assertEquals(expectedCount + 1, rsLoader.totalRowCount());
        // Result should exclude the overflow row
        VectorContainer container = rsLoader.harvest();
        BatchValidator.validate(container);
        RowSet result = fixture.wrap(container);
        assertEquals(expectedCount, result.rowCount());
        result.clear();
    }
    // Next batch should start with the overflow row
    {
        rsLoader.startBatch();
        assertEquals(1, rootWriter.rowCount());
        assertEquals(expectedCount + 1, rsLoader.totalRowCount());
        VectorContainer container = rsLoader.harvest();
        BatchValidator.validate(container);
        RowSet result = fixture.wrap(container);
        assertEquals(1, result.rowCount());
        result.clear();
    }
    rsLoader.close();
}