Examples with ResultSetOptions org.apache.drill.exec.physical.resultSet.impl.ResultSetLoaderImpl.ResultSetOptions used on opensource projects

Example 11 with ResultSetOptions

use of org.apache.drill.exec.physical.resultSet.impl.ResultSetLoaderImpl.ResultSetOptions in project drill by apache.

the class TestResultSetLoaderProjection method testDictProjection.

@Test
public void testDictProjection() {
    final String dictName1 = "d1";
    final String dictName2 = "d2";
    // There is no test for case when obtaining a value by key as this is not as simple projection
    // as it is in case of map - there is a need to find a value corresponding to a key
    // (the functionality is currently present in DictReader) and final column schema should be
    // changed from dict structure with `key` and `value` children to a simple `value`.
    List<SchemaPath> selection = RowSetTestUtils.projectList(dictName1);
    TupleMetadata schema = new SchemaBuilder().addDict(dictName1, MinorType.VARCHAR).value(MinorType.INT).resumeSchema().addDict(dictName2, MinorType.VARCHAR).value(MinorType.INT).resumeSchema().buildSchema();
    ResultSetOptions options = new ResultSetOptionBuilder().projection(Projections.parse(selection)).readerSchema(schema).build();
    ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
    RowSetLoader rootWriter = rsLoader.writer();
    // Verify the projected columns
    TupleMetadata actualSchema = rootWriter.tupleSchema();
    ColumnMetadata dictMetadata1 = actualSchema.metadata(dictName1);
    DictWriter dictWriter1 = rootWriter.dict(dictName1);
    assertTrue(dictMetadata1.isDict());
    assertTrue(dictWriter1.isProjected());
    assertEquals(2, dictMetadata1.tupleSchema().size());
    assertTrue(dictWriter1.keyWriter().isProjected());
    assertTrue(dictWriter1.valueWriter().isProjected());
    ColumnMetadata dictMetadata2 = actualSchema.metadata(dictName2);
    DictWriter dictWriter2 = rootWriter.dict(dictName2);
    assertTrue(dictMetadata2.isDict());
    assertFalse(dictWriter2.isProjected());
    assertEquals(2, dictMetadata2.tupleSchema().size());
    assertFalse(dictWriter2.keyWriter().isProjected());
    assertFalse(dictWriter2.valueWriter().isProjected());
    // Write a couple of rows.
    rsLoader.startBatch();
    rootWriter.start();
    rootWriter.addRow(map("a", 1, "b", 2), map("c", 3, "d", 4)).addRow(map("a", 11, "b", 12), map("c", 13, "d", 14));
    // Verify. Only the projected columns appear in the result set.
    TupleMetadata expectedSchema = new SchemaBuilder().addDict(dictName1, MinorType.VARCHAR).value(MinorType.INT).resumeSchema().buildSchema();
    SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).addRow(map("a", 1, "b", 2)).addRow(map("a", 11, "b", 12)).build();
    RowSetUtilities.verify(expected, fixture.wrap(rsLoader.harvest()));
    rsLoader.close();
}

Example 12 with ResultSetOptions

use of org.apache.drill.exec.physical.resultSet.impl.ResultSetLoaderImpl.ResultSetOptions in project drill by apache.

the class TestResultSetLoaderProjection method testDictNumericKeyAccess.

@Test
public void testDictNumericKeyAccess() {
    List<SchemaPath> selection = RowSetTestUtils.projectList("col[0]");
    TupleMetadata schema = new SchemaBuilder().addDict("col", MinorType.INT).value(MinorType.VARCHAR).resumeSchema().buildSchema();
    ResultSetOptions options = new ResultSetOptionBuilder().projection(Projections.parse(selection)).readerSchema(schema).build();
    // no validation error
    new ResultSetLoaderImpl(fixture.allocator(), options);
}

Example 13 with ResultSetOptions

use of org.apache.drill.exec.physical.resultSet.impl.ResultSetLoaderImpl.ResultSetOptions in project drill by apache.

the class TestResultSetLoaderOverflow method testMissingArrayValues.

/**
 * Test the case that an array has "missing values" before the overflow.
 */
@Test
public void testMissingArrayValues() {
    TupleMetadata schema = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.VARCHAR).addArray("c", MinorType.INT).buildSchema();
    ResultSetOptions options = new ResultSetOptionBuilder().rowCountLimit(ValueVector.MAX_ROW_COUNT).readerSchema(schema).build();
    ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
    RowSetLoader rootWriter = rsLoader.writer();
    byte[] value = new byte[512];
    Arrays.fill(value, (byte) 'X');
    int blankAfter = ValueVector.MAX_BUFFER_SIZE / 512 * 2 / 3;
    ScalarWriter cWriter = rootWriter.array("c").scalar();
    rsLoader.startBatch();
    int rowId = 0;
    while (rootWriter.start()) {
        rootWriter.scalar("a").setInt(rowId);
        rootWriter.scalar("b").setBytes(value, value.length);
        if (rowId < blankAfter) {
            for (int i = 0; i < 3; i++) {
                cWriter.setInt(rowId * 3 + i);
            }
        }
        rootWriter.save();
        rowId++;
    }
    VectorContainer container = rsLoader.harvest();
    BatchValidator.validate(container);
    RowSet result = fixture.wrap(container);
    assertEquals(rowId - 1, result.rowCount());
    RowSetReader reader = result.reader();
    ArrayReader cArray = reader.array("c");
    ScalarReader cReader = cArray.scalar();
    while (reader.next()) {
        assertEquals(reader.offset(), reader.scalar("a").getInt());
        assertArrayEquals(value, reader.scalar("b").getBytes());
        if (reader.offset() < blankAfter) {
            assertEquals(3, cArray.size());
            for (int i = 0; i < 3; i++) {
                assertTrue(cArray.next());
                assertEquals(reader.offset() * 3 + i, cReader.getInt());
            }
        } else {
            assertEquals(0, cArray.size());
        }
    }
    result.clear();
    rsLoader.close();
}

Example 14 with ResultSetOptions

use of org.apache.drill.exec.physical.resultSet.impl.ResultSetLoaderImpl.ResultSetOptions in project drill by apache.

the class TestResultSetLoaderOverflow method testOverflowWithNullables.

@Test
public void testOverflowWithNullables() {
    TupleMetadata schema = new SchemaBuilder().add("n", MinorType.INT).addNullable("a", MinorType.VARCHAR).addNullable("b", MinorType.VARCHAR).addNullable("c", MinorType.VARCHAR).buildSchema();
    ResultSetOptions options = new ResultSetOptionBuilder().rowCountLimit(ValueVector.MAX_ROW_COUNT).readerSchema(schema).build();
    ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
    RowSetLoader rootWriter = rsLoader.writer();
    rsLoader.startBatch();
    byte[] value = new byte[512];
    Arrays.fill(value, (byte) 'X');
    int count = 0;
    while (!rootWriter.isFull()) {
        rootWriter.start();
        rootWriter.scalar(0).setInt(count);
        rootWriter.scalar(1).setNull();
        rootWriter.scalar(2).setBytes(value, value.length);
        rootWriter.scalar(3).setNull();
        rootWriter.save();
        count++;
    }
    // Result should exclude the overflow row
    {
        VectorContainer container = rsLoader.harvest();
        BatchValidator.validate(container);
        RowSet result = fixture.wrap(container);
        assertEquals(count - 1, result.rowCount());
        RowSetReader reader = result.reader();
        while (reader.next()) {
            assertEquals(reader.offset(), reader.scalar(0).getInt());
            assertTrue(reader.scalar(1).isNull());
            assertArrayEquals(value, reader.scalar(2).getBytes());
            assertTrue(reader.scalar(3).isNull());
        }
        result.clear();
    }
    // Next batch should start with the overflow row
    rsLoader.startBatch();
    {
        VectorContainer container = rsLoader.harvest();
        BatchValidator.validate(container);
        RowSet result = fixture.wrap(container);
        RowSetReader reader = result.reader();
        assertEquals(1, result.rowCount());
        assertTrue(reader.next());
        assertEquals(count - 1, reader.scalar(0).getInt());
        assertTrue(reader.scalar(1).isNull());
        assertArrayEquals(value, reader.scalar(2).getBytes());
        assertTrue(reader.scalar(3).isNull());
        result.clear();
    }
    rsLoader.close();
}

Example 15 with ResultSetOptions

use of org.apache.drill.exec.physical.resultSet.impl.ResultSetLoaderImpl.ResultSetOptions in project drill by apache.

the class TestResultSetLoaderOverflow method testSizeLimitOnArray.

/**
 * Test a row with a single array column which overflows. Verifies
 * that all the fiddly bits about offset vectors and so on works
 * correctly. Run this test (the simplest case) if you change anything
 * about the array handling code.
 */
@Test
public void testSizeLimitOnArray() {
    TupleMetadata schema = new SchemaBuilder().addArray("s", MinorType.VARCHAR).buildSchema();
    ResultSetOptions options = new ResultSetOptionBuilder().rowCountLimit(ValueVector.MAX_ROW_COUNT).readerSchema(schema).build();
    ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
    RowSetLoader rootWriter = rsLoader.writer();
    // Fill batch with rows of with a single array, three values each. Tack on
    // a suffix to each so we can be sure the proper data is written and moved
    // to the overflow batch.
    rsLoader.startBatch();
    byte[] value = new byte[473];
    Arrays.fill(value, (byte) 'X');
    String strValue = new String(value, Charsets.UTF_8);
    int valuesPerArray = 13;
    int count = 0;
    {
        int rowSize = 0;
        int totalSize = 0;
        while (rootWriter.start()) {
            totalSize += rowSize;
            rowSize = 0;
            ScalarWriter array = rootWriter.array(0).scalar();
            for (int i = 0; i < valuesPerArray; i++) {
                String cellValue = strValue + (count + 1) + "." + i;
                array.setString(cellValue);
                rowSize += cellValue.length();
            }
            rootWriter.save();
            count++;
        }
        // Row count should include the overflow row.
        int expectedCount = count - 1;
        // Size without overflow row should fit in the vector, size
        // with overflow should not.
        assertTrue(totalSize <= ValueVector.MAX_BUFFER_SIZE);
        assertTrue(totalSize + rowSize > ValueVector.MAX_BUFFER_SIZE);
        // Result should exclude the overflow row. Last row
        // should hold the last full array.
        VectorContainer container = rsLoader.harvest();
        BatchValidator.validate(container);
        RowSet result = fixture.wrap(container);
        assertEquals(expectedCount, result.rowCount());
        RowSetReader reader = result.reader();
        reader.setPosition(expectedCount - 1);
        ArrayReader arrayReader = reader.array(0);
        ScalarReader strReader = arrayReader.scalar();
        assertEquals(valuesPerArray, arrayReader.size());
        for (int i = 0; i < valuesPerArray; i++) {
            assertTrue(arrayReader.next());
            String cellValue = strValue + (count - 1) + "." + i;
            assertEquals(cellValue, strReader.getString());
        }
        result.clear();
    }
    // Next batch should start with the overflow row.
    // The only row in this next batch should be the whole
    // array being written at the time of overflow.
    {
        rsLoader.startBatch();
        assertEquals(1, rootWriter.rowCount());
        assertEquals(count, rsLoader.totalRowCount());
        VectorContainer container = rsLoader.harvest();
        BatchValidator.validate(container);
        RowSet result = fixture.wrap(container);
        assertEquals(1, result.rowCount());
        RowSetReader reader = result.reader();
        reader.next();
        ArrayReader arrayReader = reader.array(0);
        ScalarReader strReader = arrayReader.scalar();
        assertEquals(valuesPerArray, arrayReader.size());
        for (int i = 0; i < valuesPerArray; i++) {
            assertTrue(arrayReader.next());
            String cellValue = strValue + count + "." + i;
            assertEquals(cellValue, strReader.getString());
        }
        result.clear();
    }
    rsLoader.close();
}