Examples with RowSetLoader org.apache.drill.exec.physical.resultSet.RowSetLoader used on opensource projects

Example 46 with RowSetLoader

use of org.apache.drill.exec.physical.resultSet.RowSetLoader in project drill by apache.

the class TestResultSetLoaderProjection method testNonStrictMapMemberProjectionWithSchema.

/**
 * Projection is based on both the projection list and the
 * provided schema, if strict.
 */
@Test
public void testNonStrictMapMemberProjectionWithSchema() {
    // m1 is not projected, though in the provided schema
    // m2.c is projected, in the reader schema, but not in the provided schema,
    // but schema is non-strict, so is projected
    // m2.d is projected and in both schemas
    // m3.f is projected, but m3 is not in the provided schema, but since schema is
    // non-strict, it is projected
    // m4.g is projected, is in the provided schema, but not in the reader schema
    List<SchemaPath> selection = RowSetTestUtils.projectList("m2.c", "m2.d", "m3.f", "m4.g");
    ResultSetLoader rsLoader = setupProvidedSchema(false, selection);
    RowSetLoader rootWriter = rsLoader.writer();
    // Verify the projected columns
    TupleMetadata actualSchema = rootWriter.tupleSchema();
    TupleWriter m1Writer = rootWriter.tuple("m1");
    assertFalse(m1Writer.isProjected());
    assertEquals(2, m1Writer.tupleSchema().size());
    assertFalse(m1Writer.column("a").isProjected());
    assertFalse(m1Writer.column("b").isProjected());
    TupleWriter m2Writer = rootWriter.tuple("m2");
    assertTrue(m2Writer.isProjected());
    assertEquals(2, m2Writer.tupleSchema().size());
    assertTrue(m2Writer.column("c").isProjected());
    assertTrue(m2Writer.column("d").isProjected());
    TupleWriter m3Writer = rootWriter.tuple("m3");
    assertTrue(m3Writer.isProjected());
    assertEquals(2, m3Writer.tupleSchema().size());
    assertFalse(m3Writer.column("e").isProjected());
    assertTrue(m3Writer.column("f").isProjected());
    assertNull(actualSchema.metadata("m4"));
    // Verify. Only the projected columns appear in the result set.
    TupleMetadata expectedSchema = new SchemaBuilder().addMap("m2").add("c", MinorType.INT).add("d", MinorType.INT).resumeSchema().addMap("m3").add("f", MinorType.INT).resumeSchema().build();
    SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).addRow(mapValue(3, 4), mapValue(6)).addRow(mapValue(13, 14), mapValue(16)).build();
    RowSetUtilities.verify(expected, fixture.wrap(rsLoader.harvest()));
    rsLoader.close();
}

Example 47 with RowSetLoader

use of org.apache.drill.exec.physical.resultSet.RowSetLoader in project drill by apache.

the class TestResultSetLoaderProjection method testProjectionDynamic.

@Test
public void testProjectionDynamic() {
    List<SchemaPath> selection = RowSetTestUtils.projectList("c", "b", "e");
    ResultSetOptions options = new ResultSetOptionBuilder().projection(Projections.parse(selection)).build();
    ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
    RowSetLoader rootWriter = rsLoader.writer();
    rootWriter.addColumn(SchemaBuilder.columnSchema("a", MinorType.INT, DataMode.REQUIRED));
    rootWriter.addColumn(SchemaBuilder.columnSchema("b", MinorType.INT, DataMode.REQUIRED));
    rootWriter.addColumn(SchemaBuilder.columnSchema("c", MinorType.INT, DataMode.REQUIRED));
    rootWriter.addColumn(SchemaBuilder.columnSchema("d", MinorType.INT, DataMode.REQUIRED));
    doProjectionTest(rsLoader);
}

Example 48 with RowSetLoader

use of org.apache.drill.exec.physical.resultSet.RowSetLoader in project drill by apache.

the class TestResultSetLoaderProjection method testProjectWithOverflow.

/**
 * Verify that the projection code plays nice with vector overflow. Overflow
 * is the most complex operation in this subsystem with many specialized
 * methods that must work together flawlessly. This test ensures that
 * non-projected columns stay in the background and don't interfere
 * with overflow logic.
 */
@Test
public void testProjectWithOverflow() {
    List<SchemaPath> selection = RowSetTestUtils.projectList("small", "dummy");
    TupleMetadata schema = new SchemaBuilder().add("big", MinorType.VARCHAR).add("small", MinorType.VARCHAR).buildSchema();
    ResultSetOptions options = new ResultSetOptionBuilder().rowCountLimit(ValueVector.MAX_ROW_COUNT).projection(Projections.parse(selection)).readerSchema(schema).build();
    ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
    RowSetLoader rootWriter = rsLoader.writer();
    byte[] big = new byte[600];
    Arrays.fill(big, (byte) 'X');
    byte[] small = new byte[512];
    Arrays.fill(small, (byte) 'X');
    rsLoader.startBatch();
    int count = 0;
    while (!rootWriter.isFull()) {
        rootWriter.start();
        rootWriter.scalar(0).setBytes(big, big.length);
        rootWriter.scalar(1).setBytes(small, small.length);
        rootWriter.save();
        count++;
    }
    // Number of rows should be driven by size of the
    // projected vector ("small"), not by the larger, unprojected
    // "big" vector.
    // Our row count should include the overflow row
    int expectedCount = ValueVector.MAX_BUFFER_SIZE / small.length;
    assertEquals(expectedCount + 1, count);
    // Loader's row count should include only "visible" rows
    assertEquals(expectedCount, rootWriter.rowCount());
    // Total count should include invisible and look-ahead rows.
    assertEquals(expectedCount + 1, rsLoader.totalRowCount());
    // Result should exclude the overflow row
    RowSet result = fixture.wrap(rsLoader.harvest());
    assertEquals(expectedCount, result.rowCount());
    result.clear();
    // Next batch should start with the overflow row
    rsLoader.startBatch();
    assertEquals(1, rootWriter.rowCount());
    assertEquals(expectedCount + 1, rsLoader.totalRowCount());
    result = fixture.wrap(rsLoader.harvest());
    assertEquals(1, result.rowCount());
    result.clear();
    rsLoader.close();
}

Example 49 with RowSetLoader

use of org.apache.drill.exec.physical.resultSet.RowSetLoader in project drill by apache.

the class TestResultSetLoaderProjection method testMapArrayProjection.

/**
 * Test a map array. Use the convenience methods to set values.
 * Only the projected array members should appear in the harvested
 * results.
 */
@Test
public void testMapArrayProjection() {
    List<SchemaPath> selection = RowSetTestUtils.projectList("m1", "m2.d");
    TupleMetadata schema = new SchemaBuilder().addMapArray("m1").add("a", MinorType.INT).add("b", MinorType.INT).resumeSchema().addMapArray("m2").add("c", MinorType.INT).add("d", MinorType.INT).resumeSchema().addMapArray("m3").add("e", MinorType.INT).add("f", MinorType.INT).resumeSchema().buildSchema();
    ResultSetOptions options = new ResultSetOptionBuilder().projection(Projections.parse(selection)).readerSchema(schema).build();
    ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
    RowSetLoader rootWriter = rsLoader.writer();
    // Write a couple of rows.
    rsLoader.startBatch();
    rootWriter.addRow(objArray(objArray(10, 20), objArray(11, 21)), objArray(objArray(30, 40), objArray(31, 42)), objArray(objArray(50, 60), objArray(51, 62)));
    rootWriter.addRow(objArray(objArray(110, 120), objArray(111, 121)), objArray(objArray(130, 140), objArray(131, 142)), objArray(objArray(150, 160), objArray(151, 162)));
    // Verify. Only the projected columns appear in the result set.
    TupleMetadata expectedSchema = new SchemaBuilder().addMapArray("m1").add("a", MinorType.INT).add("b", MinorType.INT).resumeSchema().addMapArray("m2").add("d", MinorType.INT).resumeSchema().buildSchema();
    SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).addRow(objArray(objArray(10, 20), objArray(11, 21)), objArray(objArray(40), objArray(42))).addRow(objArray(objArray(110, 120), objArray(111, 121)), objArray(objArray(140), objArray(142))).build();
    RowSetUtilities.verify(expected, fixture.wrap(rsLoader.harvest()));
    rsLoader.close();
}

Example 50 with RowSetLoader

use of org.apache.drill.exec.physical.resultSet.RowSetLoader in project drill by apache.

the class TestResultSetLoaderProjection method doProjectionTest.

private void doProjectionTest(ResultSetLoader rsLoader) {
    RowSetLoader rootWriter = rsLoader.writer();
    // All columns appear, including non-projected ones.
    TupleMetadata actualSchema = rootWriter.tupleSchema();
    assertEquals(4, actualSchema.size());
    assertEquals("a", actualSchema.column(0).getName());
    assertEquals("b", actualSchema.column(1).getName());
    assertEquals("c", actualSchema.column(2).getName());
    assertEquals("d", actualSchema.column(3).getName());
    assertEquals(0, actualSchema.index("A"));
    assertEquals(3, actualSchema.index("d"));
    assertEquals(-1, actualSchema.index("e"));
    // Non-projected columns identify themselves
    assertFalse(rootWriter.column("a").isProjected());
    assertTrue(rootWriter.column("b").isProjected());
    assertTrue(rootWriter.column("c").isProjected());
    assertFalse(rootWriter.column("d").isProjected());
    // Write some data. Doesn't need much.
    rsLoader.startBatch();
    for (int i = 1; i < 3; i++) {
        rootWriter.start();
        rootWriter.scalar(0).setInt(i * 5);
        rootWriter.scalar(1).setInt(i);
        rootWriter.scalar(2).setInt(i * 10);
        rootWriter.scalar(3).setInt(i * 20);
        rootWriter.save();
    }
    // Verify. Result should only have the projected
    // columns, only if defined by the loader, in the order
    // of definition.
    TupleMetadata expectedSchema = new SchemaBuilder().add("b", MinorType.INT).add("c", MinorType.INT).buildSchema();
    SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).addRow(1, 10).addRow(2, 20).build();
    RowSet actual = fixture.wrap(rsLoader.harvest());
    RowSetUtilities.verify(expected, actual);
    rsLoader.close();
}