Examples with ResultSetOptions org.apache.drill.exec.physical.rowSet.impl.ResultSetLoaderImpl.ResultSetOptions used on opensource projects

Example 11 with ResultSetOptions

use of org.apache.drill.exec.physical.rowSet.impl.ResultSetLoaderImpl.ResultSetOptions in project drill by axbaretto.

the class TestResultSetLoaderProjection method testMapProjection.

@Test
public void testMapProjection() {
    List<SchemaPath> selection = Lists.newArrayList(SchemaPath.getSimplePath("m1"), SchemaPath.getCompoundPath("m2", "d"));
    TupleMetadata schema = new SchemaBuilder().addMap("m1").add("a", MinorType.INT).add("b", MinorType.INT).resumeSchema().addMap("m2").add("c", MinorType.INT).add("d", MinorType.INT).resumeSchema().addMap("m3").add("e", MinorType.INT).add("f", MinorType.INT).resumeSchema().buildSchema();
    ResultSetOptions options = new OptionBuilder().setProjection(selection).setSchema(schema).build();
    ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
    RowSetLoader rootWriter = rsLoader.writer();
    // Verify the projected columns
    TupleMetadata actualSchema = rootWriter.schema();
    ColumnMetadata m1Md = actualSchema.metadata("m1");
    assertTrue(m1Md.isMap());
    assertTrue(m1Md.isProjected());
    assertEquals(2, m1Md.mapSchema().size());
    assertTrue(m1Md.mapSchema().metadata("a").isProjected());
    assertTrue(m1Md.mapSchema().metadata("b").isProjected());
    ColumnMetadata m2Md = actualSchema.metadata("m2");
    assertTrue(m2Md.isMap());
    assertTrue(m2Md.isProjected());
    assertEquals(2, m2Md.mapSchema().size());
    assertFalse(m2Md.mapSchema().metadata("c").isProjected());
    assertTrue(m2Md.mapSchema().metadata("d").isProjected());
    ColumnMetadata m3Md = actualSchema.metadata("m3");
    assertTrue(m3Md.isMap());
    assertFalse(m3Md.isProjected());
    assertEquals(2, m3Md.mapSchema().size());
    assertFalse(m3Md.mapSchema().metadata("e").isProjected());
    assertFalse(m3Md.mapSchema().metadata("f").isProjected());
    // Write a couple of rows.
    rsLoader.startBatch();
    rootWriter.start();
    rootWriter.tuple("m1").scalar("a").setInt(1);
    rootWriter.tuple("m1").scalar("b").setInt(2);
    rootWriter.tuple("m2").scalar("c").setInt(3);
    rootWriter.tuple("m2").scalar("d").setInt(4);
    rootWriter.tuple("m3").scalar("e").setInt(5);
    rootWriter.tuple("m3").scalar("f").setInt(6);
    rootWriter.save();
    rootWriter.start();
    rootWriter.tuple("m1").scalar("a").setInt(11);
    rootWriter.tuple("m1").scalar("b").setInt(12);
    rootWriter.tuple("m2").scalar("c").setInt(13);
    rootWriter.tuple("m2").scalar("d").setInt(14);
    rootWriter.tuple("m3").scalar("e").setInt(15);
    rootWriter.tuple("m3").scalar("f").setInt(16);
    rootWriter.save();
    // Verify. Only the projected columns appear in the result set.
    BatchSchema expectedSchema = new SchemaBuilder().addMap("m1").add("a", MinorType.INT).add("b", MinorType.INT).resumeSchema().addMap("m2").add("d", MinorType.INT).resumeSchema().build();
    SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).addRow(objArray(1, 2), objArray(4)).addRow(objArray(11, 12), objArray(14)).build();
    new RowSetComparison(expected).verifyAndClearAll(fixture.wrap(rsLoader.harvest()));
    rsLoader.close();
}

Example 12 with ResultSetOptions

use of org.apache.drill.exec.physical.rowSet.impl.ResultSetLoaderImpl.ResultSetOptions in project drill by axbaretto.

the class TestResultSetLoaderProjection method testMapArrayProjection.

/**
 * Test a map array. Use the convenience methods to set values.
 * Only the projected array members should appear in the harvested
 * results.
 */
@Test
public void testMapArrayProjection() {
    List<SchemaPath> selection = Lists.newArrayList(SchemaPath.getSimplePath("m1"), SchemaPath.getCompoundPath("m2", "d"));
    TupleMetadata schema = new SchemaBuilder().addMapArray("m1").add("a", MinorType.INT).add("b", MinorType.INT).resumeSchema().addMapArray("m2").add("c", MinorType.INT).add("d", MinorType.INT).resumeSchema().addMapArray("m3").add("e", MinorType.INT).add("f", MinorType.INT).resumeSchema().buildSchema();
    ResultSetOptions options = new OptionBuilder().setProjection(selection).setSchema(schema).build();
    ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
    RowSetLoader rootWriter = rsLoader.writer();
    // Write a couple of rows.
    rsLoader.startBatch();
    rootWriter.addRow(objArray(objArray(10, 20), objArray(11, 21)), objArray(objArray(30, 40), objArray(31, 42)), objArray(objArray(50, 60), objArray(51, 62)));
    rootWriter.addRow(objArray(objArray(110, 120), objArray(111, 121)), objArray(objArray(130, 140), objArray(131, 142)), objArray(objArray(150, 160), objArray(151, 162)));
    // Verify. Only the projected columns appear in the result set.
    BatchSchema expectedSchema = new SchemaBuilder().addMapArray("m1").add("a", MinorType.INT).add("b", MinorType.INT).resumeSchema().addMapArray("m2").add("d", MinorType.INT).resumeSchema().build();
    SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).addRow(objArray(objArray(10, 20), objArray(11, 21)), objArray(objArray(40), objArray(42))).addRow(objArray(objArray(110, 120), objArray(111, 121)), objArray(objArray(140), objArray(142))).build();
    new RowSetComparison(expected).verifyAndClearAll(fixture.wrap(rsLoader.harvest()));
    rsLoader.close();
}

Example 13 with ResultSetOptions

use of org.apache.drill.exec.physical.rowSet.impl.ResultSetLoaderImpl.ResultSetOptions in project drill by axbaretto.

the class TestResultSetLoaderProjection method testProjectWithOverflow.

/**
 * Verify that the projection code plays nice with vector overflow. Overflow
 * is the most complex operation in this subsystem with many specialized
 * methods that must work together flawlessly. This test ensures that
 * non-projected columns stay in the background and don't interfere
 * with overflow logic.
 */
@Test
public void testProjectWithOverflow() {
    List<SchemaPath> selection = Lists.newArrayList(SchemaPath.getSimplePath("small"), SchemaPath.getSimplePath("dummy"));
    TupleMetadata schema = new SchemaBuilder().add("big", MinorType.VARCHAR).add("small", MinorType.VARCHAR).buildSchema();
    ResultSetOptions options = new OptionBuilder().setRowCountLimit(ValueVector.MAX_ROW_COUNT).setProjection(selection).setSchema(schema).build();
    ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
    RowSetLoader rootWriter = rsLoader.writer();
    byte[] big = new byte[600];
    Arrays.fill(big, (byte) 'X');
    byte[] small = new byte[512];
    Arrays.fill(small, (byte) 'X');
    rsLoader.startBatch();
    int count = 0;
    while (!rootWriter.isFull()) {
        rootWriter.start();
        rootWriter.scalar(0).setBytes(big, big.length);
        rootWriter.scalar(1).setBytes(small, small.length);
        rootWriter.save();
        count++;
    }
    // Number of rows should be driven by size of the
    // projected vector ("small"), not by the larger, unprojected
    // "big" vector.
    // Our row count should include the overflow row
    int expectedCount = ValueVector.MAX_BUFFER_SIZE / small.length;
    assertEquals(expectedCount + 1, count);
    // Loader's row count should include only "visible" rows
    assertEquals(expectedCount, rootWriter.rowCount());
    // Total count should include invisible and look-ahead rows.
    assertEquals(expectedCount + 1, rsLoader.totalRowCount());
    // Result should exclude the overflow row
    RowSet result = fixture.wrap(rsLoader.harvest());
    assertEquals(expectedCount, result.rowCount());
    result.clear();
    // Next batch should start with the overflow row
    rsLoader.startBatch();
    assertEquals(1, rootWriter.rowCount());
    assertEquals(expectedCount + 1, rsLoader.totalRowCount());
    result = fixture.wrap(rsLoader.harvest());
    assertEquals(1, result.rowCount());
    result.clear();
    rsLoader.close();
}

Example 14 with ResultSetOptions

use of org.apache.drill.exec.physical.rowSet.impl.ResultSetLoaderImpl.ResultSetOptions in project drill by axbaretto.

the class TestResultSetLoaderProjection method testProjectionStatic.

/**
 * Test imposing a selection mask between the client and the underlying
 * vector container.
 */
@Test
public void testProjectionStatic() {
    List<SchemaPath> selection = Lists.newArrayList(SchemaPath.getSimplePath("c"), SchemaPath.getSimplePath("b"), SchemaPath.getSimplePath("e"));
    TupleMetadata schema = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.INT).add("c", MinorType.INT).add("d", MinorType.INT).buildSchema();
    ResultSetOptions options = new OptionBuilder().setProjection(selection).setSchema(schema).build();
    ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
    doProjectionTest(rsLoader);
}

Example 15 with ResultSetOptions

use of org.apache.drill.exec.physical.rowSet.impl.ResultSetLoaderImpl.ResultSetOptions in project drill by axbaretto.

the class TestResultSetSchemaChange method testSchemaChangeWithOverflow.

/**
 * Test a schema change on the row that overflows. If the
 * new column is added after overflow, it will appear as
 * a schema-change in the following batch. This is fine as
 * we are essentially time-shifting: pretending that the
 * overflow row was written in the next batch (which, in
 * fact, it is: that's what overflow means.)
 */
@Test
public void testSchemaChangeWithOverflow() {
    ResultSetOptions options = new OptionBuilder().setRowCountLimit(ValueVector.MAX_ROW_COUNT).build();
    ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
    RowSetLoader rootWriter = rsLoader.writer();
    rootWriter.addColumn(SchemaBuilder.columnSchema("a", MinorType.VARCHAR, DataMode.REQUIRED));
    rsLoader.startBatch();
    byte[] value = new byte[512];
    Arrays.fill(value, (byte) 'X');
    int count = 0;
    while (!rootWriter.isFull()) {
        rootWriter.start();
        rootWriter.scalar(0).setBytes(value, value.length);
        if (rootWriter.isFull()) {
            rootWriter.addColumn(SchemaBuilder.columnSchema("b", MinorType.INT, DataMode.OPTIONAL));
            rootWriter.scalar(1).setInt(count);
            // Add a Varchar to ensure its offset fiddling is done properly
            rootWriter.addColumn(SchemaBuilder.columnSchema("c", MinorType.VARCHAR, DataMode.OPTIONAL));
            rootWriter.scalar(2).setString("c-" + count);
            // Allow adding a required column at this point.
            // (Not intuitively obvious that this should work; we back-fill
            // with zeros.)
            rootWriter.addColumn(SchemaBuilder.columnSchema("d", MinorType.INT, DataMode.REQUIRED));
        }
        rootWriter.save();
        count++;
    }
    // Result should include only the first column.
    BatchSchema expectedSchema = new SchemaBuilder().add("a", MinorType.VARCHAR).build();
    RowSet result = fixture.wrap(rsLoader.harvest());
    assertTrue(result.batchSchema().isEquivalent(expectedSchema));
    assertEquals(count - 1, result.rowCount());
    result.clear();
    assertEquals(1, rsLoader.schemaVersion());
    // Double check: still can add a required column after
    // starting the next batch. (No longer in overflow state.)
    rsLoader.startBatch();
    rootWriter.addColumn(SchemaBuilder.columnSchema("e", MinorType.INT, DataMode.REQUIRED));
    // Next batch should start with the overflow row, including
    // the column added at the end of the previous batch, after
    // overflow.
    result = fixture.wrap(rsLoader.harvest());
    assertEquals(5, rsLoader.schemaVersion());
    assertEquals(1, result.rowCount());
    expectedSchema = new SchemaBuilder(expectedSchema).addNullable("b", MinorType.INT).addNullable("c", MinorType.VARCHAR).add("d", MinorType.INT).add("e", MinorType.INT).build();
    assertTrue(result.batchSchema().isEquivalent(expectedSchema));
    RowSetReader reader = result.reader();
    reader.next();
    assertEquals(count - 1, reader.scalar(1).getInt());
    assertEquals("c-" + (count - 1), reader.scalar(2).getString());
    assertEquals(0, reader.scalar("d").getInt());
    assertEquals(0, reader.scalar("e").getInt());
    result.clear();
    rsLoader.close();
}