Examples with ResultSetOptions org.apache.drill.exec.physical.resultSet.impl.ResultSetLoaderImpl.ResultSetOptions used on opensource projects

Example 26 with ResultSetOptions

use of org.apache.drill.exec.physical.resultSet.impl.ResultSetLoaderImpl.ResultSetOptions in project drill by apache.

the class TestResultSetLoaderProjection method testMapArrayConflict.

@Test
public void testMapArrayConflict() {
    List<SchemaPath> selection = RowSetTestUtils.projectList("col[0]");
    TupleMetadata schema = new SchemaBuilder().addMap("col").add("child", MinorType.VARCHAR).resumeSchema().buildSchema();
    ResultSetOptions options = new ResultSetOptionBuilder().projection(Projections.parse(selection)).readerSchema(schema).build();
    try {
        new ResultSetLoaderImpl(fixture.allocator(), options);
        fail();
    } catch (UserException e) {
        assertTrue(e.getErrorType() == ErrorType.VALIDATION);
    }
}

Example 27 with ResultSetOptions

use of org.apache.drill.exec.physical.resultSet.impl.ResultSetLoaderImpl.ResultSetOptions in project drill by apache.

the class TestResultSetLoaderProjection method setupProvidedSchema.

// Setup to test the various project/provided schema cases in
// ProjectionFilter, especially CompoundProjectionFilter
public ResultSetLoader setupProvidedSchema(boolean isStrict, List<SchemaPath> selection) {
    TupleMetadata schema = new SchemaBuilder().addMap("m1").add("a", MinorType.INT).add("b", MinorType.INT).resumeSchema().addMap("m2").add("c", MinorType.INT).add("d", MinorType.INT).resumeSchema().addMap("m3").add("e", MinorType.INT).add("f", MinorType.INT).resumeSchema().buildSchema();
    // Provided schema: disjoint set of above.
    TupleMetadata providedSchema = new SchemaBuilder().addMap(// Same
    "m1").add("a", MinorType.INT).add("b", MinorType.INT).add("z", // Add a column
    MinorType.INT).resumeSchema().addMap(// Omit c
    "m2").add("d", MinorType.INT).resumeSchema().addMap(// Add m4
    "m4").add("g", MinorType.INT).add("h", MinorType.INT).resumeSchema().build();
    if (isStrict) {
        SchemaUtils.markStrict(providedSchema);
    }
    RequestedTuple proj = Projections.parse(selection);
    CustomErrorContext errorContext = new EmptyErrorContext();
    ProjectionFilter projectionFilter = ProjectionFilter.providedSchemaFilter(proj, providedSchema, errorContext);
    ResultSetOptions options = new ResultSetOptionBuilder().projectionFilter(projectionFilter).readerSchema(schema).build();
    ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
    // Write a couple of rows.
    rsLoader.startBatch();
    RowSetLoader rootWriter = rsLoader.writer();
    rootWriter.start();
    rootWriter.addRow(mapValue(1, 2), mapValue(3, 4), mapValue(5, 6)).addRow(mapValue(11, 12), mapValue(13, 14), mapValue(15, 16));
    return rsLoader;
}

Example 28 with ResultSetOptions

use of org.apache.drill.exec.physical.resultSet.impl.ResultSetLoaderImpl.ResultSetOptions in project drill by apache.

the class TestResultSetLoaderEmptyProject method testEmptyTopSchema.

/**
 * Verify that empty projection works: allows skipping rows and
 * reporting those rows as a batch with no vectors but with the
 * desired row count.
 */
@Test
public void testEmptyTopSchema() {
    List<SchemaPath> selection = Lists.newArrayList();
    TupleMetadata schema = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.INT).buildSchema();
    ResultSetOptions options = new ResultSetOptionBuilder().projection(Projections.parse(selection)).readerSchema(schema).build();
    ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
    assertTrue(rsLoader.isProjectionEmpty());
    // Can't skip rows if batch not started.
    int rowCount = 100_000;
    try {
        rsLoader.skipRows(10);
        fail();
    } catch (IllegalStateException e) {
    // Expected
    }
    // Loop to skip 100,000 rows. Should occur in two batches.
    rsLoader.startBatch();
    int skipped = rsLoader.skipRows(rowCount);
    assertEquals(skipped, ValueVector.MAX_ROW_COUNT);
    VectorContainer output = rsLoader.harvest();
    assertEquals(skipped, output.getRecordCount());
    assertEquals(0, output.getNumberOfColumns());
    output.zeroVectors();
    // Second batch
    rowCount -= skipped;
    rsLoader.startBatch();
    skipped = rsLoader.skipRows(rowCount);
    assertEquals(skipped, rowCount);
    output = rsLoader.harvest();
    assertEquals(skipped, output.getRecordCount());
    assertEquals(0, output.getNumberOfColumns());
    output.zeroVectors();
    rsLoader.close();
}

Example 29 with ResultSetOptions

use of org.apache.drill.exec.physical.resultSet.impl.ResultSetLoaderImpl.ResultSetOptions in project drill by apache.

the class TestResultSetLoaderOverflow method testVectorSizeLimitWithAppend.

@Test
public void testVectorSizeLimitWithAppend() {
    TupleMetadata schema = new SchemaBuilder().add("s", MinorType.VARCHAR).buildSchema();
    ResultSetOptions options = new ResultSetOptionBuilder().rowCountLimit(ValueVector.MAX_ROW_COUNT).readerSchema(schema).build();
    ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
    RowSetLoader rootWriter = rsLoader.writer();
    rsLoader.startBatch();
    byte[] head = "abc".getBytes();
    byte[] tail = new byte[523];
    Arrays.fill(tail, (byte) 'X');
    String expected = new String(head, Charsets.UTF_8);
    expected += new String(tail, Charsets.UTF_8);
    expected += new String(tail, Charsets.UTF_8);
    int count = 0;
    ScalarWriter colWriter = rootWriter.scalar(0);
    while (!rootWriter.isFull()) {
        rootWriter.start();
        colWriter.setBytes(head, head.length);
        colWriter.appendBytes(tail, tail.length);
        colWriter.appendBytes(tail, tail.length);
        rootWriter.save();
        count++;
    }
    // Number of rows should be driven by vector size.
    // Our row count should include the overflow row
    int valueLength = head.length + 2 * tail.length;
    int expectedCount = ValueVector.MAX_BUFFER_SIZE / valueLength;
    assertEquals(expectedCount + 1, count);
    // Loader's row count should include only "visible" rows
    assertEquals(expectedCount, rootWriter.rowCount());
    // Total count should include invisible and look-ahead rows.
    assertEquals(expectedCount + 1, rsLoader.totalRowCount());
    // Result should exclude the overflow row
    {
        VectorContainer container = rsLoader.harvest();
        BatchValidator.validate(container);
        RowSet result = fixture.wrap(container);
        assertEquals(expectedCount, result.rowCount());
        // Verify that the values were, in fact, appended.
        RowSetReader reader = result.reader();
        while (reader.next()) {
            assertEquals(expected, reader.scalar(0).getString());
        }
        result.clear();
    }
    // Next batch should start with the overflow row
    rsLoader.startBatch();
    assertEquals(1, rootWriter.rowCount());
    assertEquals(expectedCount + 1, rsLoader.totalRowCount());
    {
        VectorContainer container = rsLoader.harvest();
        BatchValidator.validate(container);
        RowSet result = fixture.wrap(container);
        assertEquals(1, result.rowCount());
        RowSetReader reader = result.reader();
        while (reader.next()) {
            assertEquals(expected, reader.scalar(0).getString());
        }
        result.clear();
    }
    rsLoader.close();
}

Example 30 with ResultSetOptions

use of org.apache.drill.exec.physical.resultSet.impl.ResultSetLoaderImpl.ResultSetOptions in project drill by apache.

the class TestResultSetLoaderOverflow method testCloseWithOverflow.

/**
 * Load a batch to overflow. Then, close the loader with the overflow
 * batch unharvested. The Loader should release the memory allocated
 * to the unused overflow vectors.
 */
@Test
public void testCloseWithOverflow() {
    TupleMetadata schema = new SchemaBuilder().add("s", MinorType.VARCHAR).buildSchema();
    ResultSetOptions options = new ResultSetOptionBuilder().rowCountLimit(ValueVector.MAX_ROW_COUNT).readerSchema(schema).build();
    ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
    RowSetLoader rootWriter = rsLoader.writer();
    rsLoader.startBatch();
    byte[] value = new byte[512];
    Arrays.fill(value, (byte) 'X');
    int count = 0;
    while (!rootWriter.isFull()) {
        rootWriter.start();
        rootWriter.scalar(0).setBytes(value, value.length);
        rootWriter.save();
        count++;
    }
    assertTrue(count < ValueVector.MAX_ROW_COUNT);
    // Harvest the full batch
    VectorContainer container = rsLoader.harvest();
    BatchValidator.validate(container);
    RowSet result = fixture.wrap(container);
    result.clear();
    // Close without harvesting the overflow batch.
    rsLoader.close();
}