Examples with SchemaBuilder org.apache.drill.exec.record.metadata.SchemaBuilder used on opensource projects

Example 86 with SchemaBuilder

use of org.apache.drill.exec.record.metadata.SchemaBuilder in project drill by apache.

the class TestResultSetLoaderOmittedValues method testOmittedValuesAtEndWithOverflow.

/**
 * Test "holes" at the end of a batch when batch overflows. Completed
 * batch must be finalized correctly, new batch initialized correct,
 * for the missing values.
 */
@Test
public void testOmittedValuesAtEndWithOverflow() {
    TupleMetadata schema = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.VARCHAR).addNullable("c", MinorType.VARCHAR).addNullable("d", MinorType.VARCHAR).buildSchema();
    ResultSetLoaderImpl.ResultSetOptions options = new ResultSetOptionBuilder().rowCountLimit(ValueVector.MAX_ROW_COUNT).readerSchema(schema).build();
    ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
    RowSetLoader rootWriter = rsLoader.writer();
    // Fill the batch. Column d has some values. Column c is worst case: no values.
    rsLoader.startBatch();
    byte[] value = new byte[533];
    Arrays.fill(value, (byte) 'X');
    int rowNumber = 0;
    while (!rootWriter.isFull()) {
        rootWriter.start();
        rowNumber++;
        rootWriter.scalar(0).setInt(rowNumber);
        rootWriter.scalar(1).setBytes(value, value.length);
        if (rowNumber < 10_000) {
            rootWriter.scalar(3).setString("d-" + rowNumber);
        }
        rootWriter.save();
        assertEquals(rowNumber, rsLoader.totalRowCount());
    }
    // Harvest and verify
    RowSet result = fixture.wrap(rsLoader.harvest());
    assertEquals(rowNumber - 1, result.rowCount());
    RowSetReader reader = result.reader();
    int rowIndex = 0;
    while (reader.next()) {
        int expectedRowNumber = 1 + rowIndex;
        assertEquals(expectedRowNumber, reader.scalar(0).getInt());
        assertTrue(reader.scalar(2).isNull());
        if (expectedRowNumber < 10_000) {
            assertEquals("d-" + expectedRowNumber, reader.scalar(3).getString());
        } else {
            assertTrue(reader.scalar(3).isNull());
        }
        rowIndex++;
    }
    // Start count for this batch is one less than current
    // count, because of the overflow row.
    int startRowNumber = rowNumber;
    // Write a few more rows to the next batch
    rsLoader.startBatch();
    for (int i = 0; i < 10; i++) {
        rootWriter.start();
        rowNumber++;
        rootWriter.scalar(0).setInt(rowNumber);
        rootWriter.scalar(1).setBytes(value, value.length);
        if (i > 5) {
            rootWriter.scalar(3).setString("d-" + rowNumber);
        }
        rootWriter.save();
        assertEquals(rowNumber, rsLoader.totalRowCount());
    }
    // Verify that holes were preserved.
    result = fixture.wrap(rsLoader.harvest());
    assertEquals(rowNumber, rsLoader.totalRowCount());
    assertEquals(rowNumber - startRowNumber + 1, result.rowCount());
    // result.print();
    reader = result.reader();
    rowIndex = 0;
    while (reader.next()) {
        int expectedRowNumber = startRowNumber + rowIndex;
        assertEquals(expectedRowNumber, reader.scalar(0).getInt());
        assertTrue(reader.scalar(2).isNull());
        if (rowIndex > 6) {
            assertEquals("d-" + expectedRowNumber, reader.scalar(3).getString());
        } else {
            assertTrue("Row " + rowIndex + " col d should be null", reader.scalar(3).isNull());
        }
        rowIndex++;
    }
    assertEquals(rowIndex, 11);
    rsLoader.close();
}

Example 87 with SchemaBuilder

use of org.apache.drill.exec.record.metadata.SchemaBuilder in project drill by apache.

the class TestResultSetLoaderOmittedValues method testSkipOverflowRow.

/**
 * Test that discarding a row works even if that row happens to be an
 * overflow row.
 */
@Test
public void testSkipOverflowRow() {
    TupleMetadata schema = new SchemaBuilder().add("a", MinorType.INT).addNullable("b", MinorType.VARCHAR).buildSchema();
    ResultSetLoaderImpl.ResultSetOptions options = new ResultSetOptionBuilder().rowCountLimit(ValueVector.MAX_ROW_COUNT).readerSchema(schema).build();
    ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
    RowSetLoader rootWriter = rsLoader.writer();
    rsLoader.startBatch();
    byte[] value = new byte[512];
    Arrays.fill(value, (byte) 'X');
    int count = 0;
    while (!rootWriter.isFull()) {
        rootWriter.start();
        rootWriter.scalar(0).setInt(count);
        rootWriter.scalar(1).setBytes(value, value.length);
        if (!rootWriter.isFull()) {
            rootWriter.save();
        }
        count++;
    }
    // Discard the results.
    rsLoader.harvest().zeroVectors();
    // Harvest the next batch. Will be empty (because overflow row
    // was discarded.)
    rsLoader.startBatch();
    RowSet result = fixture.wrap(rsLoader.harvest());
    assertEquals(0, result.rowCount());
    result.clear();
    rsLoader.close();
}

Example 88 with SchemaBuilder

use of org.apache.drill.exec.record.metadata.SchemaBuilder in project drill by apache.

the class TestProjectionFilter method testBuilders.

@Test
public void testBuilders() {
    assertSame(ProjectionFilter.PROJECT_ALL, ProjectionFilter.projectionFilter(Projections.projectAll(), EmptyErrorContext.INSTANCE));
    assertSame(ProjectionFilter.PROJECT_NONE, ProjectionFilter.projectionFilter(Projections.projectNone(), EmptyErrorContext.INSTANCE));
    assertTrue(ProjectionFilter.projectionFilter(Projections.parse(RowSetTestUtils.projectList("a")), EmptyErrorContext.INSTANCE) instanceof DirectProjectionFilter);
    TupleMetadata schema = new SchemaBuilder().add(A_COL.copy()).add(B_COL.copy()).build();
    assertSame(ProjectionFilter.PROJECT_NONE, ProjectionFilter.definedSchemaFilter(new TupleSchema(), EmptyErrorContext.INSTANCE));
    assertTrue(ProjectionFilter.definedSchemaFilter(schema, EmptyErrorContext.INSTANCE) instanceof SchemaProjectionFilter);
    assertTrue(ProjectionFilter.providedSchemaFilter(Projections.projectAll(), schema, EmptyErrorContext.INSTANCE) instanceof CompoundProjectionFilter);
    assertSame(ProjectionFilter.PROJECT_NONE, ProjectionFilter.providedSchemaFilter(Projections.projectNone(), schema, EmptyErrorContext.INSTANCE));
    assertSame(ProjectionFilter.PROJECT_ALL, ProjectionFilter.providedSchemaFilter(Projections.projectAll(), new TupleSchema(), EmptyErrorContext.INSTANCE));
    TupleMetadata strictEmpty = new TupleSchema();
    strictEmpty.setBooleanProperty(TupleMetadata.IS_STRICT_SCHEMA_PROP, true);
    assertSame(ProjectionFilter.PROJECT_NONE, ProjectionFilter.providedSchemaFilter(Projections.projectAll(), strictEmpty, EmptyErrorContext.INSTANCE));
    assertTrue(ProjectionFilter.providedSchemaFilter(Projections.parse(RowSetTestUtils.projectList("a")), schema, EmptyErrorContext.INSTANCE) instanceof CompoundProjectionFilter);
}

Example 89 with SchemaBuilder

use of org.apache.drill.exec.record.metadata.SchemaBuilder in project drill by apache.

the class TestProjectionFilter method testTypeFilter.

@Test
public void testTypeFilter() {
    TupleMetadata schema = new SchemaBuilder().add(A_COL.copy()).add(B_COL.copy()).addMap("m").add("a", MinorType.INT).resumeSchema().build();
    ProjectionFilter filter = new TypeProjectionFilter(schema, EmptyErrorContext.INSTANCE);
    assertFalse(filter.isEmpty());
    assertTrue(filter.isProjected("a"));
    assertTrue(filter.projection(A_COL).isProjected);
    assertTrue(filter.isProjected("b"));
    assertTrue(filter.projection(B_COL).isProjected);
    assertTrue(filter.isProjected("c"));
    assertTrue(filter.projection(MetadataUtils.newScalar("c", Types.required(MinorType.BIGINT))).isProjected);
    ColumnMetadata typeConflict = MetadataUtils.newScalar("a", Types.required(MinorType.BIGINT));
    try {
        filter.projection(typeConflict);
        fail();
    } catch (UserException e) {
        assertTrue(e.getMessage().contains("conflict"));
    }
    ColumnMetadata modeConflict = MetadataUtils.newScalar("a", Types.optional(MinorType.INT));
    try {
        filter.projection(modeConflict);
        fail();
    } catch (UserException e) {
        assertTrue(e.getMessage().contains("conflict"));
    }
    ProjResult result = filter.projection(MAP_COL);
    assertTrue(result.isProjected);
    ProjectionFilter child = result.mapFilter;
    assertTrue(child.isProjected("a"));
    assertTrue(child.isProjected("b"));
    result = filter.projection(MAP_COL2);
    assertTrue(result.isProjected);
    assertSame(ProjectionFilter.PROJECT_ALL, result.mapFilter);
    try {
        ColumnMetadata aMap = MetadataUtils.newMap("a", new TupleSchema());
        filter.projection(aMap);
        fail();
    } catch (UserException e) {
        assertTrue(e.getMessage().contains("type conflict"));
    }
}

Example 90 with SchemaBuilder

use of org.apache.drill.exec.record.metadata.SchemaBuilder in project drill by apache.

the class TestResultSetLoaderDictArray method testScalarValue.

@Test
public void testScalarValue() {
    TupleMetadata schema = new SchemaBuilder().add("a", MinorType.INT).addDictArray("d", MinorType.VARCHAR).value(MinorType.INT).resumeSchema().buildSchema();
    ResultSetLoaderImpl.ResultSetOptions options = new ResultSetOptionBuilder().readerSchema(schema).build();
    ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
    RowSetLoader rootWriter = rsLoader.writer();
    // Write a couple of rows
    rsLoader.startBatch();
    rootWriter.addRow(10, objArray(map("a", 1, "b", 2, "d", 4), map("a", 2, "c", 3, "d", 1, "e", 4))).addRow(20, objArray()).addRow(30, objArray(map("a", 2, "c", 4, "d", 5, "e", 6, "f", 11), map("a", 1, "d", 6, "c", 3), map("b", 2, "a", 3)));
    // Verify the batch
    RowSet actual = fixture.wrap(rsLoader.harvest());
    SingleRowSet expected = fixture.rowSetBuilder(schema).addRow(10, objArray(map("a", 1, "b", 2, "d", 4), map("a", 2, "c", 3, "d", 1, "e", 4))).addRow(20, objArray()).addRow(30, objArray(map("a", 2, "c", 4, "d", 5, "e", 6, "f", 11), map("a", 1, "d", 6, "c", 3), map("b", 2, "a", 3))).build();
    RowSetUtilities.verify(expected, actual);
    rsLoader.close();
}