Examples with ColumnMetadata org.apache.drill.exec.record.metadata.ColumnMetadata used on opensource projects

Example 91 with ColumnMetadata

use of org.apache.drill.exec.record.metadata.ColumnMetadata in project drill by apache.

the class TestScanSchemaTrackerMaps method doTestGenericMap.

private void doTestGenericMap(ScanSchemaConfigBuilder builder, boolean isAll) {
    final ScanSchemaTracker schemaTracker = builder.build();
    // Pretend the reader discovers that m is a map.
    TupleMetadata reader1InputSchema = schemaTracker.readerInputSchema();
    assertEquals(2, reader1InputSchema.size());
    final TupleMetadata readerOutputSchema = new SchemaBuilder().add("a", MinorType.INT).addMap("m").add("x", MinorType.BIGINT).add("y", MinorType.VARCHAR).resumeSchema().buildSchema();
    {
        ProjectionFilter filter = schemaTracker.projectionFilter(ERROR_CONTEXT);
        assertTrue(filter instanceof DynamicSchemaFilter);
        assertTrue(isProjected(filter, readerOutputSchema.metadata("a")));
        assertTrue(isProjected(filter, readerOutputSchema.metadata("m")));
        assertFalse(filter.isProjected("b"));
        final ColumnMetadata mapCol = readerOutputSchema.metadata("m");
        ProjResult mapResult = filter.projection(mapCol);
        assertTrue(mapResult.isProjected);
        ProjectionFilter mapFilter = mapResult.mapFilter;
        if (isAll) {
            assertSame(ProjectionFilter.PROJECT_ALL, mapFilter);
        } else {
            assertTrue(mapFilter instanceof DynamicSchemaFilter);
        }
        final TupleMetadata mapSchema = mapCol.tupleSchema();
        assertTrue(isProjected(mapFilter, mapSchema.metadata("x")));
        assertTrue(isProjected(mapFilter, mapSchema.metadata("y")));
        assertTrue(mapFilter.isProjected("z"));
        assertSame(ProjectionFilter.PROJECT_ALL, mapProjection(mapFilter, "w").mapFilter);
    }
    schemaTracker.applyReaderSchema(readerOutputSchema, ERROR_CONTEXT);
    // Scan output schema is now resolved.
    assertTrue(schemaTracker.isResolved());
    final TupleMetadata outputSchema = schemaTracker.outputSchema();
    assertEquals(readerOutputSchema, outputSchema);
    // A second reader gets a strict filter for the row, but a
    // project-all filter for the map.
    TupleMetadata reader2InputSchema = schemaTracker.readerInputSchema();
    assertEquals(outputSchema, reader2InputSchema);
    {
        final ProjectionFilter filter = schemaTracker.projectionFilter(ERROR_CONTEXT);
        assertTrue(filter instanceof DynamicSchemaFilter);
        assertTrue(isProjected(filter, readerOutputSchema.metadata("a")));
        assertTrue(isProjected(filter, readerOutputSchema.metadata("m")));
        assertFalse(filter.isProjected("b"));
        final TupleMetadata mapSchema = readerOutputSchema.metadata("m").tupleSchema();
        ProjResult mapResult = filter.projection(readerOutputSchema.metadata("m"));
        assertTrue(mapResult.isProjected);
        ProjectionFilter mapFilter = mapResult.mapFilter;
        assertTrue(mapFilter instanceof DynamicSchemaFilter);
        assertTrue(isProjected(mapFilter, mapSchema.metadata("x")));
        assertTrue(isProjected(mapFilter, mapSchema.metadata("y")));
        assertTrue(mapFilter.isProjected("z"));
        assertSame(ProjectionFilter.PROJECT_ALL, mapFilter.projection(MetadataUtils.newMap("w")).mapFilter);
    }
}

Example 92 with ColumnMetadata

use of org.apache.drill.exec.record.metadata.ColumnMetadata in project drill by apache.

the class TestScanSchemaTrackerMaps method testGenericMapWithLenientProvidedSchema.

/**
 * Test a generic map with a lenient provided schema. The schema
 * defines those columns which do exist, but allow other map members
 * to be added.
 */
@Test
public void testGenericMapWithLenientProvidedSchema() {
    final ScanSchemaConfigBuilder builder = new ScanSchemaConfigBuilder().projection(RowSetTestUtils.projectList("a", "m"));
    final TupleMetadata providedSchema = new SchemaBuilder().add("a", MinorType.INT).addMap("m").add("x", MinorType.BIGINT).add("y", MinorType.VARCHAR).resumeSchema().buildSchema();
    builder.providedSchema(providedSchema);
    final ScanSchemaTracker schemaTracker = builder.build();
    assertTrue(schemaTracker.isResolved());
    // Pretend the reader reads one of the map columns and discovers a new one.
    TupleMetadata reader1InputSchema = schemaTracker.readerInputSchema();
    assertEquals(2, reader1InputSchema.size());
    final TupleMetadata readerOutputSchema = new SchemaBuilder().add("a", MinorType.INT).addMap("m").add("x", MinorType.BIGINT).add("z", MinorType.VARCHAR).resumeSchema().buildSchema();
    {
        ProjectionFilter filter = schemaTracker.projectionFilter(ERROR_CONTEXT);
        assertTrue(filter instanceof DynamicSchemaFilter);
        assertTrue(isProjected(filter, readerOutputSchema.metadata("a")));
        assertTrue(isProjected(filter, readerOutputSchema.metadata("m")));
        assertFalse(filter.isProjected("b"));
        final ColumnMetadata mapCol = readerOutputSchema.metadata("m");
        ProjectionFilter mapFilter = filter.projection(mapCol).mapFilter;
        assertTrue(mapFilter instanceof DynamicSchemaFilter);
        final TupleMetadata mapSchema = mapCol.tupleSchema();
        assertTrue(isProjected(mapFilter, mapSchema.metadata("x")));
        assertTrue(mapFilter.isProjected("y"));
        assertTrue(isProjected(mapFilter, mapSchema.metadata("z")));
        assertSame(ProjectionFilter.PROJECT_ALL, mapProjection(mapFilter, "w").mapFilter);
    }
    schemaTracker.applyReaderSchema(readerOutputSchema, ERROR_CONTEXT);
    // Scan output schema is now resolved.
    assertTrue(schemaTracker.isResolved());
    final TupleMetadata expectedOutput = new SchemaBuilder().add("a", MinorType.INT).addMap("m").add("x", MinorType.BIGINT).add("y", MinorType.VARCHAR).add("z", MinorType.VARCHAR).resumeSchema().buildSchema();
    assertEquals(expectedOutput, schemaTracker.outputSchema());
    // A second reader gets a strict filter for the row, but a
    // project-all filter for the map.
    TupleMetadata reader2InputSchema = schemaTracker.readerInputSchema();
    assertEquals(expectedOutput, reader2InputSchema);
    {
        ProjectionFilter filter = schemaTracker.projectionFilter(ERROR_CONTEXT);
        assertTrue(filter instanceof DynamicSchemaFilter);
        assertTrue(isProjected(filter, readerOutputSchema.metadata("a")));
        assertTrue(isProjected(filter, readerOutputSchema.metadata("m")));
        assertFalse(filter.isProjected("b"));
        final ColumnMetadata mapCol = readerOutputSchema.metadata("m");
        ProjectionFilter mapFilter = filter.projection(mapCol).mapFilter;
        assertTrue(mapFilter instanceof DynamicSchemaFilter);
        final TupleMetadata mapSchema = mapCol.tupleSchema();
        assertTrue(isProjected(mapFilter, mapSchema.metadata("x")));
        assertTrue(mapFilter.isProjected("w"));
        assertSame(ProjectionFilter.PROJECT_ALL, mapProjection(mapFilter, "w").mapFilter);
    }
}

Example 93 with ColumnMetadata

use of org.apache.drill.exec.record.metadata.ColumnMetadata in project drill by axbaretto.

the class TestResultSetLoaderProjection method testMapProjection.

@Test
public void testMapProjection() {
    List<SchemaPath> selection = Lists.newArrayList(SchemaPath.getSimplePath("m1"), SchemaPath.getCompoundPath("m2", "d"));
    TupleMetadata schema = new SchemaBuilder().addMap("m1").add("a", MinorType.INT).add("b", MinorType.INT).resumeSchema().addMap("m2").add("c", MinorType.INT).add("d", MinorType.INT).resumeSchema().addMap("m3").add("e", MinorType.INT).add("f", MinorType.INT).resumeSchema().buildSchema();
    ResultSetOptions options = new OptionBuilder().setProjection(selection).setSchema(schema).build();
    ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
    RowSetLoader rootWriter = rsLoader.writer();
    // Verify the projected columns
    TupleMetadata actualSchema = rootWriter.schema();
    ColumnMetadata m1Md = actualSchema.metadata("m1");
    assertTrue(m1Md.isMap());
    assertTrue(m1Md.isProjected());
    assertEquals(2, m1Md.mapSchema().size());
    assertTrue(m1Md.mapSchema().metadata("a").isProjected());
    assertTrue(m1Md.mapSchema().metadata("b").isProjected());
    ColumnMetadata m2Md = actualSchema.metadata("m2");
    assertTrue(m2Md.isMap());
    assertTrue(m2Md.isProjected());
    assertEquals(2, m2Md.mapSchema().size());
    assertFalse(m2Md.mapSchema().metadata("c").isProjected());
    assertTrue(m2Md.mapSchema().metadata("d").isProjected());
    ColumnMetadata m3Md = actualSchema.metadata("m3");
    assertTrue(m3Md.isMap());
    assertFalse(m3Md.isProjected());
    assertEquals(2, m3Md.mapSchema().size());
    assertFalse(m3Md.mapSchema().metadata("e").isProjected());
    assertFalse(m3Md.mapSchema().metadata("f").isProjected());
    // Write a couple of rows.
    rsLoader.startBatch();
    rootWriter.start();
    rootWriter.tuple("m1").scalar("a").setInt(1);
    rootWriter.tuple("m1").scalar("b").setInt(2);
    rootWriter.tuple("m2").scalar("c").setInt(3);
    rootWriter.tuple("m2").scalar("d").setInt(4);
    rootWriter.tuple("m3").scalar("e").setInt(5);
    rootWriter.tuple("m3").scalar("f").setInt(6);
    rootWriter.save();
    rootWriter.start();
    rootWriter.tuple("m1").scalar("a").setInt(11);
    rootWriter.tuple("m1").scalar("b").setInt(12);
    rootWriter.tuple("m2").scalar("c").setInt(13);
    rootWriter.tuple("m2").scalar("d").setInt(14);
    rootWriter.tuple("m3").scalar("e").setInt(15);
    rootWriter.tuple("m3").scalar("f").setInt(16);
    rootWriter.save();
    // Verify. Only the projected columns appear in the result set.
    BatchSchema expectedSchema = new SchemaBuilder().addMap("m1").add("a", MinorType.INT).add("b", MinorType.INT).resumeSchema().addMap("m2").add("d", MinorType.INT).resumeSchema().build();
    SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).addRow(objArray(1, 2), objArray(4)).addRow(objArray(11, 12), objArray(14)).build();
    new RowSetComparison(expected).verifyAndClearAll(fixture.wrap(rsLoader.harvest()));
    rsLoader.close();
}

Example 94 with ColumnMetadata

use of org.apache.drill.exec.record.metadata.ColumnMetadata in project drill by axbaretto.

the class BuildVectorsFromMetadata method buildMap.

/**
 * Build a map column including the members of the map given a map
 * column schema.
 *
 * @param schema the schema of the map column
 * @return the completed map vector column model
 */
private AbstractMapVector buildMap(ColumnMetadata schema) {
    // Creating the map vector will create its contained vectors if we
    // give it a materialized field with children. So, instead pass a clone
    // without children so we can add them.
    MaterializedField mapField = schema.schema();
    MaterializedField emptyClone = MaterializedField.create(mapField.getName(), mapField.getType());
    // Don't get the map vector from the vector cache. Map vectors may
    // have content that varies from batch to batch. Only the leaf
    // vectors can be cached.
    AbstractMapVector mapVector = (AbstractMapVector) TypeHelper.getNewVector(emptyClone, allocator, null);
    // Create the contents building the model as we go.
    TupleMetadata mapSchema = schema.mapSchema();
    for (int i = 0; i < mapSchema.size(); i++) {
        ColumnMetadata childSchema = mapSchema.metadata(i);
        mapVector.putChild(childSchema.name(), buildVector(childSchema));
    }
    return mapVector;
}

Example 95 with ColumnMetadata

use of org.apache.drill.exec.record.metadata.ColumnMetadata in project drill by axbaretto.

the class TupleState method buildSchema.

/**
 * When creating a schema up front, provide the schema of the desired tuple,
 * then build vectors and writers to match. Allows up-front schema definition
 * in addition to on-the-fly schema creation handled elsewhere.
 *
 * @param schema desired tuple schema to be materialized
 */
public void buildSchema(TupleMetadata schema) {
    for (int i = 0; i < schema.size(); i++) {
        ColumnMetadata colSchema = schema.metadata(i);
        AbstractObjectWriter colWriter;
        if (colSchema.isMap()) {
            colWriter = addColumn(colSchema.cloneEmpty());
            BaseMapColumnState mapColState = (BaseMapColumnState) columns.get(columns.size() - 1);
            mapColState.mapState().buildSchema(colSchema.mapSchema());
        } else {
            colWriter = addColumn(colSchema);
        }
        writer().addColumnWriter(colWriter);
    }
}