Examples with ProjectionFilter org.apache.drill.exec.physical.resultSet.impl.ProjectionFilter used on opensource projects

Example 11 with ProjectionFilter

use of org.apache.drill.exec.physical.resultSet.impl.ProjectionFilter in project drill by apache.

the class TestScanSchemaTracker method testWildcard.

/**
 * Wildcard projection, schema change allowed.
 */
@Test
public void testWildcard() {
    // Simulate SELECT * ...
    final ScanSchemaConfigBuilder builder = new ScanSchemaConfigBuilder().projection(RowSetTestUtils.projectAll());
    final ScanSchemaTracker schemaTracker = builder.build();
    assertSame(ProjectionType.ALL, schemaTracker.projectionType());
    assertFalse(schemaTracker.isResolved());
    // Reader input schema is dynamic
    final TupleMetadata reader1InputSchema = schemaTracker.readerInputSchema();
    assertTrue(reader1InputSchema.isEmpty());
    ProjectionFilter filter1 = schemaTracker.projectionFilter(ERROR_CONTEXT);
    assertSame(ProjectionFilter.PROJECT_ALL, filter1);
    // Pretend the reader discovers two columns.
    final TupleMetadata reader1OutputSchema = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.BIGINT).buildSchema();
    schemaTracker.applyReaderSchema(reader1OutputSchema, ERROR_CONTEXT);
    assertTrue(schemaTracker.isResolved());
    // Schema sent downstream after first batch
    final TupleMetadata outputSchema1 = schemaTracker.outputSchema();
    assertEquals(reader1OutputSchema, outputSchema1);
    // Next reader schema is partially defined
    final TupleMetadata reader2InputSchema = schemaTracker.readerInputSchema();
    assertEquals(reader1OutputSchema, reader2InputSchema);
    ProjectionFilter filter2 = schemaTracker.projectionFilter(ERROR_CONTEXT);
    assertTrue(filter2 instanceof DynamicSchemaFilter);
    assertTrue(filter2.projection(reader1OutputSchema.metadata("a")).isProjected);
    assertTrue(filter2.projection(reader1OutputSchema.metadata("b")).isProjected);
    assertTrue(filter2.isProjected("c"));
    try {
        filter2.projection(MetadataUtils.newScalar("a", Types.required(MinorType.VARCHAR)));
        fail();
    } catch (UserException e) {
    // Expected;
    }
    // The next reader defines another column.
    // This triggers a schema change in output.
    final TupleMetadata reader2OutputSchema = new SchemaBuilder().add("c", MinorType.VARCHAR).buildSchema();
    schemaTracker.applyReaderSchema(reader2OutputSchema, ERROR_CONTEXT);
    // Schema sent downstream after second reader
    final TupleMetadata outputSchema2 = schemaTracker.outputSchema();
    final TupleMetadata expectedOutput = new SchemaBuilder().addAll(reader1OutputSchema).addAll(reader2OutputSchema).buildSchema();
    assertEquals(expectedOutput, outputSchema2);
}

Example 12 with ProjectionFilter

use of org.apache.drill.exec.physical.resultSet.impl.ProjectionFilter in project drill by apache.

the class TestScanSchemaTracker method testStrictProvidedSchemaWithWildcard.

@Test
public void testStrictProvidedSchemaWithWildcard() {
    // Simulate SELECT * ...
    final ScanSchemaConfigBuilder builder = new ScanSchemaConfigBuilder();
    final TupleMetadata providedSchema = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.BIGINT).buildSchema();
    providedSchema.setBooleanProperty(TupleMetadata.IS_STRICT_SCHEMA_PROP, true);
    builder.providedSchema(providedSchema);
    final ScanSchemaTracker schemaTracker = builder.build();
    // Even though the project list is a wildcard, the presence
    // of a strict provided schema makes it fully defined
    assertSame(ProjectionType.SOME, schemaTracker.projectionType());
    assertTrue(schemaTracker.isResolved());
    // Reader input schema is fixed
    final TupleMetadata reader1InputSchema = schemaTracker.readerInputSchema();
    assertEquals(providedSchema, reader1InputSchema);
    ProjectionFilter filter1 = schemaTracker.projectionFilter(ERROR_CONTEXT);
    assertTrue(filter1 instanceof DynamicSchemaFilter);
    // Pretend the reader discovers two columns.
    final TupleMetadata reader1OutputSchema = new SchemaBuilder().addAll(providedSchema).buildSchema();
    assertTrue(filter1.projection(reader1OutputSchema.metadata("a")).isProjected);
    assertFalse(filter1.isProjected("c"));
    schemaTracker.applyReaderSchema(reader1OutputSchema, ERROR_CONTEXT);
    assertTrue(schemaTracker.isResolved());
    // Schema sent downstream after first batch
    final TupleMetadata outputSchema = schemaTracker.outputSchema();
    assertEquals(providedSchema, outputSchema);
}

Example 13 with ProjectionFilter

use of org.apache.drill.exec.physical.resultSet.impl.ProjectionFilter in project drill by apache.

the class TestDynamicSchemaFilter method testMapProjectList.

@Test
public void testMapProjectList() {
    ProjectionParseResult parseResult = ScanProjectionParser.parse(RowSetTestUtils.projectList("m.x"));
    ProjectionFilter filter = new DynamicTupleFilter(parseResult.dynamicSchema, EmptyErrorContext.INSTANCE);
    assertFalse(filter.isEmpty());
    assertTrue(filter.isProjected(MAP_COL.name()));
    ProjResult result = filter.projection(MAP_COL);
    assertTrue(result.isProjected);
    assertNotNull(result.projection);
    assertTrue(result.projection.isDynamic());
    assertEquals(MAP_COL.name(), result.projection.name());
    assertTrue(result.mapFilter instanceof DynamicTupleFilter);
    ProjectionFilter mapFilter = result.mapFilter;
    ColumnMetadata x_col = MAP_COL.tupleSchema().metadata("x");
    assertTrue(mapFilter.isProjected("x"));
    result = mapFilter.projection(x_col);
    assertTrue(result.isProjected);
    assertNotNull(result.projection);
    assertTrue(result.projection.isDynamic());
    assertEquals(x_col.name(), result.projection.name());
    ColumnMetadata y_col = MAP_COL.tupleSchema().metadata("y");
    assertFalse(mapFilter.isProjected("y"));
    result = mapFilter.projection(y_col);
    assertFalse(result.isProjected);
}

Example 14 with ProjectionFilter

use of org.apache.drill.exec.physical.resultSet.impl.ProjectionFilter in project drill by apache.

the class TestScanSchemaTrackerMaps method doTestGenericMap.

private void doTestGenericMap(ScanSchemaConfigBuilder builder, boolean isAll) {
    final ScanSchemaTracker schemaTracker = builder.build();
    // Pretend the reader discovers that m is a map.
    TupleMetadata reader1InputSchema = schemaTracker.readerInputSchema();
    assertEquals(2, reader1InputSchema.size());
    final TupleMetadata readerOutputSchema = new SchemaBuilder().add("a", MinorType.INT).addMap("m").add("x", MinorType.BIGINT).add("y", MinorType.VARCHAR).resumeSchema().buildSchema();
    {
        ProjectionFilter filter = schemaTracker.projectionFilter(ERROR_CONTEXT);
        assertTrue(filter instanceof DynamicSchemaFilter);
        assertTrue(isProjected(filter, readerOutputSchema.metadata("a")));
        assertTrue(isProjected(filter, readerOutputSchema.metadata("m")));
        assertFalse(filter.isProjected("b"));
        final ColumnMetadata mapCol = readerOutputSchema.metadata("m");
        ProjResult mapResult = filter.projection(mapCol);
        assertTrue(mapResult.isProjected);
        ProjectionFilter mapFilter = mapResult.mapFilter;
        if (isAll) {
            assertSame(ProjectionFilter.PROJECT_ALL, mapFilter);
        } else {
            assertTrue(mapFilter instanceof DynamicSchemaFilter);
        }
        final TupleMetadata mapSchema = mapCol.tupleSchema();
        assertTrue(isProjected(mapFilter, mapSchema.metadata("x")));
        assertTrue(isProjected(mapFilter, mapSchema.metadata("y")));
        assertTrue(mapFilter.isProjected("z"));
        assertSame(ProjectionFilter.PROJECT_ALL, mapProjection(mapFilter, "w").mapFilter);
    }
    schemaTracker.applyReaderSchema(readerOutputSchema, ERROR_CONTEXT);
    // Scan output schema is now resolved.
    assertTrue(schemaTracker.isResolved());
    final TupleMetadata outputSchema = schemaTracker.outputSchema();
    assertEquals(readerOutputSchema, outputSchema);
    // A second reader gets a strict filter for the row, but a
    // project-all filter for the map.
    TupleMetadata reader2InputSchema = schemaTracker.readerInputSchema();
    assertEquals(outputSchema, reader2InputSchema);
    {
        final ProjectionFilter filter = schemaTracker.projectionFilter(ERROR_CONTEXT);
        assertTrue(filter instanceof DynamicSchemaFilter);
        assertTrue(isProjected(filter, readerOutputSchema.metadata("a")));
        assertTrue(isProjected(filter, readerOutputSchema.metadata("m")));
        assertFalse(filter.isProjected("b"));
        final TupleMetadata mapSchema = readerOutputSchema.metadata("m").tupleSchema();
        ProjResult mapResult = filter.projection(readerOutputSchema.metadata("m"));
        assertTrue(mapResult.isProjected);
        ProjectionFilter mapFilter = mapResult.mapFilter;
        assertTrue(mapFilter instanceof DynamicSchemaFilter);
        assertTrue(isProjected(mapFilter, mapSchema.metadata("x")));
        assertTrue(isProjected(mapFilter, mapSchema.metadata("y")));
        assertTrue(mapFilter.isProjected("z"));
        assertSame(ProjectionFilter.PROJECT_ALL, mapFilter.projection(MetadataUtils.newMap("w")).mapFilter);
    }
}

Example 15 with ProjectionFilter

use of org.apache.drill.exec.physical.resultSet.impl.ProjectionFilter in project drill by apache.

the class TestScanSchemaTrackerMaps method testGenericMapWithLenientProvidedSchema.

/**
 * Test a generic map with a lenient provided schema. The schema
 * defines those columns which do exist, but allow other map members
 * to be added.
 */
@Test
public void testGenericMapWithLenientProvidedSchema() {
    final ScanSchemaConfigBuilder builder = new ScanSchemaConfigBuilder().projection(RowSetTestUtils.projectList("a", "m"));
    final TupleMetadata providedSchema = new SchemaBuilder().add("a", MinorType.INT).addMap("m").add("x", MinorType.BIGINT).add("y", MinorType.VARCHAR).resumeSchema().buildSchema();
    builder.providedSchema(providedSchema);
    final ScanSchemaTracker schemaTracker = builder.build();
    assertTrue(schemaTracker.isResolved());
    // Pretend the reader reads one of the map columns and discovers a new one.
    TupleMetadata reader1InputSchema = schemaTracker.readerInputSchema();
    assertEquals(2, reader1InputSchema.size());
    final TupleMetadata readerOutputSchema = new SchemaBuilder().add("a", MinorType.INT).addMap("m").add("x", MinorType.BIGINT).add("z", MinorType.VARCHAR).resumeSchema().buildSchema();
    {
        ProjectionFilter filter = schemaTracker.projectionFilter(ERROR_CONTEXT);
        assertTrue(filter instanceof DynamicSchemaFilter);
        assertTrue(isProjected(filter, readerOutputSchema.metadata("a")));
        assertTrue(isProjected(filter, readerOutputSchema.metadata("m")));
        assertFalse(filter.isProjected("b"));
        final ColumnMetadata mapCol = readerOutputSchema.metadata("m");
        ProjectionFilter mapFilter = filter.projection(mapCol).mapFilter;
        assertTrue(mapFilter instanceof DynamicSchemaFilter);
        final TupleMetadata mapSchema = mapCol.tupleSchema();
        assertTrue(isProjected(mapFilter, mapSchema.metadata("x")));
        assertTrue(mapFilter.isProjected("y"));
        assertTrue(isProjected(mapFilter, mapSchema.metadata("z")));
        assertSame(ProjectionFilter.PROJECT_ALL, mapProjection(mapFilter, "w").mapFilter);
    }
    schemaTracker.applyReaderSchema(readerOutputSchema, ERROR_CONTEXT);
    // Scan output schema is now resolved.
    assertTrue(schemaTracker.isResolved());
    final TupleMetadata expectedOutput = new SchemaBuilder().add("a", MinorType.INT).addMap("m").add("x", MinorType.BIGINT).add("y", MinorType.VARCHAR).add("z", MinorType.VARCHAR).resumeSchema().buildSchema();
    assertEquals(expectedOutput, schemaTracker.outputSchema());
    // A second reader gets a strict filter for the row, but a
    // project-all filter for the map.
    TupleMetadata reader2InputSchema = schemaTracker.readerInputSchema();
    assertEquals(expectedOutput, reader2InputSchema);
    {
        ProjectionFilter filter = schemaTracker.projectionFilter(ERROR_CONTEXT);
        assertTrue(filter instanceof DynamicSchemaFilter);
        assertTrue(isProjected(filter, readerOutputSchema.metadata("a")));
        assertTrue(isProjected(filter, readerOutputSchema.metadata("m")));
        assertFalse(filter.isProjected("b"));
        final ColumnMetadata mapCol = readerOutputSchema.metadata("m");
        ProjectionFilter mapFilter = filter.projection(mapCol).mapFilter;
        assertTrue(mapFilter instanceof DynamicSchemaFilter);
        final TupleMetadata mapSchema = mapCol.tupleSchema();
        assertTrue(isProjected(mapFilter, mapSchema.metadata("x")));
        assertTrue(mapFilter.isProjected("w"));
        assertSame(ProjectionFilter.PROJECT_ALL, mapProjection(mapFilter, "w").mapFilter);
    }
}