Examples with ProjectionFilter org.apache.drill.exec.physical.resultSet.impl.ProjectionFilter used on opensource projects

Example 21 with ProjectionFilter

use of org.apache.drill.exec.physical.resultSet.impl.ProjectionFilter in project drill by apache.

the class TestScanSchemaTracker method doTestEmptyProject.

private void doTestEmptyProject(ScanSchemaConfigBuilder builder) {
    final ScanSchemaTracker schemaTracker = builder.build();
    assertSame(ProjectionType.NONE, schemaTracker.projectionType());
    assertTrue(schemaTracker.isResolved());
    // Sanity check of projection; detailed testing done elsewhere
    ProjectionFilter filter1 = schemaTracker.projectionFilter(ERROR_CONTEXT);
    assertSame(ProjectionFilter.PROJECT_NONE, filter1);
    assertFalse(filter1.isProjected("a"));
    // Reader input schema is empty
    TupleMetadata readerInputSchema = schemaTracker.readerInputSchema();
    assertTrue(readerInputSchema.isEmpty());
    // Projection is empty
    ProjectionFilter filter = schemaTracker.projectionFilter(ERROR_CONTEXT);
    assertTrue(filter.isEmpty());
    assertFalse(filter.isProjected("a"));
    // Reader produces a empty schema
    final TupleMetadata readerOutputSchema = new SchemaBuilder().buildSchema();
    schemaTracker.applyReaderSchema(readerOutputSchema, ERROR_CONTEXT);
    assertTrue(schemaTracker.isResolved());
    // Schema sent downstream after first batch
    final TupleMetadata outputSchema = schemaTracker.outputSchema();
    assertTrue(outputSchema.isEmpty());
}

Example 22 with ProjectionFilter

use of org.apache.drill.exec.physical.resultSet.impl.ProjectionFilter in project drill by apache.

the class TestScanSchemaTracker method testBasics.

/**
 * Basic test: select a set of columns (a, b, c) when the
 * data source has an early schema of (a, b).
 */
@Test
public void testBasics() {
    // Simulate SELECT a, b, c ...
    final ScanSchemaConfigBuilder builder = new ScanSchemaConfigBuilder().projection(RowSetTestUtils.projectList("a", "b", "c"));
    final ScanSchemaTracker schemaTracker = builder.build();
    assertSame(ProjectionType.SOME, schemaTracker.projectionType());
    assertFalse(schemaTracker.isResolved());
    final int initVersion = schemaTracker.schemaVersion();
    assertTrue(initVersion > 0);
    // Reader input schema is dynamic
    final TupleMetadata readerInputSchema = schemaTracker.readerInputSchema();
    final TupleMetadata expected = new SchemaBuilder().addDynamic("a").addDynamic("b").addDynamic("c").build();
    assertEquals(expected, readerInputSchema);
    // Pretend the reader discovered two of the columns.
    final TupleMetadata readerOutputSchema = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.BIGINT).buildSchema();
    // Projection filter is list-based
    ProjectionFilter filter = schemaTracker.projectionFilter(ERROR_CONTEXT);
    assertTrue(filter instanceof DynamicSchemaFilter);
    assertFalse(filter.isEmpty());
    assertTrue(filter.projection(readerOutputSchema.metadata("a")).isProjected);
    assertTrue(filter.projection(readerOutputSchema.metadata("b")).isProjected);
    assertTrue(filter.isProjected("c"));
    assertFalse(filter.isProjected("d"));
    assertFalse(filter.projection(MetadataUtils.newScalar("d", Types.optional(MinorType.INT))).isProjected);
    schemaTracker.applyReaderSchema(readerOutputSchema, ERROR_CONTEXT);
    assertFalse(schemaTracker.isResolved());
    final int reader1Version = schemaTracker.schemaVersion();
    assertTrue(reader1Version > initVersion);
    // Make up a type for the missing column
    final TupleMetadata missingCols = schemaTracker.missingColumns(readerOutputSchema);
    assertEquals(1, missingCols.size());
    assertEquals(MinorType.LATE, missingCols.metadata("c").type());
    TupleMetadata missingColSchema = new SchemaBuilder().add("c", MinorType.VARCHAR).buildSchema();
    schemaTracker.resolveMissingCols(missingColSchema);
    assertTrue(schemaTracker.isResolved());
    final int missing1Version = schemaTracker.schemaVersion();
    assertTrue(missing1Version > reader1Version);
    // Second reader finds all columns
    TupleMetadata reader2OutputSchema = new SchemaBuilder().addAll(readerOutputSchema).addAll(missingColSchema).build();
    schemaTracker.applyReaderSchema(reader2OutputSchema, ERROR_CONTEXT);
    assertEquals(missing1Version, schemaTracker.schemaVersion());
    // Third reader finds two columns, treats "c" as missing again.
    schemaTracker.resolveMissingCols(missingColSchema);
    // Final schema sent downstream
    final TupleMetadata outputSchema = schemaTracker.outputSchema();
    final TupleMetadata expectedOutput = new SchemaBuilder().addAll(readerOutputSchema).addAll(missingColSchema).buildSchema();
    assertEquals(expectedOutput, outputSchema);
}

Example 23 with ProjectionFilter

use of org.apache.drill.exec.physical.resultSet.impl.ProjectionFilter in project drill by apache.

the class TestScanSchemaTracker method testProvidedSchemaWithWildcard.

@Test
public void testProvidedSchemaWithWildcard() {
    // Simulate SELECT * ...
    final ScanSchemaConfigBuilder builder = new ScanSchemaConfigBuilder();
    final TupleMetadata providedSchema = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.BIGINT).buildSchema();
    builder.providedSchema(providedSchema);
    final ScanSchemaTracker schemaTracker = builder.build();
    assertSame(ProjectionType.ALL, schemaTracker.projectionType());
    assertTrue(schemaTracker.isResolved());
    // Reader input schema is dynamic
    final TupleMetadata reader1InputSchema = schemaTracker.readerInputSchema();
    assertEquals(providedSchema, reader1InputSchema);
    ProjectionFilter filter1 = schemaTracker.projectionFilter(ERROR_CONTEXT);
    assertTrue(filter1 instanceof DynamicSchemaFilter);
    // Pretend the reader discovers two columns.
    final TupleMetadata reader1OutputSchema = new SchemaBuilder().add("a", MinorType.INT).add("c", MinorType.VARCHAR).buildSchema();
    assertTrue(filter1.projection(reader1OutputSchema.metadata("a")).isProjected);
    assertTrue(filter1.projection(reader1OutputSchema.metadata("c")).isProjected);
    assertTrue(filter1.isProjected("d"));
    schemaTracker.applyReaderSchema(reader1OutputSchema, ERROR_CONTEXT);
    assertTrue(schemaTracker.isResolved());
    // Pretend something fills in the the missing columns
    TupleMetadata missingColSchema = new SchemaBuilder().add("b", MinorType.BIGINT).buildSchema();
    schemaTracker.applyReaderSchema(missingColSchema, ERROR_CONTEXT);
    // Schema sent downstream after first batch
    final TupleMetadata outputSchema = schemaTracker.outputSchema();
    final TupleMetadata expected = new SchemaBuilder().addAll(providedSchema).add(reader1OutputSchema.metadata("c")).buildSchema();
    assertEquals(expected, outputSchema);
}

Example 24 with ProjectionFilter

use of org.apache.drill.exec.physical.resultSet.impl.ProjectionFilter in project drill by apache.

the class TestScanSchemaTracker method testWildcardWithoutSchemaChange.

/**
 * Wildcard projection, schema change not allowed; first batch
 * defines the schema and projection for later readers.
 */
@Test
public void testWildcardWithoutSchemaChange() {
    // Simulate SELECT * ...
    final ScanSchemaConfigBuilder builder = new ScanSchemaConfigBuilder().projection(RowSetTestUtils.projectAll());
    builder.allowSchemaChange(false);
    final ScanSchemaTracker schemaTracker = builder.build();
    assertSame(ProjectionType.ALL, schemaTracker.projectionType());
    assertFalse(schemaTracker.isResolved());
    // Reader input schema is dynamic
    ProjectionFilter filter1 = schemaTracker.projectionFilter(ERROR_CONTEXT);
    assertSame(ProjectionFilter.PROJECT_ALL, filter1);
    assertTrue(filter1.isProjected("z"));
    TupleMetadata readerInputSchema = schemaTracker.readerInputSchema();
    assertTrue(readerInputSchema.isEmpty());
    // Pretend the reader discovered two columns.
    final TupleMetadata reader1OutputSchema = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.BIGINT).buildSchema();
    schemaTracker.applyReaderSchema(reader1OutputSchema, ERROR_CONTEXT);
    assertTrue(schemaTracker.isResolved());
    // Next reader schema is completely defined
    assertSame(ProjectionType.SOME, schemaTracker.projectionType());
    readerInputSchema = schemaTracker.readerInputSchema();
    assertEquals(reader1OutputSchema, readerInputSchema);
    // Projection list changes to what we've seen.
    ProjectionFilter filter2 = schemaTracker.projectionFilter(ERROR_CONTEXT);
    assertTrue(filter2 instanceof DynamicSchemaFilter);
    assertTrue(filter2.isProjected("a"));
    assertTrue(filter2.isProjected("b"));
    assertFalse(filter2.isProjected("c"));
    schemaTracker.applyReaderSchema(reader1OutputSchema, ERROR_CONTEXT);
    // Schema sent downstream after first batch
    TupleMetadata outputSchema = schemaTracker.outputSchema();
    assertEquals(reader1OutputSchema, outputSchema);
    // The next reader defines another column: "c". The projection filter ignores
    // the column.
    // The schema tracker ignores this column.
    final TupleMetadata reader2OutputSchema = new SchemaBuilder().buildSchema();
    schemaTracker.applyReaderSchema(reader2OutputSchema, ERROR_CONTEXT);
    // If the reader were to bypass the projection filter and define the
    // column anyway, the query will fail because the scan framework does
    // not know what to do with the unwanted (materialized) column.
    final TupleMetadata reader3OutputSchema = new SchemaBuilder().add("c", MinorType.VARCHAR).buildSchema();
    try {
        schemaTracker.applyReaderSchema(reader3OutputSchema, ERROR_CONTEXT);
        fail();
    } catch (IllegalStateException e) {
    // Expected;
    }
    // Schema sent downstream after second reader
    outputSchema = schemaTracker.outputSchema();
    assertEquals(reader1OutputSchema, outputSchema);
}

Example 25 with ProjectionFilter

use of org.apache.drill.exec.physical.resultSet.impl.ProjectionFilter in project drill by apache.

the class TestSchemaTrackerDefined method testDefinedSchema.

/**
 * If a schema is defined, then the planner has combined the projection
 * list with schema information to produce the final output schema
 * at plan time.
 */
@Test
public void testDefinedSchema() {
    // Simulate SELECT a, b, c ...
    // With a plan-provided defined schema
    final ScanSchemaConfigBuilder builder = new ScanSchemaConfigBuilder().projection(RowSetTestUtils.projectList("a", "b", "c"));
    final TupleMetadata definedSchema = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.BIGINT).add("c", MinorType.VARCHAR).buildSchema();
    builder.definedSchema(definedSchema);
    final ScanSchemaTracker schemaTracker = builder.build();
    assertTrue(schemaTracker instanceof SchemaBasedTracker);
    assertTrue(schemaTracker.isResolved());
    assertSame(ProjectionType.SOME, schemaTracker.projectionType());
    // Reader input schema is fully defined
    TupleMetadata readerInputSchema = schemaTracker.readerInputSchema();
    assertEquals(definedSchema, readerInputSchema);
    // Pretend the reader dutifully provided two of the columns
    TupleMetadata readerOutputSchema = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.BIGINT).buildSchema();
    // Projection filter is schema-based
    ProjectionFilter filter = schemaTracker.projectionFilter(ERROR_CONTEXT);
    assertTrue(filter instanceof DynamicSchemaFilter);
    assertTrue(isProjected(filter, readerOutputSchema.metadata("a")));
    assertTrue(isProjected(filter, readerOutputSchema.metadata("b")));
    assertTrue(filter.isProjected("c"));
    assertFalse(filter.isProjected("d"));
    schemaTracker.applyReaderSchema(readerOutputSchema, ERROR_CONTEXT);
    // Reader schema is still fully resolved
    assertTrue(schemaTracker.isResolved());
    assertEquals(definedSchema, readerInputSchema);
    // Pretend a class fills in the the missing columns
    TupleMetadata missingColSchema = new SchemaBuilder().add("c", MinorType.VARCHAR).buildSchema();
    schemaTracker.applyReaderSchema(missingColSchema, ERROR_CONTEXT);
    assertEquals(definedSchema, readerInputSchema);
    // Final schema sent downstream
    final TupleMetadata outputSchema = schemaTracker.outputSchema();
    assertEquals(definedSchema, outputSchema);
}