Examples with ReaderSchemaOrchestrator org.apache.drill.exec.physical.impl.scan.project.ReaderSchemaOrchestrator used on opensource projects

Example 16 with ReaderSchemaOrchestrator

use of org.apache.drill.exec.physical.impl.scan.project.ReaderSchemaOrchestrator in project drill by apache.

the class TestScanOrchestratorLateSchema method testLateSchemaSelectDisjoint.

/**
 * Test SELECT a, c FROM table(a, b)
 */
@Test
public void testLateSchemaSelectDisjoint() {
    ScanOrchestratorBuilder builder = new MockScanBuilder();
    // SELECT a, c ...
    builder.projection(RowSetTestUtils.projectList("a", "c"));
    ScanSchemaOrchestrator orchestrator = new ScanSchemaOrchestrator(fixture.allocator(), builder);
    // ... FROM file
    ReaderSchemaOrchestrator reader = orchestrator.startReader();
    // Create the table loader
    ResultSetLoader loader = reader.makeTableLoader(null);
    // file schema (a, b)
    reader.startBatch();
    RowSetLoader writer = loader.writer();
    writer.addColumn(SchemaBuilder.columnSchema("a", MinorType.INT, DataMode.REQUIRED));
    writer.addColumn(SchemaBuilder.columnSchema("b", MinorType.VARCHAR, DataMode.REQUIRED));
    // Create a batch of data.
    writer.addRow(1, "fred").addRow(2, "wilma");
    reader.endBatch();
    // Verify
    TupleMetadata expectedSchema = new SchemaBuilder().add("a", MinorType.INT).addNullable("c", MinorType.INT).buildSchema();
    SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).addRow(1, null).addRow(2, null).build();
    new RowSetComparison(expected).verifyAndClearAll(fixture.wrap(orchestrator.output()));
    orchestrator.close();
}

Example 17 with ReaderSchemaOrchestrator

use of org.apache.drill.exec.physical.impl.scan.project.ReaderSchemaOrchestrator in project drill by apache.

the class TestScanOrchestratorLateSchema method testLateSchemaWildcard.

/**
 * Test SELECT * from an early-schema table of (a, b)
 */
@Test
public void testLateSchemaWildcard() {
    ScanOrchestratorBuilder builder = new MockScanBuilder();
    // SELECT * ...
    builder.projection(RowSetTestUtils.projectAll());
    ScanSchemaOrchestrator orchestrator = new ScanSchemaOrchestrator(fixture.allocator(), builder);
    // ... FROM table
    ReaderSchemaOrchestrator reader = orchestrator.startReader();
    // Create the table loader
    ResultSetLoader loader = reader.makeTableLoader(null);
    // Late schema: no batch provided up front.
    assertFalse(reader.hasSchema());
    // Start a batch and discover a schema: (a, b)
    reader.startBatch();
    RowSetLoader writer = loader.writer();
    writer.addColumn(SchemaBuilder.columnSchema("a", MinorType.INT, DataMode.REQUIRED));
    writer.addColumn(SchemaBuilder.columnSchema("b", MinorType.VARCHAR, DataMode.REQUIRED));
    // Create a batch of data using the discovered schema
    writer.addRow(1, "fred").addRow(2, "wilma");
    reader.endBatch();
    // Verify
    TupleMetadata tableSchema = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.VARCHAR).buildSchema();
    SingleRowSet expected = fixture.rowSetBuilder(tableSchema).addRow(1, "fred").addRow(2, "wilma").build();
    new RowSetComparison(expected).verifyAndClearAll(fixture.wrap(orchestrator.output()));
    orchestrator.close();
}

Example 18 with ReaderSchemaOrchestrator

use of org.apache.drill.exec.physical.impl.scan.project.ReaderSchemaOrchestrator in project drill by apache.

the class TestColumnsArray method testReqularCol.

/**
 * Verify that if the columns column is not required, that `columns`
 * is treated like any other column.
 */
@Test
public void testReqularCol() {
    ScanSchemaOrchestrator scanner = buildScan(false, RowSetTestUtils.projectList(ColumnsScanFramework.COLUMNS_COL));
    TupleMetadata tableSchema = new SchemaBuilder().add(ColumnsScanFramework.COLUMNS_COL, MinorType.VARCHAR).buildSchema();
    ReaderSchemaOrchestrator reader = scanner.startReader();
    ResultSetLoader rsLoader = reader.makeTableLoader(tableSchema);
    reader.defineSchema();
    reader.startBatch();
    rsLoader.writer().addRow("fred");
    reader.endBatch();
    SingleRowSet expected = fixture.rowSetBuilder(tableSchema).addRow("fred").build();
    RowSetUtilities.verify(expected, fixture.wrap(scanner.output()));
    scanner.close();
}

Example 19 with ReaderSchemaOrchestrator

use of org.apache.drill.exec.physical.impl.scan.project.ReaderSchemaOrchestrator in project drill by apache.

the class TestScanOrchestratorImplicitColumns method testMixture.

/**
 * Test SELECT dir0, b, suffix, c FROM table(a, b)
 * Full combination of metadata, table and null columns
 */
@Test
public void testMixture() {
    ScanOrchestratorBuilder builder = new MockScanBuilder();
    File file = dirTestWatcher.copyResourceToRoot(Paths.get("multilevel", "csv", "1994", "Q1", "orders_94_q1.csv"), Paths.get("x", "y", "z.csv"));
    Path filePath = new Path(file.toURI().getPath());
    ImplicitColumnManager metadataManager = new ImplicitColumnManager(fixture.getOptionManager(), standardOptions(filePath));
    builder.withImplicitColumns(metadataManager);
    // SELECT dir0, b, suffix, c ...
    builder.projection(RowSetTestUtils.projectList("dir0", "b", "suffix", "c"));
    ScanSchemaOrchestrator scanner = new ScanSchemaOrchestrator(fixture.allocator(), builder);
    // ... FROM file
    metadataManager.startFile(filePath);
    ReaderSchemaOrchestrator reader = scanner.startReader();
    // file schema (a, b)
    TupleMetadata tableSchema = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.VARCHAR).buildSchema();
    // Create the table loader
    ResultSetLoader loader = reader.makeTableLoader(tableSchema);
    TupleMetadata expectedSchema = new SchemaBuilder().addNullable("dir0", MinorType.VARCHAR).add("b", MinorType.VARCHAR).add("suffix", MinorType.VARCHAR).addNullable("c", MinorType.INT).buildSchema();
    // Create a batch of data.
    reader.startBatch();
    loader.writer().addRow(1, "fred").addRow(2, "wilma");
    reader.endBatch();
    // Verify
    SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).addRow("x", "fred", "csv", null).addRow("x", "wilma", "csv", null).build();
    RowSetUtilities.verify(expected, fixture.wrap(scanner.output()));
    scanner.close();
}

Example 20 with ReaderSchemaOrchestrator

use of org.apache.drill.exec.physical.impl.scan.project.ReaderSchemaOrchestrator in project drill by apache.

the class TestColumnsArray method testMissingColumnsColumn.

/**
 * Test attempting to use the columns array with an early schema with
 * column types not compatible with a varchar array.
 */
@Test
public void testMissingColumnsColumn() {
    ScanSchemaOrchestrator scanner = buildScan(true, RowSetTestUtils.projectList(ColumnsScanFramework.COLUMNS_COL));
    TupleMetadata tableSchema = new SchemaBuilder().add("a", MinorType.VARCHAR).buildSchema();
    try {
        ReaderSchemaOrchestrator reader = scanner.startReader();
        reader.makeTableLoader(tableSchema);
        reader.defineSchema();
        fail();
    } catch (IllegalStateException e) {
    // Expected
    }
    scanner.close();
}