Examples with ScanSchemaOrchestrator org.apache.drill.exec.physical.impl.scan.project.ScanSchemaOrchestrator used on opensource projects

Example 16 with ScanSchemaOrchestrator

use of org.apache.drill.exec.physical.impl.scan.project.ScanSchemaOrchestrator in project drill by apache.

the class TestScanOrchestratorLateSchema method testLateSchemaSelectDisjoint.

/**
 * Test SELECT a, c FROM table(a, b)
 */
@Test
public void testLateSchemaSelectDisjoint() {
    ScanOrchestratorBuilder builder = new MockScanBuilder();
    // SELECT a, c ...
    builder.projection(RowSetTestUtils.projectList("a", "c"));
    ScanSchemaOrchestrator orchestrator = new ScanSchemaOrchestrator(fixture.allocator(), builder);
    // ... FROM file
    ReaderSchemaOrchestrator reader = orchestrator.startReader();
    // Create the table loader
    ResultSetLoader loader = reader.makeTableLoader(null);
    // file schema (a, b)
    reader.startBatch();
    RowSetLoader writer = loader.writer();
    writer.addColumn(SchemaBuilder.columnSchema("a", MinorType.INT, DataMode.REQUIRED));
    writer.addColumn(SchemaBuilder.columnSchema("b", MinorType.VARCHAR, DataMode.REQUIRED));
    // Create a batch of data.
    writer.addRow(1, "fred").addRow(2, "wilma");
    reader.endBatch();
    // Verify
    TupleMetadata expectedSchema = new SchemaBuilder().add("a", MinorType.INT).addNullable("c", MinorType.INT).buildSchema();
    SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).addRow(1, null).addRow(2, null).build();
    new RowSetComparison(expected).verifyAndClearAll(fixture.wrap(orchestrator.output()));
    orchestrator.close();
}

Example 17 with ScanSchemaOrchestrator

use of org.apache.drill.exec.physical.impl.scan.project.ScanSchemaOrchestrator in project drill by apache.

the class TestScanOrchestratorLateSchema method testLateSchemaWildcard.

/**
 * Test SELECT * from an early-schema table of (a, b)
 */
@Test
public void testLateSchemaWildcard() {
    ScanOrchestratorBuilder builder = new MockScanBuilder();
    // SELECT * ...
    builder.projection(RowSetTestUtils.projectAll());
    ScanSchemaOrchestrator orchestrator = new ScanSchemaOrchestrator(fixture.allocator(), builder);
    // ... FROM table
    ReaderSchemaOrchestrator reader = orchestrator.startReader();
    // Create the table loader
    ResultSetLoader loader = reader.makeTableLoader(null);
    // Late schema: no batch provided up front.
    assertFalse(reader.hasSchema());
    // Start a batch and discover a schema: (a, b)
    reader.startBatch();
    RowSetLoader writer = loader.writer();
    writer.addColumn(SchemaBuilder.columnSchema("a", MinorType.INT, DataMode.REQUIRED));
    writer.addColumn(SchemaBuilder.columnSchema("b", MinorType.VARCHAR, DataMode.REQUIRED));
    // Create a batch of data using the discovered schema
    writer.addRow(1, "fred").addRow(2, "wilma");
    reader.endBatch();
    // Verify
    TupleMetadata tableSchema = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.VARCHAR).buildSchema();
    SingleRowSet expected = fixture.rowSetBuilder(tableSchema).addRow(1, "fred").addRow(2, "wilma").build();
    new RowSetComparison(expected).verifyAndClearAll(fixture.wrap(orchestrator.output()));
    orchestrator.close();
}

Example 18 with ScanSchemaOrchestrator

use of org.apache.drill.exec.physical.impl.scan.project.ScanSchemaOrchestrator in project drill by apache.

the class TestColumnsArray method testReqularCol.

/**
 * Verify that if the columns column is not required, that `columns`
 * is treated like any other column.
 */
@Test
public void testReqularCol() {
    ScanSchemaOrchestrator scanner = buildScan(false, RowSetTestUtils.projectList(ColumnsScanFramework.COLUMNS_COL));
    TupleMetadata tableSchema = new SchemaBuilder().add(ColumnsScanFramework.COLUMNS_COL, MinorType.VARCHAR).buildSchema();
    ReaderSchemaOrchestrator reader = scanner.startReader();
    ResultSetLoader rsLoader = reader.makeTableLoader(tableSchema);
    reader.defineSchema();
    reader.startBatch();
    rsLoader.writer().addRow("fred");
    reader.endBatch();
    SingleRowSet expected = fixture.rowSetBuilder(tableSchema).addRow("fred").build();
    RowSetUtilities.verify(expected, fixture.wrap(scanner.output()));
    scanner.close();
}

Example 19 with ScanSchemaOrchestrator

use of org.apache.drill.exec.physical.impl.scan.project.ScanSchemaOrchestrator in project drill by apache.

the class TestScanOrchestratorImplicitColumns method testMixture.

/**
 * Test SELECT dir0, b, suffix, c FROM table(a, b)
 * Full combination of metadata, table and null columns
 */
@Test
public void testMixture() {
    ScanOrchestratorBuilder builder = new MockScanBuilder();
    File file = dirTestWatcher.copyResourceToRoot(Paths.get("multilevel", "csv", "1994", "Q1", "orders_94_q1.csv"), Paths.get("x", "y", "z.csv"));
    Path filePath = new Path(file.toURI().getPath());
    ImplicitColumnManager metadataManager = new ImplicitColumnManager(fixture.getOptionManager(), standardOptions(filePath));
    builder.withImplicitColumns(metadataManager);
    // SELECT dir0, b, suffix, c ...
    builder.projection(RowSetTestUtils.projectList("dir0", "b", "suffix", "c"));
    ScanSchemaOrchestrator scanner = new ScanSchemaOrchestrator(fixture.allocator(), builder);
    // ... FROM file
    metadataManager.startFile(filePath);
    ReaderSchemaOrchestrator reader = scanner.startReader();
    // file schema (a, b)
    TupleMetadata tableSchema = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.VARCHAR).buildSchema();
    // Create the table loader
    ResultSetLoader loader = reader.makeTableLoader(tableSchema);
    TupleMetadata expectedSchema = new SchemaBuilder().addNullable("dir0", MinorType.VARCHAR).add("b", MinorType.VARCHAR).add("suffix", MinorType.VARCHAR).addNullable("c", MinorType.INT).buildSchema();
    // Create a batch of data.
    reader.startBatch();
    loader.writer().addRow(1, "fred").addRow(2, "wilma");
    reader.endBatch();
    // Verify
    SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).addRow("x", "fred", "csv", null).addRow("x", "wilma", "csv", null).build();
    RowSetUtilities.verify(expected, fixture.wrap(scanner.output()));
    scanner.close();
}

Example 20 with ScanSchemaOrchestrator

use of org.apache.drill.exec.physical.impl.scan.project.ScanSchemaOrchestrator in project drill by apache.

the class TestColumnsArray method buildScanner.

private MockScanner buildScanner(List<SchemaPath> projList) {
    MockScanner mock = new MockScanner();
    // Set up the file metadata manager
    Path filePath = new Path("hdfs:///w/x/y/z.csv");
    ImplicitColumnManager metadataManager = new ImplicitColumnManager(fixture.getOptionManager(), standardOptions(filePath));
    // ...and the columns array manager
    ColumnsArrayManager colsManager = new ColumnsArrayManager(false);
    // Configure the schema orchestrator
    ScanOrchestratorBuilder builder = new MockScanBuilder();
    builder.withImplicitColumns(metadataManager);
    builder.addParser(colsManager.projectionParser());
    builder.addResolver(colsManager.resolver());
    // SELECT <proj list> ...
    builder.projection(projList);
    mock.scanner = new ScanSchemaOrchestrator(fixture.allocator(), builder);
    // FROM z.csv
    metadataManager.startFile(filePath);
    mock.reader = mock.scanner.startReader();
    // Table schema (columns: VARCHAR[])
    TupleMetadata tableSchema = new SchemaBuilder().addArray(ColumnsScanFramework.COLUMNS_COL, MinorType.VARCHAR).buildSchema();
    mock.loader = mock.reader.makeTableLoader(tableSchema);
    // First empty batch
    mock.reader.defineSchema();
    return mock;
}