Search in sources :

Example 11 with ScanLevelProjection

use of org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection in project drill by apache.

the class TestImplicitColumnProjection method testFileMetadata.

/**
 * Test a query with explicit mention of file metadata columns.
 */
@Test
public void testFileMetadata() {
    Path filePath = new Path("hdfs:///w/x/y/z.csv");
    ImplicitColumnManager metadataManager = new ImplicitColumnManager(fixture.getOptionManager(), standardOptions(filePath));
    ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList("a", ScanTestUtils.FULLY_QUALIFIED_NAME_COL, // Sic, to test case sensitivity
    "filEPath", ScanTestUtils.FILE_NAME_COL, ScanTestUtils.SUFFIX_COL), ScanTestUtils.parsers(metadataManager.projectionParser()));
    assertEquals(5, scanProj.columns().size());
    assertEquals(ScanTestUtils.FULLY_QUALIFIED_NAME_COL, scanProj.columns().get(1).name());
    assertEquals("filEPath", scanProj.columns().get(2).name());
    assertEquals(ScanTestUtils.FILE_NAME_COL, scanProj.columns().get(3).name());
    assertEquals(ScanTestUtils.SUFFIX_COL, scanProj.columns().get(4).name());
    // Schema-level projection, fills in values.
    TupleMetadata tableSchema = new SchemaBuilder().add("a", MinorType.VARCHAR).buildSchema();
    metadataManager.startFile(filePath);
    NullColumnBuilder builder = new NullBuilderBuilder().build();
    ResolvedRow rootTuple = new ResolvedRow(builder);
    new ExplicitSchemaProjection(scanProj, tableSchema, rootTuple, ScanTestUtils.resolvers(metadataManager));
    List<ResolvedColumn> columns = rootTuple.columns();
    assertEquals(5, columns.size());
    assertEquals("/w/x/y/z.csv", ((MetadataColumn) columns.get(1)).value());
    assertEquals("/w/x/y", ((MetadataColumn) columns.get(2)).value());
    assertEquals("z.csv", ((MetadataColumn) columns.get(3)).value());
    assertEquals("csv", ((MetadataColumn) columns.get(4)).value());
}
Also used : Path(org.apache.hadoop.fs.Path) ImplicitColumnManager(org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager) NullColumnBuilder(org.apache.drill.exec.physical.impl.scan.project.NullColumnBuilder) ResolvedRow(org.apache.drill.exec.physical.impl.scan.project.ResolvedTuple.ResolvedRow) ScanLevelProjection(org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection) TupleMetadata(org.apache.drill.exec.record.metadata.TupleMetadata) SchemaBuilder(org.apache.drill.exec.record.metadata.SchemaBuilder) NullBuilderBuilder(org.apache.drill.exec.physical.impl.scan.project.NullColumnBuilder.NullBuilderBuilder) ExplicitSchemaProjection(org.apache.drill.exec.physical.impl.scan.project.ExplicitSchemaProjection) ResolvedColumn(org.apache.drill.exec.physical.impl.scan.project.ResolvedColumn) SubOperatorTest(org.apache.drill.test.SubOperatorTest) Test(org.junit.Test)

Example 12 with ScanLevelProjection

use of org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection in project drill by apache.

the class TestColumnsArrayParser method testColumnsArray.

/**
 * Test the special "columns" column that asks to return all columns
 * as an array. No need for early schema. This case is special: it actually
 * creates the one and only table column to match the desired output column.
 */
@Test
public void testColumnsArray() {
    ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList(ColumnsScanFramework.COLUMNS_COL), ScanTestUtils.parsers(new ColumnsArrayParser(true)));
    assertFalse(scanProj.projectAll());
    assertEquals(1, scanProj.requestedCols().size());
    assertEquals(1, scanProj.columns().size());
    assertEquals(ColumnsScanFramework.COLUMNS_COL, scanProj.columns().get(0).name());
    // Verify column type
    assertTrue(scanProj.columns().get(0) instanceof UnresolvedColumnsArrayColumn);
}
Also used : UnresolvedColumnsArrayColumn(org.apache.drill.exec.physical.impl.scan.columns.UnresolvedColumnsArrayColumn) ScanLevelProjection(org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection) ColumnsArrayParser(org.apache.drill.exec.physical.impl.scan.columns.ColumnsArrayParser) SubOperatorTest(org.apache.drill.test.SubOperatorTest) Test(org.junit.Test)

Example 13 with ScanLevelProjection

use of org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection in project drill by apache.

the class TestColumnsArrayParser method testRequiredWildcard.

@Test
public void testRequiredWildcard() {
    ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectAll(), ScanTestUtils.parsers(new ColumnsArrayParser(true)));
    assertTrue(scanProj.projectAll());
    assertEquals(1, scanProj.requestedCols().size());
    assertEquals(1, scanProj.columns().size());
    assertEquals(ColumnsScanFramework.COLUMNS_COL, scanProj.columns().get(0).name());
    // Verify column type
    assertTrue(scanProj.columns().get(0) instanceof UnresolvedColumnsArrayColumn);
}
Also used : UnresolvedColumnsArrayColumn(org.apache.drill.exec.physical.impl.scan.columns.UnresolvedColumnsArrayColumn) ScanLevelProjection(org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection) ColumnsArrayParser(org.apache.drill.exec.physical.impl.scan.columns.ColumnsArrayParser) SubOperatorTest(org.apache.drill.test.SubOperatorTest) Test(org.junit.Test)

Example 14 with ScanLevelProjection

use of org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection in project drill by apache.

the class TestColumnsArrayParser method testRequiredColumnsArray.

@Test
public void testRequiredColumnsArray() {
    ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList(ColumnsScanFramework.COLUMNS_COL), ScanTestUtils.parsers(new ColumnsArrayParser(true)));
    assertFalse(scanProj.projectAll());
    assertEquals(1, scanProj.requestedCols().size());
    assertEquals(1, scanProj.columns().size());
    assertEquals(ColumnsScanFramework.COLUMNS_COL, scanProj.columns().get(0).name());
    // Verify column type
    assertTrue(scanProj.columns().get(0) instanceof UnresolvedColumnsArrayColumn);
}
Also used : UnresolvedColumnsArrayColumn(org.apache.drill.exec.physical.impl.scan.columns.UnresolvedColumnsArrayColumn) ScanLevelProjection(org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection) ColumnsArrayParser(org.apache.drill.exec.physical.impl.scan.columns.ColumnsArrayParser) SubOperatorTest(org.apache.drill.test.SubOperatorTest) Test(org.junit.Test)

Example 15 with ScanLevelProjection

use of org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection in project drill by apache.

the class TestColumnsArrayParser method testWildcardAndColumns.

/**
 * If a query is of the form:
 * <pre><code>
 * select * from dfs.`multilevel/csv` where columns[1] < 1000
 * </code><pre>
 * Then the projection list passed to the scan operator
 * includes both the wildcard and the `columns` array.
 * We can ignore one of them.
 */
@Test
public void testWildcardAndColumns() {
    ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList(SchemaPath.DYNAMIC_STAR, ColumnsScanFramework.COLUMNS_COL), ScanTestUtils.parsers(new ColumnsArrayParser(true)));
    assertTrue(scanProj.projectAll());
    assertEquals(2, scanProj.requestedCols().size());
    assertEquals(1, scanProj.columns().size());
    assertEquals(ColumnsScanFramework.COLUMNS_COL, scanProj.columns().get(0).name());
    // Verify column type
    assertTrue(scanProj.columns().get(0) instanceof UnresolvedColumnsArrayColumn);
}
Also used : UnresolvedColumnsArrayColumn(org.apache.drill.exec.physical.impl.scan.columns.UnresolvedColumnsArrayColumn) ScanLevelProjection(org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection) ColumnsArrayParser(org.apache.drill.exec.physical.impl.scan.columns.ColumnsArrayParser) SubOperatorTest(org.apache.drill.test.SubOperatorTest) Test(org.junit.Test)

Aggregations

ScanLevelProjection (org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection)21 SubOperatorTest (org.apache.drill.test.SubOperatorTest)21 Test (org.junit.Test)21 ImplicitColumnManager (org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager)14 Path (org.apache.hadoop.fs.Path)14 SchemaPath (org.apache.drill.common.expression.SchemaPath)11 ColumnsArrayParser (org.apache.drill.exec.physical.impl.scan.columns.ColumnsArrayParser)8 ColumnProjection (org.apache.drill.exec.physical.impl.scan.project.ColumnProjection)8 UnresolvedColumnsArrayColumn (org.apache.drill.exec.physical.impl.scan.columns.UnresolvedColumnsArrayColumn)7 PartitionColumn (org.apache.drill.exec.physical.impl.scan.file.PartitionColumn)7 FileMetadataColumn (org.apache.drill.exec.physical.impl.scan.file.FileMetadataColumn)6 UnresolvedWildcardColumn (org.apache.drill.exec.physical.impl.scan.project.AbstractUnresolvedColumn.UnresolvedWildcardColumn)6 ImplicitColumnOptions (org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager.ImplicitColumnOptions)4 UnresolvedColumn (org.apache.drill.exec.physical.impl.scan.project.AbstractUnresolvedColumn.UnresolvedColumn)3 ExplicitSchemaProjection (org.apache.drill.exec.physical.impl.scan.project.ExplicitSchemaProjection)3 NullColumnBuilder (org.apache.drill.exec.physical.impl.scan.project.NullColumnBuilder)3 NullBuilderBuilder (org.apache.drill.exec.physical.impl.scan.project.NullColumnBuilder.NullBuilderBuilder)3 ResolvedColumn (org.apache.drill.exec.physical.impl.scan.project.ResolvedColumn)3 ResolvedRow (org.apache.drill.exec.physical.impl.scan.project.ResolvedTuple.ResolvedRow)3 SchemaBuilder (org.apache.drill.exec.record.metadata.SchemaBuilder)3