Search in sources :

Example 1 with ScanLevelProjection

use of org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection in project drill by apache.

the class TestImplicitColumnProjection method testPartitionColumnTwoDigits.

/**
 * Test the obscure case that the partition column contains two digits:
 * dir11. Also tests the obscure case that the output only has partition
 * columns.
 */
@Test
public void testPartitionColumnTwoDigits() {
    Path filePath = new Path("hdfs:///x/0/1/2/3/4/5/6/7/8/9/10/d11/z.csv");
    ImplicitColumnManager metadataManager = new ImplicitColumnManager(fixture.getOptionManager(), standardOptions(filePath));
    ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList("dir11"), ScanTestUtils.parsers(metadataManager.projectionParser()));
    TupleMetadata tableSchema = new SchemaBuilder().add("a", MinorType.VARCHAR).buildSchema();
    metadataManager.startFile(filePath);
    NullColumnBuilder builder = new NullBuilderBuilder().build();
    ResolvedRow rootTuple = new ResolvedRow(builder);
    new ExplicitSchemaProjection(scanProj, tableSchema, rootTuple, ScanTestUtils.resolvers(metadataManager));
    List<ResolvedColumn> columns = rootTuple.columns();
    assertEquals(1, columns.size());
    assertEquals("d11", ((MetadataColumn) columns.get(0)).value());
}
Also used : Path(org.apache.hadoop.fs.Path) ImplicitColumnManager(org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager) NullColumnBuilder(org.apache.drill.exec.physical.impl.scan.project.NullColumnBuilder) ResolvedRow(org.apache.drill.exec.physical.impl.scan.project.ResolvedTuple.ResolvedRow) ScanLevelProjection(org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection) TupleMetadata(org.apache.drill.exec.record.metadata.TupleMetadata) SchemaBuilder(org.apache.drill.exec.record.metadata.SchemaBuilder) NullBuilderBuilder(org.apache.drill.exec.physical.impl.scan.project.NullColumnBuilder.NullBuilderBuilder) ExplicitSchemaProjection(org.apache.drill.exec.physical.impl.scan.project.ExplicitSchemaProjection) ResolvedColumn(org.apache.drill.exec.physical.impl.scan.project.ResolvedColumn) SubOperatorTest(org.apache.drill.test.SubOperatorTest) Test(org.junit.Test)

Example 2 with ScanLevelProjection

use of org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection in project drill by apache.

the class TestImplicitColumnProjection method testProjectList.

/**
 * Test the file projection planner with metadata.
 */
@Test
public void testProjectList() {
    Path filePath = new Path("hdfs:///w/x/y/z.csv");
    ImplicitColumnManager metadataManager = new ImplicitColumnManager(fixture.getOptionManager(), standardOptions(filePath));
    ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList(ScanTestUtils.FILE_NAME_COL, "a", ScanTestUtils.partitionColName(0)), ScanTestUtils.parsers(metadataManager.projectionParser()));
    assertEquals(3, scanProj.columns().size());
    // Scan-level projection: defines the columns
    {
        assertTrue(scanProj.columns().get(0) instanceof FileMetadataColumn);
        FileMetadataColumn col0 = (FileMetadataColumn) scanProj.columns().get(0);
        assertTrue(col0 instanceof FileMetadataColumn);
        assertEquals(ScanTestUtils.FILE_NAME_COL, col0.name());
        assertEquals(MinorType.VARCHAR, col0.schema().getType().getMinorType());
        assertEquals(DataMode.REQUIRED, col0.schema().getType().getMode());
        ColumnProjection col1 = scanProj.columns().get(1);
        assertTrue(col1 instanceof UnresolvedColumn);
        assertEquals("a", col1.name());
        assertTrue(scanProj.columns().get(2) instanceof PartitionColumn);
        PartitionColumn col2 = (PartitionColumn) scanProj.columns().get(2);
        assertTrue(col2 instanceof PartitionColumn);
        assertEquals(ScanTestUtils.partitionColName(0), col2.name());
        assertEquals(MinorType.VARCHAR, col2.schema().getType().getMinorType());
        assertEquals(DataMode.OPTIONAL, col2.schema().getType().getMode());
    }
    // Schema-level projection, fills in values.
    TupleMetadata tableSchema = new SchemaBuilder().add("a", MinorType.VARCHAR).buildSchema();
    metadataManager.startFile(filePath);
    NullColumnBuilder builder = new NullBuilderBuilder().build();
    ResolvedRow rootTuple = new ResolvedRow(builder);
    new ExplicitSchemaProjection(scanProj, tableSchema, rootTuple, ScanTestUtils.resolvers(metadataManager));
    List<ResolvedColumn> columns = rootTuple.columns();
    assertEquals(3, columns.size());
    {
        assertTrue(columns.get(0) instanceof FileMetadataColumn);
        FileMetadataColumn col0 = (FileMetadataColumn) columns.get(0);
        assertTrue(col0 instanceof FileMetadataColumn);
        assertEquals(ScanTestUtils.FILE_NAME_COL, col0.name());
        assertEquals("z.csv", col0.value());
        assertEquals(MinorType.VARCHAR, col0.schema().getType().getMinorType());
        assertEquals(DataMode.REQUIRED, col0.schema().getType().getMode());
        ResolvedColumn col1 = columns.get(1);
        assertEquals("a", col1.name());
        assertTrue(columns.get(2) instanceof PartitionColumn);
        PartitionColumn col2 = (PartitionColumn) columns.get(2);
        assertTrue(col2 instanceof PartitionColumn);
        assertEquals(ScanTestUtils.partitionColName(0), col2.name());
        assertEquals("x", col2.value());
        assertEquals(MinorType.VARCHAR, col2.schema().getType().getMinorType());
        assertEquals(DataMode.OPTIONAL, col2.schema().getType().getMode());
    }
    // Verify that the file metadata columns were picked out
    assertEquals(2, metadataManager.metadataColumns().size());
    assertSame(columns.get(0), metadataManager.metadataColumns().get(0));
    assertSame(columns.get(2), metadataManager.metadataColumns().get(1));
}
Also used : Path(org.apache.hadoop.fs.Path) ImplicitColumnManager(org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager) ScanLevelProjection(org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection) PartitionColumn(org.apache.drill.exec.physical.impl.scan.file.PartitionColumn) NullBuilderBuilder(org.apache.drill.exec.physical.impl.scan.project.NullColumnBuilder.NullBuilderBuilder) ExplicitSchemaProjection(org.apache.drill.exec.physical.impl.scan.project.ExplicitSchemaProjection) ResolvedColumn(org.apache.drill.exec.physical.impl.scan.project.ResolvedColumn) NullColumnBuilder(org.apache.drill.exec.physical.impl.scan.project.NullColumnBuilder) ResolvedRow(org.apache.drill.exec.physical.impl.scan.project.ResolvedTuple.ResolvedRow) UnresolvedColumn(org.apache.drill.exec.physical.impl.scan.project.AbstractUnresolvedColumn.UnresolvedColumn) ColumnProjection(org.apache.drill.exec.physical.impl.scan.project.ColumnProjection) TupleMetadata(org.apache.drill.exec.record.metadata.TupleMetadata) SchemaBuilder(org.apache.drill.exec.record.metadata.SchemaBuilder) FileMetadataColumn(org.apache.drill.exec.physical.impl.scan.file.FileMetadataColumn) SubOperatorTest(org.apache.drill.test.SubOperatorTest) Test(org.junit.Test)

Example 3 with ScanLevelProjection

use of org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection in project drill by apache.

the class TestColumnsArrayParser method testColumnsElements.

@Test
public void testColumnsElements() {
    ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList(ColumnsScanFramework.COLUMNS_COL + "[3]", ColumnsScanFramework.COLUMNS_COL + "[1]"), ScanTestUtils.parsers(new ColumnsArrayParser(true)));
    assertFalse(scanProj.projectAll());
    assertEquals(2, scanProj.requestedCols().size());
    assertEquals(1, scanProj.columns().size());
    assertEquals(ColumnsScanFramework.COLUMNS_COL, scanProj.columns().get(0).name());
    // Verify column type
    assertTrue(scanProj.columns().get(0) instanceof UnresolvedColumnsArrayColumn);
    UnresolvedColumnsArrayColumn colsCol = (UnresolvedColumnsArrayColumn) scanProj.columns().get(0);
    boolean[] indexes = colsCol.selectedIndexes();
    assertNotNull(indexes);
    assertEquals(4, indexes.length);
    assertFalse(indexes[0]);
    assertTrue(indexes[1]);
    assertFalse(indexes[0]);
    assertTrue(indexes[1]);
}
Also used : UnresolvedColumnsArrayColumn(org.apache.drill.exec.physical.impl.scan.columns.UnresolvedColumnsArrayColumn) ScanLevelProjection(org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection) ColumnsArrayParser(org.apache.drill.exec.physical.impl.scan.columns.ColumnsArrayParser) SubOperatorTest(org.apache.drill.test.SubOperatorTest) Test(org.junit.Test)

Example 4 with ScanLevelProjection

use of org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection in project drill by apache.

the class TestColumnsArrayParser method testMetadataColumnsWithColumnsArray.

/**
 * The `columns` column is special: can't be used with other column names.
 * Make sure that the rule <i>does not</i> apply to implicit columns.
 */
@Test
public void testMetadataColumnsWithColumnsArray() {
    Path filePath = new Path("hdfs:///w/x/y/z.csv");
    ImplicitColumnManager metadataManager = new ImplicitColumnManager(fixture.getOptionManager(), standardOptions(filePath));
    ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList(ScanTestUtils.FILE_NAME_COL, ColumnsScanFramework.COLUMNS_COL, ScanTestUtils.SUFFIX_COL), ScanTestUtils.parsers(new ColumnsArrayParser(true), metadataManager.projectionParser()));
    assertFalse(scanProj.projectAll());
    assertEquals(3, scanProj.columns().size());
    assertEquals(ScanTestUtils.FILE_NAME_COL, scanProj.columns().get(0).name());
    assertEquals(ColumnsScanFramework.COLUMNS_COL, scanProj.columns().get(1).name());
    assertEquals(ScanTestUtils.SUFFIX_COL, scanProj.columns().get(2).name());
    // Verify column type
    assertTrue(scanProj.columns().get(0) instanceof FileMetadataColumn);
    assertTrue(scanProj.columns().get(1) instanceof UnresolvedColumnsArrayColumn);
    assertTrue(scanProj.columns().get(2) instanceof FileMetadataColumn);
}
Also used : SchemaPath(org.apache.drill.common.expression.SchemaPath) Path(org.apache.hadoop.fs.Path) ImplicitColumnManager(org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager) UnresolvedColumnsArrayColumn(org.apache.drill.exec.physical.impl.scan.columns.UnresolvedColumnsArrayColumn) ScanLevelProjection(org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection) ColumnsArrayParser(org.apache.drill.exec.physical.impl.scan.columns.ColumnsArrayParser) FileMetadataColumn(org.apache.drill.exec.physical.impl.scan.file.FileMetadataColumn) SubOperatorTest(org.apache.drill.test.SubOperatorTest) Test(org.junit.Test)

Example 5 with ScanLevelProjection

use of org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection in project drill by apache.

the class TestColumnsArrayParser method testTwoColumnsArray.

/**
 * Requesting `columns` twice: second is ignored.
 */
@Test
public void testTwoColumnsArray() {
    ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList(ColumnsScanFramework.COLUMNS_COL, ColumnsScanFramework.COLUMNS_COL), ScanTestUtils.parsers(new ColumnsArrayParser(false)));
    assertFalse(scanProj.projectAll());
    assertEquals(2, scanProj.requestedCols().size());
    assertEquals(1, scanProj.columns().size());
    assertEquals(ColumnsScanFramework.COLUMNS_COL, scanProj.columns().get(0).name());
}
Also used : ScanLevelProjection(org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection) ColumnsArrayParser(org.apache.drill.exec.physical.impl.scan.columns.ColumnsArrayParser) SubOperatorTest(org.apache.drill.test.SubOperatorTest) Test(org.junit.Test)

Aggregations

ScanLevelProjection (org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection)21 SubOperatorTest (org.apache.drill.test.SubOperatorTest)21 Test (org.junit.Test)21 ImplicitColumnManager (org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager)14 Path (org.apache.hadoop.fs.Path)14 SchemaPath (org.apache.drill.common.expression.SchemaPath)11 ColumnsArrayParser (org.apache.drill.exec.physical.impl.scan.columns.ColumnsArrayParser)8 ColumnProjection (org.apache.drill.exec.physical.impl.scan.project.ColumnProjection)8 UnresolvedColumnsArrayColumn (org.apache.drill.exec.physical.impl.scan.columns.UnresolvedColumnsArrayColumn)7 PartitionColumn (org.apache.drill.exec.physical.impl.scan.file.PartitionColumn)7 FileMetadataColumn (org.apache.drill.exec.physical.impl.scan.file.FileMetadataColumn)6 UnresolvedWildcardColumn (org.apache.drill.exec.physical.impl.scan.project.AbstractUnresolvedColumn.UnresolvedWildcardColumn)6 ImplicitColumnOptions (org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager.ImplicitColumnOptions)4 UnresolvedColumn (org.apache.drill.exec.physical.impl.scan.project.AbstractUnresolvedColumn.UnresolvedColumn)3 ExplicitSchemaProjection (org.apache.drill.exec.physical.impl.scan.project.ExplicitSchemaProjection)3 NullColumnBuilder (org.apache.drill.exec.physical.impl.scan.project.NullColumnBuilder)3 NullBuilderBuilder (org.apache.drill.exec.physical.impl.scan.project.NullColumnBuilder.NullBuilderBuilder)3 ResolvedColumn (org.apache.drill.exec.physical.impl.scan.project.ResolvedColumn)3 ResolvedRow (org.apache.drill.exec.physical.impl.scan.project.ResolvedTuple.ResolvedRow)3 SchemaBuilder (org.apache.drill.exec.record.metadata.SchemaBuilder)3