Search in sources :

Example 6 with PartitionColumn

use of org.apache.drill.exec.physical.impl.scan.file.PartitionColumn in project drill by apache.

the class TestImplicitColumnParser method testLegacyWildcardAndFileMetadataMixed.

/**
 * As above, but include implicit columns before and after the
 * wildcard.
 */
@Test
public void testLegacyWildcardAndFileMetadataMixed() {
    Path filePath = new Path("hdfs:///w/x/y/z.csv");
    ImplicitColumnOptions options = standardOptions(filePath);
    options.useLegacyWildcardExpansion(true);
    ImplicitColumnManager implictColManager = new ImplicitColumnManager(fixture.getOptionManager(), options);
    ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList(ScanTestUtils.FILE_NAME_COL, SchemaPath.DYNAMIC_STAR, ScanTestUtils.SUFFIX_COL), Lists.newArrayList(implictColManager.projectionParser()));
    List<ColumnProjection> cols = scanProj.columns();
    assertEquals(5, cols.size());
    assertTrue(scanProj.columns().get(0) instanceof FileMetadataColumn);
    assertTrue(scanProj.columns().get(1) instanceof UnresolvedWildcardColumn);
    assertTrue(scanProj.columns().get(2) instanceof FileMetadataColumn);
    assertTrue(scanProj.columns().get(3) instanceof PartitionColumn);
    assertTrue(scanProj.columns().get(4) instanceof PartitionColumn);
}
Also used : SchemaPath(org.apache.drill.common.expression.SchemaPath) Path(org.apache.hadoop.fs.Path) ImplicitColumnManager(org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager) ImplicitColumnOptions(org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager.ImplicitColumnOptions) ScanLevelProjection(org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection) ColumnProjection(org.apache.drill.exec.physical.impl.scan.project.ColumnProjection) UnresolvedWildcardColumn(org.apache.drill.exec.physical.impl.scan.project.AbstractUnresolvedColumn.UnresolvedWildcardColumn) PartitionColumn(org.apache.drill.exec.physical.impl.scan.file.PartitionColumn) FileMetadataColumn(org.apache.drill.exec.physical.impl.scan.file.FileMetadataColumn) SubOperatorTest(org.apache.drill.test.SubOperatorTest) Test(org.junit.Test)

Example 7 with PartitionColumn

use of org.apache.drill.exec.physical.impl.scan.file.PartitionColumn in project drill by apache.

the class TestImplicitColumnParser method testLegacyWildcardAndFileMetadata.

/**
 * Combine wildcard and file metadata columns. The wildcard expands
 * table columns but not metadata columns.
 */
@Test
public void testLegacyWildcardAndFileMetadata() {
    Path filePath = new Path("hdfs:///w/x/y/z.csv");
    ImplicitColumnOptions options = standardOptions(filePath);
    options.useLegacyWildcardExpansion(true);
    ImplicitColumnManager implictColManager = new ImplicitColumnManager(fixture.getOptionManager(), options);
    ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList(SchemaPath.DYNAMIC_STAR, ScanTestUtils.FILE_NAME_COL, ScanTestUtils.SUFFIX_COL), Lists.newArrayList(implictColManager.projectionParser()));
    List<ColumnProjection> cols = scanProj.columns();
    assertEquals(5, cols.size());
    assertTrue(scanProj.columns().get(0) instanceof UnresolvedWildcardColumn);
    assertTrue(scanProj.columns().get(1) instanceof FileMetadataColumn);
    assertTrue(scanProj.columns().get(2) instanceof FileMetadataColumn);
    assertTrue(scanProj.columns().get(3) instanceof PartitionColumn);
    assertTrue(scanProj.columns().get(4) instanceof PartitionColumn);
}
Also used : SchemaPath(org.apache.drill.common.expression.SchemaPath) Path(org.apache.hadoop.fs.Path) ImplicitColumnManager(org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager) ImplicitColumnOptions(org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager.ImplicitColumnOptions) ScanLevelProjection(org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection) ColumnProjection(org.apache.drill.exec.physical.impl.scan.project.ColumnProjection) UnresolvedWildcardColumn(org.apache.drill.exec.physical.impl.scan.project.AbstractUnresolvedColumn.UnresolvedWildcardColumn) PartitionColumn(org.apache.drill.exec.physical.impl.scan.file.PartitionColumn) FileMetadataColumn(org.apache.drill.exec.physical.impl.scan.file.FileMetadataColumn) SubOperatorTest(org.apache.drill.test.SubOperatorTest) Test(org.junit.Test)

Example 8 with PartitionColumn

use of org.apache.drill.exec.physical.impl.scan.file.PartitionColumn in project drill by apache.

the class TestConstantColumnLoader method testFileMetadata.

@Test
public void testFileMetadata() {
    FileMetadata fileInfo = new FileMetadata(new Path("hdfs:///w/x/y/z.csv"), new Path("hdfs:///w"));
    List<ConstantColumnSpec> defns = new ArrayList<>();
    FileMetadataColumnDefn iDefn = new FileMetadataColumnDefn(ScanTestUtils.SUFFIX_COL, ImplicitFileColumns.SUFFIX);
    FileMetadataColumn iCol = new FileMetadataColumn(ScanTestUtils.SUFFIX_COL, iDefn, fileInfo, null, 0);
    defns.add(iCol);
    String partColName = ScanTestUtils.partitionColName(1);
    PartitionColumn pCol = new PartitionColumn(partColName, 1, fileInfo, null, 0);
    defns.add(pCol);
    ResultVectorCacheImpl cache = new ResultVectorCacheImpl(fixture.allocator());
    ConstantColumnLoader staticLoader = new ConstantColumnLoader(cache, defns);
    // Create a batch
    staticLoader.load(2);
    // Verify
    TupleMetadata expectedSchema = new SchemaBuilder().add(ScanTestUtils.SUFFIX_COL, MinorType.VARCHAR).addNullable(partColName, MinorType.VARCHAR).buildSchema();
    SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).addRow("csv", "y").addRow("csv", "y").build();
    new RowSetComparison(expected).verifyAndClearAll(fixture.wrap(staticLoader.load(2)));
    staticLoader.close();
}
Also used : Path(org.apache.hadoop.fs.Path) FileMetadataColumnDefn(org.apache.drill.exec.physical.impl.scan.file.FileMetadataColumnDefn) SingleRowSet(org.apache.drill.exec.physical.rowSet.RowSet.SingleRowSet) FileMetadata(org.apache.drill.exec.physical.impl.scan.file.FileMetadata) ArrayList(java.util.ArrayList) PartitionColumn(org.apache.drill.exec.physical.impl.scan.file.PartitionColumn) RowSetComparison(org.apache.drill.test.rowSet.RowSetComparison) ConstantColumnSpec(org.apache.drill.exec.physical.impl.scan.project.ConstantColumnLoader.ConstantColumnSpec) TupleMetadata(org.apache.drill.exec.record.metadata.TupleMetadata) SchemaBuilder(org.apache.drill.exec.record.metadata.SchemaBuilder) FileMetadataColumn(org.apache.drill.exec.physical.impl.scan.file.FileMetadataColumn) ResultVectorCacheImpl(org.apache.drill.exec.physical.resultSet.impl.ResultVectorCacheImpl) SubOperatorTest(org.apache.drill.test.SubOperatorTest) Test(org.junit.Test)

Aggregations

PartitionColumn (org.apache.drill.exec.physical.impl.scan.file.PartitionColumn)8 SubOperatorTest (org.apache.drill.test.SubOperatorTest)8 Path (org.apache.hadoop.fs.Path)8 Test (org.junit.Test)8 ImplicitColumnManager (org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager)7 ScanLevelProjection (org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection)7 SchemaPath (org.apache.drill.common.expression.SchemaPath)6 ColumnProjection (org.apache.drill.exec.physical.impl.scan.project.ColumnProjection)6 UnresolvedWildcardColumn (org.apache.drill.exec.physical.impl.scan.project.AbstractUnresolvedColumn.UnresolvedWildcardColumn)5 FileMetadataColumn (org.apache.drill.exec.physical.impl.scan.file.FileMetadataColumn)4 ImplicitColumnOptions (org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager.ImplicitColumnOptions)4 SchemaBuilder (org.apache.drill.exec.record.metadata.SchemaBuilder)2 TupleMetadata (org.apache.drill.exec.record.metadata.TupleMetadata)2 ArrayList (java.util.ArrayList)1 FileMetadata (org.apache.drill.exec.physical.impl.scan.file.FileMetadata)1 FileMetadataColumnDefn (org.apache.drill.exec.physical.impl.scan.file.FileMetadataColumnDefn)1 UnresolvedColumn (org.apache.drill.exec.physical.impl.scan.project.AbstractUnresolvedColumn.UnresolvedColumn)1 ConstantColumnSpec (org.apache.drill.exec.physical.impl.scan.project.ConstantColumnLoader.ConstantColumnSpec)1 ExplicitSchemaProjection (org.apache.drill.exec.physical.impl.scan.project.ExplicitSchemaProjection)1 NullColumnBuilder (org.apache.drill.exec.physical.impl.scan.project.NullColumnBuilder)1