Search in sources :

Example 1 with UnresolvedWildcardColumn

use of org.apache.drill.exec.physical.impl.scan.project.AbstractUnresolvedColumn.UnresolvedWildcardColumn in project drill by apache.

the class TestImplicitColumnParser method testRevisedWildcardAndPartition.

/**
 * Include both a wildcard and a partition column. The wildcard, in
 * legacy mode, will create partition columns for any partitions not
 * mentioned in the project list.
 * <p>
 * Tests proposed functionality: included only requested partition
 * columns.
 */
@Test
public void testRevisedWildcardAndPartition() {
    Path filePath = new Path("hdfs:///w/x/y/z.csv");
    ImplicitColumnManager implictColManager = new ImplicitColumnManager(fixture.getOptionManager(), standardOptions(filePath));
    ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList(SchemaPath.DYNAMIC_STAR, ScanTestUtils.partitionColName(8)), Lists.newArrayList(implictColManager.projectionParser()));
    List<ColumnProjection> cols = scanProj.columns();
    assertEquals(2, cols.size());
    assertTrue(scanProj.columns().get(0) instanceof UnresolvedWildcardColumn);
    assertTrue(scanProj.columns().get(1) instanceof PartitionColumn);
}
Also used : SchemaPath(org.apache.drill.common.expression.SchemaPath) Path(org.apache.hadoop.fs.Path) ImplicitColumnManager(org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager) ScanLevelProjection(org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection) ColumnProjection(org.apache.drill.exec.physical.impl.scan.project.ColumnProjection) UnresolvedWildcardColumn(org.apache.drill.exec.physical.impl.scan.project.AbstractUnresolvedColumn.UnresolvedWildcardColumn) PartitionColumn(org.apache.drill.exec.physical.impl.scan.file.PartitionColumn) SubOperatorTest(org.apache.drill.test.SubOperatorTest) Test(org.junit.Test)

Example 2 with UnresolvedWildcardColumn

use of org.apache.drill.exec.physical.impl.scan.project.AbstractUnresolvedColumn.UnresolvedWildcardColumn in project drill by apache.

the class TestImplicitColumnParser method testPreferedWildcardExpansionWithOverlap.

@Test
public void testPreferedWildcardExpansionWithOverlap() {
    Path filePath = new Path("hdfs:///w/x/y/z.csv");
    ImplicitColumnOptions options = standardOptions(filePath);
    options.useLegacyWildcardExpansion(true);
    ImplicitColumnManager implictColManager = new ImplicitColumnManager(fixture.getOptionManager(), options);
    ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList(SchemaPath.DYNAMIC_STAR, ScanTestUtils.partitionColName(1)), Lists.newArrayList(implictColManager.projectionParser()));
    List<ColumnProjection> cols = scanProj.columns();
    assertEquals(3, cols.size());
    assertTrue(scanProj.columns().get(0) instanceof UnresolvedWildcardColumn);
    assertTrue(scanProj.columns().get(1) instanceof PartitionColumn);
    assertEquals(1, ((PartitionColumn) cols.get(1)).partition());
    assertTrue(scanProj.columns().get(2) instanceof PartitionColumn);
    assertEquals(0, ((PartitionColumn) cols.get(2)).partition());
}
Also used : SchemaPath(org.apache.drill.common.expression.SchemaPath) Path(org.apache.hadoop.fs.Path) ImplicitColumnManager(org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager) ImplicitColumnOptions(org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager.ImplicitColumnOptions) ScanLevelProjection(org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection) ColumnProjection(org.apache.drill.exec.physical.impl.scan.project.ColumnProjection) UnresolvedWildcardColumn(org.apache.drill.exec.physical.impl.scan.project.AbstractUnresolvedColumn.UnresolvedWildcardColumn) PartitionColumn(org.apache.drill.exec.physical.impl.scan.file.PartitionColumn) SubOperatorTest(org.apache.drill.test.SubOperatorTest) Test(org.junit.Test)

Example 3 with UnresolvedWildcardColumn

use of org.apache.drill.exec.physical.impl.scan.project.AbstractUnresolvedColumn.UnresolvedWildcardColumn in project drill by apache.

the class TestImplicitColumnParser method testPreferredPartitionExpansion.

@Test
public void testPreferredPartitionExpansion() {
    Path filePath = new Path("hdfs:///w/x/y/z.csv");
    ImplicitColumnOptions options = standardOptions(filePath);
    options.useLegacyWildcardExpansion(true);
    ImplicitColumnManager implictColManager = new ImplicitColumnManager(fixture.getOptionManager(), options);
    ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList(SchemaPath.DYNAMIC_STAR, ScanTestUtils.partitionColName(8)), Lists.newArrayList(implictColManager.projectionParser()));
    List<ColumnProjection> cols = scanProj.columns();
    assertEquals(4, cols.size());
    assertTrue(scanProj.columns().get(0) instanceof UnresolvedWildcardColumn);
    assertTrue(scanProj.columns().get(1) instanceof PartitionColumn);
    assertEquals(8, ((PartitionColumn) cols.get(1)).partition());
    assertTrue(scanProj.columns().get(2) instanceof PartitionColumn);
    assertEquals(0, ((PartitionColumn) cols.get(2)).partition());
    assertTrue(scanProj.columns().get(3) instanceof PartitionColumn);
    assertEquals(1, ((PartitionColumn) cols.get(3)).partition());
}
Also used : SchemaPath(org.apache.drill.common.expression.SchemaPath) Path(org.apache.hadoop.fs.Path) ImplicitColumnManager(org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager) ImplicitColumnOptions(org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager.ImplicitColumnOptions) ScanLevelProjection(org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection) ColumnProjection(org.apache.drill.exec.physical.impl.scan.project.ColumnProjection) UnresolvedWildcardColumn(org.apache.drill.exec.physical.impl.scan.project.AbstractUnresolvedColumn.UnresolvedWildcardColumn) PartitionColumn(org.apache.drill.exec.physical.impl.scan.file.PartitionColumn) SubOperatorTest(org.apache.drill.test.SubOperatorTest) Test(org.junit.Test)

Example 4 with UnresolvedWildcardColumn

use of org.apache.drill.exec.physical.impl.scan.project.AbstractUnresolvedColumn.UnresolvedWildcardColumn in project drill by apache.

the class TestScanLevelProjection method testWildcard.

/**
 * Simulate a SELECT * query by passing "**" (Drill's internal representation
 * of the wildcard) as a column name.
 */
@Test
public void testWildcard() {
    final ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectAll(), ScanTestUtils.parsers());
    assertTrue(scanProj.projectAll());
    assertFalse(scanProj.isEmptyProjection());
    assertEquals(1, scanProj.requestedCols().size());
    assertTrue(scanProj.requestedCols().get(0).isDynamicStar());
    assertEquals(1, scanProj.columns().size());
    assertEquals(SchemaPath.DYNAMIC_STAR, scanProj.columns().get(0).name());
    // Verify bindings
    assertEquals(scanProj.columns().get(0).name(), scanProj.requestedCols().get(0).rootName());
    // Verify column type
    assertTrue(scanProj.columns().get(0) instanceof UnresolvedWildcardColumn);
    // Verify tuple projection
    RequestedTuple outputProj = scanProj.rootProjection();
    assertEquals(1, outputProj.projections().size());
    assertNotNull(outputProj.get(SchemaPath.DYNAMIC_STAR));
    assertTrue(outputProj.get(SchemaPath.DYNAMIC_STAR).isWildcard());
    // Make up a reader schema and test the projection set.
    TupleMetadata readerSchema = new SchemaBuilder().add("a", MinorType.INT).add("c", MinorType.INT).buildSchema();
    ProjectionFilter projSet = scanProj.readerProjection();
    assertTrue(isProjected(projSet, readerSchema.metadata("a")));
    assertTrue(isProjected(projSet, readerSchema.metadata("c")));
}
Also used : ProjectionFilter(org.apache.drill.exec.physical.resultSet.impl.ProjectionFilter) RequestedTuple(org.apache.drill.exec.physical.resultSet.project.RequestedTuple) TupleMetadata(org.apache.drill.exec.record.metadata.TupleMetadata) SchemaBuilder(org.apache.drill.exec.record.metadata.SchemaBuilder) UnresolvedWildcardColumn(org.apache.drill.exec.physical.impl.scan.project.AbstractUnresolvedColumn.UnresolvedWildcardColumn) SubOperatorTest(org.apache.drill.test.SubOperatorTest) Test(org.junit.Test) EvfTest(org.apache.drill.categories.EvfTest)

Example 5 with UnresolvedWildcardColumn

use of org.apache.drill.exec.physical.impl.scan.project.AbstractUnresolvedColumn.UnresolvedWildcardColumn in project drill by apache.

the class TestImplicitColumnParser method testLegacyWildcardAndFileMetadataMixed.

/**
 * As above, but include implicit columns before and after the
 * wildcard.
 */
@Test
public void testLegacyWildcardAndFileMetadataMixed() {
    Path filePath = new Path("hdfs:///w/x/y/z.csv");
    ImplicitColumnOptions options = standardOptions(filePath);
    options.useLegacyWildcardExpansion(true);
    ImplicitColumnManager implictColManager = new ImplicitColumnManager(fixture.getOptionManager(), options);
    ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList(ScanTestUtils.FILE_NAME_COL, SchemaPath.DYNAMIC_STAR, ScanTestUtils.SUFFIX_COL), Lists.newArrayList(implictColManager.projectionParser()));
    List<ColumnProjection> cols = scanProj.columns();
    assertEquals(5, cols.size());
    assertTrue(scanProj.columns().get(0) instanceof FileMetadataColumn);
    assertTrue(scanProj.columns().get(1) instanceof UnresolvedWildcardColumn);
    assertTrue(scanProj.columns().get(2) instanceof FileMetadataColumn);
    assertTrue(scanProj.columns().get(3) instanceof PartitionColumn);
    assertTrue(scanProj.columns().get(4) instanceof PartitionColumn);
}
Also used : SchemaPath(org.apache.drill.common.expression.SchemaPath) Path(org.apache.hadoop.fs.Path) ImplicitColumnManager(org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager) ImplicitColumnOptions(org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager.ImplicitColumnOptions) ScanLevelProjection(org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection) ColumnProjection(org.apache.drill.exec.physical.impl.scan.project.ColumnProjection) UnresolvedWildcardColumn(org.apache.drill.exec.physical.impl.scan.project.AbstractUnresolvedColumn.UnresolvedWildcardColumn) PartitionColumn(org.apache.drill.exec.physical.impl.scan.file.PartitionColumn) FileMetadataColumn(org.apache.drill.exec.physical.impl.scan.file.FileMetadataColumn) SubOperatorTest(org.apache.drill.test.SubOperatorTest) Test(org.junit.Test)

Aggregations

UnresolvedWildcardColumn (org.apache.drill.exec.physical.impl.scan.project.AbstractUnresolvedColumn.UnresolvedWildcardColumn)7 SubOperatorTest (org.apache.drill.test.SubOperatorTest)7 Test (org.junit.Test)7 SchemaPath (org.apache.drill.common.expression.SchemaPath)6 ImplicitColumnManager (org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager)6 ColumnProjection (org.apache.drill.exec.physical.impl.scan.project.ColumnProjection)6 ScanLevelProjection (org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection)6 Path (org.apache.hadoop.fs.Path)6 PartitionColumn (org.apache.drill.exec.physical.impl.scan.file.PartitionColumn)5 ImplicitColumnOptions (org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager.ImplicitColumnOptions)4 FileMetadataColumn (org.apache.drill.exec.physical.impl.scan.file.FileMetadataColumn)2 EvfTest (org.apache.drill.categories.EvfTest)1 ProjectionFilter (org.apache.drill.exec.physical.resultSet.impl.ProjectionFilter)1 RequestedTuple (org.apache.drill.exec.physical.resultSet.project.RequestedTuple)1 SchemaBuilder (org.apache.drill.exec.record.metadata.SchemaBuilder)1 TupleMetadata (org.apache.drill.exec.record.metadata.TupleMetadata)1