Search in sources :

Example 1 with ImplicitColumnOptions

use of org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager.ImplicitColumnOptions in project drill by apache.

the class TestImplicitColumnProjection method standardOptions.

private ImplicitColumnOptions standardOptions(List<Path> files) {
    ImplicitColumnOptions options = new ImplicitColumnOptions();
    // Don't expand partition columns for wildcard
    options.useLegacyWildcardExpansion(false);
    options.setSelectionRoot(new Path("hdfs:///w"));
    options.setFiles(files);
    return options;
}
Also used : Path(org.apache.hadoop.fs.Path) ImplicitColumnOptions(org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager.ImplicitColumnOptions)

Example 2 with ImplicitColumnOptions

use of org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager.ImplicitColumnOptions in project drill by apache.

the class TestColumnsArray method standardOptions.

private ImplicitColumnOptions standardOptions(Path filePath) {
    ImplicitColumnOptions options = new ImplicitColumnOptions();
    // Don't expand partition columns for wildcard
    options.useLegacyWildcardExpansion(false);
    options.setSelectionRoot(new Path("hdfs:///w"));
    options.setFiles(Lists.newArrayList(filePath));
    return options;
}
Also used : Path(org.apache.hadoop.fs.Path) SchemaPath(org.apache.drill.common.expression.SchemaPath) ImplicitColumnOptions(org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager.ImplicitColumnOptions)

Example 3 with ImplicitColumnOptions

use of org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager.ImplicitColumnOptions in project drill by apache.

the class TestImplicitColumnParser method standardOptions.

private ImplicitColumnOptions standardOptions(List<Path> files) {
    ImplicitColumnOptions options = new ImplicitColumnOptions();
    // Don't expand partition columns for wildcard
    options.useLegacyWildcardExpansion(false);
    options.setSelectionRoot(new Path("hdfs:///w"));
    options.setFiles(files);
    return options;
}
Also used : SchemaPath(org.apache.drill.common.expression.SchemaPath) Path(org.apache.hadoop.fs.Path) ImplicitColumnOptions(org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager.ImplicitColumnOptions)

Example 4 with ImplicitColumnOptions

use of org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager.ImplicitColumnOptions in project drill by apache.

the class TestImplicitColumnParser method testPreferedWildcardExpansionWithOverlap.

@Test
public void testPreferedWildcardExpansionWithOverlap() {
    Path filePath = new Path("hdfs:///w/x/y/z.csv");
    ImplicitColumnOptions options = standardOptions(filePath);
    options.useLegacyWildcardExpansion(true);
    ImplicitColumnManager implictColManager = new ImplicitColumnManager(fixture.getOptionManager(), options);
    ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList(SchemaPath.DYNAMIC_STAR, ScanTestUtils.partitionColName(1)), Lists.newArrayList(implictColManager.projectionParser()));
    List<ColumnProjection> cols = scanProj.columns();
    assertEquals(3, cols.size());
    assertTrue(scanProj.columns().get(0) instanceof UnresolvedWildcardColumn);
    assertTrue(scanProj.columns().get(1) instanceof PartitionColumn);
    assertEquals(1, ((PartitionColumn) cols.get(1)).partition());
    assertTrue(scanProj.columns().get(2) instanceof PartitionColumn);
    assertEquals(0, ((PartitionColumn) cols.get(2)).partition());
}
Also used : SchemaPath(org.apache.drill.common.expression.SchemaPath) Path(org.apache.hadoop.fs.Path) ImplicitColumnManager(org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager) ImplicitColumnOptions(org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager.ImplicitColumnOptions) ScanLevelProjection(org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection) ColumnProjection(org.apache.drill.exec.physical.impl.scan.project.ColumnProjection) UnresolvedWildcardColumn(org.apache.drill.exec.physical.impl.scan.project.AbstractUnresolvedColumn.UnresolvedWildcardColumn) PartitionColumn(org.apache.drill.exec.physical.impl.scan.file.PartitionColumn) SubOperatorTest(org.apache.drill.test.SubOperatorTest) Test(org.junit.Test)

Example 5 with ImplicitColumnOptions

use of org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager.ImplicitColumnOptions in project drill by apache.

the class TestImplicitColumnParser method testPreferredPartitionExpansion.

@Test
public void testPreferredPartitionExpansion() {
    Path filePath = new Path("hdfs:///w/x/y/z.csv");
    ImplicitColumnOptions options = standardOptions(filePath);
    options.useLegacyWildcardExpansion(true);
    ImplicitColumnManager implictColManager = new ImplicitColumnManager(fixture.getOptionManager(), options);
    ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList(SchemaPath.DYNAMIC_STAR, ScanTestUtils.partitionColName(8)), Lists.newArrayList(implictColManager.projectionParser()));
    List<ColumnProjection> cols = scanProj.columns();
    assertEquals(4, cols.size());
    assertTrue(scanProj.columns().get(0) instanceof UnresolvedWildcardColumn);
    assertTrue(scanProj.columns().get(1) instanceof PartitionColumn);
    assertEquals(8, ((PartitionColumn) cols.get(1)).partition());
    assertTrue(scanProj.columns().get(2) instanceof PartitionColumn);
    assertEquals(0, ((PartitionColumn) cols.get(2)).partition());
    assertTrue(scanProj.columns().get(3) instanceof PartitionColumn);
    assertEquals(1, ((PartitionColumn) cols.get(3)).partition());
}
Also used : SchemaPath(org.apache.drill.common.expression.SchemaPath) Path(org.apache.hadoop.fs.Path) ImplicitColumnManager(org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager) ImplicitColumnOptions(org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager.ImplicitColumnOptions) ScanLevelProjection(org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection) ColumnProjection(org.apache.drill.exec.physical.impl.scan.project.ColumnProjection) UnresolvedWildcardColumn(org.apache.drill.exec.physical.impl.scan.project.AbstractUnresolvedColumn.UnresolvedWildcardColumn) PartitionColumn(org.apache.drill.exec.physical.impl.scan.file.PartitionColumn) SubOperatorTest(org.apache.drill.test.SubOperatorTest) Test(org.junit.Test)

Aggregations

ImplicitColumnOptions (org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager.ImplicitColumnOptions)10 Path (org.apache.hadoop.fs.Path)10 SchemaPath (org.apache.drill.common.expression.SchemaPath)8 ImplicitColumnManager (org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager)4 PartitionColumn (org.apache.drill.exec.physical.impl.scan.file.PartitionColumn)4 UnresolvedWildcardColumn (org.apache.drill.exec.physical.impl.scan.project.AbstractUnresolvedColumn.UnresolvedWildcardColumn)4 ColumnProjection (org.apache.drill.exec.physical.impl.scan.project.ColumnProjection)4 ScanLevelProjection (org.apache.drill.exec.physical.impl.scan.project.ScanLevelProjection)4 SubOperatorTest (org.apache.drill.test.SubOperatorTest)4 Test (org.junit.Test)4 FileMetadataColumn (org.apache.drill.exec.physical.impl.scan.file.FileMetadataColumn)2