use of org.apache.drill.exec.physical.impl.scan.project.ColumnProjection in project drill by apache.
the class TestImplicitColumnProjection method testProjectList.
/**
* Test the file projection planner with metadata.
*/
@Test
public void testProjectList() {
Path filePath = new Path("hdfs:///w/x/y/z.csv");
ImplicitColumnManager metadataManager = new ImplicitColumnManager(fixture.getOptionManager(), standardOptions(filePath));
ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList(ScanTestUtils.FILE_NAME_COL, "a", ScanTestUtils.partitionColName(0)), ScanTestUtils.parsers(metadataManager.projectionParser()));
assertEquals(3, scanProj.columns().size());
// Scan-level projection: defines the columns
{
assertTrue(scanProj.columns().get(0) instanceof FileMetadataColumn);
FileMetadataColumn col0 = (FileMetadataColumn) scanProj.columns().get(0);
assertTrue(col0 instanceof FileMetadataColumn);
assertEquals(ScanTestUtils.FILE_NAME_COL, col0.name());
assertEquals(MinorType.VARCHAR, col0.schema().getType().getMinorType());
assertEquals(DataMode.REQUIRED, col0.schema().getType().getMode());
ColumnProjection col1 = scanProj.columns().get(1);
assertTrue(col1 instanceof UnresolvedColumn);
assertEquals("a", col1.name());
assertTrue(scanProj.columns().get(2) instanceof PartitionColumn);
PartitionColumn col2 = (PartitionColumn) scanProj.columns().get(2);
assertTrue(col2 instanceof PartitionColumn);
assertEquals(ScanTestUtils.partitionColName(0), col2.name());
assertEquals(MinorType.VARCHAR, col2.schema().getType().getMinorType());
assertEquals(DataMode.OPTIONAL, col2.schema().getType().getMode());
}
// Schema-level projection, fills in values.
TupleMetadata tableSchema = new SchemaBuilder().add("a", MinorType.VARCHAR).buildSchema();
metadataManager.startFile(filePath);
NullColumnBuilder builder = new NullBuilderBuilder().build();
ResolvedRow rootTuple = new ResolvedRow(builder);
new ExplicitSchemaProjection(scanProj, tableSchema, rootTuple, ScanTestUtils.resolvers(metadataManager));
List<ResolvedColumn> columns = rootTuple.columns();
assertEquals(3, columns.size());
{
assertTrue(columns.get(0) instanceof FileMetadataColumn);
FileMetadataColumn col0 = (FileMetadataColumn) columns.get(0);
assertTrue(col0 instanceof FileMetadataColumn);
assertEquals(ScanTestUtils.FILE_NAME_COL, col0.name());
assertEquals("z.csv", col0.value());
assertEquals(MinorType.VARCHAR, col0.schema().getType().getMinorType());
assertEquals(DataMode.REQUIRED, col0.schema().getType().getMode());
ResolvedColumn col1 = columns.get(1);
assertEquals("a", col1.name());
assertTrue(columns.get(2) instanceof PartitionColumn);
PartitionColumn col2 = (PartitionColumn) columns.get(2);
assertTrue(col2 instanceof PartitionColumn);
assertEquals(ScanTestUtils.partitionColName(0), col2.name());
assertEquals("x", col2.value());
assertEquals(MinorType.VARCHAR, col2.schema().getType().getMinorType());
assertEquals(DataMode.OPTIONAL, col2.schema().getType().getMode());
}
// Verify that the file metadata columns were picked out
assertEquals(2, metadataManager.metadataColumns().size());
assertSame(columns.get(0), metadataManager.metadataColumns().get(0));
assertSame(columns.get(2), metadataManager.metadataColumns().get(1));
}
use of org.apache.drill.exec.physical.impl.scan.project.ColumnProjection in project drill by apache.
the class TestImplicitColumnParser method testRevisedWildcardAndPartition.
/**
* Include both a wildcard and a partition column. The wildcard, in
* legacy mode, will create partition columns for any partitions not
* mentioned in the project list.
* <p>
* Tests proposed functionality: included only requested partition
* columns.
*/
@Test
public void testRevisedWildcardAndPartition() {
Path filePath = new Path("hdfs:///w/x/y/z.csv");
ImplicitColumnManager implictColManager = new ImplicitColumnManager(fixture.getOptionManager(), standardOptions(filePath));
ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList(SchemaPath.DYNAMIC_STAR, ScanTestUtils.partitionColName(8)), Lists.newArrayList(implictColManager.projectionParser()));
List<ColumnProjection> cols = scanProj.columns();
assertEquals(2, cols.size());
assertTrue(scanProj.columns().get(0) instanceof UnresolvedWildcardColumn);
assertTrue(scanProj.columns().get(1) instanceof PartitionColumn);
}
use of org.apache.drill.exec.physical.impl.scan.project.ColumnProjection in project drill by apache.
the class TestImplicitColumnParser method testPreferedWildcardExpansionWithOverlap.
@Test
public void testPreferedWildcardExpansionWithOverlap() {
Path filePath = new Path("hdfs:///w/x/y/z.csv");
ImplicitColumnOptions options = standardOptions(filePath);
options.useLegacyWildcardExpansion(true);
ImplicitColumnManager implictColManager = new ImplicitColumnManager(fixture.getOptionManager(), options);
ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList(SchemaPath.DYNAMIC_STAR, ScanTestUtils.partitionColName(1)), Lists.newArrayList(implictColManager.projectionParser()));
List<ColumnProjection> cols = scanProj.columns();
assertEquals(3, cols.size());
assertTrue(scanProj.columns().get(0) instanceof UnresolvedWildcardColumn);
assertTrue(scanProj.columns().get(1) instanceof PartitionColumn);
assertEquals(1, ((PartitionColumn) cols.get(1)).partition());
assertTrue(scanProj.columns().get(2) instanceof PartitionColumn);
assertEquals(0, ((PartitionColumn) cols.get(2)).partition());
}
use of org.apache.drill.exec.physical.impl.scan.project.ColumnProjection in project drill by apache.
the class TestImplicitColumnParser method testPreferredPartitionExpansion.
@Test
public void testPreferredPartitionExpansion() {
Path filePath = new Path("hdfs:///w/x/y/z.csv");
ImplicitColumnOptions options = standardOptions(filePath);
options.useLegacyWildcardExpansion(true);
ImplicitColumnManager implictColManager = new ImplicitColumnManager(fixture.getOptionManager(), options);
ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList(SchemaPath.DYNAMIC_STAR, ScanTestUtils.partitionColName(8)), Lists.newArrayList(implictColManager.projectionParser()));
List<ColumnProjection> cols = scanProj.columns();
assertEquals(4, cols.size());
assertTrue(scanProj.columns().get(0) instanceof UnresolvedWildcardColumn);
assertTrue(scanProj.columns().get(1) instanceof PartitionColumn);
assertEquals(8, ((PartitionColumn) cols.get(1)).partition());
assertTrue(scanProj.columns().get(2) instanceof PartitionColumn);
assertEquals(0, ((PartitionColumn) cols.get(2)).partition());
assertTrue(scanProj.columns().get(3) instanceof PartitionColumn);
assertEquals(1, ((PartitionColumn) cols.get(3)).partition());
}
use of org.apache.drill.exec.physical.impl.scan.project.ColumnProjection in project drill by apache.
the class TestImplicitColumnParser method testShadowed.
/**
* Verify that names that look like metadata columns, but appear
* to be maps or arrays, are not interpreted as metadata. That is,
* the projected table map or array "shadows" the metadata column.
*/
@Test
public void testShadowed() {
Path filePath = new Path("hdfs:///w/x/y/z.csv");
ImplicitColumnManager implictColManager = new ImplicitColumnManager(fixture.getOptionManager(), standardOptions(filePath));
ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList(ScanTestUtils.FILE_NAME_COL + ".a", ScanTestUtils.FILE_PATH_COL + "[0]", ScanTestUtils.partitionColName(0) + ".b", ScanTestUtils.partitionColName(1) + "[0]", ScanTestUtils.SUFFIX_COL), Lists.newArrayList(implictColManager.projectionParser()));
List<ColumnProjection> cols = scanProj.columns();
assertEquals(5, cols.size());
for (int i = 0; i < 4; i++) {
assertTrue(scanProj.columns().get(i) instanceof UnresolvedColumn);
}
assertTrue(scanProj.columns().get(4) instanceof FileMetadataColumn);
}
Aggregations