Examples with ImplicitColumnManager org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager used on opensource projects

Example 11 with ImplicitColumnManager

use of org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager in project drill by apache.

the class TestImplicitColumnParser method testPreferedWildcardExpansionWithOverlap.

@Test
public void testPreferedWildcardExpansionWithOverlap() {
    Path filePath = new Path("hdfs:///w/x/y/z.csv");
    ImplicitColumnOptions options = standardOptions(filePath);
    options.useLegacyWildcardExpansion(true);
    ImplicitColumnManager implictColManager = new ImplicitColumnManager(fixture.getOptionManager(), options);
    ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList(SchemaPath.DYNAMIC_STAR, ScanTestUtils.partitionColName(1)), Lists.newArrayList(implictColManager.projectionParser()));
    List<ColumnProjection> cols = scanProj.columns();
    assertEquals(3, cols.size());
    assertTrue(scanProj.columns().get(0) instanceof UnresolvedWildcardColumn);
    assertTrue(scanProj.columns().get(1) instanceof PartitionColumn);
    assertEquals(1, ((PartitionColumn) cols.get(1)).partition());
    assertTrue(scanProj.columns().get(2) instanceof PartitionColumn);
    assertEquals(0, ((PartitionColumn) cols.get(2)).partition());
}

Example 12 with ImplicitColumnManager

use of org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager in project drill by apache.

the class TestImplicitColumnParser method testPreferredPartitionExpansion.

@Test
public void testPreferredPartitionExpansion() {
    Path filePath = new Path("hdfs:///w/x/y/z.csv");
    ImplicitColumnOptions options = standardOptions(filePath);
    options.useLegacyWildcardExpansion(true);
    ImplicitColumnManager implictColManager = new ImplicitColumnManager(fixture.getOptionManager(), options);
    ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList(SchemaPath.DYNAMIC_STAR, ScanTestUtils.partitionColName(8)), Lists.newArrayList(implictColManager.projectionParser()));
    List<ColumnProjection> cols = scanProj.columns();
    assertEquals(4, cols.size());
    assertTrue(scanProj.columns().get(0) instanceof UnresolvedWildcardColumn);
    assertTrue(scanProj.columns().get(1) instanceof PartitionColumn);
    assertEquals(8, ((PartitionColumn) cols.get(1)).partition());
    assertTrue(scanProj.columns().get(2) instanceof PartitionColumn);
    assertEquals(0, ((PartitionColumn) cols.get(2)).partition());
    assertTrue(scanProj.columns().get(3) instanceof PartitionColumn);
    assertEquals(1, ((PartitionColumn) cols.get(3)).partition());
}

Example 13 with ImplicitColumnManager

use of org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager in project drill by apache.

the class TestSchemaSmoothing method testLongerPartitionLength.

/**
 * If using the legacy wildcard expansion, we are able to use the same
 * schema even if the new partition path is longer than the previous.
 * Because all file names are provided up front.
 */
@Test
public void testLongerPartitionLength() {
    // Set up the file metadata manager
    Path filePathA = new Path("hdfs:///w/x/a.csv");
    Path filePathB = new Path("hdfs:///w/x/y/b.csv");
    ImplicitColumnManager metadataManager = new ImplicitColumnManager(fixture.getOptionManager(), standardOptions(Lists.newArrayList(filePathA, filePathB)));
    // Set up the scan level projection
    ScanLevelProjection scanProj = ScanLevelProjection.build(ScanTestUtils.projectAllWithAllImplicit(2), ScanTestUtils.parsers(metadataManager.projectionParser()));
    // Define the schema smoother
    SchemaSmoother smoother = new SchemaSmoother(scanProj, ScanTestUtils.resolvers(metadataManager));
    TupleMetadata tableSchema = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.VARCHAR).buildSchema();
    TupleMetadata expectedSchema = ScanTestUtils.expandImplicit(tableSchema, metadataManager, 2);
    {
        metadataManager.startFile(filePathA);
        ResolvedRow rootTuple = doResolve(smoother, tableSchema);
        assertTrue(ScanTestUtils.schema(rootTuple).isEquivalent(expectedSchema));
    }
    {
        metadataManager.startFile(filePathB);
        ResolvedRow rootTuple = doResolve(smoother, tableSchema);
        assertEquals(1, smoother.schemaVersion());
        assertTrue(ScanTestUtils.schema(rootTuple).isEquivalent(expectedSchema));
    }
}

Example 14 with ImplicitColumnManager

use of org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager in project drill by apache.

the class TestSchemaSmoothing method testShorterPartitionLength.

/**
 * If using the legacy wildcard expansion, reuse schema if the new partition path
 * is shorter than the previous. (Unneeded partitions will be set to null by the
 * scan projector.)
 */
@Test
public void testShorterPartitionLength() {
    // Set up the file metadata manager
    Path filePathA = new Path("hdfs:///w/x/y/a.csv");
    Path filePathB = new Path("hdfs:///w/x/b.csv");
    ImplicitColumnManager metadataManager = new ImplicitColumnManager(fixture.getOptionManager(), standardOptions(Lists.newArrayList(filePathA, filePathB)));
    // Set up the scan level projection
    ScanLevelProjection scanProj = ScanLevelProjection.build(ScanTestUtils.projectAllWithAllImplicit(2), ScanTestUtils.parsers(metadataManager.projectionParser()));
    // Define the schema smoother
    SchemaSmoother smoother = new SchemaSmoother(scanProj, ScanTestUtils.resolvers(metadataManager));
    TupleMetadata tableSchema = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.VARCHAR).buildSchema();
    TupleMetadata expectedSchema = ScanTestUtils.expandImplicit(tableSchema, metadataManager, 2);
    {
        metadataManager.startFile(filePathA);
        ResolvedRow rootTuple = doResolve(smoother, tableSchema);
        assertTrue(ScanTestUtils.schema(rootTuple).isEquivalent(expectedSchema));
    }
    {
        metadataManager.startFile(filePathB);
        ResolvedRow rootTuple = doResolve(smoother, tableSchema);
        assertEquals(1, smoother.schemaVersion());
        assertTrue(ScanTestUtils.schema(rootTuple).isEquivalent(expectedSchema));
    }
}

Example 15 with ImplicitColumnManager

use of org.apache.drill.exec.physical.impl.scan.file.ImplicitColumnManager in project drill by apache.

the class TestSchemaSmoothing method testSamePartitionLength.

/**
 * If using the legacy wildcard expansion, reuse schema if partition paths
 * are the same length.
 */
@Test
public void testSamePartitionLength() {
    // Set up the file metadata manager
    Path filePathA = new Path("hdfs:///w/x/y/a.csv");
    Path filePathB = new Path("hdfs:///w/x/y/b.csv");
    ImplicitColumnManager metadataManager = new ImplicitColumnManager(fixture.getOptionManager(), standardOptions(Lists.newArrayList(filePathA, filePathB)));
    // Set up the scan level projection
    ScanLevelProjection scanProj = ScanLevelProjection.build(ScanTestUtils.projectAllWithAllImplicit(2), ScanTestUtils.parsers(metadataManager.projectionParser()));
    // Define the schema smoother
    SchemaSmoother smoother = new SchemaSmoother(scanProj, ScanTestUtils.resolvers(metadataManager));
    TupleMetadata tableSchema = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.VARCHAR).buildSchema();
    TupleMetadata expectedSchema = ScanTestUtils.expandImplicit(tableSchema, metadataManager, 2);
    {
        metadataManager.startFile(filePathA);
        ResolvedRow rootTuple = doResolve(smoother, tableSchema);
        assertTrue(ScanTestUtils.schema(rootTuple).isEquivalent(expectedSchema));
    }
    {
        metadataManager.startFile(filePathB);
        ResolvedRow rootTuple = doResolve(smoother, tableSchema);
        assertEquals(1, smoother.schemaVersion());
        assertTrue(ScanTestUtils.schema(rootTuple).isEquivalent(expectedSchema));
    }
}