Examples with NullBuilderBuilder org.apache.drill.exec.physical.impl.scan.project.NullColumnBuilder.NullBuilderBuilder used on opensource projects

Example 21 with NullBuilderBuilder

use of org.apache.drill.exec.physical.impl.scan.project.NullColumnBuilder.NullBuilderBuilder in project drill by apache.

the class TestSchemaSmoothing method testDiscrete.

/**
 * Sanity test for the simple, discrete case. The purpose of
 * discrete is just to run the basic lifecycle in a way that
 * is compatible with the schema-persistence version.
 */
@Test
public void testDiscrete() {
    // Set up the file metadata manager
    Path filePathA = new Path("hdfs:///w/x/y/a.csv");
    Path filePathB = new Path("hdfs:///w/x/y/b.csv");
    ImplicitColumnManager metadataManager = new ImplicitColumnManager(fixture.getOptionManager(), standardOptions(Lists.newArrayList(filePathA, filePathB)));
    // Set up the scan level projection
    ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList(ScanTestUtils.FILE_NAME_COL, "a", "b"), ScanTestUtils.parsers(metadataManager.projectionParser()));
    {
        // Define a file a.csv
        metadataManager.startFile(filePathA);
        // Build the output schema from the (a, b) table schema
        TupleMetadata twoColSchema = new SchemaBuilder().add("a", MinorType.INT).addNullable("b", MinorType.VARCHAR, 10).buildSchema();
        final NullColumnBuilder builder = new NullBuilderBuilder().build();
        ResolvedRow rootTuple = new ResolvedRow(builder);
        new ExplicitSchemaProjection(scanProj, twoColSchema, rootTuple, ScanTestUtils.resolvers(metadataManager));
        // Verify the full output schema
        TupleMetadata expectedSchema = new SchemaBuilder().add("filename", MinorType.VARCHAR).add("a", MinorType.INT).addNullable("b", MinorType.VARCHAR, 10).buildSchema();
        // Verify
        List<ResolvedColumn> columns = rootTuple.columns();
        assertEquals(3, columns.size());
        assertTrue(ScanTestUtils.schema(rootTuple).isEquivalent(expectedSchema));
        assertEquals(ScanTestUtils.FILE_NAME_COL, columns.get(0).name());
        assertEquals("a.csv", ((FileMetadataColumn) columns.get(0)).value());
        assertTrue(columns.get(1) instanceof ResolvedTableColumn);
    }
    {
        // Define a file b.csv
        metadataManager.startFile(filePathB);
        // Build the output schema from the (a) table schema
        TupleMetadata oneColSchema = new SchemaBuilder().add("a", MinorType.INT).buildSchema();
        final NullColumnBuilder builder = new NullBuilderBuilder().build();
        ResolvedRow rootTuple = new ResolvedRow(builder);
        new ExplicitSchemaProjection(scanProj, oneColSchema, rootTuple, ScanTestUtils.resolvers(metadataManager));
        // Verify the full output schema
        // Since this mode is "discrete", we don't remember the type
        // of the missing column. (Instead, it is filled in at the
        // vector level as part of vector persistence.) During projection, it is
        // marked with type NULL so that the null column builder will fill in
        // the proper type.
        TupleMetadata expectedSchema = new SchemaBuilder().add("filename", MinorType.VARCHAR).add("a", MinorType.INT).addNullable("b", MinorType.NULL).buildSchema();
        // Verify
        List<ResolvedColumn> columns = rootTuple.columns();
        assertEquals(3, columns.size());
        assertTrue(ScanTestUtils.schema(rootTuple).isEquivalent(expectedSchema));
        assertEquals(ScanTestUtils.FILE_NAME_COL, columns.get(0).name());
        assertEquals("b.csv", ((FileMetadataColumn) columns.get(0)).value());
        assertTrue(columns.get(1) instanceof ResolvedTableColumn);
        assertTrue(columns.get(2) instanceof ResolvedNullColumn);
    }
}

Example 22 with NullBuilderBuilder

use of org.apache.drill.exec.physical.impl.scan.project.NullColumnBuilder.NullBuilderBuilder in project drill by apache.

the class TestSchemaSmoothing method testSmoothingProjection.

/**
 * Low-level test of the smoothing projection, including the exceptions
 * it throws when things are not going its way.
 */
@Test
public void testSmoothingProjection() {
    final ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectAll(), ScanTestUtils.parsers());
    // Table 1: (a: nullable bigint, b)
    final TupleMetadata schema1 = new SchemaBuilder().addNullable("a", MinorType.BIGINT).addNullable("b", MinorType.VARCHAR).add("c", MinorType.FLOAT8).buildSchema();
    ResolvedRow priorSchema;
    {
        final NullColumnBuilder builder = new NullBuilderBuilder().build();
        final ResolvedRow rootTuple = new ResolvedRow(builder);
        new WildcardProjection(scanProj, schema1, rootTuple, ScanTestUtils.resolvers());
        priorSchema = rootTuple;
    }
    // Table 2: (a: nullable bigint, c), column omitted, original schema preserved
    final TupleMetadata schema2 = new SchemaBuilder().addNullable("a", MinorType.BIGINT).add("c", MinorType.FLOAT8).buildSchema();
    try {
        final NullColumnBuilder builder = new NullBuilderBuilder().build();
        final ResolvedRow rootTuple = new ResolvedRow(builder);
        new SmoothingProjection(scanProj, schema2, priorSchema, rootTuple, ScanTestUtils.resolvers());
        assertTrue(schema1.isEquivalent(ScanTestUtils.schema(rootTuple)));
        priorSchema = rootTuple;
    } catch (final IncompatibleSchemaException e) {
        fail();
    }
    // Table 3: (a, c, d), column added, must replan schema
    final TupleMetadata schema3 = new SchemaBuilder().addNullable("a", MinorType.BIGINT).addNullable("b", MinorType.VARCHAR).add("c", MinorType.FLOAT8).add("d", MinorType.INT).buildSchema();
    try {
        final NullColumnBuilder builder = new NullBuilderBuilder().build();
        final ResolvedRow rootTuple = new ResolvedRow(builder);
        new SmoothingProjection(scanProj, schema3, priorSchema, rootTuple, ScanTestUtils.resolvers());
        fail();
    } catch (final IncompatibleSchemaException e) {
    // Expected
    }
    // Table 4: (a: double), change type must replan schema
    final TupleMetadata schema4 = new SchemaBuilder().addNullable("a", MinorType.FLOAT8).addNullable("b", MinorType.VARCHAR).add("c", MinorType.FLOAT8).buildSchema();
    try {
        final NullColumnBuilder builder = new NullBuilderBuilder().build();
        final ResolvedRow rootTuple = new ResolvedRow(builder);
        new SmoothingProjection(scanProj, schema4, priorSchema, rootTuple, ScanTestUtils.resolvers());
        fail();
    } catch (final IncompatibleSchemaException e) {
    // Expected
    }
    // Table 5: Drop a non-nullable column, must replan
    final TupleMetadata schema6 = new SchemaBuilder().addNullable("a", MinorType.BIGINT).addNullable("b", MinorType.VARCHAR).buildSchema();
    try {
        final NullColumnBuilder builder = new NullBuilderBuilder().build();
        final ResolvedRow rootTuple = new ResolvedRow(builder);
        new SmoothingProjection(scanProj, schema6, priorSchema, rootTuple, ScanTestUtils.resolvers());
        fail();
    } catch (final IncompatibleSchemaException e) {
    // Expected
    }
}

Example 23 with NullBuilderBuilder

use of org.apache.drill.exec.physical.impl.scan.project.NullColumnBuilder.NullBuilderBuilder in project drill by apache.

the class TestNullColumnLoader method testNullColumnBuilder.

/**
 * Test the shim class that adapts between the null column loader
 * and the projection mechanism. The projection mechanism uses this
 * to pull in the null columns which the null column loader has
 * created.
 */
@Test
public void testNullColumnBuilder() {
    final ResultVectorCache cache = new NullResultVectorCacheImpl(fixture.allocator());
    final NullColumnBuilder builder = new NullBuilderBuilder().build();
    builder.add("unspecified");
    builder.add("nullType", Types.optional(MinorType.NULL));
    builder.add("specifiedOpt", Types.optional(MinorType.VARCHAR));
    builder.add("specifiedReq", Types.required(MinorType.VARCHAR));
    builder.add("specifiedArray", Types.repeated(MinorType.VARCHAR));
    builder.build(cache);
    // Create a batch
    builder.load(2);
    // Verify values and types
    final TupleMetadata expectedSchema = new SchemaBuilder().add("unspecified", NullColumnLoader.DEFAULT_NULL_TYPE).add("nullType", NullColumnLoader.DEFAULT_NULL_TYPE).addNullable("specifiedOpt", MinorType.VARCHAR).addNullable("specifiedReq", MinorType.VARCHAR).addArray("specifiedArray", MinorType.VARCHAR).buildSchema();
    final SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).addRow(null, null, null, null, new String[] {}).addRow(null, null, null, null, new String[] {}).build();
    RowSetUtilities.verify(expected, fixture.wrap(builder.output()));
    builder.close();
}

Example 24 with NullBuilderBuilder

use of org.apache.drill.exec.physical.impl.scan.project.NullColumnBuilder.NullBuilderBuilder in project drill by apache.

the class TestReaderLevelProjection method testOmittedMapMembers.

/**
 * Test of a map with missing columns.
 * table of (a{b, c}), project a.c, a.d, a.e.f
 */
@Test
public void testOmittedMapMembers() {
    // Simulate SELECT a.c, a.d, a.e.f ...
    final ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList("x", "a.c", "a.d", "a.e.f", "y"), ScanTestUtils.parsers());
    assertEquals(3, scanProj.columns().size());
    // Simulate a data source, with early schema, of (x, y, a{b, c})
    final TupleMetadata tableSchema = new SchemaBuilder().add("x", MinorType.VARCHAR).add("y", MinorType.INT).addMap("a").add("b", MinorType.BIGINT).add("c", MinorType.FLOAT8).resumeSchema().buildSchema();
    final NullColumnBuilder builder = new NullBuilderBuilder().build();
    final ResolvedRow rootTuple = new ResolvedRow(builder);
    new ExplicitSchemaProjection(scanProj, tableSchema, rootTuple, ScanTestUtils.resolvers());
    final List<ResolvedColumn> columns = rootTuple.columns();
    assertEquals(3, columns.size());
    // Should have resolved a.b to a map column,
    // a.d to a missing nested map, and a.e.f to a missing
    // nested map member
    // X is projected
    final ResolvedColumn xCol = columns.get(0);
    assertEquals("x", xCol.name());
    assertTrue(xCol instanceof ResolvedTableColumn);
    assertSame(rootTuple, xCol.source());
    assertEquals(0, xCol.sourceIndex());
    // Y is projected
    final ResolvedColumn yCol = columns.get(2);
    assertEquals("y", yCol.name());
    assertTrue(yCol instanceof ResolvedTableColumn);
    assertSame(rootTuple, yCol.source());
    assertEquals(1, yCol.sourceIndex());
    // A is projected
    final ResolvedColumn aCol = columns.get(1);
    assertEquals("a", aCol.name());
    assertTrue(aCol instanceof ResolvedMapColumn);
    final ResolvedMapColumn aMap = (ResolvedMapColumn) aCol;
    final ResolvedTuple aMembers = aMap.members();
    assertFalse(aMembers.isSimpleProjection());
    assertNotNull(aMembers);
    assertEquals(3, aMembers.columns().size());
    // a.c is projected
    final ResolvedColumn acCol = aMembers.columns().get(0);
    assertEquals("c", acCol.name());
    assertTrue(acCol instanceof ResolvedTableColumn);
    assertEquals(1, acCol.sourceIndex());
    // a.d is not in the table, is null
    final ResolvedColumn adCol = aMembers.columns().get(1);
    assertEquals("d", adCol.name());
    assertTrue(adCol instanceof ResolvedNullColumn);
    // a.e is not in the table, is implicitly a map
    final ResolvedColumn aeCol = aMembers.columns().get(2);
    assertEquals("e", aeCol.name());
    assertTrue(aeCol instanceof ResolvedMapColumn);
    final ResolvedMapColumn aeMap = (ResolvedMapColumn) aeCol;
    final ResolvedTuple aeMembers = aeMap.members();
    assertFalse(aeMembers.isSimpleProjection());
    assertNotNull(aeMembers);
    assertEquals(1, aeMembers.columns().size());
    // a.d.f is a null column
    final ResolvedColumn aefCol = aeMembers.columns().get(0);
    assertEquals("f", aefCol.name());
    assertTrue(aefCol instanceof ResolvedNullColumn);
}

Example 25 with NullBuilderBuilder

use of org.apache.drill.exec.physical.impl.scan.project.NullColumnBuilder.NullBuilderBuilder in project drill by apache.

the class TestRowBatchMerger method testMapArrayRevision.

/**
 * Test that the merger mechanism can rewrite a map array to include
 * projected null columns.
 */
@Test
public void testMapArrayRevision() {
    // Create the first batch
    TupleMetadata inputSchema = new SchemaBuilder().add("b", MinorType.VARCHAR).addMapArray("a").add("c", MinorType.INT).resumeSchema().buildSchema();
    RowSetSource input = new RowSetSource(fixture.rowSetBuilder(inputSchema).addRow("barney", mapArray(singleMap(10), singleMap(11), singleMap(12))).addRow("wilma", mapArray(singleMap(20), singleMap(21))).build());
    // Create mappings
    NullColumnBuilder builder = new NullBuilderBuilder().build();
    ResolvedRow resolvedTuple = new ResolvedRow(builder);
    resolvedTuple.add(new TestProjection(resolvedTuple, 0));
    ResolvedMapColumn mapCol = new ResolvedMapColumn(resolvedTuple, inputSchema.column(1), 1);
    resolvedTuple.add(mapCol);
    ResolvedTuple map = mapCol.members();
    map.add(new TestProjection(map, 0));
    map.add(map.nullBuilder().add("null1"));
    // Build the null values
    ResultVectorCache cache = new NullResultVectorCacheImpl(fixture.allocator());
    resolvedTuple.buildNulls(cache);
    // LoadNulls
    resolvedTuple.loadNulls(input.rowSet().rowCount());
    // Do the merge
    VectorContainer output = new VectorContainer(fixture.allocator());
    resolvedTuple.project(input.rowSet().container(), output);
    output.setRecordCount(input.rowSet().rowCount());
    RowSet result = fixture.wrap(output);
    // Verify
    TupleMetadata expectedSchema = new SchemaBuilder().add("b", MinorType.VARCHAR).addMapArray("a").add("c", MinorType.INT).addNullable("null1", MinorType.INT).resumeSchema().buildSchema();
    RowSet expected = fixture.rowSetBuilder(expectedSchema).addRow("barney", mapArray(mapValue(10, null), mapValue(11, null), mapValue(12, null))).addRow("wilma", mapArray(mapValue(20, null), mapValue(21, null))).build();
    new RowSetComparison(expected).verifyAndClearAll(result);
}