use of org.apache.drill.exec.physical.impl.scan.project.NullColumnBuilder.NullBuilderBuilder in project drill by apache.
the class TestSchemaSmoothing method testDiscrete.
/**
* Sanity test for the simple, discrete case. The purpose of
* discrete is just to run the basic lifecycle in a way that
* is compatible with the schema-persistence version.
*/
@Test
public void testDiscrete() {
// Set up the file metadata manager
Path filePathA = new Path("hdfs:///w/x/y/a.csv");
Path filePathB = new Path("hdfs:///w/x/y/b.csv");
ImplicitColumnManager metadataManager = new ImplicitColumnManager(fixture.getOptionManager(), standardOptions(Lists.newArrayList(filePathA, filePathB)));
// Set up the scan level projection
ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList(ScanTestUtils.FILE_NAME_COL, "a", "b"), ScanTestUtils.parsers(metadataManager.projectionParser()));
{
// Define a file a.csv
metadataManager.startFile(filePathA);
// Build the output schema from the (a, b) table schema
TupleMetadata twoColSchema = new SchemaBuilder().add("a", MinorType.INT).addNullable("b", MinorType.VARCHAR, 10).buildSchema();
final NullColumnBuilder builder = new NullBuilderBuilder().build();
ResolvedRow rootTuple = new ResolvedRow(builder);
new ExplicitSchemaProjection(scanProj, twoColSchema, rootTuple, ScanTestUtils.resolvers(metadataManager));
// Verify the full output schema
TupleMetadata expectedSchema = new SchemaBuilder().add("filename", MinorType.VARCHAR).add("a", MinorType.INT).addNullable("b", MinorType.VARCHAR, 10).buildSchema();
// Verify
List<ResolvedColumn> columns = rootTuple.columns();
assertEquals(3, columns.size());
assertTrue(ScanTestUtils.schema(rootTuple).isEquivalent(expectedSchema));
assertEquals(ScanTestUtils.FILE_NAME_COL, columns.get(0).name());
assertEquals("a.csv", ((FileMetadataColumn) columns.get(0)).value());
assertTrue(columns.get(1) instanceof ResolvedTableColumn);
}
{
// Define a file b.csv
metadataManager.startFile(filePathB);
// Build the output schema from the (a) table schema
TupleMetadata oneColSchema = new SchemaBuilder().add("a", MinorType.INT).buildSchema();
final NullColumnBuilder builder = new NullBuilderBuilder().build();
ResolvedRow rootTuple = new ResolvedRow(builder);
new ExplicitSchemaProjection(scanProj, oneColSchema, rootTuple, ScanTestUtils.resolvers(metadataManager));
// Verify the full output schema
// Since this mode is "discrete", we don't remember the type
// of the missing column. (Instead, it is filled in at the
// vector level as part of vector persistence.) During projection, it is
// marked with type NULL so that the null column builder will fill in
// the proper type.
TupleMetadata expectedSchema = new SchemaBuilder().add("filename", MinorType.VARCHAR).add("a", MinorType.INT).addNullable("b", MinorType.NULL).buildSchema();
// Verify
List<ResolvedColumn> columns = rootTuple.columns();
assertEquals(3, columns.size());
assertTrue(ScanTestUtils.schema(rootTuple).isEquivalent(expectedSchema));
assertEquals(ScanTestUtils.FILE_NAME_COL, columns.get(0).name());
assertEquals("b.csv", ((FileMetadataColumn) columns.get(0)).value());
assertTrue(columns.get(1) instanceof ResolvedTableColumn);
assertTrue(columns.get(2) instanceof ResolvedNullColumn);
}
}
use of org.apache.drill.exec.physical.impl.scan.project.NullColumnBuilder.NullBuilderBuilder in project drill by apache.
the class TestSchemaSmoothing method testSmoothingProjection.
/**
* Low-level test of the smoothing projection, including the exceptions
* it throws when things are not going its way.
*/
@Test
public void testSmoothingProjection() {
final ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectAll(), ScanTestUtils.parsers());
// Table 1: (a: nullable bigint, b)
final TupleMetadata schema1 = new SchemaBuilder().addNullable("a", MinorType.BIGINT).addNullable("b", MinorType.VARCHAR).add("c", MinorType.FLOAT8).buildSchema();
ResolvedRow priorSchema;
{
final NullColumnBuilder builder = new NullBuilderBuilder().build();
final ResolvedRow rootTuple = new ResolvedRow(builder);
new WildcardProjection(scanProj, schema1, rootTuple, ScanTestUtils.resolvers());
priorSchema = rootTuple;
}
// Table 2: (a: nullable bigint, c), column omitted, original schema preserved
final TupleMetadata schema2 = new SchemaBuilder().addNullable("a", MinorType.BIGINT).add("c", MinorType.FLOAT8).buildSchema();
try {
final NullColumnBuilder builder = new NullBuilderBuilder().build();
final ResolvedRow rootTuple = new ResolvedRow(builder);
new SmoothingProjection(scanProj, schema2, priorSchema, rootTuple, ScanTestUtils.resolvers());
assertTrue(schema1.isEquivalent(ScanTestUtils.schema(rootTuple)));
priorSchema = rootTuple;
} catch (final IncompatibleSchemaException e) {
fail();
}
// Table 3: (a, c, d), column added, must replan schema
final TupleMetadata schema3 = new SchemaBuilder().addNullable("a", MinorType.BIGINT).addNullable("b", MinorType.VARCHAR).add("c", MinorType.FLOAT8).add("d", MinorType.INT).buildSchema();
try {
final NullColumnBuilder builder = new NullBuilderBuilder().build();
final ResolvedRow rootTuple = new ResolvedRow(builder);
new SmoothingProjection(scanProj, schema3, priorSchema, rootTuple, ScanTestUtils.resolvers());
fail();
} catch (final IncompatibleSchemaException e) {
// Expected
}
// Table 4: (a: double), change type must replan schema
final TupleMetadata schema4 = new SchemaBuilder().addNullable("a", MinorType.FLOAT8).addNullable("b", MinorType.VARCHAR).add("c", MinorType.FLOAT8).buildSchema();
try {
final NullColumnBuilder builder = new NullBuilderBuilder().build();
final ResolvedRow rootTuple = new ResolvedRow(builder);
new SmoothingProjection(scanProj, schema4, priorSchema, rootTuple, ScanTestUtils.resolvers());
fail();
} catch (final IncompatibleSchemaException e) {
// Expected
}
// Table 5: Drop a non-nullable column, must replan
final TupleMetadata schema6 = new SchemaBuilder().addNullable("a", MinorType.BIGINT).addNullable("b", MinorType.VARCHAR).buildSchema();
try {
final NullColumnBuilder builder = new NullBuilderBuilder().build();
final ResolvedRow rootTuple = new ResolvedRow(builder);
new SmoothingProjection(scanProj, schema6, priorSchema, rootTuple, ScanTestUtils.resolvers());
fail();
} catch (final IncompatibleSchemaException e) {
// Expected
}
}
use of org.apache.drill.exec.physical.impl.scan.project.NullColumnBuilder.NullBuilderBuilder in project drill by apache.
the class TestNullColumnLoader method testNullColumnBuilder.
/**
* Test the shim class that adapts between the null column loader
* and the projection mechanism. The projection mechanism uses this
* to pull in the null columns which the null column loader has
* created.
*/
@Test
public void testNullColumnBuilder() {
final ResultVectorCache cache = new NullResultVectorCacheImpl(fixture.allocator());
final NullColumnBuilder builder = new NullBuilderBuilder().build();
builder.add("unspecified");
builder.add("nullType", Types.optional(MinorType.NULL));
builder.add("specifiedOpt", Types.optional(MinorType.VARCHAR));
builder.add("specifiedReq", Types.required(MinorType.VARCHAR));
builder.add("specifiedArray", Types.repeated(MinorType.VARCHAR));
builder.build(cache);
// Create a batch
builder.load(2);
// Verify values and types
final TupleMetadata expectedSchema = new SchemaBuilder().add("unspecified", NullColumnLoader.DEFAULT_NULL_TYPE).add("nullType", NullColumnLoader.DEFAULT_NULL_TYPE).addNullable("specifiedOpt", MinorType.VARCHAR).addNullable("specifiedReq", MinorType.VARCHAR).addArray("specifiedArray", MinorType.VARCHAR).buildSchema();
final SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).addRow(null, null, null, null, new String[] {}).addRow(null, null, null, null, new String[] {}).build();
RowSetUtilities.verify(expected, fixture.wrap(builder.output()));
builder.close();
}
use of org.apache.drill.exec.physical.impl.scan.project.NullColumnBuilder.NullBuilderBuilder in project drill by apache.
the class TestReaderLevelProjection method testOmittedMapMembers.
/**
* Test of a map with missing columns.
* table of (a{b, c}), project a.c, a.d, a.e.f
*/
@Test
public void testOmittedMapMembers() {
// Simulate SELECT a.c, a.d, a.e.f ...
final ScanLevelProjection scanProj = ScanLevelProjection.build(RowSetTestUtils.projectList("x", "a.c", "a.d", "a.e.f", "y"), ScanTestUtils.parsers());
assertEquals(3, scanProj.columns().size());
// Simulate a data source, with early schema, of (x, y, a{b, c})
final TupleMetadata tableSchema = new SchemaBuilder().add("x", MinorType.VARCHAR).add("y", MinorType.INT).addMap("a").add("b", MinorType.BIGINT).add("c", MinorType.FLOAT8).resumeSchema().buildSchema();
final NullColumnBuilder builder = new NullBuilderBuilder().build();
final ResolvedRow rootTuple = new ResolvedRow(builder);
new ExplicitSchemaProjection(scanProj, tableSchema, rootTuple, ScanTestUtils.resolvers());
final List<ResolvedColumn> columns = rootTuple.columns();
assertEquals(3, columns.size());
// Should have resolved a.b to a map column,
// a.d to a missing nested map, and a.e.f to a missing
// nested map member
// X is projected
final ResolvedColumn xCol = columns.get(0);
assertEquals("x", xCol.name());
assertTrue(xCol instanceof ResolvedTableColumn);
assertSame(rootTuple, xCol.source());
assertEquals(0, xCol.sourceIndex());
// Y is projected
final ResolvedColumn yCol = columns.get(2);
assertEquals("y", yCol.name());
assertTrue(yCol instanceof ResolvedTableColumn);
assertSame(rootTuple, yCol.source());
assertEquals(1, yCol.sourceIndex());
// A is projected
final ResolvedColumn aCol = columns.get(1);
assertEquals("a", aCol.name());
assertTrue(aCol instanceof ResolvedMapColumn);
final ResolvedMapColumn aMap = (ResolvedMapColumn) aCol;
final ResolvedTuple aMembers = aMap.members();
assertFalse(aMembers.isSimpleProjection());
assertNotNull(aMembers);
assertEquals(3, aMembers.columns().size());
// a.c is projected
final ResolvedColumn acCol = aMembers.columns().get(0);
assertEquals("c", acCol.name());
assertTrue(acCol instanceof ResolvedTableColumn);
assertEquals(1, acCol.sourceIndex());
// a.d is not in the table, is null
final ResolvedColumn adCol = aMembers.columns().get(1);
assertEquals("d", adCol.name());
assertTrue(adCol instanceof ResolvedNullColumn);
// a.e is not in the table, is implicitly a map
final ResolvedColumn aeCol = aMembers.columns().get(2);
assertEquals("e", aeCol.name());
assertTrue(aeCol instanceof ResolvedMapColumn);
final ResolvedMapColumn aeMap = (ResolvedMapColumn) aeCol;
final ResolvedTuple aeMembers = aeMap.members();
assertFalse(aeMembers.isSimpleProjection());
assertNotNull(aeMembers);
assertEquals(1, aeMembers.columns().size());
// a.d.f is a null column
final ResolvedColumn aefCol = aeMembers.columns().get(0);
assertEquals("f", aefCol.name());
assertTrue(aefCol instanceof ResolvedNullColumn);
}
use of org.apache.drill.exec.physical.impl.scan.project.NullColumnBuilder.NullBuilderBuilder in project drill by apache.
the class TestRowBatchMerger method testMapArrayRevision.
/**
* Test that the merger mechanism can rewrite a map array to include
* projected null columns.
*/
@Test
public void testMapArrayRevision() {
// Create the first batch
TupleMetadata inputSchema = new SchemaBuilder().add("b", MinorType.VARCHAR).addMapArray("a").add("c", MinorType.INT).resumeSchema().buildSchema();
RowSetSource input = new RowSetSource(fixture.rowSetBuilder(inputSchema).addRow("barney", mapArray(singleMap(10), singleMap(11), singleMap(12))).addRow("wilma", mapArray(singleMap(20), singleMap(21))).build());
// Create mappings
NullColumnBuilder builder = new NullBuilderBuilder().build();
ResolvedRow resolvedTuple = new ResolvedRow(builder);
resolvedTuple.add(new TestProjection(resolvedTuple, 0));
ResolvedMapColumn mapCol = new ResolvedMapColumn(resolvedTuple, inputSchema.column(1), 1);
resolvedTuple.add(mapCol);
ResolvedTuple map = mapCol.members();
map.add(new TestProjection(map, 0));
map.add(map.nullBuilder().add("null1"));
// Build the null values
ResultVectorCache cache = new NullResultVectorCacheImpl(fixture.allocator());
resolvedTuple.buildNulls(cache);
// LoadNulls
resolvedTuple.loadNulls(input.rowSet().rowCount());
// Do the merge
VectorContainer output = new VectorContainer(fixture.allocator());
resolvedTuple.project(input.rowSet().container(), output);
output.setRecordCount(input.rowSet().rowCount());
RowSet result = fixture.wrap(output);
// Verify
TupleMetadata expectedSchema = new SchemaBuilder().add("b", MinorType.VARCHAR).addMapArray("a").add("c", MinorType.INT).addNullable("null1", MinorType.INT).resumeSchema().buildSchema();
RowSet expected = fixture.rowSetBuilder(expectedSchema).addRow("barney", mapArray(mapValue(10, null), mapValue(11, null), mapValue(12, null))).addRow("wilma", mapArray(mapValue(20, null), mapValue(21, null))).build();
new RowSetComparison(expected).verifyAndClearAll(result);
}
Aggregations