use of org.apache.drill.exec.record.metadata.SchemaBuilder in project drill by apache.
the class TestConstantColumnLoader method testConstantColumnLoader.
/**
* Test the static column loader using one column of each type.
* The null column is of type int, but the associated value is of
* type string. This is a bit odd, but works out because we detect that
* the string value is null and call setNull on the writer, and avoid
* using the actual data.
*/
@Test
public void testConstantColumnLoader() {
final MajorType aType = MajorType.newBuilder().setMinorType(MinorType.VARCHAR).setMode(DataMode.REQUIRED).build();
final MajorType bType = MajorType.newBuilder().setMinorType(MinorType.VARCHAR).setMode(DataMode.OPTIONAL).build();
final List<ConstantColumnSpec> defns = new ArrayList<>();
defns.add(new DummyColumn("a", aType, "a-value"));
defns.add(new DummyColumn("b", bType, "b-value"));
final ResultVectorCacheImpl cache = new ResultVectorCacheImpl(fixture.allocator());
final ConstantColumnLoader staticLoader = new ConstantColumnLoader(cache, defns);
// Create a batch
staticLoader.load(2);
// Verify
final TupleMetadata expectedSchema = new SchemaBuilder().add("a", aType).add("b", bType).buildSchema();
final SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).addRow("a-value", "b-value").addRow("a-value", "b-value").build();
new RowSetComparison(expected).verifyAndClearAll(fixture.wrap(staticLoader.load(2)));
staticLoader.close();
}
use of org.apache.drill.exec.record.metadata.SchemaBuilder in project drill by apache.
the class TestNullColumnLoader method testCustomNullType.
/**
* Test the ability to use a type other than nullable INT for null
* columns. This occurs, for example, in the CSV reader where no
* column is ever INT (nullable or otherwise) and we want our null
* columns to be (non-nullable) VARCHAR.
*/
@Test
public void testCustomNullType() {
final List<ResolvedNullColumn> defns = new ArrayList<>();
defns.add(makeNullCol("unspecified", null));
defns.add(makeNullCol("nullType", MajorType.newBuilder().setMinorType(MinorType.NULL).setMode(DataMode.OPTIONAL).build()));
// Null required is an oxymoron, so is not tested.
// Null type array does not make sense, so is not tested.
final ResultVectorCache cache = new NullResultVectorCacheImpl(fixture.allocator());
final MajorType nullType = MajorType.newBuilder().setMinorType(MinorType.VARCHAR).setMode(DataMode.OPTIONAL).build();
final NullColumnLoader staticLoader = new NullColumnLoader(cache, defns, nullType, false);
// Create a batch
final VectorContainer output = staticLoader.load(2);
// Verify values and types
final TupleMetadata expectedSchema = new SchemaBuilder().add("unspecified", nullType).add("nullType", nullType).buildSchema();
final SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).addRow(null, null).addRow(null, null).build();
RowSetUtilities.verify(expected, fixture.wrap(output));
staticLoader.close();
}
use of org.apache.drill.exec.record.metadata.SchemaBuilder in project drill by apache.
the class TestNullColumnLoader method testDefaultValue.
/**
* Test the ability to provide a default value for a "null" column.
* Default values are only allowed for required "null" columns. For
* nullable columns, NULL is already the default.
*/
@Test
public void testDefaultValue() {
final List<ResolvedNullColumn> defns = new ArrayList<>();
defns.add(makeNullCol("int", Types.required(MinorType.INT), "10"));
defns.add(makeNullCol("str", Types.required(MinorType.VARCHAR), "foo"));
defns.add(makeNullCol("dub", Types.required(MinorType.FLOAT8), "20.0"));
final ResultVectorCache cache = new NullResultVectorCacheImpl(fixture.allocator());
final MajorType nullType = Types.optional(MinorType.VARCHAR);
final NullColumnLoader staticLoader = new NullColumnLoader(cache, defns, nullType, false);
// Create a batch
final VectorContainer output = staticLoader.load(2);
// Verify values and types
final TupleMetadata expectedSchema = new SchemaBuilder().add("int", MinorType.INT).add("str", MinorType.VARCHAR).add("dub", MinorType.FLOAT8).buildSchema();
final SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).addRow(10, "foo", 20.0D).addRow(10, "foo", 20.0D).build();
RowSetUtilities.verify(expected, fixture.wrap(output));
staticLoader.close();
}
use of org.apache.drill.exec.record.metadata.SchemaBuilder in project drill by apache.
the class TestNullColumnLoader method testCachedTypesMapToNullable.
/**
* Drill requires "schema persistence": if a scan operator
* reads two files, F1 and F2, then the scan operator must
* provide the same vectors from both readers. Not just the
* same types, the same value vector instances (but, of course,
* populated with different data.)
* <p>
* Test the case in which the reader for F1 found columns
* (a, b, c) but, F2 found only (a, b), requiring that we
* fill in column c, filled with nulls, but of the same type that it
* was in file F1. We use a vector cache to pull off this trick.
* This test ensures that the null column mechanism looks in that
* vector cache when asked to create a nullable column.
*/
@Test
public void testCachedTypesMapToNullable() {
final List<ResolvedNullColumn> defns = new ArrayList<>();
defns.add(makeNullCol("req"));
defns.add(makeNullCol("opt"));
defns.add(makeNullCol("rep"));
defns.add(makeNullCol("unk"));
// Populate the cache with a column of each mode.
final ResultVectorCacheImpl cache = new ResultVectorCacheImpl(fixture.allocator());
cache.vectorFor(SchemaBuilder.columnSchema("req", MinorType.FLOAT8, DataMode.REQUIRED));
final ValueVector opt = cache.vectorFor(SchemaBuilder.columnSchema("opt", MinorType.FLOAT8, DataMode.OPTIONAL));
final ValueVector rep = cache.vectorFor(SchemaBuilder.columnSchema("rep", MinorType.FLOAT8, DataMode.REPEATED));
// Use nullable Varchar for unknown null columns.
final MajorType nullType = Types.optional(MinorType.VARCHAR);
final NullColumnLoader staticLoader = new NullColumnLoader(cache, defns, nullType, false);
// Create a batch
final VectorContainer output = staticLoader.load(2);
// Verify vectors are reused
assertSame(opt, output.getValueVector(1).getValueVector());
assertSame(rep, output.getValueVector(2).getValueVector());
// Verify values and types
final TupleMetadata expectedSchema = new SchemaBuilder().addNullable("req", MinorType.FLOAT8).addNullable("opt", MinorType.FLOAT8).addArray("rep", MinorType.FLOAT8).addNullable("unk", MinorType.VARCHAR).buildSchema();
final SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).addRow(null, null, new int[] {}, null).addRow(null, null, new int[] {}, null).build();
RowSetUtilities.verify(expected, fixture.wrap(output));
staticLoader.close();
}
use of org.apache.drill.exec.record.metadata.SchemaBuilder in project drill by apache.
the class TestNullColumnLoader method testSchemaWithConflicts.
/**
* Test the various conflicts that can occur:
* <ul>
* <li>Schema is required, but no default value for null column.</li>
* <li>Query wants a different type than that in the schema.</li>
* <li>Query wants a different mode than that in the schema.</li>
* <ul>
*
* The type and mode provided to the builder is that which would result from
* schema smoothing. The types and modes should usually match, but verify
* the rules when they don't.
* <p>
* Defaults for null columns are ignored: null columns use NULL as the
* null value.
*/
@Test
public void testSchemaWithConflicts() {
// Note: upper case names in schema, lower case in "projection" list
final TupleMetadata outputSchema = new SchemaBuilder().add("IntReq", MinorType.INT).add("StrReq", // No default
MinorType.VARCHAR).addNullable("IntOpt", MinorType.INT).addNullable("StrOpt", MinorType.VARCHAR).buildSchema();
outputSchema.metadata("intReq").setDefaultValue("10");
outputSchema.metadata("intOpt").setDefaultValue("20");
outputSchema.metadata("strOpt").setDefaultValue("bar");
final ResultVectorCache cache = new NullResultVectorCacheImpl(fixture.allocator());
final NullColumnBuilder builder = new NullBuilderBuilder().setNullType(Types.optional(MinorType.VARCHAR)).setOutputSchema(outputSchema).build();
// Defined, required, no default so --> optional
builder.add("strReq");
builder.add("strOpt");
// Defined, has default, but conflicting type, so default --> null, so --> optional
builder.add("intReq", Types.required(MinorType.BIGINT));
// Defined, has default, conflicting mode, so keep default
builder.add("intOpt", Types.required(MinorType.INT));
builder.build(cache);
// Create a batch
builder.load(2);
// Verify values and types
final TupleMetadata expectedSchema = new SchemaBuilder().addNullable("strReq", MinorType.VARCHAR).addNullable("strOpt", MinorType.VARCHAR).addNullable("intReq", MinorType.BIGINT).add("intOpt", MinorType.INT).buildSchema();
final SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).addRow(null, null, null, 20).addRow(null, null, null, 20).build();
RowSetUtilities.verify(expected, fixture.wrap(builder.output()));
builder.close();
}
Aggregations