Examples with MaterializedField org.apache.drill.exec.record.MaterializedField used on opensource projects

Example 6 with MaterializedField

use of org.apache.drill.exec.record.MaterializedField in project drill by apache.

the class WriterRecordBatch method setupNewSchema.

protected void setupNewSchema() throws IOException {
    try {
        // update the schema in RecordWriter
        stats.startSetup();
        recordWriter.updateSchema(incoming);
        // Create two vectors for:
        //   1. Fragment unique id.
        //   2. Summary: currently contains number of records written.
        final MaterializedField fragmentIdField = MaterializedField.create("Fragment", Types.required(MinorType.VARCHAR));
        final MaterializedField summaryField = MaterializedField.create("Number of records written", Types.required(MinorType.BIGINT));
        container.addOrGet(fragmentIdField);
        container.addOrGet(summaryField);
        container.buildSchema(BatchSchema.SelectionVectorMode.NONE);
    } finally {
        stats.stopSetup();
    }
    eventBasedRecordWriter = new EventBasedRecordWriter(incoming, recordWriter);
    container.buildSchema(SelectionVectorMode.NONE);
    schema = container.getSchema();
}

Example 7 with MaterializedField

use of org.apache.drill.exec.record.MaterializedField in project drill by apache.

the class StreamingAggBatch method createAggregatorInternal.

private StreamingAggregator createAggregatorInternal() throws SchemaChangeException, ClassTransformationException, IOException {
    ClassGenerator<StreamingAggregator> cg = CodeGenerator.getRoot(StreamingAggTemplate.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
    cg.getCodeGenerator().plainJavaCapable(true);
    // Uncomment out this line to debug the generated code.
    //    cg.getCodeGenerator().saveCodeForDebugging(true);
    container.clear();
    LogicalExpression[] keyExprs = new LogicalExpression[popConfig.getKeys().size()];
    LogicalExpression[] valueExprs = new LogicalExpression[popConfig.getExprs().size()];
    TypedFieldId[] keyOutputIds = new TypedFieldId[popConfig.getKeys().size()];
    ErrorCollector collector = new ErrorCollectorImpl();
    for (int i = 0; i < keyExprs.length; i++) {
        final NamedExpression ne = popConfig.getKeys().get(i);
        final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry());
        if (expr == null) {
            continue;
        }
        keyExprs[i] = expr;
        final MaterializedField outputField = MaterializedField.create(ne.getRef().getAsUnescapedPath(), expr.getMajorType());
        @SuppressWarnings("resource") final ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator());
        keyOutputIds[i] = container.add(vector);
    }
    for (int i = 0; i < valueExprs.length; i++) {
        final NamedExpression ne = popConfig.getExprs().get(i);
        final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry());
        if (expr instanceof IfExpression) {
            throw UserException.unsupportedError(new UnsupportedOperationException("Union type not supported in aggregate functions")).build(logger);
        }
        if (expr == null) {
            continue;
        }
        final MaterializedField outputField = MaterializedField.create(ne.getRef().getAsUnescapedPath(), expr.getMajorType());
        @SuppressWarnings("resource") ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator());
        TypedFieldId id = container.add(vector);
        valueExprs[i] = new ValueVectorWriteExpression(id, expr, true);
    }
    if (collector.hasErrors()) {
        throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString());
    }
    setupIsSame(cg, keyExprs);
    setupIsSameApart(cg, keyExprs);
    addRecordValues(cg, valueExprs);
    outputRecordKeys(cg, keyOutputIds, keyExprs);
    outputRecordKeysPrev(cg, keyOutputIds, keyExprs);
    cg.getBlock("resetValues")._return(JExpr.TRUE);
    getIndex(cg);
    container.buildSchema(SelectionVectorMode.NONE);
    StreamingAggregator agg = context.getImplementationClass(cg);
    agg.setup(oContext, incoming, this);
    return agg;
}

Example 8 with MaterializedField

use of org.apache.drill.exec.record.MaterializedField in project drill by apache.

the class ScanBatch method addImplicitVectors.

private void addImplicitVectors() {
    try {
        if (implicitVectors != null) {
            for (ValueVector v : implicitVectors.values()) {
                v.clear();
            }
        }
        implicitVectors = Maps.newHashMap();
        if (implicitValues != null) {
            for (String column : implicitValues.keySet()) {
                final MaterializedField field = MaterializedField.create(column, Types.optional(MinorType.VARCHAR));
                @SuppressWarnings("resource") final ValueVector v = mutator.addField(field, NullableVarCharVector.class);
                implicitVectors.put(column, v);
            }
        }
    } catch (SchemaChangeException e) {
        // No exception should be thrown here.
        throw UserException.systemError(e).addContext("Failure while allocating implicit vectors").build(logger);
    }
}

Example 9 with MaterializedField

use of org.apache.drill.exec.record.MaterializedField in project drill by apache.

the class TestValueVector method testNullableVarCharVectorLoad.

@Test
public void testNullableVarCharVectorLoad() {
    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE);
    // Create a new value vector for 1024 nullable variable length strings.
    final NullableVarCharVector vector1 = new NullableVarCharVector(field, allocator);
    final NullableVarCharVector.Mutator mutator = vector1.getMutator();
    vector1.allocateNew(1024 * 10, 1024);
    // Populate the vector.
    final StringBuilder stringBuilder = new StringBuilder();
    final int valueCount = 10;
    for (int i = 0; i < valueCount; ++i) {
        stringBuilder.append('x');
        mutator.set(i, stringBuilder.toString().getBytes(utf8Charset));
    }
    // Check the contents.
    final NullableVarCharVector.Accessor accessor1 = vector1.getAccessor();
    stringBuilder.setLength(0);
    for (int i = 0; i < valueCount; ++i) {
        stringBuilder.append('x');
        final Object object = accessor1.getObject(i);
        assertEquals(stringBuilder.toString(), object.toString());
    }
    mutator.setValueCount(valueCount);
    assertEquals(valueCount, vector1.getAccessor().getValueCount());
    // Still ok after setting value count?
    stringBuilder.setLength(0);
    for (int i = 0; i < valueCount; ++i) {
        stringBuilder.append('x');
        final Object object = accessor1.getObject(i);
        assertEquals(stringBuilder.toString(), object.toString());
    }
    // Combine into a single buffer so we can load it into a new vector.
    final DrillBuf[] buffers1 = vector1.getBuffers(false);
    final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
    final NullableVarCharVector vector2 = new NullableVarCharVector(field, allocator);
    vector2.load(vector1.getMetadata(), buffer1);
    // Check the vector's contents.
    final NullableVarCharVector.Accessor accessor2 = vector2.getAccessor();
    stringBuilder.setLength(0);
    for (int i = 0; i < valueCount; ++i) {
        stringBuilder.append('x');
        final Object object = accessor2.getObject(i);
        assertEquals(stringBuilder.toString(), object.toString());
    }
    vector1.close();
    vector2.close();
    buffer1.release();
}

Example 10 with MaterializedField

use of org.apache.drill.exec.record.MaterializedField in project drill by apache.

the class TestValueVector method testBitVector.

@Test
public void testBitVector() {
    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, BitHolder.TYPE);
    // Create a new value vector for 1024 integers
    try (final BitVector vector = new BitVector(field, allocator)) {
        final BitVector.Mutator m = vector.getMutator();
        vector.allocateNew(1024);
        // Put and set a few values
        m.set(0, 1);
        m.set(1, 0);
        m.set(100, 0);
        m.set(1022, 1);
        final BitVector.Accessor accessor = vector.getAccessor();
        assertEquals(1, accessor.get(0));
        assertEquals(0, accessor.get(1));
        assertEquals(0, accessor.get(100));
        assertEquals(1, accessor.get(1022));
        // test setting the same value twice
        m.set(0, 1);
        m.set(0, 1);
        m.set(1, 0);
        m.set(1, 0);
        assertEquals(1, accessor.get(0));
        assertEquals(0, accessor.get(1));
        // test toggling the values
        m.set(0, 0);
        m.set(1, 1);
        assertEquals(0, accessor.get(0));
        assertEquals(1, accessor.get(1));
        // Ensure unallocated space returns 0
        assertEquals(0, accessor.get(3));
    }
}