Examples with ValueVectorWriteExpression org.apache.drill.exec.expr.ValueVectorWriteExpression used on opensource projects

Example 1 with ValueVectorWriteExpression

use of org.apache.drill.exec.expr.ValueVectorWriteExpression in project drill by apache.

the class ChainedHashTable method setupOutputRecordKeys.

private void setupOutputRecordKeys(ClassGenerator<HashTable> cg, TypedFieldId[] htKeyFieldIds, TypedFieldId[] outKeyFieldIds) {
    cg.setMappingSet(OutputRecordKeysMapping);
    if (outKeyFieldIds != null) {
        for (int i = 0; i < outKeyFieldIds.length; i++) {
            ValueVectorReadExpression vvrExpr = new ValueVectorReadExpression(htKeyFieldIds[i]);
            boolean useSetSafe = !Types.isFixedWidthType(vvrExpr.getMajorType()) || Types.isRepeated(vvrExpr.getMajorType());
            ValueVectorWriteExpression vvwExpr = new ValueVectorWriteExpression(outKeyFieldIds[i], vvrExpr, useSetSafe);
            cg.addExpr(vvwExpr, ClassGenerator.BlkCreateMode.TRUE);
        }
    }
}

Example 2 with ValueVectorWriteExpression

use of org.apache.drill.exec.expr.ValueVectorWriteExpression in project drill by apache.

the class StreamingAggBatch method createAggregatorInternal.

private StreamingAggregator createAggregatorInternal() throws SchemaChangeException, ClassTransformationException, IOException {
    ClassGenerator<StreamingAggregator> cg = CodeGenerator.getRoot(StreamingAggTemplate.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
    cg.getCodeGenerator().plainJavaCapable(true);
    // Uncomment out this line to debug the generated code.
    //    cg.getCodeGenerator().saveCodeForDebugging(true);
    container.clear();
    LogicalExpression[] keyExprs = new LogicalExpression[popConfig.getKeys().size()];
    LogicalExpression[] valueExprs = new LogicalExpression[popConfig.getExprs().size()];
    TypedFieldId[] keyOutputIds = new TypedFieldId[popConfig.getKeys().size()];
    ErrorCollector collector = new ErrorCollectorImpl();
    for (int i = 0; i < keyExprs.length; i++) {
        final NamedExpression ne = popConfig.getKeys().get(i);
        final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry());
        if (expr == null) {
            continue;
        }
        keyExprs[i] = expr;
        final MaterializedField outputField = MaterializedField.create(ne.getRef().getAsUnescapedPath(), expr.getMajorType());
        @SuppressWarnings("resource") final ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator());
        keyOutputIds[i] = container.add(vector);
    }
    for (int i = 0; i < valueExprs.length; i++) {
        final NamedExpression ne = popConfig.getExprs().get(i);
        final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry());
        if (expr instanceof IfExpression) {
            throw UserException.unsupportedError(new UnsupportedOperationException("Union type not supported in aggregate functions")).build(logger);
        }
        if (expr == null) {
            continue;
        }
        final MaterializedField outputField = MaterializedField.create(ne.getRef().getAsUnescapedPath(), expr.getMajorType());
        @SuppressWarnings("resource") ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator());
        TypedFieldId id = container.add(vector);
        valueExprs[i] = new ValueVectorWriteExpression(id, expr, true);
    }
    if (collector.hasErrors()) {
        throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString());
    }
    setupIsSame(cg, keyExprs);
    setupIsSameApart(cg, keyExprs);
    addRecordValues(cg, valueExprs);
    outputRecordKeys(cg, keyOutputIds, keyExprs);
    outputRecordKeysPrev(cg, keyOutputIds, keyExprs);
    cg.getBlock("resetValues")._return(JExpr.TRUE);
    getIndex(cg);
    container.buildSchema(SelectionVectorMode.NONE);
    StreamingAggregator agg = context.getImplementationClass(cg);
    agg.setup(oContext, incoming, this);
    return agg;
}

Example 3 with ValueVectorWriteExpression

use of org.apache.drill.exec.expr.ValueVectorWriteExpression in project drill by apache.

the class TestEvaluationVisitor method x.

@Test
public void x() throws Exception {
    DrillConfig c = DrillConfig.create();
    FunctionImplementationRegistry reg = new FunctionImplementationRegistry(c);
    EvaluationVisitor v = new EvaluationVisitor(reg);
    CodeGenerator<?> g = CodeGenerator.get(Projector.TEMPLATE_DEFINITION, reg, null);
    SchemaPath path = (SchemaPath) getExpr("a.b[4][2].c[6]");
    TypedFieldId id = //
    TypedFieldId.newBuilder().addId(//
    1).addId(//
    3).remainder(//
    path.getRootSegment()).intermediateType(Types.optional(MinorType.MAP)).finalType(//
    Types.repeated(MinorType.MAP)).hyper().withIndex().build();
    ValueVectorReadExpression e = new ValueVectorReadExpression(id);
    TypedFieldId outId = //
    TypedFieldId.newBuilder().addId(//
    1).finalType(//
    Types.repeated(MinorType.MAP)).intermediateType(//
    Types.repeated(MinorType.MAP)).build();
    ValueVectorWriteExpression e2 = new ValueVectorWriteExpression(outId, e, true);
    v.addExpr(e2, g.getRoot());
    logger.debug(g.generateAndGet());
}

Example 4 with ValueVectorWriteExpression

use of org.apache.drill.exec.expr.ValueVectorWriteExpression in project drill by apache.

the class UnionAllRecordBatch method doWork.

@SuppressWarnings("resource")
private IterOutcome doWork() throws ClassTransformationException, IOException, SchemaChangeException {
    if (allocationVectors != null) {
        for (ValueVector v : allocationVectors) {
            v.clear();
        }
    }
    allocationVectors = Lists.newArrayList();
    transfers.clear();
    // If both sides of Union-All are empty
    if (unionAllInput.isBothSideEmpty()) {
        for (int i = 0; i < outputFields.size(); ++i) {
            final String colName = outputFields.get(i).getPath();
            final MajorType majorType = MajorType.newBuilder().setMinorType(MinorType.INT).setMode(DataMode.OPTIONAL).build();
            MaterializedField outputField = MaterializedField.create(colName, majorType);
            ValueVector vv = container.addOrGet(outputField, callBack);
            allocationVectors.add(vv);
        }
        container.buildSchema(BatchSchema.SelectionVectorMode.NONE);
        return IterOutcome.OK_NEW_SCHEMA;
    }
    final ClassGenerator<UnionAller> cg = CodeGenerator.getRoot(UnionAller.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
    cg.getCodeGenerator().plainJavaCapable(true);
    // Uncomment out this line to debug the generated code.
    //    cg.getCodeGenerator().saveCodeForDebugging(true);
    int index = 0;
    for (VectorWrapper<?> vw : current) {
        ValueVector vvIn = vw.getValueVector();
        // get the original input column names
        SchemaPath inputPath = SchemaPath.getSimplePath(vvIn.getField().getPath());
        // get the renamed column names
        SchemaPath outputPath = SchemaPath.getSimplePath(outputFields.get(index).getPath());
        final ErrorCollector collector = new ErrorCollectorImpl();
        // cast data types (Minortype or DataMode)
        if (hasSameTypeAndMode(outputFields.get(index), vw.getValueVector().getField())) {
            // Transfer column
            MajorType outputFieldType = outputFields.get(index).getType();
            MaterializedField outputField = MaterializedField.create(outputPath.getAsUnescapedPath(), outputFieldType);
            /*
          todo: Fix if condition when DRILL-4824 is merged
          If condition should be changed to:
          `if (outputFields.get(index).getPath().equals(inputPath.getAsUnescapedPath())) {`
          DRILL-5419 has changed condition to correct one but this caused regression (DRILL-5521).
          Root cause is missing indication of child column in map types when it is null.
          DRILL-4824 is re-working json reader implementation, including map types and will fix this problem.
          Reverting condition to previous one to avoid regression till DRILL-4824 is merged.
          Unit test - TestJsonReader.testKvgenWithUnionAll().
         */
            if (outputFields.get(index).getPath().equals(inputPath)) {
                ValueVector vvOut = container.addOrGet(outputField);
                TransferPair tp = vvIn.makeTransferPair(vvOut);
                transfers.add(tp);
            // Copy data in order to rename the column
            } else {
                final LogicalExpression expr = ExpressionTreeMaterializer.materialize(inputPath, current, collector, context.getFunctionRegistry());
                if (collector.hasErrors()) {
                    throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema.  Errors:\n %s.", collector.toErrorString()));
                }
                ValueVector vv = container.addOrGet(outputField, callBack);
                allocationVectors.add(vv);
                TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getPath()));
                ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, true);
                cg.addExpr(write);
            }
        // Cast is necessary
        } else {
            LogicalExpression expr = ExpressionTreeMaterializer.materialize(inputPath, current, collector, context.getFunctionRegistry());
            if (collector.hasErrors()) {
                throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema.  Errors:\n %s.", collector.toErrorString()));
            }
            // cast to the one with the least restriction
            if (vvIn.getField().getType().getMode() == DataMode.REQUIRED && outputFields.get(index).getType().getMode() != DataMode.REQUIRED) {
                expr = ExpressionTreeMaterializer.convertToNullableType(expr, vvIn.getField().getType().getMinorType(), context.getFunctionRegistry(), collector);
                if (collector.hasErrors()) {
                    throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema.  Errors:\n %s.", collector.toErrorString()));
                }
            }
            // Insert a cast before the Union operation
            if (vvIn.getField().getType().getMinorType() != outputFields.get(index).getType().getMinorType()) {
                expr = ExpressionTreeMaterializer.addCastExpression(expr, outputFields.get(index).getType(), context.getFunctionRegistry(), collector);
                if (collector.hasErrors()) {
                    throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema.  Errors:\n %s.", collector.toErrorString()));
                }
            }
            final MaterializedField outputField = MaterializedField.create(outputPath.getAsUnescapedPath(), expr.getMajorType());
            ValueVector vector = container.addOrGet(outputField, callBack);
            allocationVectors.add(vector);
            TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getPath()));
            boolean useSetSafe = !(vector instanceof FixedWidthVector);
            ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, useSetSafe);
            cg.addExpr(write);
        }
        ++index;
    }
    unionall = context.getImplementationClass(cg.getCodeGenerator());
    unionall.setup(context, current, this, transfers);
    if (!schemaAvailable) {
        container.buildSchema(BatchSchema.SelectionVectorMode.NONE);
        schemaAvailable = true;
    }
    if (!doAlloc()) {
        return IterOutcome.OUT_OF_MEMORY;
    }
    recordCount = unionall.unionRecords(0, current.getRecordCount(), 0);
    setValueCount(recordCount);
    return IterOutcome.OK;
}

Example 5 with ValueVectorWriteExpression

use of org.apache.drill.exec.expr.ValueVectorWriteExpression in project drill by apache.

the class FlattenRecordBatch method setupNewSchema.

@Override
protected boolean setupNewSchema() throws SchemaChangeException {
    this.allocationVectors = Lists.newArrayList();
    container.clear();
    final List<NamedExpression> exprs = getExpressionList();
    final ErrorCollector collector = new ErrorCollectorImpl();
    final List<TransferPair> transfers = Lists.newArrayList();
    final ClassGenerator<Flattener> cg = CodeGenerator.getRoot(Flattener.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
    cg.getCodeGenerator().plainJavaCapable(true);
    // Uncomment out this line to debug the generated code.
    //    cg.getCodeGenerator().saveCodeForDebugging(true);
    final IntHashSet transferFieldIds = new IntHashSet();
    final NamedExpression flattenExpr = new NamedExpression(popConfig.getColumn(), new FieldReference(popConfig.getColumn()));
    final ValueVectorReadExpression vectorRead = (ValueVectorReadExpression) ExpressionTreeMaterializer.materialize(flattenExpr.getExpr(), incoming, collector, context.getFunctionRegistry(), true);
    final FieldReference fieldReference = flattenExpr.getRef();
    final TransferPair transferPair = getFlattenFieldTransferPair(fieldReference);
    if (transferPair != null) {
        final ValueVector flattenVector = transferPair.getTo();
        // checks that list has only default ValueVector and replaces resulting ValueVector to INT typed ValueVector
        if (exprs.size() == 0 && flattenVector.getField().getType().equals(Types.LATE_BIND_TYPE)) {
            final MaterializedField outputField = MaterializedField.create(fieldReference.getAsNamePart().getName(), Types.OPTIONAL_INT);
            final ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator());
            container.add(vector);
        } else {
            transfers.add(transferPair);
            container.add(flattenVector);
            transferFieldIds.add(vectorRead.getFieldId().getFieldIds()[0]);
        }
    }
    logger.debug("Added transfer for project expression.");
    ClassifierResult result = new ClassifierResult();
    for (int i = 0; i < exprs.size(); i++) {
        final NamedExpression namedExpression = exprs.get(i);
        result.clear();
        String outputName = getRef(namedExpression).getRootSegment().getPath();
        if (result != null && result.outputNames != null && result.outputNames.size() > 0) {
            for (int j = 0; j < result.outputNames.size(); j++) {
                if (!result.outputNames.get(j).equals(EMPTY_STRING)) {
                    outputName = result.outputNames.get(j);
                    break;
                }
            }
        }
        final LogicalExpression expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, collector, context.getFunctionRegistry(), true);
        if (collector.hasErrors()) {
            throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema.  Errors:\n %s.", collector.toErrorString()));
        }
        if (expr instanceof DrillFuncHolderExpr && ((DrillFuncHolderExpr) expr).getHolder().isComplexWriterFuncHolder()) {
            // Lazy initialization of the list of complex writers, if not done yet.
            if (complexWriters == null) {
                complexWriters = Lists.newArrayList();
            }
            // The reference name will be passed to ComplexWriter, used as the name of the output vector from the writer.
            ((DrillFuncHolderExpr) expr).getFieldReference(namedExpression.getRef());
            cg.addExpr(expr);
        } else {
            // need to do evaluation.
            final MaterializedField outputField;
            if (expr instanceof ValueVectorReadExpression) {
                final TypedFieldId id = ValueVectorReadExpression.class.cast(expr).getFieldId();
                @SuppressWarnings("resource") final ValueVector incomingVector = incoming.getValueAccessorById(id.getIntermediateClass(), id.getFieldIds()).getValueVector();
                // when the first batch will be empty.
                if (incomingVector != null) {
                    outputField = incomingVector.getField().clone();
                } else {
                    outputField = MaterializedField.create(outputName, expr.getMajorType());
                }
            } else {
                outputField = MaterializedField.create(outputName, expr.getMajorType());
            }
            @SuppressWarnings("resource") final ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator());
            allocationVectors.add(vector);
            TypedFieldId fid = container.add(vector);
            ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, true);
            cg.addExpr(write);
            logger.debug("Added eval for project expression.");
        }
    }
    cg.rotateBlock();
    cg.getEvalBlock()._return(JExpr.TRUE);
    container.buildSchema(SelectionVectorMode.NONE);
    try {
        this.flattener = context.getImplementationClass(cg.getCodeGenerator());
        flattener.setup(context, incoming, this, transfers);
    } catch (ClassTransformationException | IOException e) {
        throw new SchemaChangeException("Failure while attempting to load generated class", e);
    }
    return true;
}