Examples with DrillFuncHolderExpr org.apache.drill.exec.expr.DrillFuncHolderExpr used on opensource projects

Example 6 with DrillFuncHolderExpr

use of org.apache.drill.exec.expr.DrillFuncHolderExpr in project drill by axbaretto.

the class FlattenRecordBatch method setupNewSchema.

@Override
protected boolean setupNewSchema() throws SchemaChangeException {
    this.allocationVectors = Lists.newArrayList();
    container.clear();
    final List<NamedExpression> exprs = getExpressionList();
    final ErrorCollector collector = new ErrorCollectorImpl();
    final List<TransferPair> transfers = Lists.newArrayList();
    final ClassGenerator<Flattener> cg = CodeGenerator.getRoot(Flattener.TEMPLATE_DEFINITION, context.getOptions());
    cg.getCodeGenerator().plainJavaCapable(true);
    final IntHashSet transferFieldIds = new IntHashSet();
    final NamedExpression flattenExpr = new NamedExpression(popConfig.getColumn(), new FieldReference(popConfig.getColumn()));
    final ValueVectorReadExpression vectorRead = (ValueVectorReadExpression) ExpressionTreeMaterializer.materialize(flattenExpr.getExpr(), incoming, collector, context.getFunctionRegistry(), true);
    final FieldReference fieldReference = flattenExpr.getRef();
    final TransferPair transferPair = getFlattenFieldTransferPair(fieldReference);
    if (transferPair != null) {
        final ValueVector flattenVector = transferPair.getTo();
        // checks that list has only default ValueVector and replaces resulting ValueVector to INT typed ValueVector
        if (exprs.size() == 0 && flattenVector.getField().getType().equals(Types.LATE_BIND_TYPE)) {
            final MaterializedField outputField = MaterializedField.create(fieldReference.getAsNamePart().getName(), Types.OPTIONAL_INT);
            final ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator());
            container.add(vector);
        } else {
            transfers.add(transferPair);
            container.add(flattenVector);
            transferFieldIds.add(vectorRead.getFieldId().getFieldIds()[0]);
        }
    }
    logger.debug("Added transfer for project expression.");
    ClassifierResult result = new ClassifierResult();
    for (NamedExpression namedExpression : exprs) {
        result.clear();
        String outputName = getRef(namedExpression).getRootSegment().getPath();
        if (result != null && result.outputNames != null && result.outputNames.size() > 0) {
            for (int j = 0; j < result.outputNames.size(); j++) {
                if (!result.outputNames.get(j).equals(EMPTY_STRING)) {
                    outputName = result.outputNames.get(j);
                    break;
                }
            }
        }
        final LogicalExpression expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, collector, context.getFunctionRegistry(), true);
        if (collector.hasErrors()) {
            throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema.  Errors:\n %s.", collector.toErrorString()));
        }
        if (expr instanceof DrillFuncHolderExpr && ((DrillFuncHolderExpr) expr).getHolder().isComplexWriterFuncHolder()) {
            // Lazy initialization of the list of complex writers, if not done yet.
            if (complexWriters == null) {
                complexWriters = Lists.newArrayList();
            }
            // The reference name will be passed to ComplexWriter, used as the name of the output vector from the writer.
            ((DrillFuncHolderExpr) expr).getFieldReference(namedExpression.getRef());
            cg.addExpr(expr);
        } else {
            // need to do evaluation.
            final MaterializedField outputField;
            if (expr instanceof ValueVectorReadExpression) {
                final TypedFieldId id = ValueVectorReadExpression.class.cast(expr).getFieldId();
                @SuppressWarnings("resource") final ValueVector incomingVector = incoming.getValueAccessorById(id.getIntermediateClass(), id.getFieldIds()).getValueVector();
                // when the first batch will be empty.
                if (incomingVector != null) {
                    outputField = incomingVector.getField().clone();
                } else {
                    outputField = MaterializedField.create(outputName, expr.getMajorType());
                }
            } else {
                outputField = MaterializedField.create(outputName, expr.getMajorType());
            }
            @SuppressWarnings("resource") final ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator());
            allocationVectors.add(vector);
            TypedFieldId fid = container.add(vector);
            ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, true);
            cg.addExpr(write);
            logger.debug("Added eval for project expression.");
        }
    }
    cg.rotateBlock();
    cg.getEvalBlock()._return(JExpr.TRUE);
    container.buildSchema(SelectionVectorMode.NONE);
    try {
        this.flattener = context.getImplementationClass(cg.getCodeGenerator());
        flattener.setup(context, incoming, this, transfers);
    } catch (ClassTransformationException | IOException e) {
        throw new SchemaChangeException("Failure while attempting to load generated class", e);
    }
    return true;
}

Example 7 with DrillFuncHolderExpr

use of org.apache.drill.exec.expr.DrillFuncHolderExpr in project drill by apache.

the class HashAggBatch method createAggregatorInternal.

protected HashAggregator createAggregatorInternal() {
    CodeGenerator<HashAggregator> top = CodeGenerator.get(HashAggregator.TEMPLATE_DEFINITION, context.getOptions());
    ClassGenerator<HashAggregator> cg = top.getRoot();
    ClassGenerator<HashAggregator> cgInner = cg.getInnerGenerator("BatchHolder");
    top.plainJavaCapable(true);
    // Uncomment the following line to allow debugging of the template code
    // top.saveCodeForDebugging(true);
    container.clear();
    numGroupByExprs = (getKeyExpressions() != null) ? getKeyExpressions().size() : 0;
    numAggrExprs = (getValueExpressions() != null) ? getValueExpressions().size() : 0;
    aggrExprs = new LogicalExpression[numAggrExprs];
    groupByOutFieldIds = new TypedFieldId[numGroupByExprs];
    aggrOutFieldIds = new TypedFieldId[numAggrExprs];
    ErrorCollector collector = new ErrorCollectorImpl();
    for (int i = 0; i < numGroupByExprs; i++) {
        NamedExpression ne = getKeyExpressions().get(i);
        final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry());
        if (expr == null) {
            continue;
        }
        final MaterializedField outputField = MaterializedField.create(ne.getRef().getAsNamePart().getName(), expr.getMajorType());
        ValueVector vv = TypeHelper.getNewVector(outputField, oContext.getAllocator());
        // add this group-by vector to the output container
        groupByOutFieldIds[i] = container.add(vv);
        columnMapping.put(outputField.getName(), ne.getExpr().toString().replace('`', ' ').trim());
    }
    // each of SUM, MAX and MIN gets an extra bigint column
    int extraNonNullColumns = 0;
    for (int i = 0; i < numAggrExprs; i++) {
        NamedExpression ne = getValueExpressions().get(i);
        final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry());
        if (expr instanceof IfExpression) {
            throw UserException.unsupportedError(new UnsupportedOperationException("Union type not supported in aggregate functions")).build(logger);
        }
        collector.reportErrors(logger);
        if (expr == null) {
            continue;
        }
        // Populate the complex writers for complex exprs
        if (expr instanceof DrillFuncHolderExpr && ((DrillFuncHolderExpr) expr).getHolder().isComplexWriterFuncHolder()) {
            if (complexWriters == null) {
                complexWriters = new ArrayList<>();
            } else {
                complexWriters.clear();
            }
            // The reference name will be passed to ComplexWriter, used as the name of the output vector from the writer.
            ((DrillFuncHolderExpr) expr).setFieldReference(ne.getRef());
            MaterializedField field = MaterializedField.create(ne.getRef().getAsNamePart().getName(), UntypedNullHolder.TYPE);
            container.add(new UntypedNullVector(field, container.getAllocator()));
            aggrExprs[i] = expr;
        } else {
            MaterializedField outputField = MaterializedField.create(ne.getRef().getAsNamePart().getName(), expr.getMajorType());
            ValueVector vv = TypeHelper.getNewVector(outputField, oContext.getAllocator());
            aggrOutFieldIds[i] = container.add(vv);
            aggrExprs[i] = new ValueVectorWriteExpression(aggrOutFieldIds[i], expr, true);
            if (expr instanceof FunctionHolderExpression) {
                String funcName = ((FunctionHolderExpression) expr).getName();
                if (funcName.equals("sum") || funcName.equals("max") || funcName.equals("min")) {
                    extraNonNullColumns++;
                }
                List<LogicalExpression> args = ((FunctionCall) ne.getExpr()).args();
                if (!args.isEmpty()) {
                    if (args.get(0) instanceof SchemaPath) {
                        columnMapping.put(outputField.getName(), ((SchemaPath) args.get(0)).getAsNamePart().getName());
                    } else if (args.get(0) instanceof FunctionCall) {
                        FunctionCall functionCall = (FunctionCall) args.get(0);
                        if (functionCall.arg(0) instanceof SchemaPath) {
                            columnMapping.put(outputField.getName(), ((SchemaPath) functionCall.arg(0)).getAsNamePart().getName());
                        }
                    }
                }
            } else {
                columnMapping.put(outputField.getName(), ne.getRef().getAsNamePart().getName());
            }
        }
    }
    setupUpdateAggrValues(cgInner);
    setupGetIndex(cg);
    cg.getBlock("resetValues")._return(JExpr.TRUE);
    container.buildSchema(SelectionVectorMode.NONE);
    HashAggregator agg = context.getImplementationClass(top);
    HashTableConfig htConfig = // TODO - fix the validator on this option
    new HashTableConfig((int) context.getOptions().getOption(ExecConstants.MIN_HASH_TABLE_SIZE), HashTable.DEFAULT_LOAD_FACTOR, getKeyExpressions(), null, /* no probe exprs */
    comparators);
    agg.setup(popConfig, htConfig, context, oContext, incoming, this, aggrExprs, cgInner.getWorkspaceTypes(), cgInner, groupByOutFieldIds, this.container, extraNonNullColumns * 8);
    return agg;
}

Example 8 with DrillFuncHolderExpr

use of org.apache.drill.exec.expr.DrillFuncHolderExpr in project drill by apache.

the class OutputWidthVisitor method visitFunctionHolderExpression.

/**
 * Handles a {@link FunctionHolderExpression}. Functions that produce
 * fixed-width output are trivially converted to a {@link FixedLenExpr}. For
 * functions that produce variable width output, the output width calculator
 * annotation is looked-up and recorded in a {@link FunctionCallExpr}. This
 * calculator will later be used to convert the FunctionCallExpr to a
 * {@link FixedLenExpr} expression
 */
@Override
public OutputWidthExpression visitFunctionHolderExpression(FunctionHolderExpression holderExpr, OutputWidthVisitorState state) throws RuntimeException {
    OutputWidthExpression fixedWidth = getFixedLenExpr(holderExpr.getMajorType());
    if (fixedWidth != null) {
        return fixedWidth;
    }
    // will default to a fixed value
    if (!(holderExpr instanceof DrillFuncHolderExpr)) {
        // Use a default if this is not a DrillFunc
        return new FixedLenExpr(OutputSizeEstimateConstants.NON_DRILL_FUNCTION_OUTPUT_SIZE_ESTIMATE);
    }
    final DrillFuncHolder holder = ((DrillFuncHolderExpr) holderExpr).getHolder();
    // If the user has provided a size estimate, use it
    int estimate = holder.variableOutputSizeEstimate();
    if (estimate != FunctionTemplate.OUTPUT_SIZE_ESTIMATE_DEFAULT) {
        return new FixedLenExpr(estimate);
    }
    // Use the calculator provided by the user or use the default
    OutputWidthCalculator widthCalculator = holder.getOutputWidthCalculator();
    final int argSize = holderExpr.args.size();
    ArrayList<OutputWidthExpression> arguments = null;
    if (argSize != 0) {
        arguments = new ArrayList<>(argSize);
        for (LogicalExpression expr : holderExpr.args) {
            arguments.add(expr.accept(this, state));
        }
    }
    return new FunctionCallExpr(holderExpr, widthCalculator, arguments);
}