Examples with ErrorCollector org.apache.drill.common.expression.ErrorCollector used on opensource projects

Example 11 with ErrorCollector

use of org.apache.drill.common.expression.ErrorCollector in project drill by apache.

the class JoinUtils method addLeastRestrictiveCasts.

/**
   * Utility method used by joins to add implicit casts on one of the sides of the join condition in case the two
   * expressions have different types.
   * @param leftExpressions array of expressions from left input into the join
   * @param leftBatch left input record batch
   * @param rightExpressions array of expressions from right input into the join
   * @param rightBatch right input record batch
   * @param context fragment context
   */
public static void addLeastRestrictiveCasts(LogicalExpression[] leftExpressions, VectorAccessible leftBatch, LogicalExpression[] rightExpressions, VectorAccessible rightBatch, FragmentContext context) {
    assert rightExpressions.length == leftExpressions.length;
    for (int i = 0; i < rightExpressions.length; i++) {
        LogicalExpression rightExpression = rightExpressions[i];
        LogicalExpression leftExpression = leftExpressions[i];
        TypeProtos.MinorType rightType = rightExpression.getMajorType().getMinorType();
        TypeProtos.MinorType leftType = leftExpression.getMajorType().getMinorType();
        if (rightType == TypeProtos.MinorType.UNION || leftType == TypeProtos.MinorType.UNION) {
            continue;
        }
        if (rightType != leftType) {
            // currently we only support implicit casts if the input types are numeric or varchar/varbinary
            if (!allowImplicitCast(rightType, leftType)) {
                throw new DrillRuntimeException(String.format("Join only supports implicit casts between " + "1. Numeric data\n 2. Varchar, Varbinary data 3. Date, Timestamp data " + "Left type: %s, Right type: %s. Add explicit casts to avoid this error", leftType, rightType));
            }
            // We need to add a cast to one of the expressions
            List<TypeProtos.MinorType> types = new LinkedList<>();
            types.add(rightType);
            types.add(leftType);
            TypeProtos.MinorType result = TypeCastRules.getLeastRestrictiveType(types);
            ErrorCollector errorCollector = new ErrorCollectorImpl();
            if (result == null) {
                throw new DrillRuntimeException(String.format("Join conditions cannot be compared failing left " + "expression:" + " %s failing right expression: %s", leftExpression.getMajorType().toString(), rightExpression.getMajorType().toString()));
            } else if (result != rightType) {
                // Add a cast expression on top of the right expression
                LogicalExpression castExpr = ExpressionTreeMaterializer.addCastExpression(rightExpression, leftExpression.getMajorType(), context.getFunctionRegistry(), errorCollector);
                // Store the newly casted expression
                rightExpressions[i] = ExpressionTreeMaterializer.materialize(castExpr, rightBatch, errorCollector, context.getFunctionRegistry());
            } else if (result != leftType) {
                // Add a cast expression on top of the left expression
                LogicalExpression castExpr = ExpressionTreeMaterializer.addCastExpression(leftExpression, rightExpression.getMajorType(), context.getFunctionRegistry(), errorCollector);
                // store the newly casted expression
                leftExpressions[i] = ExpressionTreeMaterializer.materialize(castExpr, leftBatch, errorCollector, context.getFunctionRegistry());
            }
        }
    }
}

Example 12 with ErrorCollector

use of org.apache.drill.common.expression.ErrorCollector in project drill by apache.

the class MergeJoinBatch method generateNewWorker.

private JoinWorker generateNewWorker() throws ClassTransformationException, IOException, SchemaChangeException {
    final ClassGenerator<JoinWorker> cg = CodeGenerator.getRoot(JoinWorker.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
    cg.getCodeGenerator().plainJavaCapable(true);
    // Uncomment out this line to debug the generated code.
    //    cg.getCodeGenerator().saveCodeForDebugging(true);
    final ErrorCollector collector = new ErrorCollectorImpl();
    // Generate members and initialization code
    /////////////////////////////////////////
    // declare and assign JoinStatus member
    cg.setMappingSet(setupMapping);
    JClass joinStatusClass = cg.getModel().ref(JoinStatus.class);
    JVar joinStatus = cg.clazz.field(JMod.NONE, joinStatusClass, "status");
    cg.getSetupBlock().assign(JExpr._this().ref(joinStatus), JExpr.direct("status"));
    // declare and assign outgoing VectorContainer member
    JClass vectorContainerClass = cg.getModel().ref(VectorContainer.class);
    JVar outgoingVectorContainer = cg.clazz.field(JMod.NONE, vectorContainerClass, "outgoing");
    cg.getSetupBlock().assign(JExpr._this().ref(outgoingVectorContainer), JExpr.direct("outgoing"));
    // declare and assign incoming left RecordBatch member
    JClass recordBatchClass = cg.getModel().ref(RecordIterator.class);
    JVar incomingLeftRecordBatch = cg.clazz.field(JMod.NONE, recordBatchClass, "incomingLeft");
    cg.getSetupBlock().assign(JExpr._this().ref(incomingLeftRecordBatch), joinStatus.ref("left"));
    // declare and assign incoming right RecordBatch member
    JVar incomingRightRecordBatch = cg.clazz.field(JMod.NONE, recordBatchClass, "incomingRight");
    cg.getSetupBlock().assign(JExpr._this().ref(incomingRightRecordBatch), joinStatus.ref("right"));
    // declare 'incoming' member so VVReadExpr generated code can point to the left or right batch
    JVar incomingRecordBatch = cg.clazz.field(JMod.NONE, recordBatchClass, "incoming");
    /*
     * Materialize expressions on both sides of the join condition. Check if both the sides
     * have the same return type, if not then inject casts so that comparison function will work as
     * expected
     */
    LogicalExpression[] leftExpr = new LogicalExpression[conditions.size()];
    LogicalExpression[] rightExpr = new LogicalExpression[conditions.size()];
    IterOutcome lastLeftStatus = status.getLeftStatus();
    IterOutcome lastRightStatus = status.getRightStatus();
    for (int i = 0; i < conditions.size(); i++) {
        JoinCondition condition = conditions.get(i);
        leftExpr[i] = materializeExpression(condition.getLeft(), lastLeftStatus, leftIterator, collector);
        rightExpr[i] = materializeExpression(condition.getRight(), lastRightStatus, rightIterator, collector);
    }
    // call to throw an exception. In this case we can safely skip adding the casts
    if (lastRightStatus != IterOutcome.NONE) {
        JoinUtils.addLeastRestrictiveCasts(leftExpr, leftIterator, rightExpr, rightIterator, context);
    }
    //generate doCompare() method
    /////////////////////////////////////////
    generateDoCompare(cg, incomingRecordBatch, leftExpr, incomingLeftRecordBatch, rightExpr, incomingRightRecordBatch, collector);
    // generate copyLeft()
    //////////////////////
    cg.setMappingSet(copyLeftMapping);
    int vectorId = 0;
    if (worker == null || !status.left.finished()) {
        for (VectorWrapper<?> vw : leftIterator) {
            MajorType inputType = vw.getField().getType();
            MajorType outputType;
            if (joinType == JoinRelType.RIGHT && inputType.getMode() == DataMode.REQUIRED) {
                outputType = Types.overrideMode(inputType, DataMode.OPTIONAL);
            } else {
                outputType = inputType;
            }
            // TODO (DRILL-4011): Factor out CopyUtil and use it here.
            JVar vvIn = cg.declareVectorValueSetupAndMember("incomingLeft", new TypedFieldId(inputType, vectorId));
            JVar vvOut = cg.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(outputType, vectorId));
            // todo: check result of copyFromSafe and grow allocation
            cg.getEvalBlock().add(vvOut.invoke("copyFromSafe").arg(copyLeftMapping.getValueReadIndex()).arg(copyLeftMapping.getValueWriteIndex()).arg(vvIn));
            cg.rotateBlock();
            ++vectorId;
        }
    }
    // generate copyRight()
    ///////////////////////
    cg.setMappingSet(copyRightMappping);
    int rightVectorBase = vectorId;
    if (status.getRightStatus() != IterOutcome.NONE && (worker == null || !status.right.finished())) {
        for (VectorWrapper<?> vw : rightIterator) {
            MajorType inputType = vw.getField().getType();
            MajorType outputType;
            if (joinType == JoinRelType.LEFT && inputType.getMode() == DataMode.REQUIRED) {
                outputType = Types.overrideMode(inputType, DataMode.OPTIONAL);
            } else {
                outputType = inputType;
            }
            // TODO (DRILL-4011): Factor out CopyUtil and use it here.
            JVar vvIn = cg.declareVectorValueSetupAndMember("incomingRight", new TypedFieldId(inputType, vectorId - rightVectorBase));
            JVar vvOut = cg.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(outputType, vectorId));
            // todo: check result of copyFromSafe and grow allocation
            cg.getEvalBlock().add(vvOut.invoke("copyFromSafe").arg(copyRightMappping.getValueReadIndex()).arg(copyRightMappping.getValueWriteIndex()).arg(vvIn));
            cg.rotateBlock();
            ++vectorId;
        }
    }
    JoinWorker w = context.getImplementationClass(cg);
    w.setupJoin(context, status, this.container);
    return w;
}

Example 13 with ErrorCollector

use of org.apache.drill.common.expression.ErrorCollector in project drill by apache.

the class OrderedPartitionRecordBatch method setupNewSchema.

/**
   * Sets up projection that will transfer all of the columns in batch, and also populate the partition column based on
   * which partition a record falls into in the partition table
   *
   * @param batch
   * @throws SchemaChangeException
   */
protected void setupNewSchema(VectorAccessible batch) throws SchemaChangeException {
    container.clear();
    final ErrorCollector collector = new ErrorCollectorImpl();
    final List<TransferPair> transfers = Lists.newArrayList();
    final ClassGenerator<OrderedPartitionProjector> cg = CodeGenerator.getRoot(OrderedPartitionProjector.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
    for (VectorWrapper<?> vw : batch) {
        TransferPair tp = vw.getValueVector().getTransferPair(oContext.getAllocator());
        transfers.add(tp);
        container.add(tp.getTo());
    }
    cg.setMappingSet(mainMapping);
    int count = 0;
    for (Ordering od : popConfig.getOrderings()) {
        final LogicalExpression expr = ExpressionTreeMaterializer.materialize(od.getExpr(), batch, collector, context.getFunctionRegistry());
        if (collector.hasErrors()) {
            throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString());
        }
        cg.setMappingSet(incomingMapping);
        ClassGenerator.HoldingContainer left = cg.addExpr(expr, ClassGenerator.BlkCreateMode.FALSE);
        cg.setMappingSet(partitionMapping);
        ClassGenerator.HoldingContainer right = cg.addExpr(new ValueVectorReadExpression(new TypedFieldId(expr.getMajorType(), count++)), ClassGenerator.BlkCreateMode.FALSE);
        cg.setMappingSet(mainMapping);
        // next we wrap the two comparison sides and add the expression block for the comparison.
        LogicalExpression fh = FunctionGenerationHelper.getOrderingComparator(od.nullsSortHigh(), left, right, context.getFunctionRegistry());
        ClassGenerator.HoldingContainer out = cg.addExpr(fh, ClassGenerator.BlkCreateMode.FALSE);
        JConditional jc = cg.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)));
        if (od.getDirection() == Direction.ASCENDING) {
            jc._then()._return(out.getValue());
        } else {
            jc._then()._return(out.getValue().minus());
        }
    }
    cg.getEvalBlock()._return(JExpr.lit(0));
    container.add(this.partitionKeyVector);
    container.buildSchema(batch.getSchema().getSelectionVectorMode());
    try {
        this.projector = context.getImplementationClass(cg);
        projector.setup(context, batch, this, transfers, partitionVectors, partitions, popConfig.getRef());
    } catch (ClassTransformationException | IOException e) {
        throw new SchemaChangeException("Failure while attempting to load generated class", e);
    }
}

Example 14 with ErrorCollector

use of org.apache.drill.common.expression.ErrorCollector in project drill by apache.

the class SortBatch method createNewSorter.

public static Sorter createNewSorter(FragmentContext context, List<Ordering> orderings, VectorAccessible batch, MappingSet mainMapping, MappingSet leftMapping, MappingSet rightMapping) throws ClassTransformationException, IOException, SchemaChangeException {
    CodeGenerator<Sorter> cg = CodeGenerator.get(Sorter.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
    // This operator may be deprecated. No tests exercise it.
    // There is no way, at present, to verify if the generated code
    // works with Plain-old Java.
    //    cg.plainOldJavaCapable(true);
    // Uncomment out this line to debug the generated code.
    //    cg.saveCodeForDebugging(true);
    ClassGenerator<Sorter> g = cg.getRoot();
    g.setMappingSet(mainMapping);
    for (Ordering od : orderings) {
        // first, we rewrite the evaluation stack for each side of the comparison.
        ErrorCollector collector = new ErrorCollectorImpl();
        final LogicalExpression expr = ExpressionTreeMaterializer.materialize(od.getExpr(), batch, collector, context.getFunctionRegistry());
        if (collector.hasErrors()) {
            throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString());
        }
        g.setMappingSet(leftMapping);
        HoldingContainer left = g.addExpr(expr, ClassGenerator.BlkCreateMode.FALSE);
        g.setMappingSet(rightMapping);
        HoldingContainer right = g.addExpr(expr, ClassGenerator.BlkCreateMode.FALSE);
        g.setMappingSet(mainMapping);
        // next we wrap the two comparison sides and add the expression block for the comparison.
        LogicalExpression fh = FunctionGenerationHelper.getOrderingComparator(od.nullsSortHigh(), left, right, context.getFunctionRegistry());
        HoldingContainer out = g.addExpr(fh, ClassGenerator.BlkCreateMode.FALSE);
        JConditional jc = g.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)));
        if (od.getDirection() == Direction.ASCENDING) {
            jc._then()._return(out.getValue());
        } else {
            jc._then()._return(out.getValue().minus());
        }
    }
    g.getEvalBlock()._return(JExpr.lit(0));
    return context.getImplementationClass(cg);
}

Example 15 with ErrorCollector

use of org.apache.drill.common.expression.ErrorCollector in project drill by apache.

the class MergingRecordBatch method generateComparisons.

private void generateComparisons(final ClassGenerator<?> g, final VectorAccessible batch) throws SchemaChangeException {
    g.setMappingSet(MAIN_MAPPING);
    for (final Ordering od : popConfig.getOrderings()) {
        // first, we rewrite the evaluation stack for each side of the comparison.
        final ErrorCollector collector = new ErrorCollectorImpl();
        final LogicalExpression expr = ExpressionTreeMaterializer.materialize(od.getExpr(), batch, collector, context.getFunctionRegistry());
        if (collector.hasErrors()) {
            throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString());
        }
        g.setMappingSet(LEFT_MAPPING);
        final HoldingContainer left = g.addExpr(expr, ClassGenerator.BlkCreateMode.FALSE);
        g.setMappingSet(RIGHT_MAPPING);
        final HoldingContainer right = g.addExpr(expr, ClassGenerator.BlkCreateMode.FALSE);
        g.setMappingSet(MAIN_MAPPING);
        // next we wrap the two comparison sides and add the expression block for the comparison.
        final LogicalExpression fh = FunctionGenerationHelper.getOrderingComparator(od.nullsSortHigh(), left, right, context.getFunctionRegistry());
        final HoldingContainer out = g.addExpr(fh, ClassGenerator.BlkCreateMode.FALSE);
        final JConditional jc = g.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)));
        if (od.getDirection() == Direction.ASCENDING) {
            jc._then()._return(out.getValue());
        } else {
            jc._then()._return(out.getValue().minus());
        }
    }
    g.getEvalBlock()._return(JExpr.lit(0));
}