Examples with MajorType org.apache.drill.common.types.TypeProtos.MajorType used on opensource projects

Example 6 with MajorType

use of org.apache.drill.common.types.TypeProtos.MajorType in project drill by apache.

the class CopierTemplate4 method copyRecords.

@Override
public int copyRecords(int index, int recordCount) throws SchemaChangeException {
    for (VectorWrapper<?> out : outgoing) {
        MajorType type = out.getField().getType();
        if (!Types.isFixedWidthType(type) || Types.isRepeated(type)) {
            out.getValueVector().allocateNew();
        } else {
            AllocationHelper.allocate(out.getValueVector(), recordCount, 1);
        }
    }
    int outgoingPosition = 0;
    for (int svIndex = index; svIndex < index + recordCount; svIndex++, outgoingPosition++) {
        int deRefIndex = sv4.get(svIndex);
        doEval(deRefIndex, outgoingPosition);
    }
    return outgoingPosition;
}

Example 7 with MajorType

use of org.apache.drill.common.types.TypeProtos.MajorType in project drill by apache.

the class IfExpression method getMajorType.

@Override
public MajorType getMajorType() {
    if (outputType != null) {
        return outputType;
    }
    MajorType elseType = elseExpression.getMajorType();
    MajorType ifType = ifCondition.expression.getMajorType();
    if (elseType.getMinorType() == MinorType.UNION) {
        Set<MinorType> subtypes = Sets.newHashSet();
        for (MinorType subtype : elseType.getSubTypeList()) {
            subtypes.add(subtype);
        }
        for (MinorType subtype : ifType.getSubTypeList()) {
            subtypes.add(subtype);
        }
        MajorType.Builder builder = MajorType.newBuilder().setMinorType(MinorType.UNION).setMode(DataMode.OPTIONAL);
        for (MinorType subtype : subtypes) {
            builder.addSubType(subtype);
        }
        return builder.build();
    }
    MajorType.Builder builder = MajorType.newBuilder().setMinorType(ifType.getMinorType());
    builder.setMode(elseType.getMode() == DataMode.OPTIONAL || ifType.getMode() == DataMode.OPTIONAL ? DataMode.OPTIONAL : elseType.getMode());
    builder = Types.calculateTypePrecisionAndScale(ifType, elseType, builder);
    return builder.build();
}

Example 8 with MajorType

use of org.apache.drill.common.types.TypeProtos.MajorType in project drill by apache.

the class HashJoinBatch method setupHashJoinProbe.

public HashJoinProbe setupHashJoinProbe() throws ClassTransformationException, IOException {
    final CodeGenerator<HashJoinProbe> cg = CodeGenerator.get(HashJoinProbe.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
    cg.plainJavaCapable(true);
    // Uncomment out this line to debug the generated code.
    //    cg.saveCodeForDebugging(true);
    final ClassGenerator<HashJoinProbe> g = cg.getRoot();
    // Generate the code to project build side records
    g.setMappingSet(projectBuildMapping);
    int fieldId = 0;
    final JExpression buildIndex = JExpr.direct("buildIndex");
    final JExpression outIndex = JExpr.direct("outIndex");
    g.rotateBlock();
    if (rightSchema != null) {
        for (final MaterializedField field : rightSchema) {
            final MajorType inputType = field.getType();
            final MajorType outputType;
            // not nullable so we must exclude them from the check below (see DRILL-2197).
            if ((joinType == JoinRelType.LEFT || joinType == JoinRelType.FULL) && inputType.getMode() == DataMode.REQUIRED && inputType.getMinorType() != TypeProtos.MinorType.MAP) {
                outputType = Types.overrideMode(inputType, DataMode.OPTIONAL);
            } else {
                outputType = inputType;
            }
            // make sure to project field with children for children to show up in the schema
            final MaterializedField projected = field.withType(outputType);
            // Add the vector to our output container
            container.addOrGet(projected);
            final JVar inVV = g.declareVectorValueSetupAndMember("buildBatch", new TypedFieldId(field.getType(), true, fieldId));
            final JVar outVV = g.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(outputType, false, fieldId));
            g.getEvalBlock().add(outVV.invoke("copyFromSafe").arg(buildIndex.band(JExpr.lit((int) Character.MAX_VALUE))).arg(outIndex).arg(inVV.component(buildIndex.shrz(JExpr.lit(16)))));
            g.rotateBlock();
            fieldId++;
        }
    }
    // Generate the code to project probe side records
    g.setMappingSet(projectProbeMapping);
    int outputFieldId = fieldId;
    fieldId = 0;
    final JExpression probeIndex = JExpr.direct("probeIndex");
    if (leftUpstream == IterOutcome.OK || leftUpstream == IterOutcome.OK_NEW_SCHEMA) {
        for (final VectorWrapper<?> vv : left) {
            final MajorType inputType = vv.getField().getType();
            final MajorType outputType;
            // not nullable so we must exclude them from the check below (see DRILL-2771, DRILL-2197).
            if ((joinType == JoinRelType.RIGHT || joinType == JoinRelType.FULL) && inputType.getMode() == DataMode.REQUIRED && inputType.getMinorType() != TypeProtos.MinorType.MAP) {
                outputType = Types.overrideMode(inputType, DataMode.OPTIONAL);
            } else {
                outputType = inputType;
            }
            final ValueVector v = container.addOrGet(MaterializedField.create(vv.getField().getPath(), outputType));
            if (v instanceof AbstractContainerVector) {
                vv.getValueVector().makeTransferPair(v);
                v.clear();
            }
            final JVar inVV = g.declareVectorValueSetupAndMember("probeBatch", new TypedFieldId(inputType, false, fieldId));
            final JVar outVV = g.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(outputType, false, outputFieldId));
            g.getEvalBlock().add(outVV.invoke("copyFromSafe").arg(probeIndex).arg(outIndex).arg(inVV));
            g.rotateBlock();
            fieldId++;
            outputFieldId++;
        }
    }
    final HashJoinProbe hj = context.getImplementationClass(cg);
    return hj;
}

Example 9 with MajorType

use of org.apache.drill.common.types.TypeProtos.MajorType in project drill by apache.

the class MergeJoinBatch method generateNewWorker.

private JoinWorker generateNewWorker() throws ClassTransformationException, IOException, SchemaChangeException {
    final ClassGenerator<JoinWorker> cg = CodeGenerator.getRoot(JoinWorker.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
    cg.getCodeGenerator().plainJavaCapable(true);
    // Uncomment out this line to debug the generated code.
    //    cg.getCodeGenerator().saveCodeForDebugging(true);
    final ErrorCollector collector = new ErrorCollectorImpl();
    // Generate members and initialization code
    /////////////////////////////////////////
    // declare and assign JoinStatus member
    cg.setMappingSet(setupMapping);
    JClass joinStatusClass = cg.getModel().ref(JoinStatus.class);
    JVar joinStatus = cg.clazz.field(JMod.NONE, joinStatusClass, "status");
    cg.getSetupBlock().assign(JExpr._this().ref(joinStatus), JExpr.direct("status"));
    // declare and assign outgoing VectorContainer member
    JClass vectorContainerClass = cg.getModel().ref(VectorContainer.class);
    JVar outgoingVectorContainer = cg.clazz.field(JMod.NONE, vectorContainerClass, "outgoing");
    cg.getSetupBlock().assign(JExpr._this().ref(outgoingVectorContainer), JExpr.direct("outgoing"));
    // declare and assign incoming left RecordBatch member
    JClass recordBatchClass = cg.getModel().ref(RecordIterator.class);
    JVar incomingLeftRecordBatch = cg.clazz.field(JMod.NONE, recordBatchClass, "incomingLeft");
    cg.getSetupBlock().assign(JExpr._this().ref(incomingLeftRecordBatch), joinStatus.ref("left"));
    // declare and assign incoming right RecordBatch member
    JVar incomingRightRecordBatch = cg.clazz.field(JMod.NONE, recordBatchClass, "incomingRight");
    cg.getSetupBlock().assign(JExpr._this().ref(incomingRightRecordBatch), joinStatus.ref("right"));
    // declare 'incoming' member so VVReadExpr generated code can point to the left or right batch
    JVar incomingRecordBatch = cg.clazz.field(JMod.NONE, recordBatchClass, "incoming");
    /*
     * Materialize expressions on both sides of the join condition. Check if both the sides
     * have the same return type, if not then inject casts so that comparison function will work as
     * expected
     */
    LogicalExpression[] leftExpr = new LogicalExpression[conditions.size()];
    LogicalExpression[] rightExpr = new LogicalExpression[conditions.size()];
    IterOutcome lastLeftStatus = status.getLeftStatus();
    IterOutcome lastRightStatus = status.getRightStatus();
    for (int i = 0; i < conditions.size(); i++) {
        JoinCondition condition = conditions.get(i);
        leftExpr[i] = materializeExpression(condition.getLeft(), lastLeftStatus, leftIterator, collector);
        rightExpr[i] = materializeExpression(condition.getRight(), lastRightStatus, rightIterator, collector);
    }
    // call to throw an exception. In this case we can safely skip adding the casts
    if (lastRightStatus != IterOutcome.NONE) {
        JoinUtils.addLeastRestrictiveCasts(leftExpr, leftIterator, rightExpr, rightIterator, context);
    }
    //generate doCompare() method
    /////////////////////////////////////////
    generateDoCompare(cg, incomingRecordBatch, leftExpr, incomingLeftRecordBatch, rightExpr, incomingRightRecordBatch, collector);
    // generate copyLeft()
    //////////////////////
    cg.setMappingSet(copyLeftMapping);
    int vectorId = 0;
    if (worker == null || !status.left.finished()) {
        for (VectorWrapper<?> vw : leftIterator) {
            MajorType inputType = vw.getField().getType();
            MajorType outputType;
            if (joinType == JoinRelType.RIGHT && inputType.getMode() == DataMode.REQUIRED) {
                outputType = Types.overrideMode(inputType, DataMode.OPTIONAL);
            } else {
                outputType = inputType;
            }
            // TODO (DRILL-4011): Factor out CopyUtil and use it here.
            JVar vvIn = cg.declareVectorValueSetupAndMember("incomingLeft", new TypedFieldId(inputType, vectorId));
            JVar vvOut = cg.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(outputType, vectorId));
            // todo: check result of copyFromSafe and grow allocation
            cg.getEvalBlock().add(vvOut.invoke("copyFromSafe").arg(copyLeftMapping.getValueReadIndex()).arg(copyLeftMapping.getValueWriteIndex()).arg(vvIn));
            cg.rotateBlock();
            ++vectorId;
        }
    }
    // generate copyRight()
    ///////////////////////
    cg.setMappingSet(copyRightMappping);
    int rightVectorBase = vectorId;
    if (status.getRightStatus() != IterOutcome.NONE && (worker == null || !status.right.finished())) {
        for (VectorWrapper<?> vw : rightIterator) {
            MajorType inputType = vw.getField().getType();
            MajorType outputType;
            if (joinType == JoinRelType.LEFT && inputType.getMode() == DataMode.REQUIRED) {
                outputType = Types.overrideMode(inputType, DataMode.OPTIONAL);
            } else {
                outputType = inputType;
            }
            // TODO (DRILL-4011): Factor out CopyUtil and use it here.
            JVar vvIn = cg.declareVectorValueSetupAndMember("incomingRight", new TypedFieldId(inputType, vectorId - rightVectorBase));
            JVar vvOut = cg.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(outputType, vectorId));
            // todo: check result of copyFromSafe and grow allocation
            cg.getEvalBlock().add(vvOut.invoke("copyFromSafe").arg(copyRightMappping.getValueReadIndex()).arg(copyRightMappping.getValueWriteIndex()).arg(vvIn));
            cg.rotateBlock();
            ++vectorId;
        }
    }
    JoinWorker w = context.getImplementationClass(cg);
    w.setupJoin(context, status, this.container);
    return w;
}

Example 10 with MajorType

use of org.apache.drill.common.types.TypeProtos.MajorType in project drill by apache.

the class MergeJoinBatch method allocateBatch.

private void allocateBatch(boolean newSchema) {
    boolean leftAllowed = status.getLeftStatus() != IterOutcome.NONE;
    boolean rightAllowed = status.getRightStatus() != IterOutcome.NONE;
    if (newSchema) {
        container.clear();
        // add fields from both batches
        if (leftAllowed) {
            for (VectorWrapper<?> w : leftIterator) {
                MajorType inputType = w.getField().getType();
                MajorType outputType;
                if (joinType == JoinRelType.RIGHT && inputType.getMode() == DataMode.REQUIRED) {
                    outputType = Types.overrideMode(inputType, DataMode.OPTIONAL);
                } else {
                    outputType = inputType;
                }
                MaterializedField newField = MaterializedField.create(w.getField().getPath(), outputType);
                ValueVector v = container.addOrGet(newField);
                if (v instanceof AbstractContainerVector) {
                    w.getValueVector().makeTransferPair(v);
                    v.clear();
                }
            }
        }
        if (rightAllowed) {
            for (VectorWrapper<?> w : rightIterator) {
                MajorType inputType = w.getField().getType();
                MajorType outputType;
                if (joinType == JoinRelType.LEFT && inputType.getMode() == DataMode.REQUIRED) {
                    outputType = Types.overrideMode(inputType, DataMode.OPTIONAL);
                } else {
                    outputType = inputType;
                }
                MaterializedField newField = MaterializedField.create(w.getField().getPath(), outputType);
                ValueVector v = container.addOrGet(newField);
                if (v instanceof AbstractContainerVector) {
                    w.getValueVector().makeTransferPair(v);
                    v.clear();
                }
            }
        }
    } else {
        container.zeroVectors();
    }
    for (VectorWrapper w : container) {
        AllocationHelper.allocateNew(w.getValueVector(), Character.MAX_VALUE);
    }
    container.buildSchema(BatchSchema.SelectionVectorMode.NONE);
    logger.debug("Built joined schema: {}", container.getSchema());
}