Examples with TypedFieldId org.apache.drill.exec.record.TypedFieldId used on opensource projects

Example 51 with TypedFieldId

use of org.apache.drill.exec.record.TypedFieldId in project drill by axbaretto.

the class HashJoinBatch method setupHashJoinProbe.

public HashJoinProbe setupHashJoinProbe() throws ClassTransformationException, IOException {
    final CodeGenerator<HashJoinProbe> cg = CodeGenerator.get(HashJoinProbe.TEMPLATE_DEFINITION, context.getOptions());
    cg.plainJavaCapable(true);
    final ClassGenerator<HashJoinProbe> g = cg.getRoot();
    // Generate the code to project build side records
    g.setMappingSet(projectBuildMapping);
    int fieldId = 0;
    final JExpression buildIndex = JExpr.direct("buildIndex");
    final JExpression outIndex = JExpr.direct("outIndex");
    g.rotateBlock();
    if (rightSchema != null) {
        for (final MaterializedField field : rightSchema) {
            final MajorType inputType = field.getType();
            final MajorType outputType;
            // not nullable so we must exclude them from the check below (see DRILL-2197).
            if ((joinType == JoinRelType.LEFT || joinType == JoinRelType.FULL) && inputType.getMode() == DataMode.REQUIRED && inputType.getMinorType() != TypeProtos.MinorType.MAP) {
                outputType = Types.overrideMode(inputType, DataMode.OPTIONAL);
            } else {
                outputType = inputType;
            }
            // make sure to project field with children for children to show up in the schema
            final MaterializedField projected = field.withType(outputType);
            // Add the vector to our output container
            container.addOrGet(projected);
            final JVar inVV = g.declareVectorValueSetupAndMember("buildBatch", new TypedFieldId(field.getType(), true, fieldId));
            final JVar outVV = g.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(outputType, false, fieldId));
            g.getEvalBlock().add(outVV.invoke("copyFromSafe").arg(buildIndex.band(JExpr.lit((int) Character.MAX_VALUE))).arg(outIndex).arg(inVV.component(buildIndex.shrz(JExpr.lit(16)))));
            g.rotateBlock();
            fieldId++;
        }
    }
    // Generate the code to project probe side records
    g.setMappingSet(projectProbeMapping);
    int outputFieldId = fieldId;
    fieldId = 0;
    final JExpression probeIndex = JExpr.direct("probeIndex");
    if (leftUpstream == IterOutcome.OK || leftUpstream == IterOutcome.OK_NEW_SCHEMA) {
        for (final VectorWrapper<?> vv : left) {
            final MajorType inputType = vv.getField().getType();
            final MajorType outputType;
            // not nullable so we must exclude them from the check below (see DRILL-2771, DRILL-2197).
            if ((joinType == JoinRelType.RIGHT || joinType == JoinRelType.FULL) && inputType.getMode() == DataMode.REQUIRED && inputType.getMinorType() != TypeProtos.MinorType.MAP) {
                outputType = Types.overrideMode(inputType, DataMode.OPTIONAL);
            } else {
                outputType = inputType;
            }
            final ValueVector v = container.addOrGet(MaterializedField.create(vv.getField().getName(), outputType));
            if (v instanceof AbstractContainerVector) {
                vv.getValueVector().makeTransferPair(v);
                v.clear();
            }
            final JVar inVV = g.declareVectorValueSetupAndMember("probeBatch", new TypedFieldId(inputType, false, fieldId));
            final JVar outVV = g.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(outputType, false, outputFieldId));
            g.getEvalBlock().add(outVV.invoke("copyFromSafe").arg(probeIndex).arg(outIndex).arg(inVV));
            g.rotateBlock();
            fieldId++;
            outputFieldId++;
        }
    }
    final HashJoinProbe hj = context.getImplementationClass(cg);
    return hj;
}

Example 52 with TypedFieldId

use of org.apache.drill.exec.record.TypedFieldId in project drill by axbaretto.

the class MergeJoinBatch method generateNewWorker.

private JoinWorker generateNewWorker() throws ClassTransformationException, IOException, SchemaChangeException {
    final ClassGenerator<JoinWorker> cg = CodeGenerator.getRoot(JoinWorker.TEMPLATE_DEFINITION, context.getOptions());
    cg.getCodeGenerator().plainJavaCapable(true);
    // cg.getCodeGenerator().saveCodeForDebugging(true);
    final ErrorCollector collector = new ErrorCollectorImpl();
    // Generate members and initialization code
    // ///////////////////////////////////////
    // declare and assign JoinStatus member
    cg.setMappingSet(setupMapping);
    JClass joinStatusClass = cg.getModel().ref(JoinStatus.class);
    JVar joinStatus = cg.clazz.field(JMod.NONE, joinStatusClass, "status");
    cg.getSetupBlock().assign(JExpr._this().ref(joinStatus), JExpr.direct("status"));
    // declare and assign outgoing VectorContainer member
    JClass vectorContainerClass = cg.getModel().ref(VectorContainer.class);
    JVar outgoingVectorContainer = cg.clazz.field(JMod.NONE, vectorContainerClass, "outgoing");
    cg.getSetupBlock().assign(JExpr._this().ref(outgoingVectorContainer), JExpr.direct("outgoing"));
    // declare and assign incoming left RecordBatch member
    JClass recordBatchClass = cg.getModel().ref(RecordIterator.class);
    JVar incomingLeftRecordBatch = cg.clazz.field(JMod.NONE, recordBatchClass, "incomingLeft");
    cg.getSetupBlock().assign(JExpr._this().ref(incomingLeftRecordBatch), joinStatus.ref("left"));
    // declare and assign incoming right RecordBatch member
    JVar incomingRightRecordBatch = cg.clazz.field(JMod.NONE, recordBatchClass, "incomingRight");
    cg.getSetupBlock().assign(JExpr._this().ref(incomingRightRecordBatch), joinStatus.ref("right"));
    // declare 'incoming' member so VVReadExpr generated code can point to the left or right batch
    JVar incomingRecordBatch = cg.clazz.field(JMod.NONE, recordBatchClass, "incoming");
    /*
     * Materialize expressions on both sides of the join condition. Check if both the sides
     * have the same return type, if not then inject casts so that comparison function will work as
     * expected
     */
    LogicalExpression[] leftExpr = new LogicalExpression[conditions.size()];
    LogicalExpression[] rightExpr = new LogicalExpression[conditions.size()];
    IterOutcome lastLeftStatus = status.getLeftStatus();
    IterOutcome lastRightStatus = status.getRightStatus();
    for (int i = 0; i < conditions.size(); i++) {
        JoinCondition condition = conditions.get(i);
        leftExpr[i] = materializeExpression(condition.getLeft(), lastLeftStatus, leftIterator, collector);
        rightExpr[i] = materializeExpression(condition.getRight(), lastRightStatus, rightIterator, collector);
    }
    // call to throw an exception. In this case we can safely skip adding the casts
    if (lastRightStatus != IterOutcome.NONE) {
        JoinUtils.addLeastRestrictiveCasts(leftExpr, leftIterator, rightExpr, rightIterator, context);
    }
    // generate doCompare() method
    // ///////////////////////////////////////
    generateDoCompare(cg, incomingRecordBatch, leftExpr, incomingLeftRecordBatch, rightExpr, incomingRightRecordBatch, collector);
    // generate copyLeft()
    // ////////////////////
    cg.setMappingSet(copyLeftMapping);
    int vectorId = 0;
    if (worker == null || !status.left.finished()) {
        for (VectorWrapper<?> vw : leftIterator) {
            MajorType inputType = vw.getField().getType();
            MajorType outputType;
            if (joinType == JoinRelType.RIGHT && inputType.getMode() == DataMode.REQUIRED) {
                outputType = Types.overrideMode(inputType, DataMode.OPTIONAL);
            } else {
                outputType = inputType;
            }
            // TODO (DRILL-4011): Factor out CopyUtil and use it here.
            JVar vvIn = cg.declareVectorValueSetupAndMember("incomingLeft", new TypedFieldId(inputType, vectorId));
            JVar vvOut = cg.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(outputType, vectorId));
            // todo: check result of copyFromSafe and grow allocation
            cg.getEvalBlock().add(vvOut.invoke("copyFromSafe").arg(copyLeftMapping.getValueReadIndex()).arg(copyLeftMapping.getValueWriteIndex()).arg(vvIn));
            cg.rotateBlock();
            ++vectorId;
        }
    }
    // generate copyRight()
    // /////////////////////
    cg.setMappingSet(copyRightMappping);
    int rightVectorBase = vectorId;
    if (status.getRightStatus() != IterOutcome.NONE && (worker == null || !status.right.finished())) {
        for (VectorWrapper<?> vw : rightIterator) {
            MajorType inputType = vw.getField().getType();
            MajorType outputType;
            if (joinType == JoinRelType.LEFT && inputType.getMode() == DataMode.REQUIRED) {
                outputType = Types.overrideMode(inputType, DataMode.OPTIONAL);
            } else {
                outputType = inputType;
            }
            // TODO (DRILL-4011): Factor out CopyUtil and use it here.
            JVar vvIn = cg.declareVectorValueSetupAndMember("incomingRight", new TypedFieldId(inputType, vectorId - rightVectorBase));
            JVar vvOut = cg.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(outputType, vectorId));
            // todo: check result of copyFromSafe and grow allocation
            cg.getEvalBlock().add(vvOut.invoke("copyFromSafe").arg(copyRightMappping.getValueReadIndex()).arg(copyRightMappping.getValueWriteIndex()).arg(vvIn));
            cg.rotateBlock();
            ++vectorId;
        }
    }
    JoinWorker w = context.getImplementationClass(cg);
    w.setupJoin(context, status, this.container);
    return w;
}

Example 53 with TypedFieldId

use of org.apache.drill.exec.record.TypedFieldId in project drill by axbaretto.

the class NestedLoopJoinBatch method setupWorker.

/**
 * Method generates the runtime code needed for NLJ. Other than the setup method to set the input and output value
 * vector references we implement three more methods
 * 1. doEval() -> Evaluates if record from left side matches record from the right side
 * 2. emitLeft() -> Project record from the left side
 * 3. emitRight() -> Project record from the right side (which is a hyper container)
 * @return the runtime generated class that implements the NestedLoopJoin interface
 */
private NestedLoopJoin setupWorker() throws IOException, ClassTransformationException, SchemaChangeException {
    final CodeGenerator<NestedLoopJoin> nLJCodeGenerator = CodeGenerator.get(NestedLoopJoin.TEMPLATE_DEFINITION, context.getOptions());
    nLJCodeGenerator.plainJavaCapable(true);
    // Uncomment out this line to debug the generated code.
    // nLJCodeGenerator.saveCodeForDebugging(true);
    final ClassGenerator<NestedLoopJoin> nLJClassGenerator = nLJCodeGenerator.getRoot();
    // generate doEval
    final ErrorCollector collector = new ErrorCollectorImpl();
    /*
        Logical expression may contain fields from left and right batches. During code generation (materialization)
        we need to indicate from which input field should be taken.

        Non-equality joins can belong to one of below categories. For example:
        1. Join on non-equality join predicates:
        select * from t1 inner join t2 on (t1.c1 between t2.c1 AND t2.c2) AND (...)
        2. Join with an OR predicate:
        select * from t1 inner join t2 on on t1.c1 = t2.c1 OR t1.c2 = t2.c2
     */
    Map<VectorAccessible, BatchReference> batches = ImmutableMap.<VectorAccessible, BatchReference>builder().put(left, new BatchReference("leftBatch", "leftIndex")).put(rightContainer, new BatchReference("rightContainer", "rightBatchIndex", "rightRecordIndexWithinBatch")).build();
    LogicalExpression materialize = ExpressionTreeMaterializer.materialize(popConfig.getCondition(), batches, collector, context.getFunctionRegistry(), false, false);
    if (collector.hasErrors()) {
        throw new SchemaChangeException(String.format("Failure while trying to materialize join condition. Errors:\n %s.", collector.toErrorString()));
    }
    nLJClassGenerator.addExpr(new ReturnValueExpression(materialize), ClassGenerator.BlkCreateMode.FALSE);
    // generate emitLeft
    nLJClassGenerator.setMappingSet(emitLeftMapping);
    JExpression outIndex = JExpr.direct("outIndex");
    JExpression leftIndex = JExpr.direct("leftIndex");
    int fieldId = 0;
    int outputFieldId = 0;
    if (leftSchema != null) {
        // Set the input and output value vector references corresponding to the left batch
        for (MaterializedField field : leftSchema) {
            final TypeProtos.MajorType fieldType = field.getType();
            // Add the vector to the output container
            container.addOrGet(field);
            JVar inVV = nLJClassGenerator.declareVectorValueSetupAndMember("leftBatch", new TypedFieldId(fieldType, false, fieldId));
            JVar outVV = nLJClassGenerator.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(fieldType, false, outputFieldId));
            nLJClassGenerator.getEvalBlock().add(outVV.invoke("copyFromSafe").arg(leftIndex).arg(outIndex).arg(inVV));
            nLJClassGenerator.rotateBlock();
            fieldId++;
            outputFieldId++;
        }
    }
    // generate emitRight
    fieldId = 0;
    nLJClassGenerator.setMappingSet(emitRightMapping);
    JExpression batchIndex = JExpr.direct("batchIndex");
    JExpression recordIndexWithinBatch = JExpr.direct("recordIndexWithinBatch");
    if (rightSchema != null) {
        // Set the input and output value vector references corresponding to the right batch
        for (MaterializedField field : rightSchema) {
            final TypeProtos.MajorType inputType = field.getType();
            TypeProtos.MajorType outputType;
            // if join type is LEFT, make sure right batch output fields data mode is optional
            if (popConfig.getJoinType() == JoinRelType.LEFT && inputType.getMode() == TypeProtos.DataMode.REQUIRED) {
                outputType = Types.overrideMode(inputType, TypeProtos.DataMode.OPTIONAL);
            } else {
                outputType = inputType;
            }
            MaterializedField newField = MaterializedField.create(field.getName(), outputType);
            container.addOrGet(newField);
            JVar inVV = nLJClassGenerator.declareVectorValueSetupAndMember("rightContainer", new TypedFieldId(inputType, true, fieldId));
            JVar outVV = nLJClassGenerator.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(outputType, false, outputFieldId));
            nLJClassGenerator.getEvalBlock().add(outVV.invoke("copyFromSafe").arg(recordIndexWithinBatch).arg(outIndex).arg(inVV.component(batchIndex)));
            nLJClassGenerator.rotateBlock();
            fieldId++;
            outputFieldId++;
        }
    }
    return context.getImplementationClass(nLJCodeGenerator);
}

Example 54 with TypedFieldId

use of org.apache.drill.exec.record.TypedFieldId in project drill by apache.

the class BloomFilterTest method getHash64.

private static ValueVectorHashHelper.Hash64 getHash64(FragmentContext context, RowSet.SingleRowSet probeRowSet) throws ClassTransformationException, IOException, SchemaChangeException {
    RecordBatch probeRecordBatch = new TestRecordBatch(probeRowSet.container());
    TypedFieldId probeFieldId = probeRecordBatch.getValueVectorId(SchemaPath.getSimplePath("a"));
    ValueVectorReadExpression probExp = new ValueVectorReadExpression(probeFieldId);
    LogicalExpression[] probExpressions = new LogicalExpression[1];
    probExpressions[0] = probExp;
    TypedFieldId[] probeFieldIds = new TypedFieldId[1];
    probeFieldIds[0] = probeFieldId;
    ValueVectorHashHelper probeValueVectorHashHelper = new ValueVectorHashHelper(probeRecordBatch, context);
    return probeValueVectorHashHelper.getHash64(probExpressions, probeFieldIds);
}

Example 55 with TypedFieldId

use of org.apache.drill.exec.record.TypedFieldId in project drill by apache.

the class CopyUtil method generateCopies.

public static void generateCopies(ClassGenerator<?> g, VectorAccessible batch, boolean hyper) {
    // we have parallel ids for each value vector so we don't actually have to deal with managing the ids at all.
    int fieldId = 0;
    JExpression inIndex = JExpr.direct("inIndex");
    JExpression outIndex = JExpr.direct("outIndex");
    for (VectorWrapper<?> vv : batch) {
        String copyMethod;
        if (!Types.isFixedWidthType(vv.getField().getType()) || Types.isRepeated(vv.getField().getType()) || Types.isComplex(vv.getField().getType())) {
            copyMethod = "copyFromSafe";
        } else {
            copyMethod = "copyFrom";
        }
        g.rotateBlock();
        TypedFieldId inFieldId = new TypedFieldId.Builder().finalType(vv.getField().getType()).hyper(vv.isHyper()).addId(fieldId).build();
        JVar inVV = g.declareVectorValueSetupAndMember("incoming", inFieldId);
        TypedFieldId outFieldId = new TypedFieldId.Builder().finalType(vv.getField().getType()).hyper(false).addId(fieldId).build();
        JVar outVV = g.declareVectorValueSetupAndMember("outgoing", outFieldId);
        if (hyper) {
            g.getEvalBlock().add(outVV.invoke(copyMethod).arg(inIndex.band(JExpr.lit((int) Character.MAX_VALUE))).arg(outIndex).arg(inVV.component(inIndex.shrz(JExpr.lit(16)))));
        } else {
            g.getEvalBlock().add(outVV.invoke(copyMethod).arg(inIndex).arg(outIndex).arg(inVV));
        }
        g.rotateBlock();
        fieldId++;
    }
}