Examples with TypedFieldId org.apache.drill.exec.record.TypedFieldId used on opensource projects

Example 56 with TypedFieldId

use of org.apache.drill.exec.record.TypedFieldId in project drill by apache.

the class TestEvaluationVisitor method testEvaluation.

@Test
public void testEvaluation() {
    EvaluationVisitor v = new EvaluationVisitor();
    CodeGenerator<?> g = CodeGenerator.get(Projector.TEMPLATE_DEFINITION, null);
    SchemaPath path = (SchemaPath) LogicalExpressionParser.parse("a.b[4][2].c[6]");
    TypedFieldId id = TypedFieldId.newBuilder().addId(1).addId(3).remainder(path.getRootSegment()).intermediateType(Types.optional(MinorType.MAP)).finalType(Types.repeated(MinorType.MAP)).hyper(true).withIndex().build();
    ValueVectorReadExpression e = new ValueVectorReadExpression(id);
    TypedFieldId outId = TypedFieldId.newBuilder().addId(1).finalType(Types.repeated(MinorType.MAP)).intermediateType(Types.repeated(MinorType.MAP)).build();
    ValueVectorWriteExpression e2 = new ValueVectorWriteExpression(outId, e, true);
    v.addExpr(e2, g.getRoot());
}

Example 57 with TypedFieldId

use of org.apache.drill.exec.record.TypedFieldId in project drill by apache.

the class ExpressionTest method testSchemaExpression.

@Test
public void testSchemaExpression() throws Exception {
    final RecordBatch batch = mock(RecordBatch.class);
    TypedFieldId fieldId = new TypedFieldId.Builder().finalType(Types.optional(MinorType.BIGINT)).hyper(false).addId(0).build();
    when(batch.getValueVectorId(new SchemaPath("alpha", ExpressionPosition.UNKNOWN))).thenReturn(fieldId);
    getExpressionCode("1 + alpha", batch);
}

Example 58 with TypedFieldId

use of org.apache.drill.exec.record.TypedFieldId in project drill by apache.

the class OrderedPartitionRecordBatch method getCopier.

/**
 * Creates a copier that does a project for every Nth record from a
 * VectorContainer incoming into VectorContainer outgoing. Each Ordering in
 * orderings generates a column, and evaluation of the expression associated
 * with each Ordering determines the value of each column. These records will
 * later be sorted based on the values in each column, in the same order as
 * the orderings.
 */
private SampleCopier getCopier(SelectionVector4 sv4, VectorContainer incoming, VectorContainer outgoing, List<Ordering> orderings, List<ValueVector> localAllocationVectors) {
    ErrorCollector collector = new ErrorCollectorImpl();
    ClassGenerator<SampleCopier> cg = CodeGenerator.getRoot(SampleCopier.TEMPLATE_DEFINITION, context.getOptions());
    // Note: disabled for now. This may require some debugging:
    // no tests are available for this operator.
    // cg.getCodeGenerator().plainOldJavaCapable(true);
    // Uncomment out this line to debug the generated code.
    // cg.getCodeGenerator().saveCodeForDebugging(true);
    int i = 0;
    for (Ordering od : orderings) {
        LogicalExpression expr = ExpressionTreeMaterializer.materialize(od.getExpr(), incoming, collector, context.getFunctionRegistry());
        TypeProtos.MajorType.Builder builder = TypeProtos.MajorType.newBuilder().mergeFrom(expr.getMajorType()).clearMode().setMode(TypeProtos.DataMode.REQUIRED);
        TypeProtos.MajorType newType = builder.build();
        MaterializedField outputField = MaterializedField.create("f" + i++, newType);
        collector.reportErrors(logger);
        ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator());
        localAllocationVectors.add(vector);
        TypedFieldId fid = outgoing.add(vector);
        ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, true);
        HoldingContainer hc = cg.addExpr(write);
        cg.getEvalBlock()._if(hc.getValue().eq(JExpr.lit(0)))._then()._return(JExpr.FALSE);
    }
    cg.rotateBlock();
    cg.getEvalBlock()._return(JExpr.TRUE);
    outgoing.buildSchema(BatchSchema.SelectionVectorMode.NONE);
    try {
        SampleCopier sampleCopier = context.getImplementationClass(cg);
        sampleCopier.setupCopier(context, sv4, incoming, outgoing);
        return sampleCopier;
    } catch (SchemaChangeException e) {
        throw schemaChangeException(e, logger);
    }
}

Example 59 with TypedFieldId

use of org.apache.drill.exec.record.TypedFieldId in project drill by apache.

the class NestedLoopJoinBatch method setupWorker.

/**
 * Method generates the runtime code needed for NLJ. Other than the setup method to set the input and output value
 * vector references we implement three more methods
 * 1. doEval() -> Evaluates if record from left side matches record from the right side
 * 2. emitLeft() -> Project record from the left side
 * 3. emitRight() -> Project record from the right side (which is a hyper container)
 * @return the runtime generated class that implements the NestedLoopJoin interface
 */
private NestedLoopJoin setupWorker() {
    final CodeGenerator<NestedLoopJoin> nLJCodeGenerator = CodeGenerator.get(SETUP_LEFT_MAPPING, NestedLoopJoin.TEMPLATE_DEFINITION, context.getOptions());
    nLJCodeGenerator.plainJavaCapable(true);
    // Uncomment out this line to debug the generated code.
    // nLJCodeGenerator.saveCodeForDebugging(true);
    final ClassGenerator<NestedLoopJoin> nLJClassGenerator = nLJCodeGenerator.getRoot();
    // generate doEval
    final ErrorCollector collector = new ErrorCollectorImpl();
    /*
        Logical expression may contain fields from left and right batches. During code generation (materialization)
        we need to indicate from which input field should be taken.

        Non-equality joins can belong to one of below categories. For example:
        1. Join on non-equality join predicates:
        select * from t1 inner join t2 on (t1.c1 between t2.c1 AND t2.c2) AND (...)
        2. Join with an OR predicate:
        select * from t1 inner join t2 on on t1.c1 = t2.c1 OR t1.c2 = t2.c2
     */
    Map<VectorAccessible, BatchReference> batches = ImmutableMap.<VectorAccessible, BatchReference>builder().put(left, new BatchReference("leftBatch", "leftIndex")).put(rightContainer, new BatchReference("rightContainer", "rightBatchIndex", "rightRecordIndexWithinBatch")).build();
    LogicalExpression materialize = ExpressionTreeMaterializer.materialize(popConfig.getCondition(), batches, collector, context.getFunctionRegistry(), false, false);
    collector.reportErrors(logger);
    nLJClassGenerator.addExpr(new ReturnValueExpression(materialize), ClassGenerator.BlkCreateMode.FALSE);
    // generate emitLeft
    nLJClassGenerator.setMappingSet(emitLeftMapping);
    JExpression outIndex = JExpr.direct("outIndex");
    JExpression leftIndex = JExpr.direct("leftIndex");
    int fieldId = 0;
    int outputFieldId = 0;
    if (leftSchema != null) {
        // Set the input and output value vector references corresponding to the left batch
        for (MaterializedField field : leftSchema) {
            final TypeProtos.MajorType fieldType = field.getType();
            // Add the vector to the output container
            container.addOrGet(field);
            TypedFieldId inFieldId = new TypedFieldId.Builder().finalType(fieldType).hyper(false).addId(fieldId).build();
            JVar inVV = nLJClassGenerator.declareVectorValueSetupAndMember("leftBatch", inFieldId);
            TypedFieldId outFieldId = new TypedFieldId.Builder().finalType(fieldType).hyper(false).addId(outputFieldId).build();
            JVar outVV = nLJClassGenerator.declareVectorValueSetupAndMember("outgoing", outFieldId);
            nLJClassGenerator.getEvalBlock().add(outVV.invoke("copyFromSafe").arg(leftIndex).arg(outIndex).arg(inVV));
            nLJClassGenerator.rotateBlock();
            fieldId++;
            outputFieldId++;
        }
    }
    // generate emitRight
    fieldId = 0;
    nLJClassGenerator.setMappingSet(emitRightMapping);
    JExpression batchIndex = JExpr.direct("batchIndex");
    JExpression recordIndexWithinBatch = JExpr.direct("recordIndexWithinBatch");
    if (rightSchema != null) {
        // Set the input and output value vector references corresponding to the right batch
        for (MaterializedField field : rightSchema) {
            final TypeProtos.MajorType inputType = field.getType();
            TypeProtos.MajorType outputType;
            // if join type is LEFT, make sure right batch output fields data mode is optional
            if (popConfig.getJoinType() == JoinRelType.LEFT && inputType.getMode() == TypeProtos.DataMode.REQUIRED) {
                outputType = Types.overrideMode(inputType, TypeProtos.DataMode.OPTIONAL);
            } else {
                outputType = inputType;
            }
            MaterializedField newField = MaterializedField.create(field.getName(), outputType);
            container.addOrGet(newField);
            TypedFieldId inFieldId = new TypedFieldId.Builder().finalType(inputType).hyper(true).addId(fieldId).build();
            JVar inVV = nLJClassGenerator.declareVectorValueSetupAndMember("rightContainer", inFieldId);
            TypedFieldId outFieldId = new TypedFieldId.Builder().finalType(outputType).hyper(false).addId(outputFieldId).build();
            JVar outVV = nLJClassGenerator.declareVectorValueSetupAndMember("outgoing", outFieldId);
            nLJClassGenerator.getEvalBlock().add(outVV.invoke("copyFromSafe").arg(recordIndexWithinBatch).arg(outIndex).arg(inVV.component(batchIndex)));
            nLJClassGenerator.rotateBlock();
            fieldId++;
            outputFieldId++;
        }
    }
    return context.getImplementationClass(nLJCodeGenerator);
}

Example 60 with TypedFieldId

use of org.apache.drill.exec.record.TypedFieldId in project drill by apache.

the class StatisticsAggBatch method createAggregatorInternal.

@Override
protected StreamingAggregator createAggregatorInternal() {
    List<LogicalExpression> keyExprs = Lists.newArrayList();
    List<LogicalExpression> valueExprs = Lists.newArrayList();
    List<TypedFieldId> keyOutputIds = Lists.newArrayList();
    String[] colMeta = new String[] { Statistic.COLNAME, Statistic.COLTYPE };
    container.clear();
    // the implicit columns
    for (String col : colMeta) {
        MapVector parent = new MapVector(col, oContext.getAllocator(), null);
        container.add(parent);
        for (MaterializedField mf : incoming.getSchema()) {
            LogicalExpression expr;
            if (col.equals(colMeta[0])) {
                expr = ValueExpressions.getChar(SchemaPath.getSimplePath(mf.getName()).toString(), 0);
            } else {
                try {
                    expr = ValueExpressions.getChar(DrillStatsTable.getMapper().writeValueAsString(mf.getType()), 0);
                } catch (JsonProcessingException e) {
                    throw UserException.dataWriteError(e).addContext("Failed to write statistics to JSON").build();
                }
            }
            // Ignore implicit columns
            if (!isImplicitFileOrPartitionColumn(mf, incoming.getContext().getOptions())) {
                createNestedKeyColumn(parent, SchemaPath.getSimplePath(mf.getName()).toString(), expr, keyExprs, keyOutputIds);
            }
        }
    }
    // employee NDV = 500, salary NDV = 10
    for (String func : functions) {
        MapVector parent = new MapVector(func, oContext.getAllocator(), null);
        container.add(parent);
        for (MaterializedField mf : incoming.getSchema()) {
            // such as MAP, LIST are not supported!
            if (isColMinorTypeValid(mf) && !isImplicitFileOrPartitionColumn(mf, incoming.getContext().getOptions())) {
                List<LogicalExpression> args = Lists.newArrayList();
                args.add(SchemaPath.getSimplePath(mf.getName()));
                LogicalExpression call = FunctionCallFactory.createExpression(func, args);
                addMapVector(SchemaPath.getSimplePath(mf.getName()).toString(), parent, call, valueExprs);
            }
        }
    }
    // Now generate the code for the statistics aggregate
    return codegenAggregator(keyExprs, valueExprs, keyOutputIds);
}