Examples with ErrorCollectorImpl org.apache.drill.common.expression.ErrorCollectorImpl used on opensource projects

Example 71 with ErrorCollectorImpl

use of org.apache.drill.common.expression.ErrorCollectorImpl in project drill by apache.

the class ExpressionTreeMaterializerTest method testMaterializingLateboundField.

@Test
public void testMaterializingLateboundField() throws SchemaChangeException {
    final RecordBatch batch = mock(RecordBatch.class);
    TypedFieldId fieldId = new TypedFieldId.Builder().finalType(Types.required(MinorType.BIGINT)).addId(-5).build();
    when(batch.getValueVectorId(new SchemaPath("test", ExpressionPosition.UNKNOWN))).thenReturn(fieldId);
    final SchemaBuilder builder = BatchSchema.newBuilder();
    builder.addField(getField("test", bigIntType));
    final BatchSchema schema = builder.build();
    ErrorCollector ec = new ErrorCollectorImpl();
    LogicalExpression expr = ExpressionTreeMaterializer.materialize(new FieldReference("test", ExpressionPosition.UNKNOWN), batch, ec, registry);
    assertEquals(bigIntType, expr.getMajorType());
    assertFalse(ec.hasErrors());
}

Example 72 with ErrorCollectorImpl

use of org.apache.drill.common.expression.ErrorCollectorImpl in project drill by apache.

the class ExpressionTreeMaterializerTest method testMaterializingConstantTree.

@Test
public void testMaterializingConstantTree() throws SchemaChangeException {
    final RecordBatch batch = mock(RecordBatch.class);
    ErrorCollector ec = new ErrorCollectorImpl();
    LogicalExpression expr = ExpressionTreeMaterializer.materialize(new ValueExpressions.LongExpression(1L, ExpressionPosition.UNKNOWN), batch, ec, registry);
    assertTrue(expr instanceof ValueExpressions.LongExpression);
    assertEquals(1L, ValueExpressions.LongExpression.class.cast(expr).getLong());
    assertFalse(ec.hasErrors());
}

Example 73 with ErrorCollectorImpl

use of org.apache.drill.common.expression.ErrorCollectorImpl in project drill by apache.

the class AbstractGroupScanWithMetadata method getFilterPredicate.

/**
 * Returns parquet filter predicate built from specified {@code filterExpr}.
 *
 * @param filterExpr                     filter expression to build
 * @param udfUtilities                   udf utilities
 * @param functionImplementationRegistry context to find drill function holder
 * @param optionManager                  option manager
 * @param omitUnsupportedExprs           whether expressions which cannot be converted
 *                                       may be omitted from the resulting expression
 * @param supportsFileImplicitColumns    whether implicit columns are supported
 * @param schema                         schema
 * @return parquet filter predicate
 */
public static FilterPredicate<?> getFilterPredicate(LogicalExpression filterExpr, UdfUtilities udfUtilities, FunctionLookupContext functionImplementationRegistry, OptionManager optionManager, boolean omitUnsupportedExprs, boolean supportsFileImplicitColumns, TupleMetadata schema) {
    TupleMetadata types = schema.copy();
    Set<SchemaPath> schemaPathsInExpr = filterExpr.accept(FilterEvaluatorUtils.FieldReferenceFinder.INSTANCE, null);
    // adds implicit or partition columns if they weren't added before.
    if (supportsFileImplicitColumns) {
        for (SchemaPath schemaPath : schemaPathsInExpr) {
            if (isImplicitOrPartCol(schemaPath, optionManager) && SchemaPathUtils.getColumnMetadata(schemaPath, types) == null) {
                types.add(MaterializedField.create(schemaPath.getRootSegmentPath(), Types.required(TypeProtos.MinorType.VARCHAR)));
            }
        }
    }
    ErrorCollector errorCollector = new ErrorCollectorImpl();
    LogicalExpression materializedFilter = ExpressionTreeMaterializer.materializeFilterExpr(filterExpr, types, errorCollector, functionImplementationRegistry);
    if (errorCollector.hasErrors()) {
        logger.error("{} error(s) encountered when materialize filter expression : {}", errorCollector.getErrorCount(), errorCollector.toErrorString());
        return null;
    }
    if (logger.isDebugEnabled()) {
        logger.debug("materializedFilter : {}", ExpressionStringBuilder.toString(materializedFilter));
    }
    Set<LogicalExpression> constantBoundaries = ConstantExpressionIdentifier.getConstantExpressionSet(materializedFilter);
    return FilterBuilder.buildFilterPredicate(materializedFilter, constantBoundaries, udfUtilities, omitUnsupportedExprs);
}

Example 74 with ErrorCollectorImpl

use of org.apache.drill.common.expression.ErrorCollectorImpl in project drill by apache.

the class OrderedPartitionRecordBatch method getCopier.

/**
 * Creates a copier that does a project for every Nth record from a
 * VectorContainer incoming into VectorContainer outgoing. Each Ordering in
 * orderings generates a column, and evaluation of the expression associated
 * with each Ordering determines the value of each column. These records will
 * later be sorted based on the values in each column, in the same order as
 * the orderings.
 */
private SampleCopier getCopier(SelectionVector4 sv4, VectorContainer incoming, VectorContainer outgoing, List<Ordering> orderings, List<ValueVector> localAllocationVectors) {
    ErrorCollector collector = new ErrorCollectorImpl();
    ClassGenerator<SampleCopier> cg = CodeGenerator.getRoot(SampleCopier.TEMPLATE_DEFINITION, context.getOptions());
    // Note: disabled for now. This may require some debugging:
    // no tests are available for this operator.
    // cg.getCodeGenerator().plainOldJavaCapable(true);
    // Uncomment out this line to debug the generated code.
    // cg.getCodeGenerator().saveCodeForDebugging(true);
    int i = 0;
    for (Ordering od : orderings) {
        LogicalExpression expr = ExpressionTreeMaterializer.materialize(od.getExpr(), incoming, collector, context.getFunctionRegistry());
        TypeProtos.MajorType.Builder builder = TypeProtos.MajorType.newBuilder().mergeFrom(expr.getMajorType()).clearMode().setMode(TypeProtos.DataMode.REQUIRED);
        TypeProtos.MajorType newType = builder.build();
        MaterializedField outputField = MaterializedField.create("f" + i++, newType);
        collector.reportErrors(logger);
        ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator());
        localAllocationVectors.add(vector);
        TypedFieldId fid = outgoing.add(vector);
        ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, true);
        HoldingContainer hc = cg.addExpr(write);
        cg.getEvalBlock()._if(hc.getValue().eq(JExpr.lit(0)))._then()._return(JExpr.FALSE);
    }
    cg.rotateBlock();
    cg.getEvalBlock()._return(JExpr.TRUE);
    outgoing.buildSchema(BatchSchema.SelectionVectorMode.NONE);
    try {
        SampleCopier sampleCopier = context.getImplementationClass(cg);
        sampleCopier.setupCopier(context, sv4, incoming, outgoing);
        return sampleCopier;
    } catch (SchemaChangeException e) {
        throw schemaChangeException(e, logger);
    }
}

Example 75 with ErrorCollectorImpl

use of org.apache.drill.common.expression.ErrorCollectorImpl in project drill by apache.

the class NestedLoopJoinBatch method setupWorker.

/**
 * Method generates the runtime code needed for NLJ. Other than the setup method to set the input and output value
 * vector references we implement three more methods
 * 1. doEval() -> Evaluates if record from left side matches record from the right side
 * 2. emitLeft() -> Project record from the left side
 * 3. emitRight() -> Project record from the right side (which is a hyper container)
 * @return the runtime generated class that implements the NestedLoopJoin interface
 */
private NestedLoopJoin setupWorker() {
    final CodeGenerator<NestedLoopJoin> nLJCodeGenerator = CodeGenerator.get(SETUP_LEFT_MAPPING, NestedLoopJoin.TEMPLATE_DEFINITION, context.getOptions());
    nLJCodeGenerator.plainJavaCapable(true);
    // Uncomment out this line to debug the generated code.
    // nLJCodeGenerator.saveCodeForDebugging(true);
    final ClassGenerator<NestedLoopJoin> nLJClassGenerator = nLJCodeGenerator.getRoot();
    // generate doEval
    final ErrorCollector collector = new ErrorCollectorImpl();
    /*
        Logical expression may contain fields from left and right batches. During code generation (materialization)
        we need to indicate from which input field should be taken.

        Non-equality joins can belong to one of below categories. For example:
        1. Join on non-equality join predicates:
        select * from t1 inner join t2 on (t1.c1 between t2.c1 AND t2.c2) AND (...)
        2. Join with an OR predicate:
        select * from t1 inner join t2 on on t1.c1 = t2.c1 OR t1.c2 = t2.c2
     */
    Map<VectorAccessible, BatchReference> batches = ImmutableMap.<VectorAccessible, BatchReference>builder().put(left, new BatchReference("leftBatch", "leftIndex")).put(rightContainer, new BatchReference("rightContainer", "rightBatchIndex", "rightRecordIndexWithinBatch")).build();
    LogicalExpression materialize = ExpressionTreeMaterializer.materialize(popConfig.getCondition(), batches, collector, context.getFunctionRegistry(), false, false);
    collector.reportErrors(logger);
    nLJClassGenerator.addExpr(new ReturnValueExpression(materialize), ClassGenerator.BlkCreateMode.FALSE);
    // generate emitLeft
    nLJClassGenerator.setMappingSet(emitLeftMapping);
    JExpression outIndex = JExpr.direct("outIndex");
    JExpression leftIndex = JExpr.direct("leftIndex");
    int fieldId = 0;
    int outputFieldId = 0;
    if (leftSchema != null) {
        // Set the input and output value vector references corresponding to the left batch
        for (MaterializedField field : leftSchema) {
            final TypeProtos.MajorType fieldType = field.getType();
            // Add the vector to the output container
            container.addOrGet(field);
            TypedFieldId inFieldId = new TypedFieldId.Builder().finalType(fieldType).hyper(false).addId(fieldId).build();
            JVar inVV = nLJClassGenerator.declareVectorValueSetupAndMember("leftBatch", inFieldId);
            TypedFieldId outFieldId = new TypedFieldId.Builder().finalType(fieldType).hyper(false).addId(outputFieldId).build();
            JVar outVV = nLJClassGenerator.declareVectorValueSetupAndMember("outgoing", outFieldId);
            nLJClassGenerator.getEvalBlock().add(outVV.invoke("copyFromSafe").arg(leftIndex).arg(outIndex).arg(inVV));
            nLJClassGenerator.rotateBlock();
            fieldId++;
            outputFieldId++;
        }
    }
    // generate emitRight
    fieldId = 0;
    nLJClassGenerator.setMappingSet(emitRightMapping);
    JExpression batchIndex = JExpr.direct("batchIndex");
    JExpression recordIndexWithinBatch = JExpr.direct("recordIndexWithinBatch");
    if (rightSchema != null) {
        // Set the input and output value vector references corresponding to the right batch
        for (MaterializedField field : rightSchema) {
            final TypeProtos.MajorType inputType = field.getType();
            TypeProtos.MajorType outputType;
            // if join type is LEFT, make sure right batch output fields data mode is optional
            if (popConfig.getJoinType() == JoinRelType.LEFT && inputType.getMode() == TypeProtos.DataMode.REQUIRED) {
                outputType = Types.overrideMode(inputType, TypeProtos.DataMode.OPTIONAL);
            } else {
                outputType = inputType;
            }
            MaterializedField newField = MaterializedField.create(field.getName(), outputType);
            container.addOrGet(newField);
            TypedFieldId inFieldId = new TypedFieldId.Builder().finalType(inputType).hyper(true).addId(fieldId).build();
            JVar inVV = nLJClassGenerator.declareVectorValueSetupAndMember("rightContainer", inFieldId);
            TypedFieldId outFieldId = new TypedFieldId.Builder().finalType(outputType).hyper(false).addId(outputFieldId).build();
            JVar outVV = nLJClassGenerator.declareVectorValueSetupAndMember("outgoing", outFieldId);
            nLJClassGenerator.getEvalBlock().add(outVV.invoke("copyFromSafe").arg(recordIndexWithinBatch).arg(outIndex).arg(inVV.component(batchIndex)));
            nLJClassGenerator.rotateBlock();
            fieldId++;
            outputFieldId++;
        }
    }
    return context.getImplementationClass(nLJCodeGenerator);
}