Examples with ErrorCollectorImpl org.apache.drill.common.expression.ErrorCollectorImpl used on opensource projects

Example 1 with ErrorCollectorImpl

use of org.apache.drill.common.expression.ErrorCollectorImpl in project drill by apache.

the class ExpressionTreeMaterializerTest method testMaterializingConstantTree.

@Test
public void testMaterializingConstantTree(@Injectable RecordBatch batch) throws SchemaChangeException {
    ErrorCollector ec = new ErrorCollectorImpl();
    LogicalExpression expr = ExpressionTreeMaterializer.materialize(new ValueExpressions.LongExpression(1L, ExpressionPosition.UNKNOWN), batch, ec, registry);
    assertTrue(expr instanceof ValueExpressions.LongExpression);
    assertEquals(1L, ValueExpressions.LongExpression.class.cast(expr).getLong());
    assertFalse(ec.hasErrors());
}

Example 2 with ErrorCollectorImpl

use of org.apache.drill.common.expression.ErrorCollectorImpl in project drill by apache.

the class ExpressionTreeMaterializerTest method testMaterializingLateboundTree.

@Test
public void testMaterializingLateboundTree(@Injectable final RecordBatch batch) throws SchemaChangeException {
    new NonStrictExpectations() {

        {
            batch.getValueVectorId(SchemaPath.getSimplePath("test"));
            result = new TypedFieldId(Types.required(MinorType.BIT), -4);
            batch.getValueVectorId(SchemaPath.getSimplePath("test1"));
            result = new TypedFieldId(Types.required(MinorType.BIGINT), -5);
        }
    };
    ErrorCollector ec = new ErrorCollectorImpl();
    LogicalExpression elseExpression = new IfExpression.Builder().setElse(new ValueExpressions.LongExpression(1L, ExpressionPosition.UNKNOWN)).setIfCondition(new IfExpression.IfCondition(new ValueExpressions.BooleanExpression("true", ExpressionPosition.UNKNOWN), new FieldReference("test1", ExpressionPosition.UNKNOWN))).build();
    LogicalExpression expr = new IfExpression.Builder().setIfCondition(new IfExpression.IfCondition(new FieldReference("test", ExpressionPosition.UNKNOWN), new ValueExpressions.LongExpression(2L, ExpressionPosition.UNKNOWN))).setElse(elseExpression).build();
    LogicalExpression newExpr = ExpressionTreeMaterializer.materialize(expr, batch, ec, registry);
    assertTrue(newExpr instanceof IfExpression);
    IfExpression newIfExpr = (IfExpression) newExpr;
    //assertEquals(1, newIfExpr.conditions.size());
    IfExpression.IfCondition ifCondition = newIfExpr.ifCondition;
    assertTrue(newIfExpr.elseExpression instanceof IfExpression);
    //assertEquals(1, newIfExpr.conditions.size());
    //ifCondition = newIfExpr.conditions.get(0);
    assertEquals(bigIntType, ifCondition.expression.getMajorType());
    assertEquals(true, ((ValueExpressions.BooleanExpression) ((IfExpression) (newIfExpr.elseExpression)).ifCondition.condition).value);
    if (ec.hasErrors()) {
        System.out.println(ec.toErrorString());
    }
    assertFalse(ec.hasErrors());
}

Example 3 with ErrorCollectorImpl

use of org.apache.drill.common.expression.ErrorCollectorImpl in project drill by apache.

the class ExpressionTreeMaterializerTest method testMaterializingLateboundField.

@Test
public void testMaterializingLateboundField(@Injectable final RecordBatch batch) throws SchemaChangeException {
    final SchemaBuilder builder = BatchSchema.newBuilder();
    builder.addField(getField(2, "test", bigIntType));
    final BatchSchema schema = builder.build();
    new NonStrictExpectations() {

        {
            batch.getValueVectorId(new SchemaPath("test", ExpressionPosition.UNKNOWN));
            result = new TypedFieldId(Types.required(MinorType.BIGINT), -5);
        }
    };
    ErrorCollector ec = new ErrorCollectorImpl();
    LogicalExpression expr = ExpressionTreeMaterializer.materialize(new FieldReference("test", ExpressionPosition.UNKNOWN), batch, ec, registry);
    assertEquals(bigIntType, expr.getMajorType());
    assertFalse(ec.hasErrors());
}

Example 4 with ErrorCollectorImpl

use of org.apache.drill.common.expression.ErrorCollectorImpl in project drill by apache.

the class FunctionGenerationHelper method getOrderingComparator.

/**
   * Finds ordering comparator ("compare_to...") FunctionHolderExpression with
   * a specified ordering for NULL (and considering NULLS <i>equal</i>).
   * @param  null_high  whether NULL should compare as the lowest value (if
   *                    {@code false}) or the highest value (if {@code true})
   * @param  left  ...
   * @param  right  ...
   * @param  registry  ...
   * @return
   *     FunctionHolderExpression containing the found function implementation
   */
public static LogicalExpression getOrderingComparator(boolean null_high, HoldingContainer left, HoldingContainer right, FunctionImplementationRegistry registry) {
    final String comparator_name = null_high ? COMPARE_TO_NULLS_HIGH : COMPARE_TO_NULLS_LOW;
    if (!isComparableType(left.getMajorType()) || !isComparableType(right.getMajorType())) {
        throw new UnsupportedOperationException(formatCanNotCompareMsg(left.getMajorType(), right.getMajorType()));
    }
    LogicalExpression comparisonFunctionExpression = getFunctionExpression(comparator_name, Types.required(MinorType.INT), registry, left, right);
    ErrorCollector collector = new ErrorCollectorImpl();
    if (!isUnionType(left.getMajorType()) && !isUnionType(right.getMajorType())) {
        return ExpressionTreeMaterializer.materialize(comparisonFunctionExpression, null, collector, registry);
    } else {
        LogicalExpression typeComparisonFunctionExpression = getTypeComparisonFunction(comparisonFunctionExpression, left, right);
        return ExpressionTreeMaterializer.materialize(typeComparisonFunctionExpression, null, collector, registry);
    }
}

Example 5 with ErrorCollectorImpl

use of org.apache.drill.common.expression.ErrorCollectorImpl in project drill by apache.

the class ParquetGroupScan method applyFilter.

public GroupScan applyFilter(LogicalExpression filterExpr, UdfUtilities udfUtilities, FunctionImplementationRegistry functionImplementationRegistry, OptionManager optionManager) {
    if (fileSet.size() == 1 || !(parquetTableMetadata.isRowGroupPrunable()) || rowGroupInfos.size() > optionManager.getOption(PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_THRESHOLD)) {
        //    -  # of row groups is beyond PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_THRESHOLD.
        return null;
    }
    final Set<SchemaPath> schemaPathsInExpr = filterExpr.accept(new ParquetRGFilterEvaluator.FieldReferenceFinder(), null);
    final List<RowGroupMetadata> qualifiedRGs = new ArrayList<>(parquetTableMetadata.getFiles().size());
    // HashSet keeps a fileName unique.
    Set<String> qualifiedFileNames = Sets.newHashSet();
    ParquetFilterPredicate filterPredicate = null;
    for (ParquetFileMetadata file : parquetTableMetadata.getFiles()) {
        final ImplicitColumnExplorer columnExplorer = new ImplicitColumnExplorer(optionManager, this.columns);
        Map<String, String> implicitColValues = columnExplorer.populateImplicitColumns(file.getPath(), selectionRoot);
        for (RowGroupMetadata rowGroup : file.getRowGroups()) {
            ParquetMetaStatCollector statCollector = new ParquetMetaStatCollector(parquetTableMetadata, rowGroup.getColumns(), implicitColValues);
            Map<SchemaPath, ColumnStatistics> columnStatisticsMap = statCollector.collectColStat(schemaPathsInExpr);
            if (filterPredicate == null) {
                ErrorCollector errorCollector = new ErrorCollectorImpl();
                LogicalExpression materializedFilter = ExpressionTreeMaterializer.materializeFilterExpr(filterExpr, columnStatisticsMap, errorCollector, functionImplementationRegistry);
                if (errorCollector.hasErrors()) {
                    logger.error("{} error(s) encountered when materialize filter expression : {}", errorCollector.getErrorCount(), errorCollector.toErrorString());
                    return null;
                }
                //    logger.debug("materializedFilter : {}", ExpressionStringBuilder.toString(materializedFilter));
                Set<LogicalExpression> constantBoundaries = ConstantExpressionIdentifier.getConstantExpressionSet(materializedFilter);
                filterPredicate = (ParquetFilterPredicate) ParquetFilterBuilder.buildParquetFilterPredicate(materializedFilter, constantBoundaries, udfUtilities);
                if (filterPredicate == null) {
                    return null;
                }
            }
            if (ParquetRGFilterEvaluator.canDrop(filterPredicate, columnStatisticsMap, rowGroup.getRowCount())) {
                continue;
            }
            qualifiedRGs.add(rowGroup);
            // TODO : optimize when 1 file contains m row groups.
            qualifiedFileNames.add(file.getPath());
        }
    }
    if (qualifiedFileNames.size() == fileSet.size()) {
        // There is no reduction of rowGroups. Return the original groupScan.
        logger.debug("applyFilter does not have any pruning!");
        return null;
    } else if (qualifiedFileNames.size() == 0) {
        logger.warn("All rowgroups have been filtered out. Add back one to get schema from scannner");
        qualifiedFileNames.add(fileSet.iterator().next());
    }
    try {
        FileSelection newSelection = new FileSelection(null, Lists.newArrayList(qualifiedFileNames), getSelectionRoot(), cacheFileRoot, false);
        logger.info("applyFilter {} reduce parquet file # from {} to {}", ExpressionStringBuilder.toString(filterExpr), fileSet.size(), qualifiedFileNames.size());
        return this.clone(newSelection);
    } catch (IOException e) {
        logger.warn("Could not apply filter prune due to Exception : {}", e);
        return null;
    }
}