Examples with Pair org.apache.commons.lang3.tuple.Pair used on opensource projects

Example 86 with Pair

use of org.apache.commons.lang3.tuple.Pair in project asterixdb by apache.

the class ZookeeperUtil method getProcessStreams.

private Pair<CharSequence, CharSequence> getProcessStreams(Process process) throws IOException {
    StringWriter stdout = new StringWriter();
    StringWriter stderr = new StringWriter();
    IOUtils.copy(process.getInputStream(), stdout, Charset.defaultCharset());
    IOUtils.copy(process.getErrorStream(), stderr, Charset.defaultCharset());
    return new ImmutablePair<>(stdout.getBuffer(), stderr.getBuffer());
}

Example 87 with Pair

use of org.apache.commons.lang3.tuple.Pair in project asterixdb by apache.

the class IsomorphismOperatorVisitor method visitGroupByOperator.

@Override
public Boolean visitGroupByOperator(GroupByOperator op, ILogicalOperator arg) throws AlgebricksException {
    AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
    // properties
    if (aop.getOperatorTag() != LogicalOperatorTag.GROUP || aop.getPhysicalOperator().getOperatorTag() != op.getPhysicalOperator().getOperatorTag()) {
        return Boolean.FALSE;
    }
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> keyLists = op.getGroupByList();
    GroupByOperator gbyOpArg = (GroupByOperator) copyAndSubstituteVar(op, arg);
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> keyListsArg = gbyOpArg.getGroupByList();
    List<Pair<LogicalVariable, ILogicalExpression>> listLeft = new ArrayList<Pair<LogicalVariable, ILogicalExpression>>();
    List<Pair<LogicalVariable, ILogicalExpression>> listRight = new ArrayList<Pair<LogicalVariable, ILogicalExpression>>();
    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : keyLists) {
        listLeft.add(new Pair<LogicalVariable, ILogicalExpression>(pair.first, pair.second.getValue()));
    }
    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : keyListsArg) {
        listRight.add(new Pair<LogicalVariable, ILogicalExpression>(pair.first, pair.second.getValue()));
    }
    boolean isomorphic = VariableUtilities.varListEqualUnordered(listLeft, listRight);
    if (!isomorphic) {
        return Boolean.FALSE;
    }
    int sizeOp = op.getNestedPlans().size();
    int sizeArg = gbyOpArg.getNestedPlans().size();
    if (sizeOp != sizeArg) {
        return Boolean.FALSE;
    }
    GroupByOperator argOp = (GroupByOperator) arg;
    List<ILogicalPlan> plans = op.getNestedPlans();
    List<ILogicalPlan> plansArg = argOp.getNestedPlans();
    for (int i = 0; i < plans.size(); i++) {
        List<Mutable<ILogicalOperator>> roots = plans.get(i).getRoots();
        List<Mutable<ILogicalOperator>> rootsArg = plansArg.get(i).getRoots();
        if (roots.size() != rootsArg.size()) {
            return Boolean.FALSE;
        }
        for (int j = 0; j < roots.size(); j++) {
            ILogicalOperator topOp1 = roots.get(j).getValue();
            ILogicalOperator topOp2 = rootsArg.get(j).getValue();
            isomorphic = IsomorphismUtilities.isOperatorIsomorphicPlanSegment(topOp1, topOp2);
            if (!isomorphic) {
                return Boolean.FALSE;
            }
        }
    }
    return isomorphic;
}

Example 88 with Pair

use of org.apache.commons.lang3.tuple.Pair in project asterixdb by apache.

the class IntroduceGroupByForSubplanRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
    if (op0.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
        return false;
    }
    SubplanOperator subplan = (SubplanOperator) op0;
    Iterator<ILogicalPlan> plansIter = subplan.getNestedPlans().iterator();
    ILogicalPlan p = null;
    while (plansIter.hasNext()) {
        p = plansIter.next();
    }
    if (p == null) {
        return false;
    }
    if (p.getRoots().size() != 1) {
        return false;
    }
    Mutable<ILogicalOperator> subplanRoot = p.getRoots().get(0);
    AbstractLogicalOperator op1 = (AbstractLogicalOperator) subplanRoot.getValue();
    Mutable<ILogicalOperator> botRef = subplanRoot;
    AbstractLogicalOperator op2;
    // Project is optional
    if (op1.getOperatorTag() != LogicalOperatorTag.PROJECT) {
        op2 = op1;
    } else {
        ProjectOperator project = (ProjectOperator) op1;
        botRef = project.getInputs().get(0);
        op2 = (AbstractLogicalOperator) botRef.getValue();
    }
    if (op2.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
        return false;
    }
    AggregateOperator aggregate = (AggregateOperator) op2;
    Set<LogicalVariable> free = new HashSet<LogicalVariable>();
    VariableUtilities.getUsedVariables(aggregate, free);
    Mutable<ILogicalOperator> op3Ref = aggregate.getInputs().get(0);
    AbstractLogicalOperator op3 = (AbstractLogicalOperator) op3Ref.getValue();
    while (op3.getInputs().size() == 1) {
        Set<LogicalVariable> prod = new HashSet<LogicalVariable>();
        VariableUtilities.getProducedVariables(op3, prod);
        free.removeAll(prod);
        VariableUtilities.getUsedVariables(op3, free);
        botRef = op3Ref;
        op3Ref = op3.getInputs().get(0);
        op3 = (AbstractLogicalOperator) op3Ref.getValue();
    }
    if (op3.getOperatorTag() != LogicalOperatorTag.INNERJOIN && op3.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
        return false;
    }
    AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op3;
    if (join.getCondition().getValue() == ConstantExpression.TRUE) {
        return false;
    }
    VariableUtilities.getUsedVariables(join, free);
    AbstractLogicalOperator b0 = (AbstractLogicalOperator) join.getInputs().get(0).getValue();
    // see if there's an NTS at the end of the pipeline
    NestedTupleSourceOperator outerNts = getNts(b0);
    if (outerNts == null) {
        AbstractLogicalOperator b1 = (AbstractLogicalOperator) join.getInputs().get(1).getValue();
        outerNts = getNts(b1);
        if (outerNts == null) {
            return false;
        }
    }
    Set<LogicalVariable> pkVars = computeGbyVars(outerNts, free, context);
    if (pkVars == null || pkVars.size() < 1) {
        // there is no non-trivial primary key, group-by keys are all live variables
        // that were produced by descendant or self
        ILogicalOperator subplanInput = subplan.getInputs().get(0).getValue();
        pkVars = new HashSet<LogicalVariable>();
        //get live variables
        VariableUtilities.getLiveVariables(subplanInput, pkVars);
        //get produced variables
        Set<LogicalVariable> producedVars = new HashSet<LogicalVariable>();
        VariableUtilities.getProducedVariablesInDescendantsAndSelf(subplanInput, producedVars);
        //retain the intersection
        pkVars.retainAll(producedVars);
    }
    AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Found FD for introducing group-by: " + pkVars);
    Mutable<ILogicalOperator> rightRef = join.getInputs().get(1);
    LogicalVariable testForNull = null;
    AbstractLogicalOperator right = (AbstractLogicalOperator) rightRef.getValue();
    switch(right.getOperatorTag()) {
        case UNNEST:
            {
                UnnestOperator innerUnnest = (UnnestOperator) right;
                // Select [ $y != null ]
                testForNull = innerUnnest.getVariable();
                break;
            }
        case RUNNINGAGGREGATE:
            {
                ILogicalOperator inputToRunningAggregate = right.getInputs().get(0).getValue();
                Set<LogicalVariable> producedVars = new ListSet<LogicalVariable>();
                VariableUtilities.getProducedVariables(inputToRunningAggregate, producedVars);
                if (!producedVars.isEmpty()) {
                    // Select [ $y != null ]
                    testForNull = producedVars.iterator().next();
                }
                break;
            }
        case DATASOURCESCAN:
            {
                DataSourceScanOperator innerScan = (DataSourceScanOperator) right;
                // Select [ $y != null ]
                if (innerScan.getVariables().size() == 1) {
                    testForNull = innerScan.getVariables().get(0);
                }
                break;
            }
        default:
            break;
    }
    if (testForNull == null) {
        testForNull = context.newVar();
        AssignOperator tmpAsgn = new AssignOperator(testForNull, new MutableObject<ILogicalExpression>(ConstantExpression.TRUE));
        tmpAsgn.getInputs().add(new MutableObject<ILogicalOperator>(rightRef.getValue()));
        rightRef.setValue(tmpAsgn);
        context.computeAndSetTypeEnvironmentForOperator(tmpAsgn);
    }
    IFunctionInfo finfoEq = context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.IS_MISSING);
    ILogicalExpression isNullTest = new ScalarFunctionCallExpression(finfoEq, new MutableObject<ILogicalExpression>(new VariableReferenceExpression(testForNull)));
    IFunctionInfo finfoNot = context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.NOT);
    ScalarFunctionCallExpression nonNullTest = new ScalarFunctionCallExpression(finfoNot, new MutableObject<ILogicalExpression>(isNullTest));
    SelectOperator selectNonNull = new SelectOperator(new MutableObject<ILogicalExpression>(nonNullTest), false, null);
    GroupByOperator g = new GroupByOperator();
    Mutable<ILogicalOperator> newSubplanRef = new MutableObject<ILogicalOperator>(subplan);
    NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>(g));
    opRef.setValue(g);
    selectNonNull.getInputs().add(new MutableObject<ILogicalOperator>(nts));
    List<Mutable<ILogicalOperator>> prodInpList = botRef.getValue().getInputs();
    prodInpList.clear();
    prodInpList.add(new MutableObject<ILogicalOperator>(selectNonNull));
    ILogicalPlan gPlan = new ALogicalPlanImpl(new MutableObject<ILogicalOperator>(subplanRoot.getValue()));
    g.getNestedPlans().add(gPlan);
    subplanRoot.setValue(op3Ref.getValue());
    g.getInputs().add(newSubplanRef);
    HashSet<LogicalVariable> underVars = new HashSet<LogicalVariable>();
    VariableUtilities.getLiveVariables(subplan.getInputs().get(0).getValue(), underVars);
    underVars.removeAll(pkVars);
    Map<LogicalVariable, LogicalVariable> mappedVars = buildVarExprList(pkVars, context, g, g.getGroupByList());
    context.updatePrimaryKeys(mappedVars);
    for (LogicalVariable uv : underVars) {
        g.getDecorList().add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(null, new MutableObject<ILogicalExpression>(new VariableReferenceExpression(uv))));
    }
    OperatorPropertiesUtil.typeOpRec(subplanRoot, context);
    OperatorPropertiesUtil.typeOpRec(gPlan.getRoots().get(0), context);
    context.computeAndSetTypeEnvironmentForOperator(g);
    return true;
}

Example 89 with Pair

use of org.apache.commons.lang3.tuple.Pair in project drill by apache.

the class ClassTransformer method getImplementationClass.

public Class<?> getImplementationClass(final QueryClassLoader classLoader, final TemplateClassDefinition<?> templateDefinition, final String entireClass, final String materializedClassName) throws ClassTransformationException {
    // unfortunately, this hasn't been set up at construction time, so we have to do it here
    final ScalarReplacementOption scalarReplacementOption = ScalarReplacementOption.fromString(optionManager.getOption(SCALAR_REPLACEMENT_VALIDATOR));
    try {
        final long t1 = System.nanoTime();
        final ClassSet set = new ClassSet(null, templateDefinition.getTemplateClassName(), materializedClassName);
        final byte[][] implementationClasses = classLoader.getClassByteCode(set.generated, entireClass);
        long totalBytecodeSize = 0;
        Map<String, Pair<byte[], ClassNode>> classesToMerge = Maps.newHashMap();
        for (byte[] clazz : implementationClasses) {
            totalBytecodeSize += clazz.length;
            final ClassNode node = AsmUtil.classFromBytes(clazz, ClassReader.EXPAND_FRAMES);
            if (!AsmUtil.isClassOk(logger, "implementationClasses", node)) {
                throw new IllegalStateException("Problem found with implementationClasses");
            }
            classesToMerge.put(node.name, Pair.of(clazz, node));
        }
        final LinkedList<ClassSet> names = Lists.newLinkedList();
        final Set<ClassSet> namesCompleted = Sets.newHashSet();
        names.add(set);
        while (!names.isEmpty()) {
            final ClassSet nextSet = names.removeFirst();
            if (namesCompleted.contains(nextSet)) {
                continue;
            }
            final ClassNames nextPrecompiled = nextSet.precompiled;
            final byte[] precompiledBytes = byteCodeLoader.getClassByteCodeFromPath(nextPrecompiled.clazz);
            final ClassNames nextGenerated = nextSet.generated;
            // keeps only classes that have not be merged
            Pair<byte[], ClassNode> classNodePair = classesToMerge.remove(nextGenerated.slash);
            final ClassNode generatedNode;
            if (classNodePair != null) {
                generatedNode = classNodePair.getValue();
            } else {
                generatedNode = null;
            }
            /*
         * TODO
         * We're having a problem with some cases of scalar replacement, but we want to get
         * the code in so it doesn't rot anymore.
         *
         *  Here, we use the specified replacement option. The loop will allow us to retry if
         *  we're using TRY.
         */
            MergedClassResult result = null;
            boolean scalarReplace = scalarReplacementOption != ScalarReplacementOption.OFF && entireClass.length() < MAX_SCALAR_REPLACE_CODE_SIZE;
            while (true) {
                try {
                    result = MergeAdapter.getMergedClass(nextSet, precompiledBytes, generatedNode, scalarReplace);
                    break;
                } catch (RuntimeException e) {
                    // if we had a problem without using scalar replacement, then rethrow
                    if (!scalarReplace) {
                        throw e;
                    }
                    // if we did try to use scalar replacement, decide if we need to retry or not
                    if (scalarReplacementOption == ScalarReplacementOption.ON) {
                        // option is forced on, so this is a hard error
                        throw e;
                    }
                    /*
             * We tried to use scalar replacement, with the option to fall back to not using it.
             * Log this failure before trying again without scalar replacement.
             */
                    logger.info("scalar replacement failure (retrying)\n", e);
                    scalarReplace = false;
                }
            }
            for (String s : result.innerClasses) {
                s = s.replace(FileUtils.separatorChar, '.');
                names.add(nextSet.getChild(s));
            }
            classLoader.injectByteCode(nextGenerated.dot, result.bytes);
            namesCompleted.add(nextSet);
        }
        // adds byte code of the classes that have not been merged to make them accessible for outer class
        for (Map.Entry<String, Pair<byte[], ClassNode>> clazz : classesToMerge.entrySet()) {
            classLoader.injectByteCode(clazz.getKey().replace(FileUtils.separatorChar, '.'), clazz.getValue().getKey());
        }
        Class<?> c = classLoader.findClass(set.generated.dot);
        if (templateDefinition.getExternalInterface().isAssignableFrom(c)) {
            logger.debug("Compiled and merged {}: bytecode size = {}, time = {} ms.", c.getSimpleName(), DrillStringUtils.readable(totalBytecodeSize), (System.nanoTime() - t1 + 500_000) / 1_000_000);
            return c;
        }
        throw new ClassTransformationException("The requested class did not implement the expected interface.");
    } catch (CompileException | IOException | ClassNotFoundException e) {
        throw new ClassTransformationException(String.format("Failure generating transformation classes for value: \n %s", entireClass), e);
    }
}

Example 90 with Pair

use of org.apache.commons.lang3.tuple.Pair in project drill by apache.

the class LocalFunctionRegistry method registerOperatorsWithInference.

private void registerOperatorsWithInference(DrillOperatorTable operatorTable, Map<String, Collection<DrillFuncHolder>> registeredFunctions) {
    final Map<String, DrillSqlOperator.DrillSqlOperatorBuilder> map = Maps.newHashMap();
    final Map<String, DrillSqlAggOperator.DrillSqlAggOperatorBuilder> mapAgg = Maps.newHashMap();
    for (Entry<String, Collection<DrillFuncHolder>> function : registeredFunctions.entrySet()) {
        final ArrayListMultimap<Pair<Integer, Integer>, DrillFuncHolder> functions = ArrayListMultimap.create();
        final ArrayListMultimap<Integer, DrillFuncHolder> aggregateFunctions = ArrayListMultimap.create();
        final String name = function.getKey().toUpperCase();
        boolean isDeterministic = true;
        boolean isNiladic = false;
        for (DrillFuncHolder func : function.getValue()) {
            final int paramCount = func.getParamCount();
            if (func.isAggregating()) {
                aggregateFunctions.put(paramCount, func);
            } else {
                final Pair<Integer, Integer> argNumberRange;
                if (registeredFuncNameToArgRange.containsKey(name)) {
                    argNumberRange = registeredFuncNameToArgRange.get(name);
                } else {
                    argNumberRange = Pair.of(func.getParamCount(), func.getParamCount());
                }
                functions.put(argNumberRange, func);
            }
            if (!func.isDeterministic()) {
                isDeterministic = false;
            }
            if (func.isNiladic()) {
                isNiladic = true;
            }
        }
        for (Entry<Pair<Integer, Integer>, Collection<DrillFuncHolder>> entry : functions.asMap().entrySet()) {
            final Pair<Integer, Integer> range = entry.getKey();
            final int max = range.getRight();
            final int min = range.getLeft();
            if (!map.containsKey(name)) {
                map.put(name, new DrillSqlOperator.DrillSqlOperatorBuilder().setName(name));
            }
            final DrillSqlOperator.DrillSqlOperatorBuilder drillSqlOperatorBuilder = map.get(name);
            drillSqlOperatorBuilder.addFunctions(entry.getValue()).setArgumentCount(min, max).setDeterministic(isDeterministic).setNiladic(isNiladic);
        }
        for (Entry<Integer, Collection<DrillFuncHolder>> entry : aggregateFunctions.asMap().entrySet()) {
            if (!mapAgg.containsKey(name)) {
                mapAgg.put(name, new DrillSqlAggOperator.DrillSqlAggOperatorBuilder().setName(name));
            }
            final DrillSqlAggOperator.DrillSqlAggOperatorBuilder drillSqlAggOperatorBuilder = mapAgg.get(name);
            drillSqlAggOperatorBuilder.addFunctions(entry.getValue()).setArgumentCount(entry.getKey(), entry.getKey());
        }
    }
    for (final Entry<String, DrillSqlOperator.DrillSqlOperatorBuilder> entry : map.entrySet()) {
        operatorTable.addOperatorWithInference(entry.getKey(), entry.getValue().build());
    }
    for (final Entry<String, DrillSqlAggOperator.DrillSqlAggOperatorBuilder> entry : mapAgg.entrySet()) {
        operatorTable.addOperatorWithInference(entry.getKey(), entry.getValue().build());
    }
}