Search in sources :

Example 1 with TypeComparable

use of org.apache.flink.api.common.operators.util.TypeComparable in project flink by apache.

the class CollectionExecutor method executeDeltaIteration.

@SuppressWarnings("unchecked")
private <T> List<T> executeDeltaIteration(DeltaIterationBase<?, ?> iteration) throws Exception {
    Operator<?> solutionInput = iteration.getInitialSolutionSet();
    Operator<?> worksetInput = iteration.getInitialWorkset();
    if (solutionInput == null) {
        throw new InvalidProgramException("The delta iteration " + iteration.getName() + " has no initial solution set.");
    }
    if (worksetInput == null) {
        throw new InvalidProgramException("The delta iteration " + iteration.getName() + " has no initial workset.");
    }
    if (iteration.getSolutionSetDelta() == null) {
        throw new InvalidProgramException("The iteration " + iteration.getName() + " has no solution set delta defined (is not closed).");
    }
    if (iteration.getNextWorkset() == null) {
        throw new InvalidProgramException("The iteration " + iteration.getName() + " has no workset defined (is not closed).");
    }
    List<T> solutionInputData = (List<T>) execute(solutionInput);
    List<T> worksetInputData = (List<T>) execute(worksetInput);
    // get the operators that are iterative
    Set<Operator<?>> dynamics = new LinkedHashSet<Operator<?>>();
    DynamicPathCollector dynCollector = new DynamicPathCollector(dynamics);
    iteration.getSolutionSetDelta().accept(dynCollector);
    iteration.getNextWorkset().accept(dynCollector);
    BinaryOperatorInformation<?, ?, ?> operatorInfo = iteration.getOperatorInfo();
    TypeInformation<?> solutionType = operatorInfo.getFirstInputType();
    int[] keyColumns = iteration.getSolutionSetKeyFields();
    boolean[] inputOrderings = new boolean[keyColumns.length];
    TypeComparator<T> inputComparator = ((CompositeType<T>) solutionType).createComparator(keyColumns, inputOrderings, 0, executionConfig);
    Map<TypeComparable<T>, T> solutionMap = new HashMap<TypeComparable<T>, T>(solutionInputData.size());
    // fill the solution from the initial input
    for (T delta : solutionInputData) {
        TypeComparable<T> wrapper = new TypeComparable<T>(delta, inputComparator);
        solutionMap.put(wrapper, delta);
    }
    List<?> currentWorkset = worksetInputData;
    // register the aggregators
    for (AggregatorWithName<?> a : iteration.getAggregators().getAllRegisteredAggregators()) {
        aggregators.put(a.getName(), a.getAggregator());
    }
    String convCriterionAggName = iteration.getAggregators().getConvergenceCriterionAggregatorName();
    ConvergenceCriterion<Value> convCriterion = (ConvergenceCriterion<Value>) iteration.getAggregators().getConvergenceCriterion();
    final int maxIterations = iteration.getMaximumNumberOfIterations();
    for (int superstep = 1; superstep <= maxIterations; superstep++) {
        List<T> currentSolution = new ArrayList<T>(solutionMap.size());
        currentSolution.addAll(solutionMap.values());
        // set the input to the current partial solution
        this.intermediateResults.put(iteration.getSolutionSet(), currentSolution);
        this.intermediateResults.put(iteration.getWorkset(), currentWorkset);
        // set the superstep number
        iterationSuperstep = superstep;
        // grab the current iteration result
        List<T> solutionSetDelta = (List<T>) execute(iteration.getSolutionSetDelta(), superstep);
        this.intermediateResults.put(iteration.getSolutionSetDelta(), solutionSetDelta);
        // update the solution
        for (T delta : solutionSetDelta) {
            TypeComparable<T> wrapper = new TypeComparable<T>(delta, inputComparator);
            solutionMap.put(wrapper, delta);
        }
        currentWorkset = execute(iteration.getNextWorkset(), superstep);
        if (currentWorkset.isEmpty()) {
            break;
        }
        // evaluate the aggregator convergence criterion
        if (convCriterion != null && convCriterionAggName != null) {
            Value v = aggregators.get(convCriterionAggName).getAggregate();
            if (convCriterion.isConverged(superstep, v)) {
                break;
            }
        }
        // clear the dynamic results
        for (Operator<?> o : dynamics) {
            intermediateResults.remove(o);
        }
        // set the previous iteration's aggregates and reset the aggregators
        for (Map.Entry<String, Aggregator<?>> e : aggregators.entrySet()) {
            previousAggregates.put(e.getKey(), e.getValue().getAggregate());
            e.getValue().reset();
        }
    }
    previousAggregates.clear();
    aggregators.clear();
    List<T> currentSolution = new ArrayList<T>(solutionMap.size());
    currentSolution.addAll(solutionMap.values());
    return currentSolution;
}
Also used : LinkedHashSet(java.util.LinkedHashSet) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) TypeComparable(org.apache.flink.api.common.operators.util.TypeComparable) InvalidProgramException(org.apache.flink.api.common.InvalidProgramException) ArrayList(java.util.ArrayList) List(java.util.List) Aggregator(org.apache.flink.api.common.aggregators.Aggregator) ConvergenceCriterion(org.apache.flink.api.common.aggregators.ConvergenceCriterion) Value(org.apache.flink.types.Value) HashMap(java.util.HashMap) Map(java.util.Map) CompositeType(org.apache.flink.api.common.typeutils.CompositeType)

Aggregations

ArrayList (java.util.ArrayList)1 HashMap (java.util.HashMap)1 LinkedHashSet (java.util.LinkedHashSet)1 List (java.util.List)1 Map (java.util.Map)1 InvalidProgramException (org.apache.flink.api.common.InvalidProgramException)1 Aggregator (org.apache.flink.api.common.aggregators.Aggregator)1 ConvergenceCriterion (org.apache.flink.api.common.aggregators.ConvergenceCriterion)1 TypeComparable (org.apache.flink.api.common.operators.util.TypeComparable)1 CompositeType (org.apache.flink.api.common.typeutils.CompositeType)1 Value (org.apache.flink.types.Value)1