Examples with Aggregator org.apache.flink.api.common.aggregators.Aggregator used on opensource projects

Example 11 with Aggregator

use of org.apache.flink.api.common.aggregators.Aggregator in project flink by apache.

the class EventWithAggregatorsTest method testSerializationOfEventWithAggregateValues.

@Test
public void testSerializationOfEventWithAggregateValues() {
    StringValue stringValue = new StringValue("test string");
    LongValue longValue = new LongValue(68743254);
    String stringValueName = "stringValue";
    String longValueName = "longValue";
    Aggregator<StringValue> stringAgg = new TestAggregator<StringValue>(stringValue);
    Aggregator<LongValue> longAgg = new TestAggregator<LongValue>(longValue);
    Map<String, Aggregator<?>> aggMap = new HashMap<String, Aggregator<?>>();
    aggMap.put(stringValueName, stringAgg);
    aggMap.put(longValueName, longAgg);
    Set<String> allNames = new HashSet<String>();
    allNames.add(stringValueName);
    allNames.add(longValueName);
    Set<Value> allVals = new HashSet<Value>();
    allVals.add(stringValue);
    allVals.add(longValue);
    // run the serialization
    AllWorkersDoneEvent e = new AllWorkersDoneEvent(aggMap);
    IterationEventWithAggregators deserialized = pipeThroughSerialization(e);
    // verify the result
    String[] names = deserialized.getAggregatorNames();
    Value[] aggregates = deserialized.getAggregates(cl);
    Assert.assertEquals(allNames.size(), names.length);
    Assert.assertEquals(allVals.size(), aggregates.length);
    // check that all the correct names and values are returned
    for (String s : names) {
        allNames.remove(s);
    }
    for (Value v : aggregates) {
        allVals.remove(v);
    }
    Assert.assertTrue(allNames.isEmpty());
    Assert.assertTrue(allVals.isEmpty());
}

Example 12 with Aggregator

use of org.apache.flink.api.common.aggregators.Aggregator in project flink by apache.

the class IterationSynchronizationSinkTask method checkForConvergence.

private boolean checkForConvergence() {
    if (maxNumberOfIterations == currentIteration) {
        if (log.isInfoEnabled()) {
            log.info(formatLogString("maximum number of iterations [" + currentIteration + "] reached, terminating..."));
        }
        return true;
    }
    if (convergenceAggregatorName != null) {
        @SuppressWarnings("unchecked") Aggregator<Value> aggregator = (Aggregator<Value>) aggregators.get(convergenceAggregatorName);
        if (aggregator == null) {
            throw new RuntimeException("Error: Aggregator for convergence criterion was null.");
        }
        Value aggregate = aggregator.getAggregate();
        if (convergenceCriterion.isConverged(currentIteration, aggregate)) {
            if (log.isInfoEnabled()) {
                log.info(formatLogString("convergence reached after [" + currentIteration + "] iterations, terminating..."));
            }
            return true;
        }
    }
    if (implicitConvergenceAggregatorName != null) {
        @SuppressWarnings("unchecked") Aggregator<Value> aggregator = (Aggregator<Value>) aggregators.get(implicitConvergenceAggregatorName);
        if (aggregator == null) {
            throw new RuntimeException("Error: Aggregator for default convergence criterion was null.");
        }
        Value aggregate = aggregator.getAggregate();
        if (implicitConvergenceCriterion.isConverged(currentIteration, aggregate)) {
            if (log.isInfoEnabled()) {
                log.info(formatLogString("empty workset convergence reached after [" + currentIteration + "] iterations, terminating..."));
            }
            return true;
        }
    }
    return false;
}

Example 13 with Aggregator

use of org.apache.flink.api.common.aggregators.Aggregator in project flink by splunk.

the class CollectionExecutor method executeDeltaIteration.

@SuppressWarnings("unchecked")
private <T> List<T> executeDeltaIteration(DeltaIterationBase<?, ?> iteration, JobID jobID) throws Exception {
    Operator<?> solutionInput = iteration.getInitialSolutionSet();
    Operator<?> worksetInput = iteration.getInitialWorkset();
    if (solutionInput == null) {
        throw new InvalidProgramException("The delta iteration " + iteration.getName() + " has no initial solution set.");
    }
    if (worksetInput == null) {
        throw new InvalidProgramException("The delta iteration " + iteration.getName() + " has no initial workset.");
    }
    if (iteration.getSolutionSetDelta() == null) {
        throw new InvalidProgramException("The iteration " + iteration.getName() + " has no solution set delta defined (is not closed).");
    }
    if (iteration.getNextWorkset() == null) {
        throw new InvalidProgramException("The iteration " + iteration.getName() + " has no workset defined (is not closed).");
    }
    List<T> solutionInputData = (List<T>) execute(solutionInput, jobID);
    List<T> worksetInputData = (List<T>) execute(worksetInput, jobID);
    // get the operators that are iterative
    Set<Operator<?>> dynamics = new LinkedHashSet<Operator<?>>();
    DynamicPathCollector dynCollector = new DynamicPathCollector(dynamics);
    iteration.getSolutionSetDelta().accept(dynCollector);
    iteration.getNextWorkset().accept(dynCollector);
    BinaryOperatorInformation<?, ?, ?> operatorInfo = iteration.getOperatorInfo();
    TypeInformation<?> solutionType = operatorInfo.getFirstInputType();
    int[] keyColumns = iteration.getSolutionSetKeyFields();
    boolean[] inputOrderings = new boolean[keyColumns.length];
    TypeComparator<T> inputComparator = ((CompositeType<T>) solutionType).createComparator(keyColumns, inputOrderings, 0, executionConfig);
    Map<TypeComparable<T>, T> solutionMap = new HashMap<TypeComparable<T>, T>(solutionInputData.size());
    // fill the solution from the initial input
    for (T delta : solutionInputData) {
        TypeComparable<T> wrapper = new TypeComparable<T>(delta, inputComparator);
        solutionMap.put(wrapper, delta);
    }
    List<?> currentWorkset = worksetInputData;
    // register the aggregators
    for (AggregatorWithName<?> a : iteration.getAggregators().getAllRegisteredAggregators()) {
        aggregators.put(a.getName(), a.getAggregator());
    }
    String convCriterionAggName = iteration.getAggregators().getConvergenceCriterionAggregatorName();
    ConvergenceCriterion<Value> convCriterion = (ConvergenceCriterion<Value>) iteration.getAggregators().getConvergenceCriterion();
    final int maxIterations = iteration.getMaximumNumberOfIterations();
    for (int superstep = 1; superstep <= maxIterations; superstep++) {
        List<T> currentSolution = new ArrayList<T>(solutionMap.size());
        currentSolution.addAll(solutionMap.values());
        // set the input to the current partial solution
        this.intermediateResults.put(iteration.getSolutionSet(), currentSolution);
        this.intermediateResults.put(iteration.getWorkset(), currentWorkset);
        // set the superstep number
        iterationSuperstep = superstep;
        // grab the current iteration result
        List<T> solutionSetDelta = (List<T>) execute(iteration.getSolutionSetDelta(), superstep, jobID);
        this.intermediateResults.put(iteration.getSolutionSetDelta(), solutionSetDelta);
        // update the solution
        for (T delta : solutionSetDelta) {
            TypeComparable<T> wrapper = new TypeComparable<T>(delta, inputComparator);
            solutionMap.put(wrapper, delta);
        }
        currentWorkset = execute(iteration.getNextWorkset(), superstep, jobID);
        if (currentWorkset.isEmpty()) {
            break;
        }
        // evaluate the aggregator convergence criterion
        if (convCriterion != null && convCriterionAggName != null) {
            Value v = aggregators.get(convCriterionAggName).getAggregate();
            if (convCriterion.isConverged(superstep, v)) {
                break;
            }
        }
        // clear the dynamic results
        for (Operator<?> o : dynamics) {
            intermediateResults.remove(o);
        }
        // set the previous iteration's aggregates and reset the aggregators
        for (Map.Entry<String, Aggregator<?>> e : aggregators.entrySet()) {
            previousAggregates.put(e.getKey(), e.getValue().getAggregate());
            e.getValue().reset();
        }
    }
    previousAggregates.clear();
    aggregators.clear();
    List<T> currentSolution = new ArrayList<T>(solutionMap.size());
    currentSolution.addAll(solutionMap.values());
    return currentSolution;
}

Example 14 with Aggregator

use of org.apache.flink.api.common.aggregators.Aggregator in project flink by splunk.

the class CollectionExecutor method executeBulkIteration.

@SuppressWarnings("unchecked")
private <T> List<T> executeBulkIteration(BulkIterationBase<?> iteration, JobID jobID) throws Exception {
    Operator<?> inputOp = iteration.getInput();
    if (inputOp == null) {
        throw new InvalidProgramException("The iteration " + iteration.getName() + " has no input (initial partial solution).");
    }
    if (iteration.getNextPartialSolution() == null) {
        throw new InvalidProgramException("The iteration " + iteration.getName() + " has no next partial solution defined (is not closed).");
    }
    List<T> inputData = (List<T>) execute(inputOp, jobID);
    // get the operators that are iterative
    Set<Operator<?>> dynamics = new LinkedHashSet<Operator<?>>();
    DynamicPathCollector dynCollector = new DynamicPathCollector(dynamics);
    iteration.getNextPartialSolution().accept(dynCollector);
    if (iteration.getTerminationCriterion() != null) {
        iteration.getTerminationCriterion().accept(dynCollector);
    }
    // register the aggregators
    for (AggregatorWithName<?> a : iteration.getAggregators().getAllRegisteredAggregators()) {
        aggregators.put(a.getName(), a.getAggregator());
    }
    String convCriterionAggName = iteration.getAggregators().getConvergenceCriterionAggregatorName();
    ConvergenceCriterion<Value> convCriterion = (ConvergenceCriterion<Value>) iteration.getAggregators().getConvergenceCriterion();
    List<T> currentResult = inputData;
    final int maxIterations = iteration.getMaximumNumberOfIterations();
    for (int superstep = 1; superstep <= maxIterations; superstep++) {
        // set the input to the current partial solution
        this.intermediateResults.put(iteration.getPartialSolution(), currentResult);
        // set the superstep number
        iterationSuperstep = superstep;
        // grab the current iteration result
        currentResult = (List<T>) execute(iteration.getNextPartialSolution(), superstep, jobID);
        // evaluate the termination criterion
        if (iteration.getTerminationCriterion() != null) {
            execute(iteration.getTerminationCriterion(), superstep, jobID);
        }
        // evaluate the aggregator convergence criterion
        if (convCriterion != null && convCriterionAggName != null) {
            Value v = aggregators.get(convCriterionAggName).getAggregate();
            if (convCriterion.isConverged(superstep, v)) {
                break;
            }
        }
        // clear the dynamic results
        for (Operator<?> o : dynamics) {
            intermediateResults.remove(o);
        }
        // set the previous iteration's aggregates and reset the aggregators
        for (Map.Entry<String, Aggregator<?>> e : aggregators.entrySet()) {
            previousAggregates.put(e.getKey(), e.getValue().getAggregate());
            e.getValue().reset();
        }
    }
    previousAggregates.clear();
    aggregators.clear();
    return currentResult;
}

Example 15 with Aggregator

use of org.apache.flink.api.common.aggregators.Aggregator in project flink-mirror by flink-ci.

the class CollectionExecutor method executeBulkIteration.

@SuppressWarnings("unchecked")
private <T> List<T> executeBulkIteration(BulkIterationBase<?> iteration, JobID jobID) throws Exception {
    Operator<?> inputOp = iteration.getInput();
    if (inputOp == null) {
        throw new InvalidProgramException("The iteration " + iteration.getName() + " has no input (initial partial solution).");
    }
    if (iteration.getNextPartialSolution() == null) {
        throw new InvalidProgramException("The iteration " + iteration.getName() + " has no next partial solution defined (is not closed).");
    }
    List<T> inputData = (List<T>) execute(inputOp, jobID);
    // get the operators that are iterative
    Set<Operator<?>> dynamics = new LinkedHashSet<Operator<?>>();
    DynamicPathCollector dynCollector = new DynamicPathCollector(dynamics);
    iteration.getNextPartialSolution().accept(dynCollector);
    if (iteration.getTerminationCriterion() != null) {
        iteration.getTerminationCriterion().accept(dynCollector);
    }
    // register the aggregators
    for (AggregatorWithName<?> a : iteration.getAggregators().getAllRegisteredAggregators()) {
        aggregators.put(a.getName(), a.getAggregator());
    }
    String convCriterionAggName = iteration.getAggregators().getConvergenceCriterionAggregatorName();
    ConvergenceCriterion<Value> convCriterion = (ConvergenceCriterion<Value>) iteration.getAggregators().getConvergenceCriterion();
    List<T> currentResult = inputData;
    final int maxIterations = iteration.getMaximumNumberOfIterations();
    for (int superstep = 1; superstep <= maxIterations; superstep++) {
        // set the input to the current partial solution
        this.intermediateResults.put(iteration.getPartialSolution(), currentResult);
        // set the superstep number
        iterationSuperstep = superstep;
        // grab the current iteration result
        currentResult = (List<T>) execute(iteration.getNextPartialSolution(), superstep, jobID);
        // evaluate the termination criterion
        if (iteration.getTerminationCriterion() != null) {
            execute(iteration.getTerminationCriterion(), superstep, jobID);
        }
        // evaluate the aggregator convergence criterion
        if (convCriterion != null && convCriterionAggName != null) {
            Value v = aggregators.get(convCriterionAggName).getAggregate();
            if (convCriterion.isConverged(superstep, v)) {
                break;
            }
        }
        // clear the dynamic results
        for (Operator<?> o : dynamics) {
            intermediateResults.remove(o);
        }
        // set the previous iteration's aggregates and reset the aggregators
        for (Map.Entry<String, Aggregator<?>> e : aggregators.entrySet()) {
            previousAggregates.put(e.getKey(), e.getValue().getAggregate());
            e.getValue().reset();
        }
    }
    previousAggregates.clear();
    aggregators.clear();
    return currentResult;
}