Examples with Accumulator org.apache.flink.api.common.accumulators.Accumulator used on opensource projects

Example 16 with Accumulator

use of org.apache.flink.api.common.accumulators.Accumulator in project flink by apache.

the class CoGroupOperatorCollectionTest method testExecuteOnCollection.

@Test
public void testExecuteOnCollection() {
    try {
        List<Tuple2<String, Integer>> input1 = Arrays.asList(new Tuple2Builder<String, Integer>().add("foo", 1).add("foobar", 1).add("foo", 1).add("bar", 1).add("foo", 1).add("foo", 1).build());
        List<Tuple2<String, Integer>> input2 = Arrays.asList(new Tuple2Builder<String, Integer>().add("foo", 1).add("foo", 1).add("bar", 1).add("foo", 1).add("barfoo", 1).add("foo", 1).build());
        ExecutionConfig executionConfig = new ExecutionConfig();
        final HashMap<String, Accumulator<?, ?>> accumulators = new HashMap<String, Accumulator<?, ?>>();
        final HashMap<String, Future<Path>> cpTasks = new HashMap<>();
        final TaskInfo taskInfo = new TaskInfo("Test UDF", 4, 0, 4, 0);
        final RuntimeContext ctx = new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulators, new UnregisteredMetricsGroup());
        {
            SumCoGroup udf1 = new SumCoGroup();
            SumCoGroup udf2 = new SumCoGroup();
            executionConfig.disableObjectReuse();
            List<Tuple2<String, Integer>> resultSafe = getCoGroupOperator(udf1).executeOnCollections(input1, input2, ctx, executionConfig);
            executionConfig.enableObjectReuse();
            List<Tuple2<String, Integer>> resultRegular = getCoGroupOperator(udf2).executeOnCollections(input1, input2, ctx, executionConfig);
            Assert.assertTrue(udf1.isClosed);
            Assert.assertTrue(udf2.isClosed);
            Set<Tuple2<String, Integer>> expected = new HashSet<Tuple2<String, Integer>>(Arrays.asList(new Tuple2Builder<String, Integer>().add("foo", 8).add("bar", 2).add("foobar", 1).add("barfoo", 1).build()));
            Assert.assertEquals(expected, new HashSet<Tuple2<String, Integer>>(resultSafe));
            Assert.assertEquals(expected, new HashSet<Tuple2<String, Integer>>(resultRegular));
        }
        {
            executionConfig.disableObjectReuse();
            List<Tuple2<String, Integer>> resultSafe = getCoGroupOperator(new SumCoGroup()).executeOnCollections(Collections.<Tuple2<String, Integer>>emptyList(), Collections.<Tuple2<String, Integer>>emptyList(), ctx, executionConfig);
            executionConfig.enableObjectReuse();
            List<Tuple2<String, Integer>> resultRegular = getCoGroupOperator(new SumCoGroup()).executeOnCollections(Collections.<Tuple2<String, Integer>>emptyList(), Collections.<Tuple2<String, Integer>>emptyList(), ctx, executionConfig);
            Assert.assertEquals(0, resultSafe.size());
            Assert.assertEquals(0, resultRegular.size());
        }
    } catch (Throwable t) {
        t.printStackTrace();
        Assert.fail(t.getMessage());
    }
}

Example 17 with Accumulator

use of org.apache.flink.api.common.accumulators.Accumulator in project flink by apache.

the class ExecutionGraph method deserializeAccumulators.

private Map<String, Accumulator<?, ?>> deserializeAccumulators(TaskExecutionState state) {
    AccumulatorSnapshot serializedAccumulators = state.getAccumulators();
    Map<String, Accumulator<?, ?>> accumulators = null;
    if (serializedAccumulators != null) {
        try {
            accumulators = serializedAccumulators.deserializeUserAccumulators(userClassLoader);
        } catch (Exception e) {
            LOG.error("Failed to deserialize final accumulator results.", e);
        }
    }
    return accumulators;
}

Example 18 with Accumulator

use of org.apache.flink.api.common.accumulators.Accumulator in project flink by apache.

the class AccumulatorLiveITCase method verifyResults.

private static void verifyResults() {
    new JavaTestKit(system) {

        {
            ActorGateway selfGateway = new AkkaActorGateway(getRef(), jobManagerGateway.leaderSessionID());
            // register for accumulator changes
            jobManagerGateway.tell(new TestingJobManagerMessages.NotifyWhenAccumulatorChange(jobID), selfGateway);
            expectMsgEquals(TIMEOUT, true);
            // submit job
            jobManagerGateway.tell(new JobManagerMessages.SubmitJob(jobGraph, ListeningBehaviour.EXECUTION_RESULT), selfGateway);
            expectMsgClass(TIMEOUT, JobManagerMessages.JobSubmitSuccess.class);
            TestingJobManagerMessages.UpdatedAccumulators msg = (TestingJobManagerMessages.UpdatedAccumulators) receiveOne(TIMEOUT);
            Map<String, Accumulator<?, ?>> userAccumulators = msg.userAccumulators();
            ExecutionAttemptID mapperTaskID = null;
            ExecutionAttemptID sinkTaskID = null;
            /* Check for accumulator values */
            if (checkUserAccumulators(0, userAccumulators)) {
                LOG.info("Passed initial check for map task.");
            } else {
                fail("Wrong accumulator results when map task begins execution.");
            }
            int expectedAccVal = 0;
            /* for mapper task */
            for (int i = 1; i <= NUM_ITERATIONS; i++) {
                expectedAccVal += i;
                // receive message
                msg = (TestingJobManagerMessages.UpdatedAccumulators) receiveOne(TIMEOUT);
                userAccumulators = msg.userAccumulators();
                LOG.info("{}", userAccumulators);
                if (checkUserAccumulators(expectedAccVal, userAccumulators)) {
                    LOG.info("Passed round #" + i);
                } else if (checkUserAccumulators(expectedAccVal, userAccumulators)) {
                    // we determined the wrong task id and need to switch the two here
                    ExecutionAttemptID temp = mapperTaskID;
                    mapperTaskID = sinkTaskID;
                    sinkTaskID = temp;
                    LOG.info("Passed round #" + i);
                } else {
                    fail("Failed in round #" + i);
                }
            }
            msg = (TestingJobManagerMessages.UpdatedAccumulators) receiveOne(TIMEOUT);
            userAccumulators = msg.userAccumulators();
            if (checkUserAccumulators(expectedAccVal, userAccumulators)) {
                LOG.info("Passed initial check for sink task.");
            } else {
                fail("Wrong accumulator results when sink task begins execution.");
            }
            /* for sink task */
            for (int i = 1; i <= NUM_ITERATIONS; i++) {
                // receive message
                msg = (TestingJobManagerMessages.UpdatedAccumulators) receiveOne(TIMEOUT);
                userAccumulators = msg.userAccumulators();
                LOG.info("{}", userAccumulators);
                if (checkUserAccumulators(expectedAccVal, userAccumulators)) {
                    LOG.info("Passed round #" + i);
                } else {
                    fail("Failed in round #" + i);
                }
            }
            expectMsgClass(TIMEOUT, JobManagerMessages.JobResultSuccess.class);
        }
    };
}

Example 19 with Accumulator

use of org.apache.flink.api.common.accumulators.Accumulator in project flink by apache.

the class StringifiedAccumulatorResultTest method stringifyingResultsShouldIncorporateAccumulatorLocalValueDirectly.

@Test
public void stringifyingResultsShouldIncorporateAccumulatorLocalValueDirectly() {
    final String name = "a";
    final int targetValue = 314159;
    final IntCounter acc = new IntCounter();
    acc.add(targetValue);
    final Map<String, Accumulator<?, ?>> accumulatorMap = new HashMap<>();
    accumulatorMap.put(name, acc);
    final StringifiedAccumulatorResult[] results = StringifiedAccumulatorResult.stringifyAccumulatorResults(accumulatorMap);
    assertEquals(1, results.length);
    final StringifiedAccumulatorResult firstResult = results[0];
    assertEquals(name, firstResult.getName());
    assertEquals("IntCounter", firstResult.getType());
    assertEquals(Integer.toString(targetValue), firstResult.getValue());
}

Example 20 with Accumulator

use of org.apache.flink.api.common.accumulators.Accumulator in project flink by apache.

the class ArchivedExecutionGraphTest method setupExecutionGraph.

@BeforeClass
public static void setupExecutionGraph() throws Exception {
    // -------------------------------------------------------------------------------------------------------------
    // Setup
    // -------------------------------------------------------------------------------------------------------------
    v1ID = new JobVertexID();
    v2ID = new JobVertexID();
    JobVertex v1 = new JobVertex("v1", v1ID);
    JobVertex v2 = new JobVertex("v2", v2ID);
    v1.setParallelism(1);
    v2.setParallelism(2);
    v1.setInvokableClass(AbstractInvokable.class);
    v2.setInvokableClass(AbstractInvokable.class);
    List<JobVertex> vertices = new ArrayList<JobVertex>(Arrays.asList(v1, v2));
    ExecutionConfig config = new ExecutionConfig();
    config.setExecutionMode(ExecutionMode.BATCH_FORCED);
    config.setRestartStrategy(new RestartStrategies.NoRestartStrategyConfiguration());
    config.setParallelism(4);
    config.enableObjectReuse();
    config.setGlobalJobParameters(new TestJobParameters());
    runtimeGraph = new ExecutionGraph(TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), new JobID(), "test job", new Configuration(), new SerializedValue<>(config), AkkaUtils.getDefaultTimeout(), new NoRestartStrategy(), mock(SlotProvider.class));
    runtimeGraph.attachJobGraph(vertices);
    List<ExecutionJobVertex> jobVertices = new ArrayList<>();
    jobVertices.add(runtimeGraph.getJobVertex(v1ID));
    jobVertices.add(runtimeGraph.getJobVertex(v2ID));
    CheckpointStatsTracker statsTracker = new CheckpointStatsTracker(0, jobVertices, mock(JobSnapshottingSettings.class), new UnregisteredMetricsGroup());
    runtimeGraph.enableCheckpointing(100, 100, 100, 1, ExternalizedCheckpointSettings.none(), Collections.<ExecutionJobVertex>emptyList(), Collections.<ExecutionJobVertex>emptyList(), Collections.<ExecutionJobVertex>emptyList(), new StandaloneCheckpointIDCounter(), new StandaloneCompletedCheckpointStore(1), null, null, statsTracker);
    Map<String, Accumulator<?, ?>> userAccumulators = new HashMap<>();
    userAccumulators.put("userAcc", new LongCounter(64));
    Execution executionWithAccumulators = runtimeGraph.getJobVertex(v1ID).getTaskVertices()[0].getCurrentExecutionAttempt();
    runtimeGraph.getJobVertex(v2ID).getTaskVertices()[0].getCurrentExecutionAttempt().fail(new RuntimeException("This exception was thrown on purpose."));
}