Examples with Accumulator org.apache.flink.api.common.accumulators.Accumulator used on opensource projects

Example 11 with Accumulator

use of org.apache.flink.api.common.accumulators.Accumulator in project flink by apache.

the class RuntimeUDFContextTest method testBroadcastVariableWithInitializer.

@Test
public void testBroadcastVariableWithInitializer() {
    try {
        RuntimeUDFContext ctx = new RuntimeUDFContext(taskInfo, getClass().getClassLoader(), new ExecutionConfig(), new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>(), new UnregisteredMetricsGroup());
        ctx.setBroadcastVariable("name", Arrays.asList(1, 2, 3, 4));
        // access it the first time with an initializer
        List<Double> list = ctx.getBroadcastVariableWithInitializer("name", new ConvertingInitializer());
        assertEquals(Arrays.asList(1.0, 2.0, 3.0, 4.0), list);
        // access it the second time with an initializer (which might not get executed)
        List<Double> list2 = ctx.getBroadcastVariableWithInitializer("name", new ConvertingInitializer());
        assertEquals(Arrays.asList(1.0, 2.0, 3.0, 4.0), list2);
        // access it the third time without an initializer (should work by "chance", because the result is a list)
        List<Double> list3 = ctx.getBroadcastVariable("name");
        assertEquals(Arrays.asList(1.0, 2.0, 3.0, 4.0), list3);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 12 with Accumulator

use of org.apache.flink.api.common.accumulators.Accumulator in project flink by apache.

the class RuntimeUDFContextTest method testBroadcastVariableSimple.

@Test
public void testBroadcastVariableSimple() {
    try {
        RuntimeUDFContext ctx = new RuntimeUDFContext(taskInfo, getClass().getClassLoader(), new ExecutionConfig(), new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>(), new UnregisteredMetricsGroup());
        ctx.setBroadcastVariable("name1", Arrays.asList(1, 2, 3, 4));
        ctx.setBroadcastVariable("name2", Arrays.asList(1.0, 2.0, 3.0, 4.0));
        assertTrue(ctx.hasBroadcastVariable("name1"));
        assertTrue(ctx.hasBroadcastVariable("name2"));
        List<Integer> list1 = ctx.getBroadcastVariable("name1");
        List<Double> list2 = ctx.getBroadcastVariable("name2");
        assertEquals(Arrays.asList(1, 2, 3, 4), list1);
        assertEquals(Arrays.asList(1.0, 2.0, 3.0, 4.0), list2);
        // access again
        List<Integer> list3 = ctx.getBroadcastVariable("name1");
        List<Double> list4 = ctx.getBroadcastVariable("name2");
        assertEquals(Arrays.asList(1, 2, 3, 4), list3);
        assertEquals(Arrays.asList(1.0, 2.0, 3.0, 4.0), list4);
        // and again ;-)
        List<Integer> list5 = ctx.getBroadcastVariable("name1");
        List<Double> list6 = ctx.getBroadcastVariable("name2");
        assertEquals(Arrays.asList(1, 2, 3, 4), list5);
        assertEquals(Arrays.asList(1.0, 2.0, 3.0, 4.0), list6);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 13 with Accumulator

use of org.apache.flink.api.common.accumulators.Accumulator in project flink by apache.

the class RuntimeUDFContextTest method testBroadcastVariableNotFound.

@Test
public void testBroadcastVariableNotFound() {
    try {
        RuntimeUDFContext ctx = new RuntimeUDFContext(taskInfo, getClass().getClassLoader(), new ExecutionConfig(), new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>(), new UnregisteredMetricsGroup());
        assertFalse(ctx.hasBroadcastVariable("some name"));
        try {
            ctx.getBroadcastVariable("some name");
            fail("should throw an exception");
        } catch (IllegalArgumentException e) {
        // expected
        }
        try {
            ctx.getBroadcastVariableWithInitializer("some name", new BroadcastVariableInitializer<Object, Object>() {

                public Object initializeBroadcastVariable(Iterable<Object> data) {
                    return null;
                }
            });
            fail("should throw an exception");
        } catch (IllegalArgumentException e) {
        // expected
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 14 with Accumulator

use of org.apache.flink.api.common.accumulators.Accumulator in project flink by apache.

the class RuntimeUDFContextTest method testResetBroadcastVariableWithInitializer.

@Test
public void testResetBroadcastVariableWithInitializer() {
    try {
        RuntimeUDFContext ctx = new RuntimeUDFContext(taskInfo, getClass().getClassLoader(), new ExecutionConfig(), new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>(), new UnregisteredMetricsGroup());
        ctx.setBroadcastVariable("name", Arrays.asList(1, 2, 3, 4));
        // access it the first time with an initializer
        List<Double> list = ctx.getBroadcastVariableWithInitializer("name", new ConvertingInitializer());
        assertEquals(Arrays.asList(1.0, 2.0, 3.0, 4.0), list);
        // set it again to something different
        ctx.setBroadcastVariable("name", Arrays.asList(2, 3, 4, 5));
        List<Double> list2 = ctx.getBroadcastVariableWithInitializer("name", new ConvertingInitializer());
        assertEquals(Arrays.asList(2.0, 3.0, 4.0, 5.0), list2);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 15 with Accumulator

use of org.apache.flink.api.common.accumulators.Accumulator in project flink by apache.

the class MapOperatorTest method testMapWithRuntimeContext.

@Test
public void testMapWithRuntimeContext() {
    try {
        final String taskName = "Test Task";
        final AtomicBoolean opened = new AtomicBoolean();
        final AtomicBoolean closed = new AtomicBoolean();
        final MapFunction<String, Integer> parser = new RichMapFunction<String, Integer>() {

            @Override
            public void open(Configuration parameters) throws Exception {
                opened.set(true);
                RuntimeContext ctx = getRuntimeContext();
                assertEquals(0, ctx.getIndexOfThisSubtask());
                assertEquals(1, ctx.getNumberOfParallelSubtasks());
                assertEquals(taskName, ctx.getTaskName());
            }

            @Override
            public Integer map(String value) {
                return Integer.parseInt(value);
            }

            @Override
            public void close() throws Exception {
                closed.set(true);
            }
        };
        MapOperatorBase<String, Integer, MapFunction<String, Integer>> op = new MapOperatorBase<String, Integer, MapFunction<String, Integer>>(parser, new UnaryOperatorInformation<String, Integer>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO), taskName);
        List<String> input = new ArrayList<String>(asList("1", "2", "3", "4", "5", "6"));
        final HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<String, Accumulator<?, ?>>();
        final HashMap<String, Future<Path>> cpTasks = new HashMap<>();
        final TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);
        ExecutionConfig executionConfig = new ExecutionConfig();
        executionConfig.disableObjectReuse();
        List<Integer> resultMutableSafe = op.executeOnCollections(input, new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap, new UnregisteredMetricsGroup()), executionConfig);
        executionConfig.enableObjectReuse();
        List<Integer> resultRegular = op.executeOnCollections(input, new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap, new UnregisteredMetricsGroup()), executionConfig);
        assertEquals(asList(1, 2, 3, 4, 5, 6), resultMutableSafe);
        assertEquals(asList(1, 2, 3, 4, 5, 6), resultRegular);
        assertTrue(opened.get());
        assertTrue(closed.get());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}