Examples with Accumulator org.apache.flink.api.common.accumulators.Accumulator used on opensource projects

Example 6 with Accumulator

use of org.apache.flink.api.common.accumulators.Accumulator in project flink by apache.

the class RuntimeUDFContextTest method testBroadcastVariableWithInitializerAndMismatch.

@Test
public void testBroadcastVariableWithInitializerAndMismatch() {
    try {
        RuntimeUDFContext ctx = new RuntimeUDFContext(taskInfo, getClass().getClassLoader(), new ExecutionConfig(), new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>(), new UnregisteredMetricsGroup());
        ctx.setBroadcastVariable("name", Arrays.asList(1, 2, 3, 4));
        // access it the first time with an initializer
        int sum = ctx.getBroadcastVariableWithInitializer("name", new SumInitializer());
        assertEquals(10, sum);
        // access it the second time with no initializer -> should fail due to type mismatch
        try {
            ctx.getBroadcastVariable("name");
            fail("should throw an exception");
        } catch (IllegalStateException e) {
        // expected
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 7 with Accumulator

use of org.apache.flink.api.common.accumulators.Accumulator in project flink by apache.

the class InnerJoinOperatorBaseTest method testJoinRich.

@Test
public void testJoinRich() {
    final AtomicBoolean opened = new AtomicBoolean(false);
    final AtomicBoolean closed = new AtomicBoolean(false);
    final String taskName = "Test rich join function";
    final RichFlatJoinFunction<String, String, Integer> joiner = new RichFlatJoinFunction<String, String, Integer>() {

        @Override
        public void open(Configuration parameters) throws Exception {
            opened.compareAndSet(false, true);
            assertEquals(0, getRuntimeContext().getIndexOfThisSubtask());
            assertEquals(1, getRuntimeContext().getNumberOfParallelSubtasks());
        }

        @Override
        public void close() throws Exception {
            closed.compareAndSet(false, true);
        }

        @Override
        public void join(String first, String second, Collector<Integer> out) throws Exception {
            out.collect(first.length());
            out.collect(second.length());
        }
    };
    InnerJoinOperatorBase<String, String, Integer, RichFlatJoinFunction<String, String, Integer>> base = new InnerJoinOperatorBase<String, String, Integer, RichFlatJoinFunction<String, String, Integer>>(joiner, new BinaryOperatorInformation<String, String, Integer>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO), new int[0], new int[0], taskName);
    final List<String> inputData1 = new ArrayList<String>(Arrays.asList("foo", "bar", "foobar"));
    final List<String> inputData2 = new ArrayList<String>(Arrays.asList("foobar", "foo"));
    final List<Integer> expected = new ArrayList<Integer>(Arrays.asList(3, 3, 6, 6));
    try {
        final TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);
        final HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<String, Accumulator<?, ?>>();
        final HashMap<String, Future<Path>> cpTasks = new HashMap<>();
        ExecutionConfig executionConfig = new ExecutionConfig();
        executionConfig.disableObjectReuse();
        List<Integer> resultSafe = base.executeOnCollections(inputData1, inputData2, new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap, new UnregisteredMetricsGroup()), executionConfig);
        executionConfig.enableObjectReuse();
        List<Integer> resultRegular = base.executeOnCollections(inputData1, inputData2, new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap, new UnregisteredMetricsGroup()), executionConfig);
        assertEquals(expected, resultSafe);
        assertEquals(expected, resultRegular);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
    assertTrue(opened.get());
    assertTrue(closed.get());
}

Example 8 with Accumulator

use of org.apache.flink.api.common.accumulators.Accumulator in project flink by apache.

the class OuterJoinOperatorBaseTest method setup.

@SuppressWarnings({ "rawtypes", "unchecked" })
@Before
public void setup() {
    joiner = new MockRichFlatJoinFunction();
    baseOperator = new OuterJoinOperatorBase(joiner, new BinaryOperatorInformation(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO), new int[0], new int[0], "TestJoiner", null);
    executionConfig = new ExecutionConfig();
    String taskName = "Test rich outer join function";
    TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);
    HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<>();
    HashMap<String, Future<Path>> cpTasks = new HashMap<>();
    runtimeContext = new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap, new UnregisteredMetricsGroup());
}

Example 9 with Accumulator

use of org.apache.flink.api.common.accumulators.Accumulator in project flink by apache.

the class GenericDataSinkBaseTest method testDataSourceWithRuntimeContext.

@Test
public void testDataSourceWithRuntimeContext() {
    try {
        TestRichOutputFormat out = new TestRichOutputFormat();
        GenericDataSinkBase<String> sink = new GenericDataSinkBase<String>(out, new UnaryOperatorInformation<String, Nothing>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.getInfoFor(Nothing.class)), "test_sink");
        sink.setInput(source);
        ExecutionConfig executionConfig = new ExecutionConfig();
        final HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<String, Accumulator<?, ?>>();
        final HashMap<String, Future<Path>> cpTasks = new HashMap<>();
        final TaskInfo taskInfo = new TaskInfo("test_sink", 1, 0, 1, 0);
        executionConfig.disableObjectReuse();
        in.reset();
        sink.executeOnCollections(asList(TestIOData.NAMES), new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap, new UnregisteredMetricsGroup()), executionConfig);
        assertEquals(out.output, asList(TestIOData.RICH_NAMES));
        executionConfig.enableObjectReuse();
        out.clear();
        in.reset();
        sink.executeOnCollections(asList(TestIOData.NAMES), new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap, new UnregisteredMetricsGroup()), executionConfig);
        assertEquals(out.output, asList(TestIOData.RICH_NAMES));
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 10 with Accumulator

use of org.apache.flink.api.common.accumulators.Accumulator in project flink by apache.

the class GenericDataSourceBaseTest method testDataSourceWithRuntimeContext.

@Test
public void testDataSourceWithRuntimeContext() {
    try {
        TestRichInputFormat in = new TestRichInputFormat();
        GenericDataSourceBase<String, TestRichInputFormat> source = new GenericDataSourceBase<String, TestRichInputFormat>(in, new OperatorInformation<String>(BasicTypeInfo.STRING_TYPE_INFO), "testSource");
        final HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<String, Accumulator<?, ?>>();
        final HashMap<String, Future<Path>> cpTasks = new HashMap<>();
        final TaskInfo taskInfo = new TaskInfo("test_source", 1, 0, 1, 0);
        ExecutionConfig executionConfig = new ExecutionConfig();
        executionConfig.disableObjectReuse();
        assertEquals(false, in.hasBeenClosed());
        assertEquals(false, in.hasBeenOpened());
        List<String> resultMutableSafe = source.executeOnCollections(new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap, new UnregisteredMetricsGroup()), executionConfig);
        assertEquals(true, in.hasBeenClosed());
        assertEquals(true, in.hasBeenOpened());
        in.reset();
        executionConfig.enableObjectReuse();
        assertEquals(false, in.hasBeenClosed());
        assertEquals(false, in.hasBeenOpened());
        List<String> resultRegular = source.executeOnCollections(new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap, new UnregisteredMetricsGroup()), executionConfig);
        assertEquals(true, in.hasBeenClosed());
        assertEquals(true, in.hasBeenOpened());
        assertEquals(asList(TestIOData.RICH_NAMES), resultMutableSafe);
        assertEquals(asList(TestIOData.RICH_NAMES), resultRegular);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}