Examples with FromElementsFunction org.apache.flink.streaming.api.functions.source.FromElementsFunction used on opensource projects

Example 6 with FromElementsFunction

use of org.apache.flink.streaming.api.functions.source.FromElementsFunction in project flink by apache.

the class FromElementsFunctionTest method testCheckpointAndRestore.

@Test
public void testCheckpointAndRestore() {
    try {
        final int numElements = 10000;
        List<Integer> data = new ArrayList<Integer>(numElements);
        List<Integer> result = new ArrayList<Integer>(numElements);
        for (int i = 0; i < numElements; i++) {
            data.add(i);
        }
        final FromElementsFunction<Integer> source = new FromElementsFunction<>(IntSerializer.INSTANCE, data);
        StreamSource<Integer, FromElementsFunction<Integer>> src = new StreamSource<>(source);
        AbstractStreamOperatorTestHarness<Integer> testHarness = new AbstractStreamOperatorTestHarness<>(src, 1, 1, 0);
        testHarness.open();
        final SourceFunction.SourceContext<Integer> ctx = new ListSourceContext<Integer>(result, 2L);
        final Throwable[] error = new Throwable[1];
        // run the source asynchronously
        Thread runner = new Thread() {

            @Override
            public void run() {
                try {
                    source.run(ctx);
                } catch (Throwable t) {
                    error[0] = t;
                }
            }
        };
        runner.start();
        // wait for a bit
        Thread.sleep(1000);
        // make a checkpoint
        List<Integer> checkpointData = new ArrayList<>(numElements);
        OperatorSubtaskState handles = null;
        synchronized (ctx.getCheckpointLock()) {
            handles = testHarness.snapshot(566, System.currentTimeMillis());
            checkpointData.addAll(result);
        }
        // cancel the source
        source.cancel();
        runner.join();
        // check for errors
        if (error[0] != null) {
            System.err.println("Error in asynchronous source runner");
            error[0].printStackTrace();
            fail("Error in asynchronous source runner");
        }
        final FromElementsFunction<Integer> sourceCopy = new FromElementsFunction<>(IntSerializer.INSTANCE, data);
        StreamSource<Integer, FromElementsFunction<Integer>> srcCopy = new StreamSource<>(sourceCopy);
        AbstractStreamOperatorTestHarness<Integer> testHarnessCopy = new AbstractStreamOperatorTestHarness<>(srcCopy, 1, 1, 0);
        testHarnessCopy.setup();
        testHarnessCopy.initializeState(handles);
        testHarnessCopy.open();
        // recovery run
        SourceFunction.SourceContext<Integer> newCtx = new ListSourceContext<>(checkpointData);
        sourceCopy.run(newCtx);
        assertEquals(data, checkpointData);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 7 with FromElementsFunction

use of org.apache.flink.streaming.api.functions.source.FromElementsFunction in project flink by apache.

the class FromElementsFunctionTest method testSetOutputTypeWithExistingBrokenSerializer.

@Test
public void testSetOutputTypeWithExistingBrokenSerializer() throws Exception {
    // the original serializer throws an exception
    TypeInformation<DeserializeTooMuchType> info = new ValueTypeInfo<>(DeserializeTooMuchType.class);
    FromElementsFunction<DeserializeTooMuchType> source = new FromElementsFunction<>(info.createSerializer(new ExecutionConfig()), new DeserializeTooMuchType());
    TypeSerializer<DeserializeTooMuchType> existingSerializer = source.getSerializer();
    source.setOutputType(new GenericTypeInfo<>(DeserializeTooMuchType.class), new ExecutionConfig());
    TypeSerializer<DeserializeTooMuchType> newSerializer = source.getSerializer();
    assertNotEquals(existingSerializer, newSerializer);
    List<DeserializeTooMuchType> result = runSource(source);
    assertThat(result, hasSize(1));
    assertThat(result.get(0), instanceOf(DeserializeTooMuchType.class));
}

Example 8 with FromElementsFunction

use of org.apache.flink.streaming.api.functions.source.FromElementsFunction in project flink by apache.

the class FromElementsFunctionTest method testSetOutputTypeWithNoSerializer.

@Test
public void testSetOutputTypeWithNoSerializer() throws Exception {
    FromElementsFunction<String> source = new FromElementsFunction<>(STRING_ARRAY_DATA);
    assertNull(source.getSerializer());
    source.setOutputType(BasicTypeInfo.STRING_TYPE_INFO, new ExecutionConfig());
    assertNotNull(source.getSerializer());
    assertEquals(BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), source.getSerializer());
    List<String> result = runSource(source);
    assertEquals(STRING_LIST_DATA, result);
}

Example 9 with FromElementsFunction

use of org.apache.flink.streaming.api.functions.source.FromElementsFunction in project flink by apache.

the class StreamExecutionEnvironmentTest method testFromElementsDeducedType.

@Test
public void testFromElementsDeducedType() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStreamSource<String> source = env.fromElements("a", "b");
    FromElementsFunction<String> elementsFunction = (FromElementsFunction<String>) getFunctionFromDataSource(source);
    assertEquals(BasicTypeInfo.STRING_TYPE_INFO.createSerializer(env.getConfig()), elementsFunction.getSerializer());
}

Example 10 with FromElementsFunction

use of org.apache.flink.streaming.api.functions.source.FromElementsFunction in project flink by apache.

the class StreamExecutionEnvironmentTest method testFromElementsPostConstructionType.

@Test
public void testFromElementsPostConstructionType() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStreamSource<String> source = env.fromElements("a", "b");
    TypeInformation<String> customType = new GenericTypeInfo<>(String.class);
    source.returns(customType);
    FromElementsFunction<String> elementsFunction = (FromElementsFunction<String>) getFunctionFromDataSource(source);
    assertNotEquals(BasicTypeInfo.STRING_TYPE_INFO.createSerializer(env.getConfig()), elementsFunction.getSerializer());
    assertEquals(customType.createSerializer(env.getConfig()), elementsFunction.getSerializer());
}