Examples with TypeInformationSerializationSchema org.apache.flink.streaming.util.serialization.TypeInformationSerializationSchema used on opensource projects

Example 11 with TypeInformationSerializationSchema

use of org.apache.flink.streaming.util.serialization.TypeInformationSerializationSchema in project flink by apache.

the class KafkaConsumerTestBase method runMultipleSourcesOnePartitionExactlyOnceTest.

/**
	 * Tests the proper consumption when having more Flink sources than Kafka partitions, which means
	 * that some Flink sources will read no partitions.
	 */
public void runMultipleSourcesOnePartitionExactlyOnceTest() throws Exception {
    final String topic = "manyToOneTopic";
    final int numPartitions = 5;
    final int numElementsPerPartition = 1000;
    final int totalElements = numPartitions * numElementsPerPartition;
    final int failAfterElements = numElementsPerPartition / 3;
    final int parallelism = 8;
    createTestTopic(topic, numPartitions, 1);
    DataGenerators.generateRandomizedIntegerSequence(StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort), kafkaServer, topic, numPartitions, numElementsPerPartition, true);
    // run the topology that fails and recovers
    DeserializationSchema<Integer> schema = new TypeInformationSerializationSchema<>(BasicTypeInfo.INT_TYPE_INFO, new ExecutionConfig());
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
    env.enableCheckpointing(500);
    env.setParallelism(parallelism);
    // set the number of restarts to one. The failing mapper will fail once, then it's only success exceptions.
    env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
    env.getConfig().disableSysoutLogging();
    env.setBufferTimeout(0);
    Properties props = new Properties();
    props.putAll(standardProps);
    props.putAll(secureProps);
    FlinkKafkaConsumerBase<Integer> kafkaSource = kafkaServer.getConsumer(topic, schema, props);
    env.addSource(kafkaSource).map(new PartitionValidatingMapper(numPartitions, 1)).map(new FailingIdentityMapper<Integer>(failAfterElements)).addSink(new ValidatingExactlyOnceSink(totalElements)).setParallelism(1);
    FailingIdentityMapper.failedBefore = false;
    tryExecute(env, "multi-source-one-partitions exactly once test");
    deleteTestTopic(topic);
}

Example 12 with TypeInformationSerializationSchema

use of org.apache.flink.streaming.util.serialization.TypeInformationSerializationSchema in project flink by apache.

the class DataGenerators method generateRandomizedIntegerSequence.

public static void generateRandomizedIntegerSequence(StreamExecutionEnvironment env, KafkaTestEnvironment testServer, String topic, final int numPartitions, final int numElements, final boolean randomizeOrder) throws Exception {
    env.setParallelism(numPartitions);
    env.getConfig().disableSysoutLogging();
    env.setRestartStrategy(RestartStrategies.noRestart());
    DataStream<Integer> stream = env.addSource(new RichParallelSourceFunction<Integer>() {

        private volatile boolean running = true;

        @Override
        public void run(SourceContext<Integer> ctx) {
            // create a sequence
            int[] elements = new int[numElements];
            for (int i = 0, val = getRuntimeContext().getIndexOfThisSubtask(); i < numElements; i++, val += getRuntimeContext().getNumberOfParallelSubtasks()) {
                elements[i] = val;
            }
            // scramble the sequence
            if (randomizeOrder) {
                Random rnd = new Random();
                for (int i = 0; i < elements.length; i++) {
                    int otherPos = rnd.nextInt(elements.length);
                    int tmp = elements[i];
                    elements[i] = elements[otherPos];
                    elements[otherPos] = tmp;
                }
            }
            // emit the sequence
            int pos = 0;
            while (running && pos < elements.length) {
                ctx.collect(elements[pos++]);
            }
        }

        @Override
        public void cancel() {
            running = false;
        }
    });
    Properties props = new Properties();
    props.putAll(FlinkKafkaProducerBase.getPropertiesFromBrokerList(testServer.getBrokerConnectionString()));
    Properties secureProps = testServer.getSecureProperties();
    if (secureProps != null) {
        props.putAll(testServer.getSecureProperties());
    }
    stream = stream.rebalance();
    testServer.produceIntoKafka(stream, topic, new KeyedSerializationSchemaWrapper<>(new TypeInformationSerializationSchema<>(BasicTypeInfo.INT_TYPE_INFO, env.getConfig())), props, new KafkaPartitioner<Integer>() {

        @Override
        public int partition(Integer next, byte[] serializedKey, byte[] serializedValue, int numPartitions) {
            return next % numPartitions;
        }
    });
    env.execute("Scrambles int sequence generator");
}

Example 13 with TypeInformationSerializationSchema

use of org.apache.flink.streaming.util.serialization.TypeInformationSerializationSchema in project flink by apache.

the class TypeInformationSerializationSchemaTest method testDeSerialization.

@Test
public void testDeSerialization() {
    try {
        TypeInformation<MyPOJO> info = TypeExtractor.getForClass(MyPOJO.class);
        TypeInformationSerializationSchema<MyPOJO> schema = new TypeInformationSerializationSchema<MyPOJO>(info, new ExecutionConfig());
        MyPOJO[] types = { new MyPOJO(72, new Date(763784523L), new Date(88234L)), new MyPOJO(-1, new Date(11111111111111L)), new MyPOJO(42), new MyPOJO(17, new Date(222763784523L)) };
        for (MyPOJO val : types) {
            byte[] serialized = schema.serialize(val);
            MyPOJO deser = schema.deserialize(serialized);
            assertEquals(val, deser);
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 14 with TypeInformationSerializationSchema

use of org.apache.flink.streaming.util.serialization.TypeInformationSerializationSchema in project flink by apache.

the class TypeInformationSerializationSchemaTest method testSerializability.

@Test
public void testSerializability() {
    try {
        TypeInformation<MyPOJO> info = TypeExtractor.getForClass(MyPOJO.class);
        TypeInformationSerializationSchema<MyPOJO> schema = new TypeInformationSerializationSchema<MyPOJO>(info, new ExecutionConfig());
        // this needs to succeed
        CommonTestUtils.createCopySerializable(schema);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}