Examples with SimpleStringSchema org.apache.flink.streaming.util.serialization.SimpleStringSchema used on opensource projects

Example 11 with SimpleStringSchema

use of org.apache.flink.streaming.util.serialization.SimpleStringSchema in project flink by apache.

the class KafkaConsumer08Test method testAtLeastOneBootstrapServerHostIsValid.

@Test
public void testAtLeastOneBootstrapServerHostIsValid() {
    try {
        String zookeeperConnect = "localhost:56794";
        String unknownHost = "foobar:11111";
        // we declare one valid bootstrap server, namely the one with
        // 'localhost'
        String bootstrapServers = unknownHost + ", localhost:22222";
        URL unknownHostURL = NetUtils.getCorrectHostnamePort(unknownHost);
        PowerMockito.mockStatic(InetAddress.class);
        when(InetAddress.getByName(Matchers.eq(unknownHostURL.getHost()))).thenThrow(new UnknownHostException("Test exception"));
        String groupId = "non-existent-group";
        Properties props = createKafkaProps(zookeeperConnect, bootstrapServers, groupId);
        FlinkKafkaConsumer08<String> consumer = new FlinkKafkaConsumer08<>(Collections.singletonList("no op topic"), new SimpleStringSchema(), props);
        consumer.open(new Configuration());
        fail();
    } catch (Exception e) {
        // test is not failing because we have one valid boostrap server
        assertTrue("The cause of the exception should not be 'all boostrap server are invalid'!", !e.getMessage().contains("All the hosts provided in: " + ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG + " config are invalid"));
    }
}

Example 12 with SimpleStringSchema

use of org.apache.flink.streaming.util.serialization.SimpleStringSchema in project flink by apache.

the class KafkaConsumer08Test method testAllBoostrapServerHostsAreInvalid.

@Test
public void testAllBoostrapServerHostsAreInvalid() {
    try {
        String unknownHost = "foobar:11111";
        URL unknownHostURL = NetUtils.getCorrectHostnamePort(unknownHost);
        PowerMockito.mockStatic(InetAddress.class);
        when(InetAddress.getByName(Matchers.eq(unknownHostURL.getHost()))).thenThrow(new UnknownHostException("Test exception"));
        String zookeeperConnect = "localhost:56794";
        String groupId = "non-existent-group";
        Properties props = createKafkaProps(zookeeperConnect, unknownHost, groupId);
        FlinkKafkaConsumer08<String> consumer = new FlinkKafkaConsumer08<>(Collections.singletonList("no op topic"), new SimpleStringSchema(), props);
        StreamingRuntimeContext mockRuntimeContext = mock(StreamingRuntimeContext.class);
        Mockito.when(mockRuntimeContext.isCheckpointingEnabled()).thenReturn(true);
        consumer.setRuntimeContext(mockRuntimeContext);
        consumer.open(new Configuration());
        fail();
    } catch (Exception e) {
        assertTrue("Exception should be thrown containing 'all bootstrap servers invalid' message!", e.getMessage().contains("All the servers provided in: '" + ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG + "' config are invalid"));
    }
}

Example 13 with SimpleStringSchema

use of org.apache.flink.streaming.util.serialization.SimpleStringSchema in project flink by apache.

the class FlinkKinesisConsumerTest method testSnapshotStateShouldBeNullIfSourceNotRun.

@Test
public void testSnapshotStateShouldBeNullIfSourceNotRun() throws Exception {
    Properties config = new Properties();
    config.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
    config.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, "accessKeyId");
    config.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, "secretKey");
    FlinkKinesisConsumer<String> consumer = new FlinkKinesisConsumer<>("fakeStream", new SimpleStringSchema(), config);
    // only opened, not run
    consumer.open(new Configuration());
    //arbitrary checkpoint id and timestamp
    assertTrue(consumer.snapshotState(123, 123) == null);
}

Example 14 with SimpleStringSchema

use of org.apache.flink.streaming.util.serialization.SimpleStringSchema in project flink by apache.

the class WriteIntoKafka method main.

public static void main(String[] args) throws Exception {
    ParameterTool parameterTool = ParameterTool.fromArgs(args);
    if (parameterTool.getNumberOfParameters() < 2) {
        System.out.println("Missing parameters!");
        System.out.println("Usage: Kafka --topic <topic> --bootstrap.servers <kafka brokers>");
        return;
    }
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().disableSysoutLogging();
    env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 10000));
    // very simple data generator
    DataStream<String> messageStream = env.addSource(new SourceFunction<String>() {

        private static final long serialVersionUID = 6369260445318862378L;

        public boolean running = true;

        @Override
        public void run(SourceContext<String> ctx) throws Exception {
            long i = 0;
            while (this.running) {
                ctx.collect("Element - " + i++);
                Thread.sleep(500);
            }
        }

        @Override
        public void cancel() {
            running = false;
        }
    });
    // write data into Kafka
    messageStream.addSink(new FlinkKafkaProducer08<>(parameterTool.getRequired("topic"), new SimpleStringSchema(), parameterTool.getProperties()));
    env.execute("Write into Kafka example");
}

Example 15 with SimpleStringSchema

use of org.apache.flink.streaming.util.serialization.SimpleStringSchema in project flink by apache.

the class KafkaProducerTest method testPropagateExceptions.

@Test
@SuppressWarnings("unchecked")
public void testPropagateExceptions() {
    try {
        // mock kafka producer
        KafkaProducer<?, ?> kafkaProducerMock = mock(KafkaProducer.class);
        // partition setup
        when(kafkaProducerMock.partitionsFor(anyString())).thenReturn(// returning a unmodifiable list to mimic KafkaProducer#partitionsFor() behaviour
        Collections.singletonList(new PartitionInfo("mock_topic", 42, null, null, null)));
        // failure when trying to send an element
        when(kafkaProducerMock.send(any(ProducerRecord.class), any(Callback.class))).thenAnswer(new Answer<Future<RecordMetadata>>() {

            @Override
            public Future<RecordMetadata> answer(InvocationOnMock invocation) throws Throwable {
                Callback callback = (Callback) invocation.getArguments()[1];
                callback.onCompletion(null, new Exception("Test error"));
                return null;
            }
        });
        // make sure the FlinkKafkaProducer instantiates our mock producer
        whenNew(KafkaProducer.class).withAnyArguments().thenReturn(kafkaProducerMock);
        // (1) producer that propagates errors
        FlinkKafkaProducer09<String> producerPropagating = new FlinkKafkaProducer09<>("mock_topic", new SimpleStringSchema(), FakeStandardProducerConfig.get(), null);
        OneInputStreamOperatorTestHarness<String, Object> testHarness = new OneInputStreamOperatorTestHarness<>(new StreamSink(producerPropagating));
        testHarness.open();
        try {
            testHarness.processElement(new StreamRecord<>("value"));
            testHarness.processElement(new StreamRecord<>("value"));
            fail("This should fail with an exception");
        } catch (Exception e) {
            assertNotNull(e.getCause());
            assertNotNull(e.getCause().getMessage());
            assertTrue(e.getCause().getMessage().contains("Test error"));
        }
        // (2) producer that only logs errors
        FlinkKafkaProducer09<String> producerLogging = new FlinkKafkaProducer09<>("mock_topic", new SimpleStringSchema(), FakeStandardProducerConfig.get(), null);
        producerLogging.setLogFailuresOnly(true);
        testHarness = new OneInputStreamOperatorTestHarness<>(new StreamSink(producerLogging));
        testHarness.open();
        testHarness.processElement(new StreamRecord<>("value"));
        testHarness.processElement(new StreamRecord<>("value"));
        testHarness.close();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}