Examples with SimpleStringSchema org.apache.flink.api.common.serialization.SimpleStringSchema used on opensource projects

Example 46 with SimpleStringSchema

use of org.apache.flink.api.common.serialization.SimpleStringSchema in project flink by apache.

the class FlinkKinesisProducerTest method testAsyncErrorRethrownOnCheckpoint.

/**
 * Test ensuring that if a snapshot call happens right after an async exception is caught, it
 * should be rethrown.
 */
@SuppressWarnings("ResultOfMethodCallIgnored")
@Test
public void testAsyncErrorRethrownOnCheckpoint() throws Throwable {
    final DummyFlinkKinesisProducer<String> producer = new DummyFlinkKinesisProducer<>(new SimpleStringSchema());
    OneInputStreamOperatorTestHarness<String, Object> testHarness = new OneInputStreamOperatorTestHarness<>(new StreamSink<>(producer));
    testHarness.open();
    testHarness.processElement(new StreamRecord<>("msg-1"));
    producer.getPendingRecordFutures().get(0).setException(new Exception("artificial async exception"));
    try {
        testHarness.snapshot(123L, 123L);
    } catch (Exception e) {
        // the next checkpoint should rethrow the async exception
        Assert.assertTrue(ExceptionUtils.findThrowableWithMessage(e, "artificial async exception").isPresent());
        // test succeeded
        return;
    }
    Assert.fail();
}

Example 47 with SimpleStringSchema

use of org.apache.flink.api.common.serialization.SimpleStringSchema in project flink by apache.

the class FlinkKinesisProducerTest method testAsyncErrorRethrownOnInvoke.

// ----------------------------------------------------------------------
// Tests to verify at-least-once guarantee
// ----------------------------------------------------------------------
/**
 * Test ensuring that if an invoke call happens right after an async exception is caught, it
 * should be rethrown.
 */
@SuppressWarnings("ResultOfMethodCallIgnored")
@Test
public void testAsyncErrorRethrownOnInvoke() throws Throwable {
    final DummyFlinkKinesisProducer<String> producer = new DummyFlinkKinesisProducer<>(new SimpleStringSchema());
    OneInputStreamOperatorTestHarness<String, Object> testHarness = new OneInputStreamOperatorTestHarness<>(new StreamSink<>(producer));
    testHarness.open();
    testHarness.processElement(new StreamRecord<>("msg-1"));
    producer.getPendingRecordFutures().get(0).setException(new Exception("artificial async exception"));
    try {
        testHarness.processElement(new StreamRecord<>("msg-2"));
    } catch (Exception e) {
        // the next invoke should rethrow the async exception
        Assert.assertTrue(ExceptionUtils.findThrowableWithMessage(e, "artificial async exception").isPresent());
        // test succeeded
        return;
    }
    Assert.fail();
}

Example 48 with SimpleStringSchema

use of org.apache.flink.api.common.serialization.SimpleStringSchema in project flink by apache.

the class FlinkKinesisProducerTest method testBackpressure.

/**
 * Test ensuring that the producer blocks if the queue limit is exceeded, until the queue length
 * drops below the limit; we set a timeout because the test will not finish if the logic is
 * broken.
 */
@Test(timeout = 10000)
public void testBackpressure() throws Throwable {
    final Deadline deadline = Deadline.fromNow(Duration.ofSeconds(10));
    final DummyFlinkKinesisProducer<String> producer = new DummyFlinkKinesisProducer<>(new SimpleStringSchema());
    producer.setQueueLimit(1);
    OneInputStreamOperatorTestHarness<String, Object> testHarness = new OneInputStreamOperatorTestHarness<>(new StreamSink<>(producer));
    testHarness.open();
    UserRecordResult result = mock(UserRecordResult.class);
    when(result.isSuccessful()).thenReturn(true);
    CheckedThread msg1 = new CheckedThread() {

        @Override
        public void go() throws Exception {
            testHarness.processElement(new StreamRecord<>("msg-1"));
        }
    };
    msg1.start();
    msg1.trySync(deadline.timeLeftIfAny().toMillis());
    assertFalse("Flush triggered before reaching queue limit", msg1.isAlive());
    // consume msg-1 so that queue is empty again
    producer.getPendingRecordFutures().get(0).set(result);
    CheckedThread msg2 = new CheckedThread() {

        @Override
        public void go() throws Exception {
            testHarness.processElement(new StreamRecord<>("msg-2"));
        }
    };
    msg2.start();
    msg2.trySync(deadline.timeLeftIfAny().toMillis());
    assertFalse("Flush triggered before reaching queue limit", msg2.isAlive());
    CheckedThread moreElementsThread = new CheckedThread() {

        @Override
        public void go() throws Exception {
            // this should block until msg-2 is consumed
            testHarness.processElement(new StreamRecord<>("msg-3"));
            // this should block until msg-3 is consumed
            testHarness.processElement(new StreamRecord<>("msg-4"));
        }
    };
    moreElementsThread.start();
    assertTrue("Producer should still block, but doesn't", moreElementsThread.isAlive());
    // consume msg-2 from the queue, leaving msg-3 in the queue and msg-4 blocked
    while (producer.getPendingRecordFutures().size() < 2) {
        Thread.sleep(50);
    }
    producer.getPendingRecordFutures().get(1).set(result);
    assertTrue("Producer should still block, but doesn't", moreElementsThread.isAlive());
    // consume msg-3, blocked msg-4 can be inserted into the queue and block is released
    while (producer.getPendingRecordFutures().size() < 3) {
        Thread.sleep(50);
    }
    producer.getPendingRecordFutures().get(2).set(result);
    moreElementsThread.trySync(deadline.timeLeftIfAny().toMillis());
    assertFalse("Prodcuer still blocks although the queue is flushed", moreElementsThread.isAlive());
    producer.getPendingRecordFutures().get(3).set(result);
    testHarness.close();
}

Example 49 with SimpleStringSchema

use of org.apache.flink.api.common.serialization.SimpleStringSchema in project flink by apache.

the class ConsumeFromDynamoDBStreams method main.

public static void main(String[] args) throws Exception {
    ParameterTool pt = ParameterTool.fromArgs(args);
    StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
    see.setParallelism(1);
    Properties dynamodbStreamsConsumerConfig = new Properties();
    final String streamName = pt.getRequired(DYNAMODB_STREAM_NAME);
    dynamodbStreamsConsumerConfig.setProperty(ConsumerConfigConstants.AWS_REGION, pt.getRequired("region"));
    dynamodbStreamsConsumerConfig.setProperty(ConsumerConfigConstants.AWS_ACCESS_KEY_ID, pt.getRequired("accesskey"));
    dynamodbStreamsConsumerConfig.setProperty(ConsumerConfigConstants.AWS_SECRET_ACCESS_KEY, pt.getRequired("secretkey"));
    DataStream<String> dynamodbStreams = see.addSource(new FlinkDynamoDBStreamsConsumer<>(streamName, new SimpleStringSchema(), dynamodbStreamsConsumerConfig));
    dynamodbStreams.print();
    see.execute();
}

Example 50 with SimpleStringSchema

use of org.apache.flink.api.common.serialization.SimpleStringSchema in project flink by apache.

the class KinesisDataFetcherTest method testCancelDuringDiscovery.

@Test
public void testCancelDuringDiscovery() throws Exception {
    final String stream = "fakeStream";
    final int numShards = 3;
    Properties standardProperties = TestUtils.getStandardProperties();
    standardProperties.setProperty(SHARD_DISCOVERY_INTERVAL_MILLIS, "10000000");
    final LinkedList<KinesisStreamShardState> testShardStates = new LinkedList<>();
    final TestSourceContext<String> sourceContext = new TestSourceContext<>();
    TestableKinesisDataFetcher<String> fetcher = new TestableKinesisDataFetcher<String>(singletonList(stream), sourceContext, standardProperties, new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()), 1, 0, new AtomicReference<>(), testShardStates, new HashMap<>(), FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(Collections.singletonMap(stream, numShards)));
    // FlinkKinesisConsumer is responsible for setting up the fetcher before it can be run;
    // run the consumer until it reaches the point where the fetcher starts to run
    final DummyFlinkKinesisConsumer<String> consumer = new DummyFlinkKinesisConsumer<>(TestUtils.getStandardProperties(), fetcher, 1, 0);
    CheckedThread consumerThread = new CheckedThread() {

        @Override
        public void go() throws Exception {
            consumer.run(new TestSourceContext<>());
        }
    };
    consumerThread.start();
    // wait for the second discovery to be triggered, that has a high probability to be inside
    // discovery sleep (10k s)
    fetcher.waitUntilDiscovery(2);
    Thread.sleep(1000);
    consumer.cancel();
    consumerThread.sync();
}