Examples with SequenceNumberRange com.amazonaws.services.kinesis.model.SequenceNumberRange used on opensource projects

Example 6 with SequenceNumberRange

use of com.amazonaws.services.kinesis.model.SequenceNumberRange in project flink by apache.

the class FlinkKinesisConsumerMigrationTest method testRestore.

@Test
public void testRestore() throws Exception {
    final List<StreamShardHandle> initialDiscoveryShards = new ArrayList<>(TEST_STATE.size());
    for (StreamShardMetadata shardMetadata : TEST_STATE.keySet()) {
        Shard shard = new Shard();
        shard.setShardId(shardMetadata.getShardId());
        SequenceNumberRange sequenceNumberRange = new SequenceNumberRange();
        sequenceNumberRange.withStartingSequenceNumber("1");
        shard.setSequenceNumberRange(sequenceNumberRange);
        initialDiscoveryShards.add(new StreamShardHandle(shardMetadata.getStreamName(), shard));
    }
    final TestFetcher<String> fetcher = new TestFetcher<>(Collections.singletonList(TEST_STREAM_NAME), new TestSourceContext<>(), new TestRuntimeContext(true, 1, 0), TestUtils.getStandardProperties(), new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()), null, initialDiscoveryShards);
    final DummyFlinkKinesisConsumer<String> consumerFunction = new DummyFlinkKinesisConsumer<>(fetcher, new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()));
    StreamSource<String, DummyFlinkKinesisConsumer<String>> consumerOperator = new StreamSource<>(consumerFunction);
    final AbstractStreamOperatorTestHarness<String> testHarness = new AbstractStreamOperatorTestHarness<>(consumerOperator, 1, 1, 0);
    testHarness.setup();
    testHarness.initializeState(OperatorSnapshotUtil.getResourceFilename("kinesis-consumer-migration-test-flink" + testMigrateVersion + "-snapshot"));
    testHarness.open();
    consumerFunction.run(new TestSourceContext<>());
    // assert that state is correctly restored
    assertNotEquals(null, consumerFunction.getRestoredState());
    assertEquals(1, consumerFunction.getRestoredState().size());
    assertEquals(TEST_STATE, removeEquivalenceWrappers(consumerFunction.getRestoredState()));
    assertEquals(1, fetcher.getSubscribedShardsState().size());
    assertEquals(TEST_SEQUENCE_NUMBER, fetcher.getSubscribedShardsState().get(0).getLastProcessedSequenceNum());
    KinesisStreamShardState restoredShardState = fetcher.getSubscribedShardsState().get(0);
    assertEquals(TEST_STREAM_NAME, restoredShardState.getStreamShardHandle().getStreamName());
    assertEquals(TEST_SHARD_ID, restoredShardState.getStreamShardHandle().getShard().getShardId());
    assertFalse(restoredShardState.getStreamShardHandle().isClosed());
    assertEquals(TEST_SEQUENCE_NUMBER, restoredShardState.getLastProcessedSequenceNum());
    consumerOperator.close();
    consumerOperator.cancel();
}

Example 7 with SequenceNumberRange

use of com.amazonaws.services.kinesis.model.SequenceNumberRange in project flink by apache.

the class FlinkKinesisConsumerTest method testFindSequenceNumberToRestoreFromIfTheShardHasBeenClosedSinceTheStateWasStored.

/**
 * FLINK-8484: ensure that a state change in the StreamShardMetadata other than {@link
 * StreamShardMetadata#getShardId()} or {@link StreamShardMetadata#getStreamName()} does not
 * result in the shard not being able to be restored. This handles the corner case where the
 * stored shard metadata is open (no ending sequence number), but after the job restore, the
 * shard has been closed (ending number set) due to re-sharding, and we can no longer rely on
 * {@link StreamShardMetadata#equals(Object)} to find back the sequence number in the collection
 * of restored shard metadata.
 *
 * <p>Therefore, we will rely on synchronizing the snapshot's state with the Kinesis shard
 * before attempting to find back the sequence number to restore.
 */
@Test
public void testFindSequenceNumberToRestoreFromIfTheShardHasBeenClosedSinceTheStateWasStored() throws Exception {
    // ----------------------------------------------------------------------
    // setup initial state
    // ----------------------------------------------------------------------
    HashMap<StreamShardHandle, SequenceNumber> fakeRestoredState = getFakeRestoredStore("all");
    // ----------------------------------------------------------------------
    // mock operator state backend and initial state for initializeState()
    // ----------------------------------------------------------------------
    TestingListState<Tuple2<StreamShardMetadata, SequenceNumber>> listState = new TestingListState<>();
    for (Map.Entry<StreamShardHandle, SequenceNumber> state : fakeRestoredState.entrySet()) {
        listState.add(Tuple2.of(KinesisDataFetcher.convertToStreamShardMetadata(state.getKey()), state.getValue()));
    }
    OperatorStateStore operatorStateStore = mock(OperatorStateStore.class);
    when(operatorStateStore.getUnionListState(Matchers.any(ListStateDescriptor.class))).thenReturn(listState);
    StateInitializationContext initializationContext = mock(StateInitializationContext.class);
    when(initializationContext.getOperatorStateStore()).thenReturn(operatorStateStore);
    when(initializationContext.isRestored()).thenReturn(true);
    // ----------------------------------------------------------------------
    // mock fetcher
    // ----------------------------------------------------------------------
    KinesisDataFetcher mockedFetcher = mockKinesisDataFetcher();
    List<StreamShardHandle> shards = new ArrayList<>();
    // create a fake stream shard handle based on the first entry in the restored state
    final StreamShardHandle originalStreamShardHandle = fakeRestoredState.keySet().iterator().next();
    final StreamShardHandle closedStreamShardHandle = new StreamShardHandle(originalStreamShardHandle.getStreamName(), originalStreamShardHandle.getShard());
    // close the shard handle by setting an ending sequence number
    final SequenceNumberRange sequenceNumberRange = new SequenceNumberRange();
    sequenceNumberRange.setEndingSequenceNumber("1293844");
    closedStreamShardHandle.getShard().setSequenceNumberRange(sequenceNumberRange);
    shards.add(closedStreamShardHandle);
    when(mockedFetcher.discoverNewShardsToSubscribe()).thenReturn(shards);
    // assume the given config is correct
    PowerMockito.mockStatic(KinesisConfigUtil.class);
    PowerMockito.doNothing().when(KinesisConfigUtil.class);
    // ----------------------------------------------------------------------
    // start to test fetcher's initial state seeding
    // ----------------------------------------------------------------------
    TestableFlinkKinesisConsumer consumer = new TestableFlinkKinesisConsumer("fakeStream", new Properties(), 10, 2);
    consumer.initializeState(initializationContext);
    consumer.open(new Configuration());
    consumer.run(Mockito.mock(SourceFunction.SourceContext.class));
    Mockito.verify(mockedFetcher).registerNewSubscribedShardState(new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(closedStreamShardHandle), closedStreamShardHandle, fakeRestoredState.get(closedStreamShardHandle)));
}

Example 8 with SequenceNumberRange

use of com.amazonaws.services.kinesis.model.SequenceNumberRange in project flink by apache.

the class KinesisDataFetcherTest method testStreamShardMetadataAndHandleConversion.

@Test
public void testStreamShardMetadataAndHandleConversion() {
    String streamName = "fakeStream1";
    String shardId = "shard-000001";
    String parentShardId = "shard-000002";
    String adjacentParentShardId = "shard-000003";
    String startingHashKey = "key-000001";
    String endingHashKey = "key-000010";
    String startingSequenceNumber = "seq-0000021";
    String endingSequenceNumber = "seq-00000031";
    StreamShardMetadata kinesisStreamShard = new StreamShardMetadata();
    kinesisStreamShard.setStreamName(streamName);
    kinesisStreamShard.setShardId(shardId);
    kinesisStreamShard.setParentShardId(parentShardId);
    kinesisStreamShard.setAdjacentParentShardId(adjacentParentShardId);
    kinesisStreamShard.setStartingHashKey(startingHashKey);
    kinesisStreamShard.setEndingHashKey(endingHashKey);
    kinesisStreamShard.setStartingSequenceNumber(startingSequenceNumber);
    kinesisStreamShard.setEndingSequenceNumber(endingSequenceNumber);
    Shard shard = new Shard().withShardId(shardId).withParentShardId(parentShardId).withAdjacentParentShardId(adjacentParentShardId).withHashKeyRange(new HashKeyRange().withStartingHashKey(startingHashKey).withEndingHashKey(endingHashKey)).withSequenceNumberRange(new SequenceNumberRange().withStartingSequenceNumber(startingSequenceNumber).withEndingSequenceNumber(endingSequenceNumber));
    StreamShardHandle streamShardHandle = new StreamShardHandle(streamName, shard);
    assertEquals(kinesisStreamShard, KinesisDataFetcher.convertToStreamShardMetadata(streamShardHandle));
    assertEquals(streamShardHandle, KinesisDataFetcher.convertToStreamShardHandle(kinesisStreamShard));
}

Example 9 with SequenceNumberRange

use of com.amazonaws.services.kinesis.model.SequenceNumberRange in project druid by druid-io.

the class KinesisSupervisorTest method testGetIgnorablePartitionIds.

@Test
public void testGetIgnorablePartitionIds() {
    supervisor = getTestableSupervisor(1, 2, true, "PT1H", null, null);
    supervisor.setupRecordSupplier();
    supervisor.tryInit();
    String stream = supervisor.getKinesisSupervisorSpec().getSource();
    SequenceNumberRange openShardRange = new SequenceNumberRange().withEndingSequenceNumber(null);
    SequenceNumberRange closedShardRange = new SequenceNumberRange().withEndingSequenceNumber("non-null");
    Shard openShard = new Shard().withShardId("openShard").withSequenceNumberRange(openShardRange);
    Shard emptyClosedShard = new Shard().withShardId("emptyClosedShard").withSequenceNumberRange(closedShardRange);
    Shard nonEmptyClosedShard = new Shard().withShardId("nonEmptyClosedShard").withSequenceNumberRange(closedShardRange);
    EasyMock.expect(supervisorRecordSupplier.getShards(stream)).andReturn(ImmutableSet.of(openShard, nonEmptyClosedShard, emptyClosedShard)).once().andReturn(ImmutableSet.of(openShard, nonEmptyClosedShard, emptyClosedShard)).once().andReturn(ImmutableSet.of(openShard, emptyClosedShard)).once().andReturn(ImmutableSet.of(openShard)).once().andReturn(ImmutableSet.of(openShard, nonEmptyClosedShard, emptyClosedShard)).once();
    // The following calls happen twice, once during the first call since there was no cache,
    // and once during the last since the cache was cleared prior to it
    EasyMock.expect(supervisorRecordSupplier.isClosedShardEmpty(stream, emptyClosedShard.getShardId())).andReturn(true).times(2);
    EasyMock.expect(supervisorRecordSupplier.isClosedShardEmpty(stream, nonEmptyClosedShard.getShardId())).andReturn(false).times(2);
    EasyMock.replay(supervisorRecordSupplier);
    // ActiveShards = {open, empty-closed, nonEmpty-closed}, IgnorableShards = {empty-closed}
    // {empty-closed, nonEmpty-closed} added to cache
    Assert.assertEquals(Collections.singleton(emptyClosedShard.getShardId()), supervisor.computeIgnorablePartitionIds());
    // ActiveShards = {open, empty-closed, nonEmpty-closed}, IgnorableShards = {empty-closed}
    Assert.assertEquals(Collections.singleton(emptyClosedShard.getShardId()), supervisor.computeIgnorablePartitionIds());
    // ActiveShards = {open, empty-closed}, IgnorableShards = {empty-closed}
    // {nonEmpty-closed} removed from cache
    Assert.assertEquals(Collections.singleton(emptyClosedShard.getShardId()), supervisor.computeIgnorablePartitionIds());
    // ActiveShards = {open}, IgnorableShards = {}
    // {empty-closed} removed from cache
    Assert.assertEquals(new HashSet<>(), supervisor.computeIgnorablePartitionIds());
    // ActiveShards = {open, empty-closed, nonEmpty-closed}, IgnorableShards = {empty-closed}
    // {empty-closed, nonEmpty-closed} re-added to cache
    Assert.assertEquals(Collections.singleton(emptyClosedShard.getShardId()), supervisor.computeIgnorablePartitionIds());
}