Examples with LinkedMap org.apache.commons.collections.map.LinkedMap used on opensource projects

Example 6 with LinkedMap

use of org.apache.commons.collections.map.LinkedMap in project flink by apache.

the class FlinkKafkaConsumerBaseTest method testSnapshotStateWithCommitOnCheckpointsEnabled.

@Test
@SuppressWarnings("unchecked")
public void testSnapshotStateWithCommitOnCheckpointsEnabled() throws Exception {
    // --------------------------------------------------------------------
    //   prepare fake states
    // --------------------------------------------------------------------
    final HashMap<KafkaTopicPartition, Long> state1 = new HashMap<>();
    state1.put(new KafkaTopicPartition("abc", 13), 16768L);
    state1.put(new KafkaTopicPartition("def", 7), 987654321L);
    final HashMap<KafkaTopicPartition, Long> state2 = new HashMap<>();
    state2.put(new KafkaTopicPartition("abc", 13), 16770L);
    state2.put(new KafkaTopicPartition("def", 7), 987654329L);
    final HashMap<KafkaTopicPartition, Long> state3 = new HashMap<>();
    state3.put(new KafkaTopicPartition("abc", 13), 16780L);
    state3.put(new KafkaTopicPartition("def", 7), 987654377L);
    // --------------------------------------------------------------------
    final AbstractFetcher<String, ?> fetcher = mock(AbstractFetcher.class);
    when(fetcher.snapshotCurrentState()).thenReturn(state1, state2, state3);
    final LinkedMap pendingOffsetsToCommit = new LinkedMap();
    FlinkKafkaConsumerBase<String> consumer = getConsumer(fetcher, pendingOffsetsToCommit, true);
    StreamingRuntimeContext mockRuntimeContext = mock(StreamingRuntimeContext.class);
    // enable checkpointing
    when(mockRuntimeContext.isCheckpointingEnabled()).thenReturn(true);
    consumer.setRuntimeContext(mockRuntimeContext);
    assertEquals(0, pendingOffsetsToCommit.size());
    OperatorStateStore backend = mock(OperatorStateStore.class);
    TestingListState<Serializable> listState = new TestingListState<>();
    when(backend.getSerializableListState(Matchers.any(String.class))).thenReturn(listState);
    StateInitializationContext initializationContext = mock(StateInitializationContext.class);
    when(initializationContext.getOperatorStateStore()).thenReturn(backend);
    when(initializationContext.isRestored()).thenReturn(false, true, true, true);
    consumer.initializeState(initializationContext);
    consumer.open(new Configuration());
    // checkpoint 1
    consumer.snapshotState(new StateSnapshotContextSynchronousImpl(138, 138));
    HashMap<KafkaTopicPartition, Long> snapshot1 = new HashMap<>();
    for (Serializable serializable : listState.get()) {
        Tuple2<KafkaTopicPartition, Long> kafkaTopicPartitionLongTuple2 = (Tuple2<KafkaTopicPartition, Long>) serializable;
        snapshot1.put(kafkaTopicPartitionLongTuple2.f0, kafkaTopicPartitionLongTuple2.f1);
    }
    assertEquals(state1, snapshot1);
    assertEquals(1, pendingOffsetsToCommit.size());
    assertEquals(state1, pendingOffsetsToCommit.get(138L));
    // checkpoint 2
    consumer.snapshotState(new StateSnapshotContextSynchronousImpl(140, 140));
    HashMap<KafkaTopicPartition, Long> snapshot2 = new HashMap<>();
    for (Serializable serializable : listState.get()) {
        Tuple2<KafkaTopicPartition, Long> kafkaTopicPartitionLongTuple2 = (Tuple2<KafkaTopicPartition, Long>) serializable;
        snapshot2.put(kafkaTopicPartitionLongTuple2.f0, kafkaTopicPartitionLongTuple2.f1);
    }
    assertEquals(state2, snapshot2);
    assertEquals(2, pendingOffsetsToCommit.size());
    assertEquals(state2, pendingOffsetsToCommit.get(140L));
    // ack checkpoint 1
    consumer.notifyCheckpointComplete(138L);
    assertEquals(1, pendingOffsetsToCommit.size());
    assertTrue(pendingOffsetsToCommit.containsKey(140L));
    // checkpoint 3
    consumer.snapshotState(new StateSnapshotContextSynchronousImpl(141, 141));
    HashMap<KafkaTopicPartition, Long> snapshot3 = new HashMap<>();
    for (Serializable serializable : listState.get()) {
        Tuple2<KafkaTopicPartition, Long> kafkaTopicPartitionLongTuple2 = (Tuple2<KafkaTopicPartition, Long>) serializable;
        snapshot3.put(kafkaTopicPartitionLongTuple2.f0, kafkaTopicPartitionLongTuple2.f1);
    }
    assertEquals(state3, snapshot3);
    assertEquals(2, pendingOffsetsToCommit.size());
    assertEquals(state3, pendingOffsetsToCommit.get(141L));
    // ack checkpoint 3, subsumes number 2
    consumer.notifyCheckpointComplete(141L);
    assertEquals(0, pendingOffsetsToCommit.size());
    // invalid checkpoint
    consumer.notifyCheckpointComplete(666);
    assertEquals(0, pendingOffsetsToCommit.size());
    OperatorStateStore operatorStateStore = mock(OperatorStateStore.class);
    listState = new TestingListState<>();
    when(operatorStateStore.getOperatorState(Matchers.any(ListStateDescriptor.class))).thenReturn(listState);
    // create 500 snapshots
    for (int i = 100; i < 600; i++) {
        consumer.snapshotState(new StateSnapshotContextSynchronousImpl(i, i));
        listState.clear();
    }
    assertEquals(FlinkKafkaConsumerBase.MAX_NUM_PENDING_CHECKPOINTS, pendingOffsetsToCommit.size());
    // commit only the second last
    consumer.notifyCheckpointComplete(598);
    assertEquals(1, pendingOffsetsToCommit.size());
    // access invalid checkpoint
    consumer.notifyCheckpointComplete(590);
    // and the last
    consumer.notifyCheckpointComplete(599);
    assertEquals(0, pendingOffsetsToCommit.size());
}

Example 7 with LinkedMap

use of org.apache.commons.collections.map.LinkedMap in project flink by apache.

the class FlinkKafkaConsumerBaseTest method checkRestoredNullCheckpointWhenFetcherNotReady.

/**
	 * Tests that no checkpoints happen when the fetcher is not running.
	 */
@Test
public void checkRestoredNullCheckpointWhenFetcherNotReady() throws Exception {
    FlinkKafkaConsumerBase<String> consumer = getConsumer(null, new LinkedMap(), true);
    OperatorStateStore operatorStateStore = mock(OperatorStateStore.class);
    TestingListState<Serializable> listState = new TestingListState<>();
    when(operatorStateStore.getSerializableListState(Matchers.any(String.class))).thenReturn(listState);
    StateInitializationContext initializationContext = mock(StateInitializationContext.class);
    when(initializationContext.getOperatorStateStore()).thenReturn(operatorStateStore);
    when(initializationContext.isRestored()).thenReturn(false);
    consumer.initializeState(initializationContext);
    consumer.open(new Configuration());
    consumer.snapshotState(new StateSnapshotContextSynchronousImpl(17, 17));
    assertFalse(listState.get().iterator().hasNext());
}

Example 8 with LinkedMap

use of org.apache.commons.collections.map.LinkedMap in project flink by apache.

the class FlinkKafkaConsumerBaseTest method testSnapshotStateWithCommitOnCheckpointsDisabled.

@Test
@SuppressWarnings("unchecked")
public void testSnapshotStateWithCommitOnCheckpointsDisabled() throws Exception {
    // --------------------------------------------------------------------
    //   prepare fake states
    // --------------------------------------------------------------------
    final HashMap<KafkaTopicPartition, Long> state1 = new HashMap<>();
    state1.put(new KafkaTopicPartition("abc", 13), 16768L);
    state1.put(new KafkaTopicPartition("def", 7), 987654321L);
    final HashMap<KafkaTopicPartition, Long> state2 = new HashMap<>();
    state2.put(new KafkaTopicPartition("abc", 13), 16770L);
    state2.put(new KafkaTopicPartition("def", 7), 987654329L);
    final HashMap<KafkaTopicPartition, Long> state3 = new HashMap<>();
    state3.put(new KafkaTopicPartition("abc", 13), 16780L);
    state3.put(new KafkaTopicPartition("def", 7), 987654377L);
    // --------------------------------------------------------------------
    final AbstractFetcher<String, ?> fetcher = mock(AbstractFetcher.class);
    when(fetcher.snapshotCurrentState()).thenReturn(state1, state2, state3);
    final LinkedMap pendingOffsetsToCommit = new LinkedMap();
    FlinkKafkaConsumerBase<String> consumer = getConsumer(fetcher, pendingOffsetsToCommit, true);
    StreamingRuntimeContext mockRuntimeContext = mock(StreamingRuntimeContext.class);
    // enable checkpointing
    when(mockRuntimeContext.isCheckpointingEnabled()).thenReturn(true);
    consumer.setRuntimeContext(mockRuntimeContext);
    // disable offset committing
    consumer.setCommitOffsetsOnCheckpoints(false);
    assertEquals(0, pendingOffsetsToCommit.size());
    OperatorStateStore backend = mock(OperatorStateStore.class);
    TestingListState<Serializable> listState = new TestingListState<>();
    when(backend.getSerializableListState(Matchers.any(String.class))).thenReturn(listState);
    StateInitializationContext initializationContext = mock(StateInitializationContext.class);
    when(initializationContext.getOperatorStateStore()).thenReturn(backend);
    when(initializationContext.isRestored()).thenReturn(false, true, true, true);
    consumer.initializeState(initializationContext);
    consumer.open(new Configuration());
    // checkpoint 1
    consumer.snapshotState(new StateSnapshotContextSynchronousImpl(138, 138));
    HashMap<KafkaTopicPartition, Long> snapshot1 = new HashMap<>();
    for (Serializable serializable : listState.get()) {
        Tuple2<KafkaTopicPartition, Long> kafkaTopicPartitionLongTuple2 = (Tuple2<KafkaTopicPartition, Long>) serializable;
        snapshot1.put(kafkaTopicPartitionLongTuple2.f0, kafkaTopicPartitionLongTuple2.f1);
    }
    assertEquals(state1, snapshot1);
    // pending offsets to commit should not be updated
    assertEquals(0, pendingOffsetsToCommit.size());
    // checkpoint 2
    consumer.snapshotState(new StateSnapshotContextSynchronousImpl(140, 140));
    HashMap<KafkaTopicPartition, Long> snapshot2 = new HashMap<>();
    for (Serializable serializable : listState.get()) {
        Tuple2<KafkaTopicPartition, Long> kafkaTopicPartitionLongTuple2 = (Tuple2<KafkaTopicPartition, Long>) serializable;
        snapshot2.put(kafkaTopicPartitionLongTuple2.f0, kafkaTopicPartitionLongTuple2.f1);
    }
    assertEquals(state2, snapshot2);
    // pending offsets to commit should not be updated
    assertEquals(0, pendingOffsetsToCommit.size());
    // ack checkpoint 1
    consumer.notifyCheckpointComplete(138L);
    // not offsets should be committed
    verify(fetcher, never()).commitInternalOffsetsToKafka(anyMap());
    // checkpoint 3
    consumer.snapshotState(new StateSnapshotContextSynchronousImpl(141, 141));
    HashMap<KafkaTopicPartition, Long> snapshot3 = new HashMap<>();
    for (Serializable serializable : listState.get()) {
        Tuple2<KafkaTopicPartition, Long> kafkaTopicPartitionLongTuple2 = (Tuple2<KafkaTopicPartition, Long>) serializable;
        snapshot3.put(kafkaTopicPartitionLongTuple2.f0, kafkaTopicPartitionLongTuple2.f1);
    }
    assertEquals(state3, snapshot3);
    // pending offsets to commit should not be updated
    assertEquals(0, pendingOffsetsToCommit.size());
    // ack checkpoint 3, subsumes number 2
    consumer.notifyCheckpointComplete(141L);
    // not offsets should be committed
    verify(fetcher, never()).commitInternalOffsetsToKafka(anyMap());
    // invalid checkpoint
    consumer.notifyCheckpointComplete(666);
    // not offsets should be committed
    verify(fetcher, never()).commitInternalOffsetsToKafka(anyMap());
    OperatorStateStore operatorStateStore = mock(OperatorStateStore.class);
    listState = new TestingListState<>();
    when(operatorStateStore.getOperatorState(Matchers.any(ListStateDescriptor.class))).thenReturn(listState);
    // create 500 snapshots
    for (int i = 100; i < 600; i++) {
        consumer.snapshotState(new StateSnapshotContextSynchronousImpl(i, i));
        listState.clear();
    }
    // pending offsets to commit should not be updated
    assertEquals(0, pendingOffsetsToCommit.size());
    // commit only the second last
    consumer.notifyCheckpointComplete(598);
    // not offsets should be committed
    verify(fetcher, never()).commitInternalOffsetsToKafka(anyMap());
    // access invalid checkpoint
    consumer.notifyCheckpointComplete(590);
    // not offsets should be committed
    verify(fetcher, never()).commitInternalOffsetsToKafka(anyMap());
    // and the last
    consumer.notifyCheckpointComplete(599);
    // not offsets should be committed
    verify(fetcher, never()).commitInternalOffsetsToKafka(anyMap());
}

Example 9 with LinkedMap

use of org.apache.commons.collections.map.LinkedMap in project hadoop by apache.

the class TestEnhancedByteBufferAccess method testZeroCopyMmapCache.

@Test
public void testZeroCopyMmapCache() throws Exception {
    HdfsConfiguration conf = initZeroCopyTest();
    MiniDFSCluster cluster = null;
    final Path TEST_PATH = new Path("/a");
    final int TEST_FILE_LENGTH = 5 * BLOCK_SIZE;
    final int RANDOM_SEED = 23453;
    final String CONTEXT = "testZeroCopyMmapCacheContext";
    FSDataInputStream fsIn = null;
    ByteBuffer[] results = { null, null, null, null };
    DistributedFileSystem fs = null;
    conf.set(HdfsClientConfigKeys.DFS_CLIENT_CONTEXT, CONTEXT);
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    fs = cluster.getFileSystem();
    DFSTestUtil.createFile(fs, TEST_PATH, TEST_FILE_LENGTH, (short) 1, RANDOM_SEED);
    try {
        DFSTestUtil.waitReplication(fs, TEST_PATH, (short) 1);
    } catch (InterruptedException e) {
        Assert.fail("unexpected InterruptedException during " + "waitReplication: " + e);
    } catch (TimeoutException e) {
        Assert.fail("unexpected TimeoutException during " + "waitReplication: " + e);
    }
    fsIn = fs.open(TEST_PATH);
    byte[] original = new byte[TEST_FILE_LENGTH];
    IOUtils.readFully(fsIn, original, 0, TEST_FILE_LENGTH);
    fsIn.close();
    fsIn = fs.open(TEST_PATH);
    final ShortCircuitCache cache = ClientContext.get(CONTEXT, conf).getShortCircuitCache();
    cache.accept(new CountingVisitor(0, 5, 5, 0));
    results[0] = fsIn.read(null, BLOCK_SIZE, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
    fsIn.seek(0);
    results[1] = fsIn.read(null, BLOCK_SIZE, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
    // The mmap should be of the first block of the file.
    final ExtendedBlock firstBlock = DFSTestUtil.getFirstBlock(fs, TEST_PATH);
    cache.accept(new CacheVisitor() {

        @Override
        public void visit(int numOutstandingMmaps, Map<ExtendedBlockId, ShortCircuitReplica> replicas, Map<ExtendedBlockId, InvalidToken> failedLoads, LinkedMap evictable, LinkedMap evictableMmapped) {
            ShortCircuitReplica replica = replicas.get(new ExtendedBlockId(firstBlock.getBlockId(), firstBlock.getBlockPoolId()));
            Assert.assertNotNull(replica);
            Assert.assertTrue(replica.hasMmap());
            // The replica should not yet be evictable, since we have it open.
            Assert.assertNull(replica.getEvictableTimeNs());
        }
    });
    // Read more blocks.
    results[2] = fsIn.read(null, BLOCK_SIZE, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
    results[3] = fsIn.read(null, BLOCK_SIZE, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
    // we should have 3 mmaps, 1 evictable
    cache.accept(new CountingVisitor(3, 5, 2, 0));
    // using a very quick timeout)
    for (ByteBuffer buffer : results) {
        if (buffer != null) {
            fsIn.releaseBuffer(buffer);
        }
    }
    fsIn.close();
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        public Boolean get() {
            final MutableBoolean finished = new MutableBoolean(false);
            cache.accept(new CacheVisitor() {

                @Override
                public void visit(int numOutstandingMmaps, Map<ExtendedBlockId, ShortCircuitReplica> replicas, Map<ExtendedBlockId, InvalidToken> failedLoads, LinkedMap evictable, LinkedMap evictableMmapped) {
                    finished.setValue(evictableMmapped.isEmpty());
                }
            });
            return finished.booleanValue();
        }
    }, 10, 60000);
    cache.accept(new CountingVisitor(0, -1, -1, -1));
    fs.close();
    cluster.shutdown();
}

Example 10 with LinkedMap

use of org.apache.commons.collections.map.LinkedMap in project hadoop by apache.

the class TestEnhancedByteBufferAccess method waitForReplicaAnchorStatus.

private void waitForReplicaAnchorStatus(final ShortCircuitCache cache, final ExtendedBlock block, final boolean expectedIsAnchorable, final boolean expectedIsAnchored, final int expectedOutstandingMmaps) throws Exception {
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            final MutableBoolean result = new MutableBoolean(false);
            cache.accept(new CacheVisitor() {

                @Override
                public void visit(int numOutstandingMmaps, Map<ExtendedBlockId, ShortCircuitReplica> replicas, Map<ExtendedBlockId, InvalidToken> failedLoads, LinkedMap evictable, LinkedMap evictableMmapped) {
                    Assert.assertEquals(expectedOutstandingMmaps, numOutstandingMmaps);
                    ShortCircuitReplica replica = replicas.get(ExtendedBlockId.fromExtendedBlock(block));
                    Assert.assertNotNull(replica);
                    Slot slot = replica.getSlot();
                    if ((expectedIsAnchorable != slot.isAnchorable()) || (expectedIsAnchored != slot.isAnchored())) {
                        LOG.info("replica " + replica + " has isAnchorable = " + slot.isAnchorable() + ", isAnchored = " + slot.isAnchored() + ".  Waiting for isAnchorable = " + expectedIsAnchorable + ", isAnchored = " + expectedIsAnchored);
                        return;
                    }
                    result.setValue(true);
                }
            });
            return result.toBoolean();
        }
    }, 10, 60000);
}