Examples with StateTransferStartCommand org.infinispan.commands.statetransfer.StateTransferStartCommand used on opensource projects

Example 1 with StateTransferStartCommand

use of org.infinispan.commands.statetransfer.StateTransferStartCommand in project infinispan by infinispan.

the class BaseTxStateTransferOverwriteTest method doTestWhereCommitOccursAfterStateTransferBeginsBeforeCompletion.

/**
 * When L1 is enabled this test should not be ran when a previous value is present as it will cause timeouts.  Due
 * to how locking works with L1 this cannot occur when the previous value exists.
 */
protected void doTestWhereCommitOccursAfterStateTransferBeginsBeforeCompletion(final TestWriteOperation op) throws Exception {
    if (l1Enabled() && op.getPreviousValue() != null) {
        fail("This test cannot be ran with L1 when a previous value is set");
    }
    // Test scenario:
    // cache0,1,2 are in the cluster, an owner leaves
    // Key k is in the cache, and is transferred to the non owner
    // A user operation also modifies key k causing an invalidation
    // on the non owner which is getting the state transfer
    final AdvancedCache<Object, Object> primaryOwnerCache = cache(0, cacheName).getAdvancedCache();
    final AdvancedCache<Object, Object> backupOwnerCache = cache(1, cacheName).getAdvancedCache();
    final AdvancedCache<Object, Object> nonOwnerCache = cache(2, cacheName).getAdvancedCache();
    final MagicKey key = new MagicKey(primaryOwnerCache, backupOwnerCache);
    // Prepare for replace/remove: put a previous value in cache0
    final Object previousValue = op.getPreviousValue();
    if (previousValue != null) {
        primaryOwnerCache.put(key, previousValue);
        assertEquals(previousValue, primaryOwnerCache.get(key));
        log.tracef("Previous value inserted: %s = %s", key, previousValue);
        assertEquals(previousValue, nonOwnerCache.get(key));
        if (l1Enabled()) {
            assertIsInL1(nonOwnerCache, key);
        }
    }
    int preJoinTopologyId = primaryOwnerCache.getDistributionManager().getCacheTopology().getTopologyId();
    // Block any state response commands on cache0
    CheckPoint checkPoint = new CheckPoint();
    ControlledRpcManager blockingRpcManager0 = ControlledRpcManager.replaceRpcManager(primaryOwnerCache);
    ControlledRpcManager blockingRpcManager2 = ControlledRpcManager.replaceRpcManager(nonOwnerCache);
    // The execution of the write/prepare/commit commands is controlled with the BlockingInterceptor
    blockingRpcManager0.excludeCommands(BackupWriteCommand.class, PrepareCommand.class, CommitCommand.class, TxCompletionNotificationCommand.class);
    blockingRpcManager2.excludeCommands(BackupAckCommand.class);
    // Block the rebalance confirmation on cache0
    int rebalanceTopologyId = preJoinTopologyId + 2;
    blockRebalanceConfirmation(primaryOwnerCache.getCacheManager(), checkPoint, rebalanceTopologyId);
    assertEquals(primaryOwnerCache.getCacheManager().getCoordinator(), primaryOwnerCache.getCacheManager().getAddress());
    // Remove the leaver
    log.trace("Stopping the cache");
    backupOwnerCache.getCacheManager().stop();
    // Wait for the write CH to contain the joiner everywhere
    eventuallyEquals(2, () -> primaryOwnerCache.getRpcManager().getMembers().size());
    eventuallyEquals(2, () -> nonOwnerCache.getRpcManager().getMembers().size());
    assertEquals(primaryOwnerCache.getCacheManager().getCoordinator(), primaryOwnerCache.getCacheManager().getAddress());
    // Wait for both nodes to start state transfer
    if (transactional) {
        blockingRpcManager0.expectCommand(StateTransferGetTransactionsCommand.class).send().receiveAll();
        blockingRpcManager2.expectCommand(StateTransferGetTransactionsCommand.class).send().receiveAll();
    }
    ControlledRpcManager.BlockedRequest<StateTransferStartCommand> blockedStateRequest0 = blockingRpcManager0.expectCommand(StateTransferStartCommand.class);
    ControlledRpcManager.BlockedRequest<StateTransferStartCommand> blockedStateRequest2 = blockingRpcManager2.expectCommand(StateTransferStartCommand.class);
    // Unblock the state request from node 2
    // Don't wait for response, because node 2 might be sending the first state response on the request thread
    blockedStateRequest2.send().receiveAllAsync();
    // Wait for cache0 to collect the state to send to node 2 (including our previous value).
    ControlledRpcManager.BlockedRequest<StateResponseCommand> blockedStateResponse0 = blockingRpcManager0.expectCommand(StateResponseCommand.class);
    // Every PutKeyValueCommand will be blocked before committing the entry on cache1
    CyclicBarrier beforeCommitCache1Barrier = new CyclicBarrier(2);
    BlockingInterceptor<?> blockingInterceptor1 = new BlockingInterceptor<>(beforeCommitCache1Barrier, op.getCommandClass(), true, false);
    extractInterceptorChain(nonOwnerCache).addInterceptorAfter(blockingInterceptor1, EntryWrappingInterceptor.class);
    // Put/Replace/Remove from cache0 with cache0 as primary owner, cache1 will become a backup owner for the retry
    // The put command will be blocked on cache1 just before committing the entry.
    Future<Object> future = fork(() -> op.perform(primaryOwnerCache, key));
    // Wait for the entry to be wrapped on node 2
    // The replicated command could be either a non-tx BackupWriteCommand or a PrepareCommand
    beforeCommitCache1Barrier.await(10, TimeUnit.SECONDS);
    // Remove the interceptor so we don't mess up any other state transfer puts
    removeAllBlockingInterceptorsFromCache(nonOwnerCache);
    // Allow the state to be applied on cache1 (writing the old value for our entry)
    blockedStateResponse0.send().receiveAll();
    // Wait for second in line to finish applying the state, but don't allow the rebalance confirmation to be processed.
    // (It would change the topology and it would trigger a retry for the command.)
    // Don't wait for response, because node 2 might be sending the first state response on the request thread
    blockedStateRequest0.send().receiveAllAsync();
    blockingRpcManager2.expectCommand(StateResponseCommand.class).send().receiveAll();
    checkPoint.awaitStrict("pre_rebalance_confirmation_" + rebalanceTopologyId + "_from_" + primaryOwnerCache.getCacheManager().getAddress(), 10, SECONDS);
    // Now allow the command to commit on cache1
    beforeCommitCache1Barrier.await(10, TimeUnit.SECONDS);
    // Wait for the command to finish and check that it didn't fail
    Object result = future.get(10, TimeUnit.SECONDS);
    assertEquals(op.getReturnValue(), result);
    log.tracef("%s operation is done", op);
    // Allow the rebalance confirmation to proceed and wait for the topology to change everywhere
    checkPoint.trigger("resume_rebalance_confirmation_" + rebalanceTopologyId + "_from_" + primaryOwnerCache.getCacheManager().getAddress());
    checkPoint.trigger("resume_rebalance_confirmation_" + rebalanceTopologyId + "_from_" + nonOwnerCache.getCacheManager().getAddress());
    TestingUtil.waitForNoRebalance(primaryOwnerCache, nonOwnerCache);
    switch(op) {
        case REMOVE:
        case REMOVE_EXACT:
            break;
        default:
            assertIsInContainerImmortal(primaryOwnerCache, key);
            assertIsInContainerImmortal(nonOwnerCache, key);
            break;
    }
    // Check the value to make sure data container contains correct value
    assertEquals(op.getValue(), primaryOwnerCache.get(key));
    assertEquals(op.getValue(), nonOwnerCache.get(key));
}

Example 2 with StateTransferStartCommand

use of org.infinispan.commands.statetransfer.StateTransferStartCommand in project infinispan by infinispan.

the class StateResponseOrderingTest method testSimulatedOldStateResponse.

public void testSimulatedOldStateResponse() throws Throwable {
    // Initial owners for both segments are cache 1, 2, and 3
    // Start a rebalance, with cache 0 becoming an owner of both CH segments
    // Block the first StateTransferStartCommand on cache 0
    // While state transfer is blocked, simulate an old state response command on cache 0
    // Check that the old command is ignored and state transfer completes successfully
    StateSequencer sequencer = new StateSequencer();
    sequencer.logicalThread("st", "st:block_state_request", "st:simulate_old_response", "st:resume_state_request");
    cache(1).put("k1", "v1");
    cache(2).put("k2", "v2");
    cache(3).put("k3", "v3");
    DistributionManager dm0 = advancedCache(0).getDistributionManager();
    final int initialTopologyId = dm0.getCacheTopology().getTopologyId();
    assertEquals(Arrays.asList(address(1), address(2), address(3)), dm0.getCacheTopology().getDistribution("k1").readOwners());
    assertNull(dm0.getCacheTopology().getPendingCH());
    // Block when cache 0 sends the first state request to cache 1
    CommandMatcher segmentRequestMatcher = command -> command instanceof StateTransferStartCommand && ((StateTransferStartCommand) command).getTopologyId() == initialTopologyId + 1;
    advanceOnOutboundRpc(sequencer, cache(0), segmentRequestMatcher).before("st:block_state_request", "st:resume_state_request");
    // Cache 0 will become an owner and will request state from cache 1
    consistentHashFactory.setOwnerIndexes(new int[][] { { 0, 1, 2 }, { 0, 1, 2 } });
    consistentHashFactory.triggerRebalance(cache(0));
    sequencer.enter("st:simulate_old_response");
    assertNotNull(dm0.getCacheTopology().getPendingCH());
    assertEquals(Arrays.asList(address(0), address(1), address(2)), dm0.getCacheTopology().getPendingCH().locateOwnersForSegment(0));
    assertEquals(Arrays.asList(address(1), address(2), address(3), address(0)), dm0.getCacheTopology().getDistribution("k1").writeOwners());
    // Cache 0 didn't manage to request any segments yet, but it has registered all the inbound transfer tasks.
    // We'll pretend it got a StateResponseCommand with an older topology id.
    PerCacheInboundInvocationHandler handler = TestingUtil.extractComponent(cache(0), PerCacheInboundInvocationHandler.class);
    StateChunk stateChunk0 = new StateChunk(0, Arrays.asList(new ImmortalCacheEntry("k0", "v0")), true);
    StateChunk stateChunk1 = new StateChunk(1, Arrays.asList(new ImmortalCacheEntry("k0", "v0")), true);
    String cacheName = manager(0).getCacheManagerConfiguration().defaultCacheName().get();
    StateResponseCommand stateResponseCommand = new StateResponseCommand(ByteString.fromString(cacheName), initialTopologyId, Arrays.asList(stateChunk0, stateChunk1), true, false);
    // Call with preserveOrder = true to force the execution in the same thread
    stateResponseCommand.setOrigin(address(3));
    handler.handle(stateResponseCommand, Reply.NO_OP, DeliverOrder.PER_SENDER);
    sequencer.exit("st:simulate_old_response");
    waitForNoRebalance(cache(0), cache(1), cache(2), cache(3));
    // Check that state wasn't lost
    assertTrue(dm0.getCacheTopology().isReadOwner("k1"));
    assertTrue(dm0.getCacheTopology().isReadOwner("k2"));
    assertTrue(dm0.getCacheTopology().isReadOwner("k3"));
    assertEquals("v1", cache(0).get("k1"));
    assertEquals("v2", cache(0).get("k2"));
    assertEquals("v3", cache(0).get("k3"));
    // Check that the old state response was ignored
    assertNull(cache(0).get("k0"));
}

Example 3 with StateTransferStartCommand

use of org.infinispan.commands.statetransfer.StateTransferStartCommand in project infinispan by infinispan.

the class StateConsumerTest method test1.

public void test1() throws Exception {
    // create cache configuration
    ConfigurationBuilder cb = new ConfigurationBuilder();
    cb.clustering().invocationBatching().enable().clustering().cacheMode(CacheMode.DIST_SYNC).clustering().stateTransfer().timeout(30000).locking().lockAcquisitionTimeout(TestingUtil.shortTimeoutMillis()).locking().isolationLevel(IsolationLevel.REPEATABLE_READ);
    Configuration configuration = cb.build();
    PersistentUUIDManager persistentUUIDManager = new PersistentUUIDManagerImpl();
    // create list of 6 members
    Address[] addresses = new Address[4];
    for (int i = 0; i < 4; i++) {
        addresses[i] = new TestAddress(i);
        persistentUUIDManager.addPersistentAddressMapping(addresses[i], PersistentUUID.randomUUID());
    }
    List<Address> members1 = Arrays.asList(addresses[0], addresses[1], addresses[2], addresses[3]);
    List<Address> members2 = Arrays.asList(addresses[0], addresses[1], addresses[2]);
    // create CHes
    DefaultConsistentHashFactory chf = new DefaultConsistentHashFactory();
    KeyPartitioner keyPartitioner = new HashFunctionPartitioner(40);
    DefaultConsistentHash ch1 = chf.create(2, 40, members1, null);
    final DefaultConsistentHash ch2 = chf.updateMembers(ch1, members2, null);
    DefaultConsistentHash ch3 = chf.rebalance(ch2);
    DefaultConsistentHash ch23 = chf.union(ch2, ch3);
    log.debug(ch1);
    log.debug(ch2);
    // create dependencies
    Cache cache = mock(Cache.class);
    when(cache.getName()).thenReturn("testCache");
    when(cache.getStatus()).thenReturn(ComponentStatus.RUNNING);
    pooledExecutorService = new ThreadPoolExecutor(0, 20, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>(), getTestThreadFactory("Worker"), new ThreadPoolExecutor.CallerRunsPolicy());
    LocalTopologyManager localTopologyManager = mock(LocalTopologyManager.class);
    CacheNotifier cacheNotifier = mock(CacheNotifier.class);
    RpcManager rpcManager = mock(RpcManager.class);
    Transport transport = mock(Transport.class);
    CommandsFactory commandsFactory = mock(CommandsFactory.class);
    PersistenceManager persistenceManager = mock(PersistenceManager.class);
    InternalDataContainer dataContainer = mock(InternalDataContainer.class);
    TransactionTable transactionTable = mock(TransactionTable.class);
    StateTransferLock stateTransferLock = mock(StateTransferLock.class);
    AsyncInterceptorChain interceptorChain = mock(AsyncInterceptorChain.class);
    InvocationContextFactory icf = mock(InvocationContextFactory.class);
    InternalConflictManager conflictManager = mock(InternalConflictManager.class);
    DistributionManager distributionManager = mock(DistributionManager.class);
    LocalPublisherManager localPublisherManager = mock(LocalPublisherManager.class);
    PerCacheInboundInvocationHandler invocationHandler = mock(PerCacheInboundInvocationHandler.class);
    XSiteStateTransferManager xSiteStateTransferManager = mock(XSiteStateTransferManager.class);
    when(persistenceManager.removeSegments(any())).thenReturn(CompletableFuture.completedFuture(false));
    when(persistenceManager.addSegments(any())).thenReturn(CompletableFuture.completedFuture(false));
    when(persistenceManager.publishKeys(any(), any())).thenReturn(Flowable.empty());
    when(commandsFactory.buildStateTransferStartCommand(anyInt(), any(IntSet.class))).thenAnswer(invocation -> new StateTransferStartCommand(ByteString.fromString("cache1"), (Integer) invocation.getArguments()[0], (IntSet) invocation.getArguments()[1]));
    when(commandsFactory.buildStateTransferGetTransactionsCommand(anyInt(), any(IntSet.class))).thenAnswer(invocation -> new StateTransferGetTransactionsCommand(ByteString.fromString("cache1"), (Integer) invocation.getArguments()[0], (IntSet) invocation.getArguments()[1]));
    when(commandsFactory.buildStateTransferCancelCommand(anyInt(), any(IntSet.class))).thenAnswer(invocation -> new StateTransferCancelCommand(ByteString.fromString("cache1"), (Integer) invocation.getArguments()[0], (IntSet) invocation.getArguments()[1]));
    when(transport.getViewId()).thenReturn(1);
    when(rpcManager.getAddress()).thenReturn(addresses[0]);
    when(rpcManager.getTransport()).thenReturn(transport);
    final Map<Address, Set<Integer>> requestedSegments = new ConcurrentHashMap<>();
    final Set<Integer> flatRequestedSegments = new ConcurrentSkipListSet<>();
    when(rpcManager.invokeCommand(any(Address.class), any(StateTransferGetTransactionsCommand.class), any(ResponseCollector.class), any(RpcOptions.class))).thenAnswer(invocation -> {
        Address recipient = invocation.getArgument(0);
        StateTransferGetTransactionsCommand cmd = invocation.getArgument(1);
        Set<Integer> segments = cmd.getSegments();
        requestedSegments.put(recipient, segments);
        flatRequestedSegments.addAll(segments);
        return CompletableFuture.completedFuture(SuccessfulResponse.create(new ArrayList<TransactionInfo>()));
    });
    Answer<?> successfulResponse = invocation -> CompletableFuture.completedFuture(SuccessfulResponse.SUCCESSFUL_EMPTY_RESPONSE);
    when(rpcManager.invokeCommand(any(Address.class), any(StateTransferStartCommand.class), any(ResponseCollector.class), any(RpcOptions.class))).thenAnswer(successfulResponse);
    when(rpcManager.invokeCommand(any(Address.class), any(StateTransferCancelCommand.class), any(ResponseCollector.class), any(RpcOptions.class))).thenAnswer(successfulResponse);
    when(rpcManager.getSyncRpcOptions()).thenReturn(new RpcOptions(DeliverOrder.NONE, 10000, TimeUnit.MILLISECONDS));
    when(rpcManager.blocking(any(CompletionStage.class))).thenAnswer(invocation -> ((CompletionStage) invocation.getArgument(0)).toCompletableFuture().join());
    doNothing().when(xSiteStateTransferManager).onTopologyUpdated(any(CacheTopology.class), anyBoolean());
    // create state provider
    final StateConsumerImpl stateConsumer = new StateConsumerImpl();
    TestingUtil.inject(stateConsumer, cache, TestingUtil.named(NON_BLOCKING_EXECUTOR, pooledExecutorService), localTopologyManager, interceptorChain, icf, configuration, rpcManager, commandsFactory, persistenceManager, dataContainer, transactionTable, stateTransferLock, cacheNotifier, new CommitManager(), new CommandAckCollector(), new TriangleOrderManager(0), new HashFunctionPartitioner(), conflictManager, distributionManager, localPublisherManager, invocationHandler, xSiteStateTransferManager);
    stateConsumer.start();
    final List<InternalCacheEntry> cacheEntries = new ArrayList<>();
    Object key1 = new TestKey("key1", 0, keyPartitioner);
    Object key2 = new TestKey("key2", 0, keyPartitioner);
    cacheEntries.add(new ImmortalCacheEntry(key1, "value1"));
    cacheEntries.add(new ImmortalCacheEntry(key2, "value2"));
    when(dataContainer.iterator()).thenAnswer(invocation -> cacheEntries.iterator());
    when(transactionTable.getLocalTransactions()).thenReturn(Collections.emptyList());
    when(transactionTable.getRemoteTransactions()).thenReturn(Collections.emptyList());
    assertFalse(stateConsumer.hasActiveTransfers());
    // node 4 leaves
    stateConsumer.onTopologyUpdate(new CacheTopology(1, 1, ch2, null, CacheTopology.Phase.NO_REBALANCE, ch2.getMembers(), persistentUUIDManager.mapAddresses(ch2.getMembers())), false);
    assertFalse(stateConsumer.hasActiveTransfers());
    // start a rebalance
    stateConsumer.onTopologyUpdate(new CacheTopology(2, 2, ch2, ch3, ch23, CacheTopology.Phase.READ_OLD_WRITE_ALL, ch23.getMembers(), persistentUUIDManager.mapAddresses(ch23.getMembers())), true);
    assertTrue(stateConsumer.hasActiveTransfers());
    // check that all segments have been requested
    Set<Integer> oldSegments = ch2.getSegmentsForOwner(addresses[0]);
    final Set<Integer> newSegments = ch3.getSegmentsForOwner(addresses[0]);
    newSegments.removeAll(oldSegments);
    log.debugf("Rebalancing. Added segments=%s, old segments=%s", newSegments, oldSegments);
    assertEquals(flatRequestedSegments, newSegments);
    // simulate a cluster state recovery and return to ch2
    Future<Object> future = fork(() -> {
        stateConsumer.onTopologyUpdate(new CacheTopology(3, 2, ch2, null, CacheTopology.Phase.NO_REBALANCE, ch2.getMembers(), persistentUUIDManager.mapAddresses(ch2.getMembers())), false);
        return null;
    });
    stateConsumer.onTopologyUpdate(new CacheTopology(3, 2, ch2, null, CacheTopology.Phase.NO_REBALANCE, ch2.getMembers(), persistentUUIDManager.mapAddresses(ch2.getMembers())), false);
    future.get();
    assertFalse(stateConsumer.hasActiveTransfers());
    // restart the rebalance
    requestedSegments.clear();
    stateConsumer.onTopologyUpdate(new CacheTopology(4, 4, ch2, ch3, ch23, CacheTopology.Phase.READ_OLD_WRITE_ALL, ch23.getMembers(), persistentUUIDManager.mapAddresses(ch23.getMembers())), true);
    assertTrue(stateConsumer.hasActiveTransfers());
    assertEquals(flatRequestedSegments, newSegments);
    // apply state
    ArrayList<StateChunk> stateChunks = new ArrayList<>();
    for (Integer segment : newSegments) {
        stateChunks.add(new StateChunk(segment, Collections.emptyList(), true));
    }
    stateConsumer.applyState(addresses[1], 2, false, stateChunks);
    stateConsumer.stop();
    assertFalse(stateConsumer.hasActiveTransfers());
}

Example 4 with StateTransferStartCommand

use of org.infinispan.commands.statetransfer.StateTransferStartCommand in project infinispan by infinispan.

the class VersionAwareMarshallerTest method testStateTransferControlCommand.

public void testStateTransferControlCommand() throws Exception {
    ByteString cacheName = ByteString.fromString(TestingUtil.getDefaultCacheName(cm));
    ImmortalCacheEntry entry1 = (ImmortalCacheEntry) TestInternalCacheEntryFactory.create("key", "value", System.currentTimeMillis() - 1000, -1, System.currentTimeMillis(), -1);
    Collection<InternalCacheEntry> state = new ArrayList<>();
    state.add(entry1);
    Address a1 = new JGroupsAddress(UUID.randomUUID());
    Address a2 = new JGroupsAddress(UUID.randomUUID());
    Address a3 = new JGroupsAddress(UUID.randomUUID());
    List<Address> oldAddresses = new ArrayList<>();
    oldAddresses.add(a1);
    oldAddresses.add(a2);
    List<Address> newAddresses = new ArrayList<>();
    newAddresses.add(a1);
    newAddresses.add(a2);
    newAddresses.add(a3);
    CacheRpcCommand c14 = new StateTransferStartCommand(cacheName, 99, IntSets.mutableEmptySet());
    byte[] bytes = marshaller.objectToByteBuffer(c14);
    marshaller.objectFromByteBuffer(bytes);
    bytes = marshaller.objectToByteBuffer(c14);
    marshaller.objectFromByteBuffer(bytes);
    bytes = marshaller.objectToByteBuffer(c14);
    marshaller.objectFromByteBuffer(bytes);
}