Search in sources :

Example 1 with TO_SLAVE

use of org.neo4j.kernel.ha.cluster.HighAvailabilityMemberState.TO_SLAVE in project neo4j by neo4j.

the class HighAvailabilityModeSwitcherTest method shouldRecognizeNewMasterIfNewMasterBecameAvailableDuringSwitch.

@Test
public void shouldRecognizeNewMasterIfNewMasterBecameAvailableDuringSwitch() throws Throwable {
    // When messages coming in the following ordering, the slave should detect that the master id has changed
    // M1: Get masterIsAvailable for instance 1 at PENDING state, changing PENDING -> TO_SLAVE
    // M2: Get masterIsAvailable for instance 2 at TO_SLAVE state, changing TO_SLAVE -> TO_SLAVE
    // Given
    final CountDownLatch firstMasterAvailableHandled = new CountDownLatch(1);
    final CountDownLatch secondMasterAvailableComes = new CountDownLatch(1);
    final CountDownLatch secondMasterAvailableHandled = new CountDownLatch(1);
    SwitchToSlaveCopyThenBranch switchToSlave = mock(SwitchToSlaveCopyThenBranch.class);
    HighAvailabilityModeSwitcher modeSwitcher = new HighAvailabilityModeSwitcher(switchToSlave, mock(SwitchToMaster.class), mock(Election.class), mock(ClusterMemberAvailability.class), mock(ClusterClient.class), mock(Supplier.class), new InstanceId(4), new ComponentSwitcherContainer(), neoStoreDataSourceSupplierMock(), NullLogService.getInstance()) {

        @Override
        ScheduledExecutorService createExecutor() {
            final ScheduledExecutorService executor = mock(ScheduledExecutorService.class);
            final ExecutorService realExecutor = Executors.newSingleThreadExecutor();
            when(executor.submit(any(Runnable.class))).thenAnswer((Answer<Future<?>>) invocation -> realExecutor.submit((Runnable) () -> ((Runnable) invocation.getArguments()[0]).run()));
            when(executor.schedule(any(Runnable.class), anyLong(), any(TimeUnit.class))).thenAnswer((Answer<Future<?>>) invocation -> {
                realExecutor.submit((Callable<Void>) () -> {
                    firstMasterAvailableHandled.countDown();
                    secondMasterAvailableComes.await();
                    ((Runnable) invocation.getArguments()[0]).run();
                    secondMasterAvailableHandled.countDown();
                    return null;
                });
                return mock(ScheduledFuture.class);
            });
            return executor;
        }
    };
    modeSwitcher.init();
    modeSwitcher.start();
    modeSwitcher.listeningAt(URI.create("ha://server3?serverId=3"));
    // When
    // masterIsAvailable for instance 1
    URI uri1 = URI.create("ha://server1");
    // The first masterIsAvailable should fail so that the slave instance stops at TO_SLAVE state
    doThrow(new ComException("Fail to switch to slave and reschedule to retry")).when(switchToSlave).switchToSlave(any(LifeSupport.class), any(URI.class), eq(uri1), any(CancellationRequest.class));
    modeSwitcher.masterIsAvailable(new HighAvailabilityMemberChangeEvent(PENDING, TO_SLAVE, new InstanceId(1), uri1));
    // wait until the first masterIsAvailable triggers the exception handling
    firstMasterAvailableHandled.await();
    verify(switchToSlave).switchToSlave(any(LifeSupport.class), any(URI.class), eq(uri1), any(CancellationRequest.class));
    // masterIsAvailable for instance 2
    URI uri2 = URI.create("ha://server2");
    modeSwitcher.masterIsAvailable(new HighAvailabilityMemberChangeEvent(TO_SLAVE, TO_SLAVE, new InstanceId(2), uri2));
    secondMasterAvailableComes.countDown();
    // wait until switchToSlave method is invoked again
    secondMasterAvailableHandled.await();
    // Then
    // switchToSlave should be retried with new master id
    verify(switchToSlave).switchToSlave(any(LifeSupport.class), any(URI.class), eq(uri2), any(CancellationRequest.class));
}
Also used : InstanceId(org.neo4j.cluster.InstanceId) StoreId(org.neo4j.kernel.impl.store.StoreId) ScheduledFuture(java.util.concurrent.ScheduledFuture) NeoStoreDataSource(org.neo4j.kernel.NeoStoreDataSource) SwitchToMaster(org.neo4j.kernel.ha.cluster.SwitchToMaster) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) TO_SLAVE(org.neo4j.kernel.ha.cluster.HighAvailabilityMemberState.TO_SLAVE) Callable(java.util.concurrent.Callable) LifeSupport(org.neo4j.kernel.lifecycle.LifeSupport) NullLogProvider(org.neo4j.logging.NullLogProvider) Supplier(java.util.function.Supplier) Mockito.verifyZeroInteractions(org.mockito.Mockito.verifyZeroInteractions) Answer(org.mockito.stubbing.Answer) Mockito.doThrow(org.mockito.Mockito.doThrow) Future(java.util.concurrent.Future) PENDING(org.neo4j.kernel.ha.cluster.HighAvailabilityMemberState.PENDING) AssertableLogProvider(org.neo4j.logging.AssertableLogProvider) Matchers.eq(org.mockito.Matchers.eq) CancellationRequest(org.neo4j.helpers.CancellationRequest) Mockito.doAnswer(org.mockito.Mockito.doAnswer) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) Matchers.anyLong(org.mockito.Matchers.anyLong) URI(java.net.URI) MismatchingStoreIdException(org.neo4j.kernel.impl.store.MismatchingStoreIdException) ExecutorService(java.util.concurrent.ExecutorService) HighAvailabilityMemberState(org.neo4j.kernel.ha.cluster.HighAvailabilityMemberState) InOrder(org.mockito.InOrder) ComException(org.neo4j.com.ComException) SwitchToSlaveCopyThenBranch(org.neo4j.kernel.ha.cluster.SwitchToSlaveCopyThenBranch) Test(org.junit.Test) Mockito.times(org.mockito.Mockito.times) AssertableLogProvider.inLog(org.neo4j.logging.AssertableLogProvider.inLog) Election(org.neo4j.cluster.protocol.election.Election) Mockito.when(org.mockito.Mockito.when) DataSourceManager(org.neo4j.kernel.impl.transaction.state.DataSourceManager) Executors(java.util.concurrent.Executors) Mockito.verify(org.mockito.Mockito.verify) TimeUnit(java.util.concurrent.TimeUnit) Matchers.any(org.mockito.Matchers.any) CountDownLatch(java.util.concurrent.CountDownLatch) NullLogService(org.neo4j.kernel.impl.logging.NullLogService) ClusterMemberAvailability(org.neo4j.cluster.member.ClusterMemberAvailability) Mockito.inOrder(org.mockito.Mockito.inOrder) SimpleLogService(org.neo4j.kernel.impl.logging.SimpleLogService) ClusterClient(org.neo4j.cluster.client.ClusterClient) HighAvailabilityMemberChangeEvent(org.neo4j.kernel.ha.cluster.HighAvailabilityMemberChangeEvent) Mockito.reset(org.mockito.Mockito.reset) Mockito.mock(org.mockito.Mockito.mock) ComException(org.neo4j.com.ComException) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) InstanceId(org.neo4j.cluster.InstanceId) ClusterMemberAvailability(org.neo4j.cluster.member.ClusterMemberAvailability) CountDownLatch(java.util.concurrent.CountDownLatch) Election(org.neo4j.cluster.protocol.election.Election) URI(java.net.URI) Callable(java.util.concurrent.Callable) ScheduledFuture(java.util.concurrent.ScheduledFuture) ClusterClient(org.neo4j.cluster.client.ClusterClient) SwitchToSlaveCopyThenBranch(org.neo4j.kernel.ha.cluster.SwitchToSlaveCopyThenBranch) HighAvailabilityMemberChangeEvent(org.neo4j.kernel.ha.cluster.HighAvailabilityMemberChangeEvent) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ExecutorService(java.util.concurrent.ExecutorService) ScheduledFuture(java.util.concurrent.ScheduledFuture) Future(java.util.concurrent.Future) TimeUnit(java.util.concurrent.TimeUnit) LifeSupport(org.neo4j.kernel.lifecycle.LifeSupport) Supplier(java.util.function.Supplier) SwitchToMaster(org.neo4j.kernel.ha.cluster.SwitchToMaster) CancellationRequest(org.neo4j.helpers.CancellationRequest) Test(org.junit.Test)

Example 2 with TO_SLAVE

use of org.neo4j.kernel.ha.cluster.HighAvailabilityMemberState.TO_SLAVE in project neo4j by neo4j.

the class HighAvailabilityModeSwitcherTest method shouldNotResetAvailableMasterURIIfElectionResultReceived.

@Test
public void shouldNotResetAvailableMasterURIIfElectionResultReceived() throws Throwable {
    /*
         * It is possible that a masterIsElected nulls out the current available master URI in the HAMS. That can
         * be a problem if handing the mIE event is concurrent with an ongoing switch which re-runs because
         * the store was incompatible or a log was missing. In such a case it will find a null master URI on
         * rerun and it will fail.
         */
    // Given
    SwitchToSlaveCopyThenBranch switchToSlave = mock(SwitchToSlaveCopyThenBranch.class);
    // The fist run through switchToSlave
    final CountDownLatch firstCallMade = new CountDownLatch(1);
    // The second run through switchToSlave
    final CountDownLatch secondCallMade = new CountDownLatch(1);
    // The latch for waiting for the masterIsElected to come through
    final CountDownLatch waitForSecondMessage = new CountDownLatch(1);
    HighAvailabilityModeSwitcher toTest = new HighAvailabilityModeSwitcher(switchToSlave, mock(SwitchToMaster.class), mock(Election.class), mock(ClusterMemberAvailability.class), mock(ClusterClient.class), storeSupplierMock(), new InstanceId(1), new ComponentSwitcherContainer(), neoStoreDataSourceSupplierMock(), NullLogService.getInstance());
    URI uri1 = URI.create("ha://server1");
    toTest.init();
    toTest.start();
    toTest.listeningAt(URI.create("ha://server3?serverId=3"));
    when(switchToSlave.switchToSlave(any(LifeSupport.class), any(URI.class), any(URI.class), any(CancellationRequest.class))).thenAnswer(invocation -> {
        firstCallMade.countDown();
        waitForSecondMessage.await();
        throw new MismatchingStoreIdException(StoreId.DEFAULT, StoreId.DEFAULT);
    }).thenAnswer(invocation -> {
        secondCallMade.countDown();
        return URI.create("ha://server3");
    });
    // When
    // The first message goes through, start the first run
    toTest.masterIsAvailable(new HighAvailabilityMemberChangeEvent(PENDING, TO_SLAVE, new InstanceId(1), uri1));
    // Wait for it to be processed but get just before the exception
    firstCallMade.await();
    // It is just about to throw the exception, i.e. rerun. Send in the event
    toTest.masterIsElected(new HighAvailabilityMemberChangeEvent(TO_SLAVE, TO_SLAVE, new InstanceId(1), null));
    // Allow to continue and do the second run
    waitForSecondMessage.countDown();
    // Wait for the call to finish
    secondCallMade.await();
    // Then
    verify(switchToSlave, times(2)).switchToSlave(any(LifeSupport.class), any(URI.class), eq(uri1), any(CancellationRequest.class));
}
Also used : InstanceId(org.neo4j.cluster.InstanceId) StoreId(org.neo4j.kernel.impl.store.StoreId) ScheduledFuture(java.util.concurrent.ScheduledFuture) NeoStoreDataSource(org.neo4j.kernel.NeoStoreDataSource) SwitchToMaster(org.neo4j.kernel.ha.cluster.SwitchToMaster) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) TO_SLAVE(org.neo4j.kernel.ha.cluster.HighAvailabilityMemberState.TO_SLAVE) Callable(java.util.concurrent.Callable) LifeSupport(org.neo4j.kernel.lifecycle.LifeSupport) NullLogProvider(org.neo4j.logging.NullLogProvider) Supplier(java.util.function.Supplier) Mockito.verifyZeroInteractions(org.mockito.Mockito.verifyZeroInteractions) Answer(org.mockito.stubbing.Answer) Mockito.doThrow(org.mockito.Mockito.doThrow) Future(java.util.concurrent.Future) PENDING(org.neo4j.kernel.ha.cluster.HighAvailabilityMemberState.PENDING) AssertableLogProvider(org.neo4j.logging.AssertableLogProvider) Matchers.eq(org.mockito.Matchers.eq) CancellationRequest(org.neo4j.helpers.CancellationRequest) Mockito.doAnswer(org.mockito.Mockito.doAnswer) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) Matchers.anyLong(org.mockito.Matchers.anyLong) URI(java.net.URI) MismatchingStoreIdException(org.neo4j.kernel.impl.store.MismatchingStoreIdException) ExecutorService(java.util.concurrent.ExecutorService) HighAvailabilityMemberState(org.neo4j.kernel.ha.cluster.HighAvailabilityMemberState) InOrder(org.mockito.InOrder) ComException(org.neo4j.com.ComException) SwitchToSlaveCopyThenBranch(org.neo4j.kernel.ha.cluster.SwitchToSlaveCopyThenBranch) Test(org.junit.Test) Mockito.times(org.mockito.Mockito.times) AssertableLogProvider.inLog(org.neo4j.logging.AssertableLogProvider.inLog) Election(org.neo4j.cluster.protocol.election.Election) Mockito.when(org.mockito.Mockito.when) DataSourceManager(org.neo4j.kernel.impl.transaction.state.DataSourceManager) Executors(java.util.concurrent.Executors) Mockito.verify(org.mockito.Mockito.verify) TimeUnit(java.util.concurrent.TimeUnit) Matchers.any(org.mockito.Matchers.any) CountDownLatch(java.util.concurrent.CountDownLatch) NullLogService(org.neo4j.kernel.impl.logging.NullLogService) ClusterMemberAvailability(org.neo4j.cluster.member.ClusterMemberAvailability) Mockito.inOrder(org.mockito.Mockito.inOrder) SimpleLogService(org.neo4j.kernel.impl.logging.SimpleLogService) ClusterClient(org.neo4j.cluster.client.ClusterClient) HighAvailabilityMemberChangeEvent(org.neo4j.kernel.ha.cluster.HighAvailabilityMemberChangeEvent) Mockito.reset(org.mockito.Mockito.reset) Mockito.mock(org.mockito.Mockito.mock) InstanceId(org.neo4j.cluster.InstanceId) ClusterMemberAvailability(org.neo4j.cluster.member.ClusterMemberAvailability) MismatchingStoreIdException(org.neo4j.kernel.impl.store.MismatchingStoreIdException) CountDownLatch(java.util.concurrent.CountDownLatch) Election(org.neo4j.cluster.protocol.election.Election) URI(java.net.URI) ClusterClient(org.neo4j.cluster.client.ClusterClient) SwitchToSlaveCopyThenBranch(org.neo4j.kernel.ha.cluster.SwitchToSlaveCopyThenBranch) HighAvailabilityMemberChangeEvent(org.neo4j.kernel.ha.cluster.HighAvailabilityMemberChangeEvent) LifeSupport(org.neo4j.kernel.lifecycle.LifeSupport) SwitchToMaster(org.neo4j.kernel.ha.cluster.SwitchToMaster) CancellationRequest(org.neo4j.helpers.CancellationRequest) Test(org.junit.Test)

Aggregations

URI (java.net.URI)2 Callable (java.util.concurrent.Callable)2 CountDownLatch (java.util.concurrent.CountDownLatch)2 ExecutorService (java.util.concurrent.ExecutorService)2 Executors (java.util.concurrent.Executors)2 Future (java.util.concurrent.Future)2 ScheduledExecutorService (java.util.concurrent.ScheduledExecutorService)2 ScheduledFuture (java.util.concurrent.ScheduledFuture)2 TimeUnit (java.util.concurrent.TimeUnit)2 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)2 Supplier (java.util.function.Supplier)2 Test (org.junit.Test)2 InOrder (org.mockito.InOrder)2 Matchers.any (org.mockito.Matchers.any)2 Matchers.anyLong (org.mockito.Matchers.anyLong)2 Matchers.eq (org.mockito.Matchers.eq)2 Mockito.doAnswer (org.mockito.Mockito.doAnswer)2 Mockito.doThrow (org.mockito.Mockito.doThrow)2 Mockito.inOrder (org.mockito.Mockito.inOrder)2 Mockito.mock (org.mockito.Mockito.mock)2