Search in sources :

Example 11 with StuckProcessor

use of com.hazelcast.jet.core.TestProcessors.StuckProcessor in project hazelcast-jet by hazelcast.

the class SplitBrainTest method when_quorumIsLostOnMinority_then_jobRestartsUntilMerge.

@Test
public void when_quorumIsLostOnMinority_then_jobRestartsUntilMerge() {
    int firstSubClusterSize = 3;
    int secondSubClusterSize = 2;
    int clusterSize = firstSubClusterSize + secondSubClusterSize;
    StuckProcessor.executionStarted = new CountDownLatch(clusterSize * PARALLELISM);
    Job[] jobRef = new Job[1];
    Consumer<JetInstance[]> beforeSplit = instances -> {
        MockPS processorSupplier = new MockPS(StuckProcessor::new, clusterSize);
        DAG dag = new DAG().vertex(new Vertex("test", processorSupplier));
        jobRef[0] = instances[0].newJob(dag, new JobConfig().setSplitBrainProtection(true));
        assertOpenEventually(StuckProcessor.executionStarted);
    };
    Future[] minorityJobFutureRef = new Future[1];
    BiConsumer<JetInstance[], JetInstance[]> onSplit = (firstSubCluster, secondSubCluster) -> {
        StuckProcessor.proceedLatch.countDown();
        assertTrueEventually(() -> assertEquals(clusterSize + firstSubClusterSize, MockPS.initCount.get()));
        long jobId = jobRef[0].getId();
        assertTrueEventually(() -> {
            JetService service = getJetService(firstSubCluster[0]);
            assertEquals(COMPLETED, service.getJobCoordinationService().getJobStatus(jobId));
        });
        JetService service2 = getJetService(secondSubCluster[0]);
        assertTrueEventually(() -> {
            assertEquals(STARTING, service2.getJobCoordinationService().getJobStatus(jobId));
        });
        MasterContext masterContext = service2.getJobCoordinationService().getMasterContext(jobId);
        assertNotNull(masterContext);
        minorityJobFutureRef[0] = masterContext.completionFuture();
        assertTrueAllTheTime(() -> {
            assertEquals(STARTING, service2.getJobCoordinationService().getJobStatus(jobId));
        }, 20);
    };
    Consumer<JetInstance[]> afterMerge = instances -> {
        assertTrueEventually(() -> {
            assertEquals(clusterSize + firstSubClusterSize, MockPS.initCount.get());
            assertEquals(clusterSize + firstSubClusterSize, MockPS.closeCount.get());
        });
        assertEquals(clusterSize, MockPS.receivedCloseErrors.size());
        MockPS.receivedCloseErrors.forEach(t -> assertTrue(t instanceof TopologyChangedException));
        try {
            minorityJobFutureRef[0].get();
            fail();
        } catch (CancellationException ignored) {
        } catch (Exception e) {
            throw new AssertionError(e);
        }
    };
    testSplitBrain(firstSubClusterSize, secondSubClusterSize, beforeSplit, onSplit, afterMerge);
}
Also used : MasterContext(com.hazelcast.jet.impl.MasterContext) JetInstance(com.hazelcast.jet.JetInstance) RunWith(org.junit.runner.RunWith) HazelcastSerialClassRunner(com.hazelcast.test.HazelcastSerialClassRunner) Future(java.util.concurrent.Future) STARTING(com.hazelcast.jet.core.JobStatus.STARTING) BiConsumer(java.util.function.BiConsumer) Assert.fail(org.junit.Assert.fail) ExpectedException(org.junit.rules.ExpectedException) Job(com.hazelcast.jet.Job) JobRepository(com.hazelcast.jet.impl.JobRepository) JetConfig(com.hazelcast.jet.config.JetConfig) MAX_BACKUP_COUNT(com.hazelcast.spi.partition.IPartition.MAX_BACKUP_COUNT) MockPS(com.hazelcast.jet.core.TestProcessors.MockPS) CancellationException(java.util.concurrent.CancellationException) RESTARTING(com.hazelcast.jet.core.JobStatus.RESTARTING) Assert.assertNotNull(org.junit.Assert.assertNotNull) JobConfig(com.hazelcast.jet.config.JobConfig) StuckProcessor(com.hazelcast.jet.core.TestProcessors.StuckProcessor) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) CountDownLatch(java.util.concurrent.CountDownLatch) Rule(org.junit.Rule) JetService(com.hazelcast.jet.impl.JetService) COMPLETED(com.hazelcast.jet.core.JobStatus.COMPLETED) JobRecord(com.hazelcast.jet.impl.JobRecord) Assert.assertEquals(org.junit.Assert.assertEquals) MockPS(com.hazelcast.jet.core.TestProcessors.MockPS) JetService(com.hazelcast.jet.impl.JetService) CountDownLatch(java.util.concurrent.CountDownLatch) JobConfig(com.hazelcast.jet.config.JobConfig) ExpectedException(org.junit.rules.ExpectedException) CancellationException(java.util.concurrent.CancellationException) CancellationException(java.util.concurrent.CancellationException) Future(java.util.concurrent.Future) Job(com.hazelcast.jet.Job) MasterContext(com.hazelcast.jet.impl.MasterContext) Test(org.junit.Test)

Example 12 with StuckProcessor

use of com.hazelcast.jet.core.TestProcessors.StuckProcessor in project hazelcast-jet by hazelcast.

the class SplitBrainTest method when_quorumIsLostOnBothSides_then_jobRestartsUntilMerge.

@Test
public void when_quorumIsLostOnBothSides_then_jobRestartsUntilMerge() {
    int firstSubClusterSize = 2;
    int secondSubClusterSize = 2;
    int clusterSize = firstSubClusterSize + secondSubClusterSize;
    StuckProcessor.executionStarted = new CountDownLatch(clusterSize * PARALLELISM);
    Job[] jobRef = new Job[1];
    Consumer<JetInstance[]> beforeSplit = instances -> {
        MockPS processorSupplier = new MockPS(StuckProcessor::new, clusterSize);
        DAG dag = new DAG().vertex(new Vertex("test", processorSupplier));
        jobRef[0] = instances[0].newJob(dag, new JobConfig().setSplitBrainProtection(true));
        assertOpenEventually(StuckProcessor.executionStarted);
    };
    BiConsumer<JetInstance[], JetInstance[]> onSplit = (firstSubCluster, secondSubCluster) -> {
        StuckProcessor.proceedLatch.countDown();
        long jobId = jobRef[0].getId();
        assertTrueEventually(() -> {
            JetService service1 = getJetService(firstSubCluster[0]);
            JetService service2 = getJetService(secondSubCluster[0]);
            assertEquals(RESTARTING, service1.getJobCoordinationService().getJobStatus(jobId));
            assertEquals(STARTING, service2.getJobCoordinationService().getJobStatus(jobId));
        });
        assertTrueAllTheTime(() -> {
            JetService service1 = getJetService(firstSubCluster[0]);
            JetService service2 = getJetService(secondSubCluster[0]);
            assertEquals(RESTARTING, service1.getJobCoordinationService().getJobStatus(jobId));
            assertEquals(STARTING, service2.getJobCoordinationService().getJobStatus(jobId));
        }, 20);
    };
    Consumer<JetInstance[]> afterMerge = instances -> {
        assertTrueEventually(() -> {
            assertEquals(clusterSize * 2, MockPS.initCount.get());
            assertEquals(clusterSize * 2, MockPS.closeCount.get());
        });
        assertEquals(clusterSize, MockPS.receivedCloseErrors.size());
        MockPS.receivedCloseErrors.forEach(t -> assertTrue(t instanceof TopologyChangedException));
    };
    testSplitBrain(firstSubClusterSize, secondSubClusterSize, beforeSplit, onSplit, afterMerge);
}
Also used : MasterContext(com.hazelcast.jet.impl.MasterContext) JetInstance(com.hazelcast.jet.JetInstance) RunWith(org.junit.runner.RunWith) HazelcastSerialClassRunner(com.hazelcast.test.HazelcastSerialClassRunner) Future(java.util.concurrent.Future) STARTING(com.hazelcast.jet.core.JobStatus.STARTING) BiConsumer(java.util.function.BiConsumer) Assert.fail(org.junit.Assert.fail) ExpectedException(org.junit.rules.ExpectedException) Job(com.hazelcast.jet.Job) JobRepository(com.hazelcast.jet.impl.JobRepository) JetConfig(com.hazelcast.jet.config.JetConfig) MAX_BACKUP_COUNT(com.hazelcast.spi.partition.IPartition.MAX_BACKUP_COUNT) MockPS(com.hazelcast.jet.core.TestProcessors.MockPS) CancellationException(java.util.concurrent.CancellationException) RESTARTING(com.hazelcast.jet.core.JobStatus.RESTARTING) Assert.assertNotNull(org.junit.Assert.assertNotNull) JobConfig(com.hazelcast.jet.config.JobConfig) StuckProcessor(com.hazelcast.jet.core.TestProcessors.StuckProcessor) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) CountDownLatch(java.util.concurrent.CountDownLatch) Rule(org.junit.Rule) JetService(com.hazelcast.jet.impl.JetService) COMPLETED(com.hazelcast.jet.core.JobStatus.COMPLETED) JobRecord(com.hazelcast.jet.impl.JobRecord) Assert.assertEquals(org.junit.Assert.assertEquals) MockPS(com.hazelcast.jet.core.TestProcessors.MockPS) JetService(com.hazelcast.jet.impl.JetService) CountDownLatch(java.util.concurrent.CountDownLatch) JobConfig(com.hazelcast.jet.config.JobConfig) Job(com.hazelcast.jet.Job) Test(org.junit.Test)

Example 13 with StuckProcessor

use of com.hazelcast.jet.core.TestProcessors.StuckProcessor in project hazelcast-jet by hazelcast.

the class SplitBrainTest method when_newMemberJoinsToCluster_then_jobQuorumSizeIsUpdated.

@Test
public void when_newMemberJoinsToCluster_then_jobQuorumSizeIsUpdated() {
    int clusterSize = 3;
    JetConfig jetConfig = new JetConfig();
    JetInstance[] instances = new JetInstance[clusterSize];
    for (int i = 0; i < clusterSize; i++) {
        instances[i] = createJetMember(jetConfig);
    }
    StuckProcessor.executionStarted = new CountDownLatch(clusterSize * PARALLELISM);
    MockPS processorSupplier = new MockPS(StuckProcessor::new, clusterSize);
    DAG dag = new DAG().vertex(new Vertex("test", processorSupplier));
    Job job = instances[0].newJob(dag, new JobConfig().setSplitBrainProtection(true));
    assertOpenEventually(StuckProcessor.executionStarted);
    createJetMember(jetConfig);
    assertTrueEventually(() -> {
        JobRepository jobRepository = getJetService(instances[0]).getJobRepository();
        JobRecord jobRecord = jobRepository.getJobRecord(job.getId());
        assertEquals(3, jobRecord.getQuorumSize());
    });
    StuckProcessor.proceedLatch.countDown();
}
Also used : MockPS(com.hazelcast.jet.core.TestProcessors.MockPS) JetInstance(com.hazelcast.jet.JetInstance) CountDownLatch(java.util.concurrent.CountDownLatch) JobRepository(com.hazelcast.jet.impl.JobRepository) JetConfig(com.hazelcast.jet.config.JetConfig) JobConfig(com.hazelcast.jet.config.JobConfig) JobRecord(com.hazelcast.jet.impl.JobRecord) StuckProcessor(com.hazelcast.jet.core.TestProcessors.StuckProcessor) Job(com.hazelcast.jet.Job) Test(org.junit.Test)

Example 14 with StuckProcessor

use of com.hazelcast.jet.core.TestProcessors.StuckProcessor in project hazelcast-jet by hazelcast.

the class SplitBrainTest method when_newMemberIsAddedAfterClusterSizeFallsBelowQuorumSize_then_jobRestartDoesNotSucceed.

@Test
public void when_newMemberIsAddedAfterClusterSizeFallsBelowQuorumSize_then_jobRestartDoesNotSucceed() {
    int clusterSize = 5;
    JetConfig jetConfig = new JetConfig();
    JetInstance[] instances = new JetInstance[clusterSize];
    for (int i = 0; i < clusterSize; i++) {
        instances[i] = createJetMember(jetConfig);
    }
    StuckProcessor.executionStarted = new CountDownLatch(clusterSize * PARALLELISM);
    MockPS processorSupplier = new MockPS(StuckProcessor::new, clusterSize);
    DAG dag = new DAG().vertex(new Vertex("test", processorSupplier));
    Job job = instances[0].newJob(dag, new JobConfig().setSplitBrainProtection(true));
    assertOpenEventually(StuckProcessor.executionStarted);
    for (int i = 1; i < clusterSize; i++) {
        instances[i].getHazelcastInstance().getLifecycleService().terminate();
    }
    StuckProcessor.proceedLatch.countDown();
    assertTrueEventually(() -> assertEquals(RESTARTING, job.getStatus()));
    createJetMember(jetConfig);
    assertTrueAllTheTime(() -> assertEquals(RESTARTING, job.getStatus()), 5);
}
Also used : MockPS(com.hazelcast.jet.core.TestProcessors.MockPS) JetInstance(com.hazelcast.jet.JetInstance) StuckProcessor(com.hazelcast.jet.core.TestProcessors.StuckProcessor) CountDownLatch(java.util.concurrent.CountDownLatch) Job(com.hazelcast.jet.Job) JetConfig(com.hazelcast.jet.config.JetConfig) JobConfig(com.hazelcast.jet.config.JobConfig) Test(org.junit.Test)

Example 15 with StuckProcessor

use of com.hazelcast.jet.core.TestProcessors.StuckProcessor in project hazelcast-jet by hazelcast.

the class TopologyChangeDuringJobSubmissionTest method when_jobIsCompletedBeforeSubmissionCallReturns_then_jobRunsOnlyOnce.

@Test
public void when_jobIsCompletedBeforeSubmissionCallReturns_then_jobRunsOnlyOnce() throws Throwable {
    // Given that the job is already completed
    dropOperationsBetween(instance1.getHazelcastInstance(), instance2.getHazelcastInstance(), SpiDataSerializerHook.F_ID, singletonList(SpiDataSerializerHook.NORMAL_RESPONSE));
    String jobName = "job1";
    Future<Job> future = spawn(() -> {
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS(StuckProcessor::new, 1)));
        return instance2.newJob(dag, new JobConfig().setName(jobName));
    });
    StuckProcessor.executionStarted.await();
    Job submittedJob = instance1.getJob(jobName);
    assertNotNull(submittedJob);
    StuckProcessor.proceedLatch.countDown();
    submittedJob.join();
    // When the coordinator leaves before the submission response is received
    instance1.getHazelcastInstance().getLifecycleService().terminate();
    Job job = future.get();
    // Then the job does not run for the second time
    job.join();
    assertEquals(1, MockPS.initCount.get());
}
Also used : MockPS(com.hazelcast.jet.core.TestProcessors.MockPS) StuckProcessor(com.hazelcast.jet.core.TestProcessors.StuckProcessor) Job(com.hazelcast.jet.Job) JobConfig(com.hazelcast.jet.config.JobConfig) Test(org.junit.Test)

Aggregations

MockPS (com.hazelcast.jet.core.TestProcessors.MockPS)37 StuckProcessor (com.hazelcast.jet.core.TestProcessors.StuckProcessor)37 Job (com.hazelcast.jet.Job)36 Test (org.junit.Test)32 JobConfig (com.hazelcast.jet.config.JobConfig)20 JetInstance (com.hazelcast.jet.JetInstance)12 JetConfig (com.hazelcast.jet.config.JetConfig)8 JobRepository (com.hazelcast.jet.impl.JobRepository)8 CountDownLatch (java.util.concurrent.CountDownLatch)8 JetService (com.hazelcast.jet.impl.JetService)7 STARTING (com.hazelcast.jet.core.JobStatus.STARTING)6 MasterContext (com.hazelcast.jet.impl.MasterContext)6 Future (java.util.concurrent.Future)6 Assert.assertEquals (org.junit.Assert.assertEquals)6 Assert.assertNotNull (org.junit.Assert.assertNotNull)6 Assert.assertTrue (org.junit.Assert.assertTrue)6 Assert.fail (org.junit.Assert.fail)6 Rule (org.junit.Rule)6 ExpectedException (org.junit.rules.ExpectedException)6 RunWith (org.junit.runner.RunWith)6