use of com.hazelcast.jet.core.TestProcessors.StuckProcessor in project hazelcast-jet by hazelcast.
the class SplitBrainTest method when_quorumIsLostOnMinority_then_jobRestartsUntilMerge.
@Test
public void when_quorumIsLostOnMinority_then_jobRestartsUntilMerge() {
int firstSubClusterSize = 3;
int secondSubClusterSize = 2;
int clusterSize = firstSubClusterSize + secondSubClusterSize;
StuckProcessor.executionStarted = new CountDownLatch(clusterSize * PARALLELISM);
Job[] jobRef = new Job[1];
Consumer<JetInstance[]> beforeSplit = instances -> {
MockPS processorSupplier = new MockPS(StuckProcessor::new, clusterSize);
DAG dag = new DAG().vertex(new Vertex("test", processorSupplier));
jobRef[0] = instances[0].newJob(dag, new JobConfig().setSplitBrainProtection(true));
assertOpenEventually(StuckProcessor.executionStarted);
};
Future[] minorityJobFutureRef = new Future[1];
BiConsumer<JetInstance[], JetInstance[]> onSplit = (firstSubCluster, secondSubCluster) -> {
StuckProcessor.proceedLatch.countDown();
assertTrueEventually(() -> assertEquals(clusterSize + firstSubClusterSize, MockPS.initCount.get()));
long jobId = jobRef[0].getId();
assertTrueEventually(() -> {
JetService service = getJetService(firstSubCluster[0]);
assertEquals(COMPLETED, service.getJobCoordinationService().getJobStatus(jobId));
});
JetService service2 = getJetService(secondSubCluster[0]);
assertTrueEventually(() -> {
assertEquals(STARTING, service2.getJobCoordinationService().getJobStatus(jobId));
});
MasterContext masterContext = service2.getJobCoordinationService().getMasterContext(jobId);
assertNotNull(masterContext);
minorityJobFutureRef[0] = masterContext.completionFuture();
assertTrueAllTheTime(() -> {
assertEquals(STARTING, service2.getJobCoordinationService().getJobStatus(jobId));
}, 20);
};
Consumer<JetInstance[]> afterMerge = instances -> {
assertTrueEventually(() -> {
assertEquals(clusterSize + firstSubClusterSize, MockPS.initCount.get());
assertEquals(clusterSize + firstSubClusterSize, MockPS.closeCount.get());
});
assertEquals(clusterSize, MockPS.receivedCloseErrors.size());
MockPS.receivedCloseErrors.forEach(t -> assertTrue(t instanceof TopologyChangedException));
try {
minorityJobFutureRef[0].get();
fail();
} catch (CancellationException ignored) {
} catch (Exception e) {
throw new AssertionError(e);
}
};
testSplitBrain(firstSubClusterSize, secondSubClusterSize, beforeSplit, onSplit, afterMerge);
}
use of com.hazelcast.jet.core.TestProcessors.StuckProcessor in project hazelcast-jet by hazelcast.
the class SplitBrainTest method when_quorumIsLostOnBothSides_then_jobRestartsUntilMerge.
@Test
public void when_quorumIsLostOnBothSides_then_jobRestartsUntilMerge() {
int firstSubClusterSize = 2;
int secondSubClusterSize = 2;
int clusterSize = firstSubClusterSize + secondSubClusterSize;
StuckProcessor.executionStarted = new CountDownLatch(clusterSize * PARALLELISM);
Job[] jobRef = new Job[1];
Consumer<JetInstance[]> beforeSplit = instances -> {
MockPS processorSupplier = new MockPS(StuckProcessor::new, clusterSize);
DAG dag = new DAG().vertex(new Vertex("test", processorSupplier));
jobRef[0] = instances[0].newJob(dag, new JobConfig().setSplitBrainProtection(true));
assertOpenEventually(StuckProcessor.executionStarted);
};
BiConsumer<JetInstance[], JetInstance[]> onSplit = (firstSubCluster, secondSubCluster) -> {
StuckProcessor.proceedLatch.countDown();
long jobId = jobRef[0].getId();
assertTrueEventually(() -> {
JetService service1 = getJetService(firstSubCluster[0]);
JetService service2 = getJetService(secondSubCluster[0]);
assertEquals(RESTARTING, service1.getJobCoordinationService().getJobStatus(jobId));
assertEquals(STARTING, service2.getJobCoordinationService().getJobStatus(jobId));
});
assertTrueAllTheTime(() -> {
JetService service1 = getJetService(firstSubCluster[0]);
JetService service2 = getJetService(secondSubCluster[0]);
assertEquals(RESTARTING, service1.getJobCoordinationService().getJobStatus(jobId));
assertEquals(STARTING, service2.getJobCoordinationService().getJobStatus(jobId));
}, 20);
};
Consumer<JetInstance[]> afterMerge = instances -> {
assertTrueEventually(() -> {
assertEquals(clusterSize * 2, MockPS.initCount.get());
assertEquals(clusterSize * 2, MockPS.closeCount.get());
});
assertEquals(clusterSize, MockPS.receivedCloseErrors.size());
MockPS.receivedCloseErrors.forEach(t -> assertTrue(t instanceof TopologyChangedException));
};
testSplitBrain(firstSubClusterSize, secondSubClusterSize, beforeSplit, onSplit, afterMerge);
}
use of com.hazelcast.jet.core.TestProcessors.StuckProcessor in project hazelcast-jet by hazelcast.
the class SplitBrainTest method when_newMemberJoinsToCluster_then_jobQuorumSizeIsUpdated.
@Test
public void when_newMemberJoinsToCluster_then_jobQuorumSizeIsUpdated() {
int clusterSize = 3;
JetConfig jetConfig = new JetConfig();
JetInstance[] instances = new JetInstance[clusterSize];
for (int i = 0; i < clusterSize; i++) {
instances[i] = createJetMember(jetConfig);
}
StuckProcessor.executionStarted = new CountDownLatch(clusterSize * PARALLELISM);
MockPS processorSupplier = new MockPS(StuckProcessor::new, clusterSize);
DAG dag = new DAG().vertex(new Vertex("test", processorSupplier));
Job job = instances[0].newJob(dag, new JobConfig().setSplitBrainProtection(true));
assertOpenEventually(StuckProcessor.executionStarted);
createJetMember(jetConfig);
assertTrueEventually(() -> {
JobRepository jobRepository = getJetService(instances[0]).getJobRepository();
JobRecord jobRecord = jobRepository.getJobRecord(job.getId());
assertEquals(3, jobRecord.getQuorumSize());
});
StuckProcessor.proceedLatch.countDown();
}
use of com.hazelcast.jet.core.TestProcessors.StuckProcessor in project hazelcast-jet by hazelcast.
the class SplitBrainTest method when_newMemberIsAddedAfterClusterSizeFallsBelowQuorumSize_then_jobRestartDoesNotSucceed.
@Test
public void when_newMemberIsAddedAfterClusterSizeFallsBelowQuorumSize_then_jobRestartDoesNotSucceed() {
int clusterSize = 5;
JetConfig jetConfig = new JetConfig();
JetInstance[] instances = new JetInstance[clusterSize];
for (int i = 0; i < clusterSize; i++) {
instances[i] = createJetMember(jetConfig);
}
StuckProcessor.executionStarted = new CountDownLatch(clusterSize * PARALLELISM);
MockPS processorSupplier = new MockPS(StuckProcessor::new, clusterSize);
DAG dag = new DAG().vertex(new Vertex("test", processorSupplier));
Job job = instances[0].newJob(dag, new JobConfig().setSplitBrainProtection(true));
assertOpenEventually(StuckProcessor.executionStarted);
for (int i = 1; i < clusterSize; i++) {
instances[i].getHazelcastInstance().getLifecycleService().terminate();
}
StuckProcessor.proceedLatch.countDown();
assertTrueEventually(() -> assertEquals(RESTARTING, job.getStatus()));
createJetMember(jetConfig);
assertTrueAllTheTime(() -> assertEquals(RESTARTING, job.getStatus()), 5);
}
use of com.hazelcast.jet.core.TestProcessors.StuckProcessor in project hazelcast-jet by hazelcast.
the class TopologyChangeDuringJobSubmissionTest method when_jobIsCompletedBeforeSubmissionCallReturns_then_jobRunsOnlyOnce.
@Test
public void when_jobIsCompletedBeforeSubmissionCallReturns_then_jobRunsOnlyOnce() throws Throwable {
// Given that the job is already completed
dropOperationsBetween(instance1.getHazelcastInstance(), instance2.getHazelcastInstance(), SpiDataSerializerHook.F_ID, singletonList(SpiDataSerializerHook.NORMAL_RESPONSE));
String jobName = "job1";
Future<Job> future = spawn(() -> {
DAG dag = new DAG().vertex(new Vertex("test", new MockPS(StuckProcessor::new, 1)));
return instance2.newJob(dag, new JobConfig().setName(jobName));
});
StuckProcessor.executionStarted.await();
Job submittedJob = instance1.getJob(jobName);
assertNotNull(submittedJob);
StuckProcessor.proceedLatch.countDown();
submittedJob.join();
// When the coordinator leaves before the submission response is received
instance1.getHazelcastInstance().getLifecycleService().terminate();
Job job = future.get();
// Then the job does not run for the second time
job.join();
assertEquals(1, MockPS.initCount.get());
}
Aggregations