use of com.hazelcast.jet.JetInstance in project hazelcast-jet by hazelcast.
the class SplitBrainTest method when_splitBrainProtectionIsDisabled_then_jobCompletesOnBothSides.
@Test
public void when_splitBrainProtectionIsDisabled_then_jobCompletesOnBothSides() {
int firstSubClusterSize = 2;
int secondSubClusterSize = 2;
int clusterSize = firstSubClusterSize + secondSubClusterSize;
StuckProcessor.executionStarted = new CountDownLatch(clusterSize * PARALLELISM);
Job[] jobRef = new Job[1];
Consumer<JetInstance[]> beforeSplit = instances -> {
MockPS processorSupplier = new MockPS(StuckProcessor::new, clusterSize);
DAG dag = new DAG().vertex(new Vertex("test", processorSupplier));
jobRef[0] = instances[0].newJob(dag);
assertOpenEventually(StuckProcessor.executionStarted);
};
BiConsumer<JetInstance[], JetInstance[]> onSplit = (firstSubCluster, secondSubCluster) -> {
StuckProcessor.proceedLatch.countDown();
long jobId = jobRef[0].getId();
assertTrueEventually(() -> {
JetService service1 = getJetService(firstSubCluster[0]);
JetService service2 = getJetService(secondSubCluster[0]);
assertEquals(COMPLETED, service1.getJobCoordinationService().getJobStatus(jobId));
assertEquals(COMPLETED, service2.getJobCoordinationService().getJobStatus(jobId));
});
};
Consumer<JetInstance[]> afterMerge = instances -> {
assertTrueEventually(() -> {
assertEquals(clusterSize * 2, MockPS.initCount.get());
assertEquals(clusterSize * 2, MockPS.closeCount.get());
});
assertEquals(clusterSize, MockPS.receivedCloseErrors.size());
MockPS.receivedCloseErrors.forEach(t -> assertTrue(t instanceof TopologyChangedException));
};
testSplitBrain(firstSubClusterSize, secondSubClusterSize, beforeSplit, onSplit, afterMerge);
}
use of com.hazelcast.jet.JetInstance in project hazelcast-jet by hazelcast.
the class TopologyChangeTest method when_coordinatorLeavesDuringExecutionAndNoRestartConfigured_then_jobFails.
@Test
public void when_coordinatorLeavesDuringExecutionAndNoRestartConfigured_then_jobFails() throws Throwable {
// Given
JetInstance client = createJetClient();
DAG dag = new DAG().vertex(new Vertex("test", new MockPS(StuckProcessor::new, nodeCount)));
JobConfig config = new JobConfig().setAutoRestartOnMemberFailure(false);
// When
Job job = client.newJob(dag, config);
StuckProcessor.executionStarted.await();
instances[0].getHazelcastInstance().getLifecycleService().terminate();
StuckProcessor.proceedLatch.countDown();
Throwable ex = job.getFuture().handle((r, e) -> e).get();
assertInstanceOf(TopologyChangedException.class, ex);
}
use of com.hazelcast.jet.JetInstance in project hazelcast-jet by hazelcast.
the class TopologyChangeTest method when_nonCoordinatorLeavesDuringExecution_then_clientStillGetsJobResult.
@Test
public void when_nonCoordinatorLeavesDuringExecution_then_clientStillGetsJobResult() throws Throwable {
// Given
JetInstance client = createJetClient();
DAG dag = new DAG().vertex(new Vertex("test", new MockPS(StuckProcessor::new, nodeCount)));
// When
Job job = client.newJob(dag);
StuckProcessor.executionStarted.await();
instances[2].getHazelcastInstance().getLifecycleService().terminate();
StuckProcessor.proceedLatch.countDown();
job.join();
}
use of com.hazelcast.jet.JetInstance in project hazelcast-jet by hazelcast.
the class TopologyChangeTest method when_jobParticipantReceivesStaleInitOperation_then_jobRestarts.
@Test
public void when_jobParticipantReceivesStaleInitOperation_then_jobRestarts() {
// Given
JetInstance newInstance = createJetMember(config);
for (JetInstance instance : instances) {
assertClusterSizeEventually(NODE_COUNT + 1, instance.getHazelcastInstance());
}
rejectOperationsBetween(instances[0].getHazelcastInstance(), instances[2].getHazelcastInstance(), JetInitDataSerializerHook.FACTORY_ID, singletonList(INIT_EXECUTION_OP));
DAG dag = new DAG().vertex(new Vertex("test", new MockPS(TestProcessors.Identity::new, nodeCount + 1)));
Job job = instances[0].newJob(dag);
JetService jetService = getJetService(instances[0]);
assertTrueEventually(() -> assertFalse(jetService.getJobCoordinationService().getMasterContexts().isEmpty()));
MasterContext masterContext = jetService.getJobCoordinationService().getMasterContext(job.getId());
assertTrueEventually(() -> {
assertEquals(STARTING, masterContext.jobStatus());
assertNotEquals(0, masterContext.getExecutionId());
});
// When
long executionId = masterContext.getExecutionId();
assertTrueEventually(() -> {
Arrays.stream(instances).filter(instance -> !instance.getHazelcastInstance().getCluster().getLocalMember().isLiteMember()).filter(instance -> instance != instances[2]).map(JetTestSupport::getJetService).map(service -> service.getJobExecutionService().getExecutionContext(executionId)).forEach(Assert::assertNotNull);
});
newInstance.getHazelcastInstance().getLifecycleService().terminate();
for (JetInstance instance : instances) {
assertClusterSizeEventually(NODE_COUNT, instance.getHazelcastInstance());
}
resetPacketFiltersFrom(instances[0].getHazelcastInstance());
// Then
job.join();
assertNotEquals(executionId, masterContext.getExecutionId());
}
use of com.hazelcast.jet.JetInstance in project hazelcast-jet by hazelcast.
the class WriteHdfsPTest method testWriteFile.
@Test
public void testWriteFile() throws Exception {
int messageCount = 20;
String mapName = randomMapName();
JetInstance instance = createJetMember();
createJetMember();
Map<IntWritable, IntWritable> map = IntStream.range(0, messageCount).boxed().collect(toMap(IntWritable::new, IntWritable::new));
instance.getMap(mapName).putAll(map);
Path path = getPath();
JobConf conf = new JobConf();
conf.setOutputFormat(outputFormatClass);
conf.setOutputCommitter(FileOutputCommitter.class);
conf.setOutputKeyClass(IntWritable.class);
conf.setOutputValueClass(IntWritable.class);
FileOutputFormat.setOutputPath(conf, path);
Pipeline p = Pipeline.create();
p.drawFrom(Sources.map(mapName)).drainTo(HdfsSinks.hdfs(conf));
Future<Void> future = instance.newJob(p).getFuture();
assertCompletesEventually(future);
JobConf readJobConf = new JobConf();
readJobConf.setInputFormat(inputFormatClass);
FileInputFormat.addInputPath(readJobConf, path);
p = Pipeline.create();
p.drawFrom(HdfsSources.hdfs(readJobConf)).drainTo(Sinks.list("results"));
future = instance.newJob(p).getFuture();
assertCompletesEventually(future);
IList<Object> results = instance.getList("results");
assertEquals(messageCount, results.size());
}
Aggregations