Search in sources :

Example 66 with JetInstance

use of com.hazelcast.jet.JetInstance in project hazelcast-jet by hazelcast.

the class SplitBrainTest method when_splitBrainProtectionIsDisabled_then_jobCompletesOnBothSides.

@Test
public void when_splitBrainProtectionIsDisabled_then_jobCompletesOnBothSides() {
    int firstSubClusterSize = 2;
    int secondSubClusterSize = 2;
    int clusterSize = firstSubClusterSize + secondSubClusterSize;
    StuckProcessor.executionStarted = new CountDownLatch(clusterSize * PARALLELISM);
    Job[] jobRef = new Job[1];
    Consumer<JetInstance[]> beforeSplit = instances -> {
        MockPS processorSupplier = new MockPS(StuckProcessor::new, clusterSize);
        DAG dag = new DAG().vertex(new Vertex("test", processorSupplier));
        jobRef[0] = instances[0].newJob(dag);
        assertOpenEventually(StuckProcessor.executionStarted);
    };
    BiConsumer<JetInstance[], JetInstance[]> onSplit = (firstSubCluster, secondSubCluster) -> {
        StuckProcessor.proceedLatch.countDown();
        long jobId = jobRef[0].getId();
        assertTrueEventually(() -> {
            JetService service1 = getJetService(firstSubCluster[0]);
            JetService service2 = getJetService(secondSubCluster[0]);
            assertEquals(COMPLETED, service1.getJobCoordinationService().getJobStatus(jobId));
            assertEquals(COMPLETED, service2.getJobCoordinationService().getJobStatus(jobId));
        });
    };
    Consumer<JetInstance[]> afterMerge = instances -> {
        assertTrueEventually(() -> {
            assertEquals(clusterSize * 2, MockPS.initCount.get());
            assertEquals(clusterSize * 2, MockPS.closeCount.get());
        });
        assertEquals(clusterSize, MockPS.receivedCloseErrors.size());
        MockPS.receivedCloseErrors.forEach(t -> assertTrue(t instanceof TopologyChangedException));
    };
    testSplitBrain(firstSubClusterSize, secondSubClusterSize, beforeSplit, onSplit, afterMerge);
}
Also used : MasterContext(com.hazelcast.jet.impl.MasterContext) JetInstance(com.hazelcast.jet.JetInstance) RunWith(org.junit.runner.RunWith) HazelcastSerialClassRunner(com.hazelcast.test.HazelcastSerialClassRunner) Future(java.util.concurrent.Future) STARTING(com.hazelcast.jet.core.JobStatus.STARTING) BiConsumer(java.util.function.BiConsumer) Assert.fail(org.junit.Assert.fail) ExpectedException(org.junit.rules.ExpectedException) Job(com.hazelcast.jet.Job) JobRepository(com.hazelcast.jet.impl.JobRepository) JetConfig(com.hazelcast.jet.config.JetConfig) MAX_BACKUP_COUNT(com.hazelcast.spi.partition.IPartition.MAX_BACKUP_COUNT) MockPS(com.hazelcast.jet.core.TestProcessors.MockPS) CancellationException(java.util.concurrent.CancellationException) RESTARTING(com.hazelcast.jet.core.JobStatus.RESTARTING) Assert.assertNotNull(org.junit.Assert.assertNotNull) JobConfig(com.hazelcast.jet.config.JobConfig) StuckProcessor(com.hazelcast.jet.core.TestProcessors.StuckProcessor) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) CountDownLatch(java.util.concurrent.CountDownLatch) Rule(org.junit.Rule) JetService(com.hazelcast.jet.impl.JetService) COMPLETED(com.hazelcast.jet.core.JobStatus.COMPLETED) JobRecord(com.hazelcast.jet.impl.JobRecord) Assert.assertEquals(org.junit.Assert.assertEquals) MockPS(com.hazelcast.jet.core.TestProcessors.MockPS) JetService(com.hazelcast.jet.impl.JetService) CountDownLatch(java.util.concurrent.CountDownLatch) Job(com.hazelcast.jet.Job) Test(org.junit.Test)

Example 67 with JetInstance

use of com.hazelcast.jet.JetInstance in project hazelcast-jet by hazelcast.

the class TopologyChangeTest method when_coordinatorLeavesDuringExecutionAndNoRestartConfigured_then_jobFails.

@Test
public void when_coordinatorLeavesDuringExecutionAndNoRestartConfigured_then_jobFails() throws Throwable {
    // Given
    JetInstance client = createJetClient();
    DAG dag = new DAG().vertex(new Vertex("test", new MockPS(StuckProcessor::new, nodeCount)));
    JobConfig config = new JobConfig().setAutoRestartOnMemberFailure(false);
    // When
    Job job = client.newJob(dag, config);
    StuckProcessor.executionStarted.await();
    instances[0].getHazelcastInstance().getLifecycleService().terminate();
    StuckProcessor.proceedLatch.countDown();
    Throwable ex = job.getFuture().handle((r, e) -> e).get();
    assertInstanceOf(TopologyChangedException.class, ex);
}
Also used : HazelcastInstanceNotActiveException(com.hazelcast.core.HazelcastInstanceNotActiveException) Arrays(java.util.Arrays) INIT_EXECUTION_OP(com.hazelcast.jet.impl.execution.init.JetInitDataSerializerHook.INIT_EXECUTION_OP) Collections.singletonList(java.util.Collections.singletonList) Assert.assertThat(org.junit.Assert.assertThat) PacketFiltersUtil.dropOperationsBetween(com.hazelcast.test.PacketFiltersUtil.dropOperationsBetween) PacketFiltersUtil.rejectOperationsBetween(com.hazelcast.test.PacketFiltersUtil.rejectOperationsBetween) Future(java.util.concurrent.Future) MemberInfo(com.hazelcast.internal.cluster.MemberInfo) STARTING(com.hazelcast.jet.core.JobStatus.STARTING) Assert.fail(org.junit.Assert.fail) ClusterDataSerializerHook(com.hazelcast.internal.cluster.impl.ClusterDataSerializerHook) Parameterized(org.junit.runners.Parameterized) MockPS(com.hazelcast.jet.core.TestProcessors.MockPS) MEMBER_INFO_UPDATE(com.hazelcast.internal.cluster.impl.ClusterDataSerializerHook.MEMBER_INFO_UPDATE) Collection(java.util.Collection) START_EXECUTION_OP(com.hazelcast.jet.impl.execution.init.JetInitDataSerializerHook.START_EXECUTION_OP) JobConfig(com.hazelcast.jet.config.JobConfig) StuckProcessor(com.hazelcast.jet.core.TestProcessors.StuckProcessor) Set(java.util.Set) PartitionDataSerializerHook(com.hazelcast.internal.partition.impl.PartitionDataSerializerHook) JobResult(com.hazelcast.jet.impl.JobResult) CountDownLatch(java.util.concurrent.CountDownLatch) Assert.assertFalse(org.junit.Assert.assertFalse) MasterContext(com.hazelcast.jet.impl.MasterContext) JetInstance(com.hazelcast.jet.JetInstance) RunWith(org.junit.runner.RunWith) JetInitDataSerializerHook(com.hazelcast.jet.impl.execution.init.JetInitDataSerializerHook) HashSet(java.util.HashSet) InitExecutionOperation(com.hazelcast.jet.impl.operation.InitExecutionOperation) ExpectedException(org.junit.rules.ExpectedException) Job(com.hazelcast.jet.Job) Before(org.junit.Before) JobRepository(com.hazelcast.jet.impl.JobRepository) HazelcastInstance(com.hazelcast.core.HazelcastInstance) JetConfig(com.hazelcast.jet.config.JetConfig) Matchers.empty(org.hamcrest.Matchers.empty) Assert.assertNotNull(org.junit.Assert.assertNotNull) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) Assert.assertNotEquals(org.junit.Assert.assertNotEquals) ExecutionException(java.util.concurrent.ExecutionException) Rule(org.junit.Rule) JetService(com.hazelcast.jet.impl.JetService) HazelcastParametersRunnerFactory(com.hazelcast.test.HazelcastParametersRunnerFactory) RUNNING(com.hazelcast.jet.core.JobStatus.RUNNING) Assert(org.junit.Assert) PacketFiltersUtil.resetPacketFiltersFrom(com.hazelcast.test.PacketFiltersUtil.resetPacketFiltersFrom) Assert.assertEquals(org.junit.Assert.assertEquals) SHUTDOWN_REQUEST(com.hazelcast.internal.partition.impl.PartitionDataSerializerHook.SHUTDOWN_REQUEST) MockPS(com.hazelcast.jet.core.TestProcessors.MockPS) JetInstance(com.hazelcast.jet.JetInstance) StuckProcessor(com.hazelcast.jet.core.TestProcessors.StuckProcessor) Job(com.hazelcast.jet.Job) JobConfig(com.hazelcast.jet.config.JobConfig) Test(org.junit.Test)

Example 68 with JetInstance

use of com.hazelcast.jet.JetInstance in project hazelcast-jet by hazelcast.

the class TopologyChangeTest method when_nonCoordinatorLeavesDuringExecution_then_clientStillGetsJobResult.

@Test
public void when_nonCoordinatorLeavesDuringExecution_then_clientStillGetsJobResult() throws Throwable {
    // Given
    JetInstance client = createJetClient();
    DAG dag = new DAG().vertex(new Vertex("test", new MockPS(StuckProcessor::new, nodeCount)));
    // When
    Job job = client.newJob(dag);
    StuckProcessor.executionStarted.await();
    instances[2].getHazelcastInstance().getLifecycleService().terminate();
    StuckProcessor.proceedLatch.countDown();
    job.join();
}
Also used : MockPS(com.hazelcast.jet.core.TestProcessors.MockPS) JetInstance(com.hazelcast.jet.JetInstance) StuckProcessor(com.hazelcast.jet.core.TestProcessors.StuckProcessor) Job(com.hazelcast.jet.Job) Test(org.junit.Test)

Example 69 with JetInstance

use of com.hazelcast.jet.JetInstance in project hazelcast-jet by hazelcast.

the class TopologyChangeTest method when_jobParticipantReceivesStaleInitOperation_then_jobRestarts.

@Test
public void when_jobParticipantReceivesStaleInitOperation_then_jobRestarts() {
    // Given
    JetInstance newInstance = createJetMember(config);
    for (JetInstance instance : instances) {
        assertClusterSizeEventually(NODE_COUNT + 1, instance.getHazelcastInstance());
    }
    rejectOperationsBetween(instances[0].getHazelcastInstance(), instances[2].getHazelcastInstance(), JetInitDataSerializerHook.FACTORY_ID, singletonList(INIT_EXECUTION_OP));
    DAG dag = new DAG().vertex(new Vertex("test", new MockPS(TestProcessors.Identity::new, nodeCount + 1)));
    Job job = instances[0].newJob(dag);
    JetService jetService = getJetService(instances[0]);
    assertTrueEventually(() -> assertFalse(jetService.getJobCoordinationService().getMasterContexts().isEmpty()));
    MasterContext masterContext = jetService.getJobCoordinationService().getMasterContext(job.getId());
    assertTrueEventually(() -> {
        assertEquals(STARTING, masterContext.jobStatus());
        assertNotEquals(0, masterContext.getExecutionId());
    });
    // When
    long executionId = masterContext.getExecutionId();
    assertTrueEventually(() -> {
        Arrays.stream(instances).filter(instance -> !instance.getHazelcastInstance().getCluster().getLocalMember().isLiteMember()).filter(instance -> instance != instances[2]).map(JetTestSupport::getJetService).map(service -> service.getJobExecutionService().getExecutionContext(executionId)).forEach(Assert::assertNotNull);
    });
    newInstance.getHazelcastInstance().getLifecycleService().terminate();
    for (JetInstance instance : instances) {
        assertClusterSizeEventually(NODE_COUNT, instance.getHazelcastInstance());
    }
    resetPacketFiltersFrom(instances[0].getHazelcastInstance());
    // Then
    job.join();
    assertNotEquals(executionId, masterContext.getExecutionId());
}
Also used : HazelcastInstanceNotActiveException(com.hazelcast.core.HazelcastInstanceNotActiveException) Arrays(java.util.Arrays) INIT_EXECUTION_OP(com.hazelcast.jet.impl.execution.init.JetInitDataSerializerHook.INIT_EXECUTION_OP) Collections.singletonList(java.util.Collections.singletonList) Assert.assertThat(org.junit.Assert.assertThat) PacketFiltersUtil.dropOperationsBetween(com.hazelcast.test.PacketFiltersUtil.dropOperationsBetween) PacketFiltersUtil.rejectOperationsBetween(com.hazelcast.test.PacketFiltersUtil.rejectOperationsBetween) Future(java.util.concurrent.Future) MemberInfo(com.hazelcast.internal.cluster.MemberInfo) STARTING(com.hazelcast.jet.core.JobStatus.STARTING) Assert.fail(org.junit.Assert.fail) ClusterDataSerializerHook(com.hazelcast.internal.cluster.impl.ClusterDataSerializerHook) Parameterized(org.junit.runners.Parameterized) MockPS(com.hazelcast.jet.core.TestProcessors.MockPS) MEMBER_INFO_UPDATE(com.hazelcast.internal.cluster.impl.ClusterDataSerializerHook.MEMBER_INFO_UPDATE) Collection(java.util.Collection) START_EXECUTION_OP(com.hazelcast.jet.impl.execution.init.JetInitDataSerializerHook.START_EXECUTION_OP) JobConfig(com.hazelcast.jet.config.JobConfig) StuckProcessor(com.hazelcast.jet.core.TestProcessors.StuckProcessor) Set(java.util.Set) PartitionDataSerializerHook(com.hazelcast.internal.partition.impl.PartitionDataSerializerHook) JobResult(com.hazelcast.jet.impl.JobResult) CountDownLatch(java.util.concurrent.CountDownLatch) Assert.assertFalse(org.junit.Assert.assertFalse) MasterContext(com.hazelcast.jet.impl.MasterContext) JetInstance(com.hazelcast.jet.JetInstance) RunWith(org.junit.runner.RunWith) JetInitDataSerializerHook(com.hazelcast.jet.impl.execution.init.JetInitDataSerializerHook) HashSet(java.util.HashSet) InitExecutionOperation(com.hazelcast.jet.impl.operation.InitExecutionOperation) ExpectedException(org.junit.rules.ExpectedException) Job(com.hazelcast.jet.Job) Before(org.junit.Before) JobRepository(com.hazelcast.jet.impl.JobRepository) HazelcastInstance(com.hazelcast.core.HazelcastInstance) JetConfig(com.hazelcast.jet.config.JetConfig) Matchers.empty(org.hamcrest.Matchers.empty) Assert.assertNotNull(org.junit.Assert.assertNotNull) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) Assert.assertNotEquals(org.junit.Assert.assertNotEquals) ExecutionException(java.util.concurrent.ExecutionException) Rule(org.junit.Rule) JetService(com.hazelcast.jet.impl.JetService) HazelcastParametersRunnerFactory(com.hazelcast.test.HazelcastParametersRunnerFactory) RUNNING(com.hazelcast.jet.core.JobStatus.RUNNING) Assert(org.junit.Assert) PacketFiltersUtil.resetPacketFiltersFrom(com.hazelcast.test.PacketFiltersUtil.resetPacketFiltersFrom) Assert.assertEquals(org.junit.Assert.assertEquals) SHUTDOWN_REQUEST(com.hazelcast.internal.partition.impl.PartitionDataSerializerHook.SHUTDOWN_REQUEST) MockPS(com.hazelcast.jet.core.TestProcessors.MockPS) JetService(com.hazelcast.jet.impl.JetService) Assert(org.junit.Assert) JetInstance(com.hazelcast.jet.JetInstance) Job(com.hazelcast.jet.Job) MasterContext(com.hazelcast.jet.impl.MasterContext) Test(org.junit.Test)

Example 70 with JetInstance

use of com.hazelcast.jet.JetInstance in project hazelcast-jet by hazelcast.

the class WriteHdfsPTest method testWriteFile.

@Test
public void testWriteFile() throws Exception {
    int messageCount = 20;
    String mapName = randomMapName();
    JetInstance instance = createJetMember();
    createJetMember();
    Map<IntWritable, IntWritable> map = IntStream.range(0, messageCount).boxed().collect(toMap(IntWritable::new, IntWritable::new));
    instance.getMap(mapName).putAll(map);
    Path path = getPath();
    JobConf conf = new JobConf();
    conf.setOutputFormat(outputFormatClass);
    conf.setOutputCommitter(FileOutputCommitter.class);
    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(IntWritable.class);
    FileOutputFormat.setOutputPath(conf, path);
    Pipeline p = Pipeline.create();
    p.drawFrom(Sources.map(mapName)).drainTo(HdfsSinks.hdfs(conf));
    Future<Void> future = instance.newJob(p).getFuture();
    assertCompletesEventually(future);
    JobConf readJobConf = new JobConf();
    readJobConf.setInputFormat(inputFormatClass);
    FileInputFormat.addInputPath(readJobConf, path);
    p = Pipeline.create();
    p.drawFrom(HdfsSources.hdfs(readJobConf)).drainTo(Sinks.list("results"));
    future = instance.newJob(p).getFuture();
    assertCompletesEventually(future);
    IList<Object> results = instance.getList("results");
    assertEquals(messageCount, results.size());
}
Also used : Path(org.apache.hadoop.fs.Path) JetInstance(com.hazelcast.jet.JetInstance) JobConf(org.apache.hadoop.mapred.JobConf) IntWritable(org.apache.hadoop.io.IntWritable) Pipeline(com.hazelcast.jet.pipeline.Pipeline) Test(org.junit.Test) ParallelTest(com.hazelcast.test.annotation.ParallelTest)

Aggregations

JetInstance (com.hazelcast.jet.JetInstance)84 Test (org.junit.Test)45 Job (com.hazelcast.jet.Job)32 JetConfig (com.hazelcast.jet.config.JetConfig)22 MockPS (com.hazelcast.jet.core.TestProcessors.MockPS)14 Pipeline (com.hazelcast.jet.pipeline.Pipeline)14 JobConfig (com.hazelcast.jet.config.JobConfig)13 StuckProcessor (com.hazelcast.jet.core.TestProcessors.StuckProcessor)12 DAG (com.hazelcast.jet.core.DAG)10 ExpectedException (org.junit.rules.ExpectedException)10 Jet (com.hazelcast.jet.Jet)9 Assert.assertEquals (org.junit.Assert.assertEquals)9 JetService (com.hazelcast.jet.impl.JetService)8 Map (java.util.Map)8 CountDownLatch (java.util.concurrent.CountDownLatch)8 JobRepository (com.hazelcast.jet.impl.JobRepository)7 Sources (com.hazelcast.jet.pipeline.Sources)7 CancellationException (java.util.concurrent.CancellationException)7 Assert.assertNotNull (org.junit.Assert.assertNotNull)7 Before (org.junit.Before)7