Search in sources :

Example 1 with INIT_EXECUTION_OP

use of com.hazelcast.jet.impl.execution.init.JetInitDataSerializerHook.INIT_EXECUTION_OP in project hazelcast by hazelcast.

the class TopologyChangeTest method when_jobParticipantReceivesStaleInitOperation_then_jobRestarts.

@Test
public void when_jobParticipantReceivesStaleInitOperation_then_jobRestarts() {
    // Given
    HazelcastInstance newInstance = createHazelcastInstance(config);
    for (HazelcastInstance instance : instances) {
        assertClusterSizeEventually(NODE_COUNT + 1, instance);
    }
    rejectOperationsBetween(instances[0], instances[2], JetInitDataSerializerHook.FACTORY_ID, singletonList(INIT_EXECUTION_OP));
    DAG dag = new DAG().vertex(new Vertex("test", new MockPS(TestProcessors.Identity::new, nodeCount + 1)));
    Job job = instances[0].getJet().newJob(dag);
    JetServiceBackend jetServiceBackend = getJetServiceBackend(instances[0]);
    assertTrueEventually(() -> assertFalse(jetServiceBackend.getJobCoordinationService().getMasterContexts().isEmpty()));
    MasterContext masterContext = jetServiceBackend.getJobCoordinationService().getMasterContext(job.getId());
    assertTrueEventually(() -> {
        assertEquals(STARTING, masterContext.jobStatus());
        assertNotEquals(0, masterContext.executionId());
    });
    // When
    long executionId = masterContext.executionId();
    assertTrueEventually(() -> {
        Arrays.stream(instances).filter(instance -> !instance.getCluster().getLocalMember().isLiteMember()).filter(instance -> instance != instances[2]).map(JetTestSupport::getJetServiceBackend).map(service -> service.getJobExecutionService().getExecutionContext(executionId)).forEach(Assert::assertNotNull);
    });
    newInstance.getLifecycleService().terminate();
    for (HazelcastInstance instance : instances) {
        assertClusterSizeEventually(NODE_COUNT, instance);
    }
    resetPacketFiltersFrom(instances[0]);
    // Then
    job.join();
    assertNotEquals(executionId, masterContext.executionId());
}
Also used : HazelcastInstanceNotActiveException(com.hazelcast.core.HazelcastInstanceNotActiveException) ParallelJVMTest(com.hazelcast.test.annotation.ParallelJVMTest) Arrays(java.util.Arrays) INIT_EXECUTION_OP(com.hazelcast.jet.impl.execution.init.JetInitDataSerializerHook.INIT_EXECUTION_OP) Collections.singletonList(java.util.Collections.singletonList) Assert.assertThat(org.junit.Assert.assertThat) PacketFiltersUtil.dropOperationsBetween(com.hazelcast.test.PacketFiltersUtil.dropOperationsBetween) PacketFiltersUtil.rejectOperationsBetween(com.hazelcast.test.PacketFiltersUtil.rejectOperationsBetween) Future(java.util.concurrent.Future) MemberInfo(com.hazelcast.internal.cluster.MemberInfo) STARTING(com.hazelcast.jet.core.JobStatus.STARTING) SUSPENDED(com.hazelcast.jet.core.JobStatus.SUSPENDED) Assert.fail(org.junit.Assert.fail) ClusterDataSerializerHook(com.hazelcast.internal.cluster.impl.ClusterDataSerializerHook) Parameterized(org.junit.runners.Parameterized) MockPS(com.hazelcast.jet.core.TestProcessors.MockPS) HazelcastParametrizedRunner(com.hazelcast.test.HazelcastParametrizedRunner) CancellationException(java.util.concurrent.CancellationException) MEMBER_INFO_UPDATE(com.hazelcast.internal.cluster.impl.ClusterDataSerializerHook.MEMBER_INFO_UPDATE) Collection(java.util.Collection) START_EXECUTION_OP(com.hazelcast.jet.impl.execution.init.JetInitDataSerializerHook.START_EXECUTION_OP) JobConfig(com.hazelcast.jet.config.JobConfig) Set(java.util.Set) PartitionDataSerializerHook(com.hazelcast.internal.partition.impl.PartitionDataSerializerHook) JobResult(com.hazelcast.jet.impl.JobResult) TargetNotMemberException(com.hazelcast.spi.exception.TargetNotMemberException) Category(org.junit.experimental.categories.Category) NoOutputSourceP(com.hazelcast.jet.core.TestProcessors.NoOutputSourceP) Assert.assertFalse(org.junit.Assert.assertFalse) MasterContext(com.hazelcast.jet.impl.MasterContext) RunWith(org.junit.runner.RunWith) HazelcastSerialParametersRunnerFactory(com.hazelcast.test.HazelcastSerialParametersRunnerFactory) JetInitDataSerializerHook(com.hazelcast.jet.impl.execution.init.JetInitDataSerializerHook) Accessors(com.hazelcast.test.Accessors) HashSet(java.util.HashSet) InitExecutionOperation(com.hazelcast.jet.impl.operation.InitExecutionOperation) Version(com.hazelcast.version.Version) ExpectedException(org.junit.rules.ExpectedException) Job(com.hazelcast.jet.Job) Before(org.junit.Before) UseParametersRunnerFactory(org.junit.runners.Parameterized.UseParametersRunnerFactory) JobRepository(com.hazelcast.jet.impl.JobRepository) Config(com.hazelcast.config.Config) HazelcastInstance(com.hazelcast.core.HazelcastInstance) Matchers.empty(org.hamcrest.Matchers.empty) Assert.assertNotNull(org.junit.Assert.assertNotNull) EXACTLY_ONCE(com.hazelcast.jet.config.ProcessingGuarantee.EXACTLY_ONCE) MemberLeftException(com.hazelcast.core.MemberLeftException) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) SlowTest(com.hazelcast.test.annotation.SlowTest) NONE(com.hazelcast.jet.config.ProcessingGuarantee.NONE) Assert.assertNotEquals(org.junit.Assert.assertNotEquals) ExecutionException(java.util.concurrent.ExecutionException) Rule(org.junit.Rule) FAILED(com.hazelcast.jet.core.JobStatus.FAILED) JOB_RECORDS_MAP_NAME(com.hazelcast.jet.impl.JobRepository.JOB_RECORDS_MAP_NAME) RUNNING(com.hazelcast.jet.core.JobStatus.RUNNING) Assert(org.junit.Assert) PacketFiltersUtil.resetPacketFiltersFrom(com.hazelcast.test.PacketFiltersUtil.resetPacketFiltersFrom) Collections(java.util.Collections) JobRecord(com.hazelcast.jet.impl.JobRecord) Assert.assertEquals(org.junit.Assert.assertEquals) JetServiceBackend(com.hazelcast.jet.impl.JetServiceBackend) SHUTDOWN_REQUEST(com.hazelcast.internal.partition.impl.PartitionDataSerializerHook.SHUTDOWN_REQUEST) MockPS(com.hazelcast.jet.core.TestProcessors.MockPS) HazelcastInstance(com.hazelcast.core.HazelcastInstance) Assert(org.junit.Assert) Job(com.hazelcast.jet.Job) MasterContext(com.hazelcast.jet.impl.MasterContext) JetServiceBackend(com.hazelcast.jet.impl.JetServiceBackend) ParallelJVMTest(com.hazelcast.test.annotation.ParallelJVMTest) Test(org.junit.Test) SlowTest(com.hazelcast.test.annotation.SlowTest)

Example 2 with INIT_EXECUTION_OP

use of com.hazelcast.jet.impl.execution.init.JetInitDataSerializerHook.INIT_EXECUTION_OP in project hazelcast-jet by hazelcast.

the class TopologyChangeTest method when_jobParticipantReceivesStaleInitOperation_then_jobRestarts.

@Test
public void when_jobParticipantReceivesStaleInitOperation_then_jobRestarts() {
    // Given
    JetInstance newInstance = createJetMember(config);
    for (JetInstance instance : instances) {
        assertClusterSizeEventually(NODE_COUNT + 1, instance.getHazelcastInstance());
    }
    rejectOperationsBetween(instances[0].getHazelcastInstance(), instances[2].getHazelcastInstance(), JetInitDataSerializerHook.FACTORY_ID, singletonList(INIT_EXECUTION_OP));
    DAG dag = new DAG().vertex(new Vertex("test", new MockPS(TestProcessors.Identity::new, nodeCount + 1)));
    Job job = instances[0].newJob(dag);
    JetService jetService = getJetService(instances[0]);
    assertTrueEventually(() -> assertFalse(jetService.getJobCoordinationService().getMasterContexts().isEmpty()));
    MasterContext masterContext = jetService.getJobCoordinationService().getMasterContext(job.getId());
    assertTrueEventually(() -> {
        assertEquals(STARTING, masterContext.jobStatus());
        assertNotEquals(0, masterContext.getExecutionId());
    });
    // When
    long executionId = masterContext.getExecutionId();
    assertTrueEventually(() -> {
        Arrays.stream(instances).filter(instance -> !instance.getHazelcastInstance().getCluster().getLocalMember().isLiteMember()).filter(instance -> instance != instances[2]).map(JetTestSupport::getJetService).map(service -> service.getJobExecutionService().getExecutionContext(executionId)).forEach(Assert::assertNotNull);
    });
    newInstance.getHazelcastInstance().getLifecycleService().terminate();
    for (JetInstance instance : instances) {
        assertClusterSizeEventually(NODE_COUNT, instance.getHazelcastInstance());
    }
    resetPacketFiltersFrom(instances[0].getHazelcastInstance());
    // Then
    job.join();
    assertNotEquals(executionId, masterContext.getExecutionId());
}
Also used : HazelcastInstanceNotActiveException(com.hazelcast.core.HazelcastInstanceNotActiveException) Arrays(java.util.Arrays) INIT_EXECUTION_OP(com.hazelcast.jet.impl.execution.init.JetInitDataSerializerHook.INIT_EXECUTION_OP) Collections.singletonList(java.util.Collections.singletonList) Assert.assertThat(org.junit.Assert.assertThat) PacketFiltersUtil.dropOperationsBetween(com.hazelcast.test.PacketFiltersUtil.dropOperationsBetween) PacketFiltersUtil.rejectOperationsBetween(com.hazelcast.test.PacketFiltersUtil.rejectOperationsBetween) Future(java.util.concurrent.Future) MemberInfo(com.hazelcast.internal.cluster.MemberInfo) STARTING(com.hazelcast.jet.core.JobStatus.STARTING) Assert.fail(org.junit.Assert.fail) ClusterDataSerializerHook(com.hazelcast.internal.cluster.impl.ClusterDataSerializerHook) Parameterized(org.junit.runners.Parameterized) MockPS(com.hazelcast.jet.core.TestProcessors.MockPS) MEMBER_INFO_UPDATE(com.hazelcast.internal.cluster.impl.ClusterDataSerializerHook.MEMBER_INFO_UPDATE) Collection(java.util.Collection) START_EXECUTION_OP(com.hazelcast.jet.impl.execution.init.JetInitDataSerializerHook.START_EXECUTION_OP) JobConfig(com.hazelcast.jet.config.JobConfig) StuckProcessor(com.hazelcast.jet.core.TestProcessors.StuckProcessor) Set(java.util.Set) PartitionDataSerializerHook(com.hazelcast.internal.partition.impl.PartitionDataSerializerHook) JobResult(com.hazelcast.jet.impl.JobResult) CountDownLatch(java.util.concurrent.CountDownLatch) Assert.assertFalse(org.junit.Assert.assertFalse) MasterContext(com.hazelcast.jet.impl.MasterContext) JetInstance(com.hazelcast.jet.JetInstance) RunWith(org.junit.runner.RunWith) JetInitDataSerializerHook(com.hazelcast.jet.impl.execution.init.JetInitDataSerializerHook) HashSet(java.util.HashSet) InitExecutionOperation(com.hazelcast.jet.impl.operation.InitExecutionOperation) ExpectedException(org.junit.rules.ExpectedException) Job(com.hazelcast.jet.Job) Before(org.junit.Before) JobRepository(com.hazelcast.jet.impl.JobRepository) HazelcastInstance(com.hazelcast.core.HazelcastInstance) JetConfig(com.hazelcast.jet.config.JetConfig) Matchers.empty(org.hamcrest.Matchers.empty) Assert.assertNotNull(org.junit.Assert.assertNotNull) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) Assert.assertNotEquals(org.junit.Assert.assertNotEquals) ExecutionException(java.util.concurrent.ExecutionException) Rule(org.junit.Rule) JetService(com.hazelcast.jet.impl.JetService) HazelcastParametersRunnerFactory(com.hazelcast.test.HazelcastParametersRunnerFactory) RUNNING(com.hazelcast.jet.core.JobStatus.RUNNING) Assert(org.junit.Assert) PacketFiltersUtil.resetPacketFiltersFrom(com.hazelcast.test.PacketFiltersUtil.resetPacketFiltersFrom) Assert.assertEquals(org.junit.Assert.assertEquals) SHUTDOWN_REQUEST(com.hazelcast.internal.partition.impl.PartitionDataSerializerHook.SHUTDOWN_REQUEST) MockPS(com.hazelcast.jet.core.TestProcessors.MockPS) JetService(com.hazelcast.jet.impl.JetService) Assert(org.junit.Assert) JetInstance(com.hazelcast.jet.JetInstance) Job(com.hazelcast.jet.Job) MasterContext(com.hazelcast.jet.impl.MasterContext) Test(org.junit.Test)

Aggregations

HazelcastInstance (com.hazelcast.core.HazelcastInstance)2 HazelcastInstanceNotActiveException (com.hazelcast.core.HazelcastInstanceNotActiveException)2 MemberInfo (com.hazelcast.internal.cluster.MemberInfo)2 ClusterDataSerializerHook (com.hazelcast.internal.cluster.impl.ClusterDataSerializerHook)2 MEMBER_INFO_UPDATE (com.hazelcast.internal.cluster.impl.ClusterDataSerializerHook.MEMBER_INFO_UPDATE)2 PartitionDataSerializerHook (com.hazelcast.internal.partition.impl.PartitionDataSerializerHook)2 SHUTDOWN_REQUEST (com.hazelcast.internal.partition.impl.PartitionDataSerializerHook.SHUTDOWN_REQUEST)2 Job (com.hazelcast.jet.Job)2 JobConfig (com.hazelcast.jet.config.JobConfig)2 RUNNING (com.hazelcast.jet.core.JobStatus.RUNNING)2 STARTING (com.hazelcast.jet.core.JobStatus.STARTING)2 MockPS (com.hazelcast.jet.core.TestProcessors.MockPS)2 JobRepository (com.hazelcast.jet.impl.JobRepository)2 JobResult (com.hazelcast.jet.impl.JobResult)2 MasterContext (com.hazelcast.jet.impl.MasterContext)2 JetInitDataSerializerHook (com.hazelcast.jet.impl.execution.init.JetInitDataSerializerHook)2 INIT_EXECUTION_OP (com.hazelcast.jet.impl.execution.init.JetInitDataSerializerHook.INIT_EXECUTION_OP)2 START_EXECUTION_OP (com.hazelcast.jet.impl.execution.init.JetInitDataSerializerHook.START_EXECUTION_OP)2 InitExecutionOperation (com.hazelcast.jet.impl.operation.InitExecutionOperation)2 PacketFiltersUtil.dropOperationsBetween (com.hazelcast.test.PacketFiltersUtil.dropOperationsBetween)2