Examples with MockPS com.hazelcast.jet.core.TestProcessors.MockPS used on opensource projects

Example 31 with MockPS

use of com.hazelcast.jet.core.TestProcessors.MockPS in project hazelcast by hazelcast.

the class SplitBrainTest method when_jobIsSubmittedToMinoritySide_then_jobIsCancelledDuringMerge.

@Test
public void when_jobIsSubmittedToMinoritySide_then_jobIsCancelledDuringMerge() {
    int firstSubClusterSize = 3;
    int secondSubClusterSize = 2;
    NoOutputSourceP.executionStarted = new CountDownLatch(secondSubClusterSize * PARALLELISM);
    Job[] jobRef = new Job[1];
    BiConsumer<HazelcastInstance[], HazelcastInstance[]> onSplit = (firstSubCluster, secondSubCluster) -> {
        MockPS processorSupplier = new MockPS(NoOutputSourceP::new, secondSubClusterSize);
        DAG dag = new DAG().vertex(new Vertex("test", processorSupplier));
        jobRef[0] = secondSubCluster[0].getJet().newJob(dag, new JobConfig().setSplitBrainProtection(true));
        assertOpenEventually(NoOutputSourceP.executionStarted);
    };
    Consumer<HazelcastInstance[]> afterMerge = instances -> {
        assertTrueEventually(() -> assertEquals(secondSubClusterSize, MockPS.receivedCloseErrors.size()), 20);
        MockPS.receivedCloseErrors.forEach(t -> assertTrue("received: " + t, t instanceof CancellationException));
        try {
            jobRef[0].getFuture().get(30, TimeUnit.SECONDS);
            fail();
        } catch (CancellationException ignored) {
        } catch (Exception e) {
            throw new AssertionError(e);
        }
    };
    testSplitBrain(firstSubClusterSize, secondSubClusterSize, null, onSplit, afterMerge);
}

Example 32 with MockPS

use of com.hazelcast.jet.core.TestProcessors.MockPS in project hazelcast by hazelcast.

the class SplitBrainTest method when_quorumIsLostOnMinority_then_jobDoesNotRestartOnMinorityAndCancelledAfterMerge.

@Test
public void when_quorumIsLostOnMinority_then_jobDoesNotRestartOnMinorityAndCancelledAfterMerge() {
    int firstSubClusterSize = 3;
    int secondSubClusterSize = 2;
    int clusterSize = firstSubClusterSize + secondSubClusterSize;
    NoOutputSourceP.executionStarted = new CountDownLatch(clusterSize * PARALLELISM);
    Job[] jobRef = new Job[1];
    Consumer<HazelcastInstance[]> beforeSplit = instances -> {
        MockPS processorSupplier = new MockPS(NoOutputSourceP::new, clusterSize);
        DAG dag = new DAG().vertex(new Vertex("test", processorSupplier));
        jobRef[0] = instances[0].getJet().newJob(dag, new JobConfig().setSplitBrainProtection(true));
        assertOpenEventually(NoOutputSourceP.executionStarted);
    };
    Future[] minorityJobFutureRef = new Future[1];
    BiConsumer<HazelcastInstance[], HazelcastInstance[]> onSplit = (firstSubCluster, secondSubCluster) -> {
        NoOutputSourceP.proceedLatch.countDown();
        assertTrueEventually(() -> assertEquals(clusterSize + firstSubClusterSize, MockPS.initCount.get()));
        long jobId = jobRef[0].getId();
        assertTrueEventually(() -> {
            JetServiceBackend service = getJetServiceBackend(firstSubCluster[0]);
            assertEquals(COMPLETED, service.getJobCoordinationService().getJobStatus(jobId).get());
        });
        JetServiceBackend service2 = getJetServiceBackend(secondSubCluster[0]);
        assertTrueEventually(() -> {
            MasterContext masterContext = service2.getJobCoordinationService().getMasterContext(jobId);
            assertNotNull(masterContext);
            minorityJobFutureRef[0] = masterContext.jobContext().jobCompletionFuture();
        });
        assertTrueAllTheTime(() -> {
            assertStatusNotRunningOrStarting(service2.getJobCoordinationService().getJobStatus(jobId).get());
        }, 20);
    };
    Consumer<HazelcastInstance[]> afterMerge = instances -> {
        assertTrueEventually(() -> {
            assertEquals(clusterSize + firstSubClusterSize, MockPS.initCount.get());
            assertEquals(clusterSize + firstSubClusterSize, MockPS.closeCount.get());
        });
        assertEquals(clusterSize, MockPS.receivedCloseErrors.size());
        MockPS.receivedCloseErrors.forEach(t -> assertTrue("received " + t, t instanceof CancellationException));
        try {
            minorityJobFutureRef[0].get();
            fail();
        } catch (CancellationException expected) {
        } catch (Exception e) {
            throw new AssertionError(e);
        }
    };
    testSplitBrain(firstSubClusterSize, secondSubClusterSize, beforeSplit, onSplit, afterMerge);
}

Example 33 with MockPS

use of com.hazelcast.jet.core.TestProcessors.MockPS in project hazelcast by hazelcast.

the class SuspendResumeTest method before.

@Before
public void before() {
    TestProcessors.reset(NODE_COUNT * PARALLELISM);
    instances = new HazelcastInstance[NODE_COUNT];
    config = smallInstanceConfig();
    config.getJetConfig().setCooperativeThreadCount(PARALLELISM);
    for (int i = 0; i < NODE_COUNT; i++) {
        instances[i] = createHazelcastInstance(config);
    }
    dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT)));
}

Example 34 with MockPS

use of com.hazelcast.jet.core.TestProcessors.MockPS in project hazelcast by hazelcast.

the class JobLifecycleMetricsTest method multipleJobsSubmittedAndCompleted.

@Test
public void multipleJobsSubmittedAndCompleted() {
    // when
    Job job1 = hzInstances[0].getJet().newJob(batchPipeline());
    job1.join();
    job1.cancel();
    // then
    assertTrueEventually(() -> assertJobStats(1, 1, 1, 1, 0));
    // given
    DAG dag = new DAG();
    Throwable e = new AssertionError("mock error");
    Vertex source = dag.newVertex("source", ListSource.supplier(singletonList(1)));
    Vertex process = dag.newVertex("faulty", new MockPMS(() -> new MockPS(() -> new MockP().setProcessError(e), MEMBER_COUNT)));
    dag.edge(between(source, process));
    // when
    Job job2 = hzInstances[0].getJet().newJob(dag);
    try {
        job2.join();
        fail("Expected exception not thrown!");
    } catch (Exception ex) {
    // ignore
    }
    // then
    assertTrueEventually(() -> assertJobStats(2, 2, 2, 1, 1));
}

Example 35 with MockPS

use of com.hazelcast.jet.core.TestProcessors.MockPS in project hazelcast by hazelcast.

the class TopologyChangeTest method when_nodeIsShuttingDownDuringInit_then_jobRestarts.

@Test
public void when_nodeIsShuttingDownDuringInit_then_jobRestarts() {
    // Given that newInstance will have a long shutdown process
    for (HazelcastInstance instance : instances) {
        warmUpPartitions(instance);
    }
    dropOperationsBetween(instances[2], instances[0], PartitionDataSerializerHook.F_ID, singletonList(SHUTDOWN_REQUEST));
    rejectOperationsBetween(instances[0], instances[2], JetInitDataSerializerHook.FACTORY_ID, singletonList(INIT_EXECUTION_OP));
    // When a job participant starts its shutdown after the job is submitted
    DAG dag = new DAG().vertex(new Vertex("test", new MockPS(TestProcessors.Identity::new, nodeCount - 1)));
    Job job = instances[0].getJet().newJob(dag);
    JetServiceBackend jetServiceBackend = getJetServiceBackend(instances[0]);
    assertTrueEventually(() -> assertFalse(jetServiceBackend.getJobCoordinationService().getMasterContexts().isEmpty()));
    spawn(() -> instances[2].shutdown());
    // Then, it restarts until the shutting down node is gone
    assertJobStatusEventually(job, STARTING);
    assertTrueAllTheTime(() -> assertEquals(STARTING, job.getStatus()), 5);
    resetPacketFiltersFrom(instances[2]);
    job.join();
}