Examples with NoOutputSourceP com.hazelcast.jet.core.TestProcessors.NoOutputSourceP used on opensource projects

Example 1 with NoOutputSourceP

use of com.hazelcast.jet.core.TestProcessors.NoOutputSourceP in project hazelcast by hazelcast.

the class OperationLossTest method when_terminalSnapshotOperationLost_then_jobRestarts.

@Test
public void when_terminalSnapshotOperationLost_then_jobRestarts() {
    PacketFiltersUtil.dropOperationsFrom(instance(), JetInitDataSerializerHook.FACTORY_ID, singletonList(JetInitDataSerializerHook.SNAPSHOT_PHASE1_OPERATION));
    DAG dag = new DAG();
    Vertex v1 = dag.newVertex("v1", () -> new NoOutputSourceP()).localParallelism(1);
    Vertex v2 = dag.newVertex("v2", mapP(identity())).localParallelism(1);
    dag.edge(between(v1, v2).distributed());
    Job job = instance().getJet().newJob(dag, new JobConfig().setProcessingGuarantee(EXACTLY_ONCE));
    assertJobStatusEventually(job, RUNNING, 20);
    job.restart();
    // sleep so that the SnapshotOperation is sent out, but lost
    sleepSeconds(1);
    // reset filters so that the situation can resolve
    PacketFiltersUtil.resetPacketFiltersFrom(instance());
    // Then
    assertTrueEventually(() -> assertEquals(4, NoOutputSourceP.initCount.get()));
    NoOutputSourceP.proceedLatch.countDown();
    job.join();
}

Example 2 with NoOutputSourceP

use of com.hazelcast.jet.core.TestProcessors.NoOutputSourceP in project hazelcast by hazelcast.

the class OperationLossTest method when_operationLost_then_jobRestarts.

private void when_operationLost_then_jobRestarts(int operationId, JobStatus expectedStatus) {
    PacketFiltersUtil.dropOperationsFrom(instance(), JetInitDataSerializerHook.FACTORY_ID, singletonList(operationId));
    DAG dag = new DAG();
    Vertex v1 = dag.newVertex("v1", () -> new NoOutputSourceP()).localParallelism(1);
    Vertex v2 = dag.newVertex("v2", mapP(identity())).localParallelism(1);
    dag.edge(between(v1, v2).distributed());
    Job job = instance().getJet().newJob(dag);
    assertJobStatusEventually(job, expectedStatus);
    // NOT_RUNNING will occur briefly, we might miss to observe it. But restart occurs every
    // second (that's the operation heartbeat timeout) so hopefully we'll eventually succeed.
    assertJobStatusEventually(job, NOT_RUNNING);
    // now allow the job to complete normally
    PacketFiltersUtil.resetPacketFiltersFrom(instance());
    NoOutputSourceP.proceedLatch.countDown();
    job.join();
}

Example 3 with NoOutputSourceP

use of com.hazelcast.jet.core.TestProcessors.NoOutputSourceP in project hazelcast by hazelcast.

the class OperationLossTest method when_initExecutionOperationLost_then_initRetried_lightJob.

@Test
public void when_initExecutionOperationLost_then_initRetried_lightJob() {
    PacketFiltersUtil.dropOperationsFrom(instance(), JetInitDataSerializerHook.FACTORY_ID, singletonList(JetInitDataSerializerHook.INIT_EXECUTION_OP));
    DAG dag = new DAG();
    Vertex v1 = dag.newVertex("v1", () -> new NoOutputSourceP());
    Vertex v2 = dag.newVertex("v2", mapP(identity())).localParallelism(1);
    dag.edge(between(v1, v2).distributed());
    Job job = instance().getJet().newLightJob(dag);
    // we assert that the PMS is initialized, but the PS isn't
    JobExecutionService jobExecutionService = getNodeEngineImpl(instances()[1]).<JetServiceBackend>getService(JetServiceBackend.SERVICE_NAME).getJobExecutionService();
    // assert that the execution doesn't start on the 2nd member. For light jobs, jobId == executionId
    assertTrueAllTheTime(() -> assertNull(jobExecutionService.getExecutionContext(job.getId())), 1);
    // now allow the job to complete normally
    PacketFiltersUtil.resetPacketFiltersFrom(instance());
    NoOutputSourceP.proceedLatch.countDown();
    job.join();
}

Example 4 with NoOutputSourceP

use of com.hazelcast.jet.core.TestProcessors.NoOutputSourceP in project hazelcast by hazelcast.

the class Job_SeparateClusterTest method stressTest_getJobStatus.

private void stressTest_getJobStatus(Supplier<HazelcastInstance> submitterSupplier) throws Exception {
    DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT * 2)));
    AtomicReference<Job> job = new AtomicReference<>(submitterSupplier.get().getJet().newJob(dag));
    AtomicBoolean done = new AtomicBoolean();
    List<Runnable> actions = asList(() -> job.get().getStatus(), () -> job.get().getMetrics(), () -> job.get().getConfig());
    List<Future> checkerFutures = new ArrayList<>();
    for (Runnable action : actions) {
        checkerFutures.add(spawn(() -> {
            while (!done.get()) {
                action.run();
            }
        }));
    }
    for (int i = 0; i < 5; i++) {
        instance1.shutdown();
        instance1 = createHazelcastInstance();
        job.set(submitterSupplier.get().getJet().getJob(job.get().getId()));
        assertJobStatusEventually(job.get(), RUNNING);
        instance2.shutdown();
        instance2 = createHazelcastInstance();
        job.set(submitterSupplier.get().getJet().getJob(job.get().getId()));
        assertJobStatusEventually(job.get(), RUNNING);
        sleepSeconds(1);
        if (checkerFutures.stream().anyMatch(Future::isDone)) {
            break;
        }
    }
    done.set(true);
    for (Future future : checkerFutures) {
        future.get();
    }
}

Example 5 with NoOutputSourceP

use of com.hazelcast.jet.core.TestProcessors.NoOutputSourceP in project hazelcast by hazelcast.

the class Job_SeparateClusterTest method when_joinFromClientTimesOut_then_futureShouldNotBeCompletedEarly.

@Test
public void when_joinFromClientTimesOut_then_futureShouldNotBeCompletedEarly() throws InterruptedException {
    DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT)));
    int timeoutSecs = 1;
    ClientConfig config = new ClientConfig().setProperty(ClientProperty.INVOCATION_TIMEOUT_SECONDS.getName(), Integer.toString(timeoutSecs));
    HazelcastInstance client = createHazelcastClient(config);
    // join request is sent along with job submission
    Job job = client.getJet().newJob(dag);
    NoOutputSourceP.executionStarted.await();
    // wait for join invocation to timeout
    Thread.sleep(TimeUnit.SECONDS.toMillis(timeoutSecs));
    // When
    NoOutputSourceP.initCount.set(0);
    instance1.getLifecycleService().terminate();
    // wait for job to be restarted on remaining node
    assertTrueEventually(() -> assertEquals(LOCAL_PARALLELISM, NoOutputSourceP.initCount.get()));
    RuntimeException ex = new RuntimeException("Faulty job");
    NoOutputSourceP.failure.set(ex);
    // Then
    expectedException.expectMessage(Matchers.containsString(ex.getMessage()));
    job.join();
}