Examples with EXACTLY_ONCE com.hazelcast.jet.config.ProcessingGuarantee.EXACTLY_ONCE used on opensource projects

Example 11 with EXACTLY_ONCE

use of com.hazelcast.jet.config.ProcessingGuarantee.EXACTLY_ONCE in project hazelcast by hazelcast.

the class WriteKafkaP method init.

@Override
public void init(@Nonnull Outbox outbox, @Nonnull Context context) {
    this.context = context;
    ProcessingGuarantee guarantee = context.processingGuarantee() == EXACTLY_ONCE && !exactlyOnce ? AT_LEAST_ONCE : context.processingGuarantee();
    snapshotUtility = new TransactionPoolSnapshotUtility<>(outbox, context, false, guarantee, TXN_POOL_SIZE, (processorIndex, txnIndex) -> new KafkaTransactionId(context.jobId(), context.jobConfig().getName(), context.vertexName(), processorIndex, txnIndex), txnId -> {
        if (txnId != null) {
            properties.put("transactional.id", txnId.getKafkaId());
        }
        return new KafkaTransaction<>(txnId, properties, context.logger());
    }, txnId -> {
        try {
            recoverTransaction(txnId, true);
        } catch (ProducerFencedException e) {
            context.logger().warning("Failed to finish the commit of a transaction ID saved in the " + "snapshot, data loss can occur. Transaction id: " + txnId.getKafkaId(), e);
        }
    }, txnId -> recoverTransaction(txnId, false));
}

Example 12 with EXACTLY_ONCE

use of com.hazelcast.jet.config.ProcessingGuarantee.EXACTLY_ONCE in project hazelcast by hazelcast.

the class SourceBuilder_TopologyChangeTest method testTopologyChange.

private void testTopologyChange(Supplier<HazelcastInstance> secondMemberSupplier, Consumer<HazelcastInstance> changeTopologyFn, boolean assertMonotonicity) {
    stateRestored = false;
    StreamSource<Integer> source = SourceBuilder.timestampedStream("src", ctx -> new NumberGeneratorContext()).<Integer>fillBufferFn((src, buffer) -> {
        long expectedCount = NANOSECONDS.toMillis(System.nanoTime() - src.startTime);
        expectedCount = Math.min(expectedCount, src.current + 100);
        while (src.current < expectedCount) {
            buffer.add(src.current, src.current);
            src.current++;
        }
    }).createSnapshotFn(src -> {
        System.out.println("Will save " + src.current + " to snapshot");
        return src;
    }).restoreSnapshotFn((src, states) -> {
        stateRestored = true;
        assert states.size() == 1;
        src.restore(states.get(0));
        System.out.println("Restored " + src.current + " from snapshot");
    }).build();
    Config config = smallInstanceConfig();
    // restart sooner after member add
    config.getJetConfig().setScaleUpDelayMillis(1000);
    HazelcastInstance hz = createHazelcastInstance(config);
    HazelcastInstance possibleSecondNode = secondMemberSupplier.get();
    long windowSize = 100;
    IList<WindowResult<Long>> result = hz.getList("result-" + UuidUtil.newUnsecureUuidString());
    Pipeline p = Pipeline.create();
    p.readFrom(source).withNativeTimestamps(0).window(tumbling(windowSize)).aggregate(AggregateOperations.counting()).peek().writeTo(Sinks.list(result));
    Job job = hz.getJet().newJob(p, new JobConfig().setProcessingGuarantee(EXACTLY_ONCE).setSnapshotIntervalMillis(500));
    assertTrueEventually(() -> assertFalse("result list is still empty", result.isEmpty()));
    assertJobStatusEventually(job, JobStatus.RUNNING);
    JobRepository jr = new JobRepository(hz);
    waitForFirstSnapshot(jr, job.getId(), 10, false);
    assertFalse(stateRestored);
    changeTopologyFn.accept(possibleSecondNode);
    assertTrueEventually(() -> assertTrue("restoreSnapshotFn was not called", stateRestored));
    // wait until more results are added
    int oldSize = result.size();
    assertTrueEventually(() -> assertTrue("no more results added to the list", result.size() > oldSize));
    cancelAndJoin(job);
    // results should contain sequence of results, each with count=windowSize, monotonic, if job was
    // allowed to terminate gracefully
    Iterator<WindowResult<Long>> iterator = result.iterator();
    for (int i = 0; i < result.size(); i++) {
        WindowResult<Long> next = iterator.next();
        assertEquals(windowSize, (long) next.result());
        if (assertMonotonicity) {
            assertEquals(i * windowSize, next.start());
        }
    }
}

Example 13 with EXACTLY_ONCE

use of com.hazelcast.jet.config.ProcessingGuarantee.EXACTLY_ONCE in project hazelcast by hazelcast.

the class GracefulShutdownTest method when_shutDown.

private void when_shutDown(boolean shutdownCoordinator, boolean snapshotted) {
    DAG dag = new DAG();
    final int numItems = 50_000;
    Vertex source = dag.newVertex("source", throttle(() -> new EmitIntegersP(numItems), 10_000)).localParallelism(1);
    Vertex sink = dag.newVertex("sink", SinkProcessors.writeListP("sink"));
    dag.edge(between(source, sink));
    Job job = client.getJet().newJob(dag, new JobConfig().setProcessingGuarantee(snapshotted ? EXACTLY_ONCE : NONE).setSnapshotIntervalMillis(HOURS.toMillis(1)));
    assertJobStatusEventually(job, JobStatus.RUNNING);
    logger.info("sleeping 1 sec");
    sleepSeconds(1);
    int shutDownInstance = shutdownCoordinator ? 0 : 1;
    int liveInstance = shutdownCoordinator ? 1 : 0;
    // When
    logger.info("Shutting down instance...");
    instances[shutDownInstance].shutdown();
    logger.info("Joining job...");
    job.join();
    logger.info("Joined");
    // Then
    // If the shutdown was graceful, output items must not be duplicated
    Map<Integer, Integer> expected;
    Map<Integer, Integer> actual = new ArrayList<>(instances[liveInstance].<Integer>getList("sink")).stream().collect(Collectors.toMap(Function.identity(), item -> 1, Integer::sum));
    if (snapshotted) {
        logger.info("savedCounters=" + EmitIntegersP.savedCounters);
        assertEquals(EmitIntegersP.savedCounters.toString(), 2, EmitIntegersP.savedCounters.size());
        int minCounter = EmitIntegersP.savedCounters.values().stream().mapToInt(Integer::intValue).min().getAsInt();
        expected = IntStream.range(0, numItems).boxed().collect(Collectors.toMap(Function.identity(), item -> item < minCounter ? 2 : 1));
        assertEquals(expected, actual);
    } else {
        // Items 0, 1, ... up to the point when the member was shut down should be 3 times
        // in the output: twice from before shutdown and one from after it, because it will start
        // from the beginning when the job is not snapshotted.
        assertEquals(3, actual.get(0).intValue());
        assertEquals(3, actual.get(1).intValue());
        assertEquals(1, actual.get(numItems - 1).intValue());
    }
}

Example 14 with EXACTLY_ONCE

use of com.hazelcast.jet.config.ProcessingGuarantee.EXACTLY_ONCE in project hazelcast by hazelcast.

the class AsyncTransformUsingServiceP_IntegrationTest method before.

@Before
public void before() {
    journaledMap = instance().getMap(randomMapName("journaledMap"));
    journaledMap.putAll(IntStream.range(0, NUM_ITEMS).boxed().collect(toMap(i -> i, i -> i)));
    sinkList = instance().getList(randomMapName("sinkList"));
    jobConfig = new JobConfig().setProcessingGuarantee(EXACTLY_ONCE).setSnapshotIntervalMillis(0);
    serviceFactory = sharedService(pctx -> Executors.newFixedThreadPool(8), ExecutorService::shutdown);
}

Example 15 with EXACTLY_ONCE

use of com.hazelcast.jet.config.ProcessingGuarantee.EXACTLY_ONCE in project hazelcast by hazelcast.

the class VertexDef_HigherPrioritySourceTest method assertHigherPriorityVertices.

private void assertHigherPriorityVertices(Vertex... vertices) {
    JobConfig jobConfig = new JobConfig();
    Map<MemberInfo, ExecutionPlan> executionPlans = createExecutionPlans(nodeEngineImpl, membersView, dag, 0, 0, jobConfig, 0, false, null);
    ExecutionPlan plan = executionPlans.values().iterator().next();
    SnapshotContext ssContext = new SnapshotContext(mock(ILogger.class), "job", 0, EXACTLY_ONCE);
    // In the production code the plan#initialize is only called from places where we have already set up the
    // processor classloaders
    JetServiceBackend jetService = nodeEngineImpl.getService(JetServiceBackend.SERVICE_NAME);
    jetService.getJobClassLoaderService().getOrCreateClassLoader(jobConfig, 0, JobPhase.EXECUTION);
    try {
        jetService.getJobClassLoaderService().prepareProcessorClassLoaders(0);
        plan.initialize(nodeEngineImpl, 0, 0, ssContext, null, (InternalSerializationService) nodeEngineImpl.getSerializationService());
    } finally {
        jetService.getJobClassLoaderService().clearProcessorClassLoaders();
    }
    jetService.getJobClassLoaderService().tryRemoveClassloadersForJob(0, JobPhase.EXECUTION);
    Set<Integer> higherPriorityVertices = VertexDef.getHigherPriorityVertices(plan.getVertices());
    String actualHigherPriorityVertices = plan.getVertices().stream().filter(v -> higherPriorityVertices.contains(v.vertexId())).map(VertexDef::name).sorted().collect(joining("\n"));
    String expectedVertices = Arrays.stream(vertices).map(Vertex::getName).sorted().collect(joining("\n"));
    assertEquals(expectedVertices, actualHigherPriorityVertices);
}