Examples with LongAccumulator com.hazelcast.jet.accumulator.LongAccumulator used on opensource projects

Example 11 with LongAccumulator

use of com.hazelcast.jet.accumulator.LongAccumulator in project hazelcast by hazelcast.

the class PostgresCdcWhiteBlackListIntegrationTest method pipeline.

private Pipeline pipeline(StreamSource<ChangeRecord> source) {
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(source).withNativeTimestamps(0).filter(t -> t.schema().startsWith(SCHEMA_PREFIX)).setLocalParallelism(1).<ChangeRecord>customTransform("filter_timestamps", filterTimestampsProcessorSupplier()).setLocalParallelism(1).groupingKey(record -> (Integer) record.key().toMap().get("id")).mapStateful(LongAccumulator::new, (accumulator, rowId, record) -> {
        long count = accumulator.get();
        accumulator.add(1);
        Operation operation = record.operation();
        RecordPart value = record.value();
        TableRow row = value.toObject(TableRow.class);
        return entry(rowId + "/" + count, operation + ":" + row);
    }).setLocalParallelism(1).peek().writeTo(Sinks.map(SINK_MAP_NAME));
    return pipeline;
}

Example 12 with LongAccumulator

use of com.hazelcast.jet.accumulator.LongAccumulator in project hazelcast by hazelcast.

the class MetricsTest method customUnit.

@Test
public void customUnit() {
    pipeline.readFrom(TestSources.items(0L, 1L, 2L, 3L, 4L)).mapStateful(LongAccumulator::new, (acc, i) -> {
        acc.add(i);
        Metric metric = Metrics.metric("sum", Unit.COUNT);
        metric.set(acc.get());
        return acc.get();
    }).writeTo(Sinks.noop());
    Job job = runPipeline(pipeline.toDag());
    JobMetricsChecker checker = new JobMetricsChecker(job);
    checker.assertSummedMetricValue("sum", 10);
}

Example 13 with LongAccumulator

use of com.hazelcast.jet.accumulator.LongAccumulator in project hazelcast by hazelcast.

the class Processors_slidingWindowingIntegrationTest method smokeTest.

@Test
public void smokeTest() throws Exception {
    HazelcastInstance instance = createHazelcastInstance();
    SlidingWindowPolicy wDef = slidingWinPolicy(2000, 1000);
    DAG dag = new DAG();
    // to prevent serialization of whole class
    boolean isBatchLocal = isBatch;
    FunctionEx<? super MyEvent, ?> keyFn = MyEvent::getKey;
    ToLongFunctionEx<? super MyEvent> timestampFn = MyEvent::getTimestamp;
    List<MyEvent> inputData = singletonList(new MyEvent(10, "a", 1L));
    Vertex source = dag.newVertex("source", () -> new EmitListP(inputData, isBatchLocal)).localParallelism(1);
    Vertex insertPP = dag.newVertex("insertWmP", insertWatermarksP(eventTimePolicy(timestampFn, limitingLag(0), wDef.frameSize(), wDef.frameOffset(), 0))).localParallelism(1);
    Vertex sink = dag.newVertex("sink", SinkProcessors.writeListP("sink"));
    dag.edge(between(source, insertPP).isolated());
    AggregateOperation<LongAccumulator, Long> counting = counting();
    if (singleStageProcessor) {
        Vertex slidingWin = dag.newVertex("slidingWin", Processors.aggregateToSlidingWindowP(singletonList(keyFn), singletonList(timestampFn), TimestampKind.EVENT, wDef, 0L, counting, KeyedWindowResult::new));
        dag.edge(between(insertPP, slidingWin).partitioned(MyEvent::getKey).distributed()).edge(between(slidingWin, sink));
    } else {
        Vertex accumulateByFrame = dag.newVertex("accumulateByFrame", Processors.accumulateByFrameP(singletonList(keyFn), singletonList(timestampFn), TimestampKind.EVENT, wDef, counting.withIdentityFinish()));
        Vertex slidingWin = dag.newVertex("slidingWin", combineToSlidingWindowP(wDef, counting, KeyedWindowResult::new));
        dag.edge(between(insertPP, accumulateByFrame).partitioned(keyFn)).edge(between(accumulateByFrame, slidingWin).partitioned(entryKey()).distributed()).edge(between(slidingWin, sink));
    }
    Job job = instance.getJet().newJob(dag);
    if (isBatch) {
        job.join();
    }
    IList<MyEvent> sinkList = instance.getList("sink");
    List<KeyedWindowResult<String, Long>> expectedOutput = asList(new KeyedWindowResult<>(-1000, 1000, "a", 1L), new KeyedWindowResult<>(0, 2000, "a", 1L));
    assertTrueEventually(() -> assertEquals(streamToString(expectedOutput.stream()), streamToString(new ArrayList<>(sinkList).stream())), 5);
    // wait a little more and make sure, that there are no more frames
    Thread.sleep(1000);
    assertEquals(expectedOutput.size(), sinkList.size());
}

Example 14 with LongAccumulator

use of com.hazelcast.jet.accumulator.LongAccumulator in project hazelcast by hazelcast.

the class SessionWindowP_failoverTest method init.

private void init(ProcessingGuarantee guarantee) throws Exception {
    AggregateOperation1<Object, LongAccumulator, Long> aggrOp = counting();
    p = new SessionWindowP<>(5000, 0L, singletonList((ToLongFunctionEx<Entry<?, Long>>) Entry::getValue), singletonList(entryKey()), aggrOp, KeyedWindowResult::new);
    Outbox outbox = new TestOutbox(128);
    Context context = new TestProcessorContext().setProcessingGuarantee(guarantee);
    p.init(outbox, context);
}

Example 15 with LongAccumulator

use of com.hazelcast.jet.accumulator.LongAccumulator in project hazelcast by hazelcast.

the class OrderedBatchParallelismTest method applyTransformAndGetDag.

private DAG applyTransformAndGetDag(FunctionEx<BatchStage<Long>, BatchStage<Long>> transform) {
    PipelineImpl p = (PipelineImpl) Pipeline.create().setPreserveOrder(true);
    BatchStage<Long> source = p.readFrom(TestSources.items(1L)).setLocalParallelism(UPSTREAM_PARALLELISM);
    BatchStage<Long> applied = source.apply(transform);
    applied.mapStateful(LongAccumulator::new, (s, x) -> x).writeTo(Sinks.noop());
    return p.toDag(PIPELINE_CTX);
}