Examples with Tuple3.tuple3 com.hazelcast.jet.datamodel.Tuple3.tuple3 used on opensource projects

Example 1 with Tuple3.tuple3

use of com.hazelcast.jet.datamodel.Tuple3.tuple3 in project hazelcast by hazelcast.

the class MasterSnapshotContext method tryBeginSnapshot.

void tryBeginSnapshot() {
    mc.coordinationService().submitToCoordinatorThread(() -> {
        boolean isTerminal;
        String snapshotMapName;
        CompletableFuture<Void> future;
        mc.lock();
        long localExecutionId;
        try {
            if (mc.jobStatus() != RUNNING) {
                logger.fine("Not beginning snapshot, " + mc.jobIdString() + " is not RUNNING, but " + mc.jobStatus());
                return;
            }
            if (snapshotInProgress) {
                logger.fine("Not beginning snapshot since one is already in progress " + mc.jobIdString());
                return;
            }
            if (terminalSnapshotFuture.isDone()) {
                logger.fine("Not beginning snapshot since terminal snapshot is already completed " + mc.jobIdString());
                return;
            }
            Tuple3<String, Boolean, CompletableFuture<Void>> requestedSnapshot = snapshotQueue.poll();
            if (requestedSnapshot == null) {
                return;
            }
            snapshotInProgress = true;
            snapshotMapName = requestedSnapshot.f0();
            assert requestedSnapshot.f1() != null;
            isTerminal = requestedSnapshot.f1();
            future = requestedSnapshot.f2();
            mc.jobExecutionRecord().startNewSnapshot(snapshotMapName);
            localExecutionId = mc.executionId();
        } finally {
            mc.unlock();
        }
        mc.writeJobExecutionRecord(false);
        long newSnapshotId = mc.jobExecutionRecord().ongoingSnapshotId();
        boolean isExport = snapshotMapName != null;
        int snapshotFlags = SnapshotFlags.create(isTerminal, isExport);
        String finalMapName = isExport ? exportedSnapshotMapName(snapshotMapName) : snapshotDataMapName(mc.jobId(), mc.jobExecutionRecord().ongoingDataMapIndex());
        mc.nodeEngine().getHazelcastInstance().getMap(finalMapName).clear();
        logFine(logger, "Starting snapshot %d for %s, flags: %s, writing to: %s", newSnapshotId, jobNameAndExecutionId(mc.jobName(), localExecutionId), SnapshotFlags.toString(snapshotFlags), snapshotMapName);
        Function<ExecutionPlan, Operation> factory = plan -> new SnapshotPhase1Operation(mc.jobId(), localExecutionId, newSnapshotId, finalMapName, snapshotFlags);
        // Need to take a copy of executionId: we don't cancel the scheduled task when the execution
        // finalizes. If a new execution is started in the meantime, we'll use the execution ID to detect it.
        mc.invokeOnParticipants(factory, responses -> onSnapshotPhase1Complete(responses, localExecutionId, newSnapshotId, finalMapName, snapshotFlags, future), null, true);
    });
}

Example 2 with Tuple3.tuple3

use of com.hazelcast.jet.datamodel.Tuple3.tuple3 in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method when_rebalanceAndGroupAggregateBuilderWithComplexAggrOp_then_singleStageAggregation.

@Test
public void when_rebalanceAndGroupAggregateBuilderWithComplexAggrOp_then_singleStageAggregation() {
    // Given
    FunctionEx<Integer, Integer> keyFn = i -> i % 10;
    Iterator<Integer> sequence = IntStream.iterate(0, i -> i + 1).boxed().iterator();
    List<Integer> input0 = streamFromIterator(sequence).limit(itemCount).collect(toList());
    List<Integer> input1 = streamFromIterator(sequence).limit(itemCount).collect(toList());
    List<Integer> input2 = streamFromIterator(sequence).limit(itemCount).collect(toList());
    BatchStageWithKey<Integer, Integer> stage0 = batchStageFromList(input0).groupingKey(keyFn);
    BatchStageWithKey<Integer, Integer> stage1 = batchStageFromList(input1).groupingKey(keyFn);
    BatchStageWithKey<Integer, Integer> stage2Rebalanced = batchStageFromList(input2).rebalance().groupingKey(keyFn);
    // When
    GroupAggregateBuilder1<Integer, Integer> b = stage0.aggregateBuilder();
    Tag<Integer> tag0_in = b.tag0();
    Tag<Integer> tag1_in = b.add(stage1);
    Tag<Integer> tag2_in = b.add(stage2Rebalanced);
    CoAggregateOperationBuilder agb = coAggregateOperationBuilder();
    Tag<Long> tag0 = agb.add(tag0_in, SUMMING);
    Tag<Long> tag1 = agb.add(tag1_in, SUMMING);
    Tag<Long> tag2 = agb.add(tag2_in, SUMMING);
    AggregateOperation<Object[], ItemsByTag> aggrOp = agb.build();
    BatchStage<Entry<Integer, ItemsByTag>> aggregated = b.build(aggrOp);
    // Then
    aggregated.writeTo(sink);
    assertSingleStageAggregation();
    execute();
    Collector<Integer, ?, Map<Integer, Long>> groupAndSum = groupingBy(keyFn, summingLong(i -> i));
    Map<Integer, Long> expectedMap0 = input0.stream().collect(groupAndSum);
    Map<Integer, Long> expectedMap1 = input1.stream().collect(groupAndSum);
    Map<Integer, Long> expectedMap2 = input2.stream().collect(groupAndSum);
    assertEquals(streamToString(expectedMap0.entrySet().stream(), e -> FORMAT_FN_3.apply(e.getKey(), tuple3(e.getValue(), expectedMap1.get(e.getKey()), expectedMap2.get(e.getKey())))), streamToString(this.<Integer, ItemsByTag>sinkStreamOfEntry(), e -> FORMAT_FN_3.apply(e.getKey(), tuple3(e.getValue().get(tag0), e.getValue().get(tag1), e.getValue().get(tag2)))));
}

Example 3 with Tuple3.tuple3

use of com.hazelcast.jet.datamodel.Tuple3.tuple3 in project hazelcast by hazelcast.

the class BatchAggregateTest method groupAggregateBuilder_withComplexAggrOp.

@Test
public void groupAggregateBuilder_withComplexAggrOp() {
    // Given
    GroupAggregateFixture fx = new GroupAggregateFixture();
    BatchStageWithKey<Integer, Integer> stage0 = fx.srcStage0.groupingKey(fx.keyFn);
    BatchStageWithKey<Integer, Integer> stage1 = fx.srcStage1().groupingKey(fx.keyFn);
    BatchStageWithKey<Integer, Integer> stage2 = fx.srcStage2().groupingKey(fx.keyFn);
    // When
    GroupAggregateBuilder1<Integer, Integer> b = stage0.aggregateBuilder();
    Tag<Integer> tag0_in = b.tag0();
    Tag<Integer> tag1_in = b.add(stage1);
    Tag<Integer> tag2_in = b.add(stage2);
    CoAggregateOperationBuilder agb = coAggregateOperationBuilder();
    Tag<Long> tag0 = agb.add(tag0_in, SUMMING);
    Tag<Long> tag1 = agb.add(tag1_in, SUMMING);
    Tag<Long> tag2 = agb.add(tag2_in, SUMMING);
    AggregateOperation<Object[], ItemsByTag> aggrOp = agb.build();
    BatchStage<Entry<Integer, ItemsByTag>> aggregated = b.build(aggrOp);
    // Then
    aggregated.writeTo(sink);
    execute();
    Map<Integer, Long> expectedMap0 = input.stream().collect(groupingBy(fx.keyFn, fx.collectOp));
    Map<Integer, Long> expectedMap1 = input.stream().map(fx.mapFn1).collect(groupingBy(fx.keyFn, fx.collectOp));
    Map<Integer, Long> expectedMap2 = input.stream().map(fx.mapFn2).collect(groupingBy(fx.keyFn, fx.collectOp));
    assertEquals(streamToString(expectedMap0.entrySet().stream(), e -> FORMAT_FN_3.apply(e.getKey(), tuple3(e.getValue(), expectedMap1.get(e.getKey()), expectedMap2.get(e.getKey())))), streamToString(this.<Integer, ItemsByTag>sinkStreamOfEntry(), e -> FORMAT_FN_3.apply(e.getKey(), tuple3(e.getValue().get(tag0), e.getValue().get(tag1), e.getValue().get(tag2)))));
}

Example 4 with Tuple3.tuple3

use of com.hazelcast.jet.datamodel.Tuple3.tuple3 in project hazelcast-jet-reference-manual by hazelcast.

the class BuildComputation method s10.

static void s10() {
    // tag::s10[]
    Pipeline p = Pipeline.create();
    // The stream to be enriched: trades
    StreamStage<Trade> trades = p.drawFrom(Sources.mapJournal("trades", mapPutEvents(), mapEventNewValue(), START_FROM_CURRENT));
    // The enriching streams: products and brokers
    BatchStage<Entry<Integer, Product>> prodEntries = p.drawFrom(Sources.map("products"));
    BatchStage<Entry<Integer, Broker>> brokEntries = p.drawFrom(Sources.map("brokers"));
    // Join the trade stream with the product and broker streams
    StreamStage<Tuple3<Trade, Product, Broker>> joined = trades.hashJoin2(prodEntries, joinMapEntries(Trade::productId), brokEntries, joinMapEntries(Trade::brokerId), Tuple3::tuple3);
// end::s10[]
}