Examples with Tuple2 com.hazelcast.jet.datamodel.Tuple2 used on opensource projects

Example 6 with Tuple2

use of com.hazelcast.jet.datamodel.Tuple2 in project hazelcast by hazelcast.

the class WindowAggregateTest method aggregateBuilder_withComplexAggrOp.

@Test
public void aggregateBuilder_withComplexAggrOp() {
    // Given
    CoAggregateFixture fx = new CoAggregateFixture();
    // When
    WindowAggregateBuilder1<Integer> b = fx.stage0.aggregateBuilder();
    Tag<Integer> tag0_in = b.tag0();
    Tag<Integer> tag1_in = b.add(fx.newStage());
    CoAggregateOperationBuilder b2 = coAggregateOperationBuilder();
    Tag<Long> tag0 = b2.add(tag0_in, SUMMING);
    Tag<Long> tag1 = b2.add(tag1_in, SUMMING);
    StreamStage<WindowResult<ItemsByTag>> aggregated = b.build(b2.build());
    // Then
    aggregated.writeTo(sink);
    execute();
    assertEquals(fx.expectedString2, streamToString(this.<ItemsByTag>sinkStreamOfWinResult(), wr -> FORMAT_FN_2.apply(wr.end(), tuple2(wr.result().get(tag0), wr.result().get(tag1)))));
}

Example 7 with Tuple2

use of com.hazelcast.jet.datamodel.Tuple2 in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method when_hashJoinRebalanceMainStage_then_distributedEdge.

@Test
public void when_hashJoinRebalanceMainStage_then_distributedEdge() {
    // Given
    List<Integer> input = sequence(itemCount);
    String prefix = "value-";
    BatchStage<Integer> stage0Rebalanced = batchStageFromList(input).rebalance();
    BatchStage<Entry<Integer, String>> enrichingStage = batchStageFromList(input).map(i -> entry(i, prefix + i));
    // When
    BatchStage<Entry<Integer, String>> joined = stage0Rebalanced.hashJoin(enrichingStage, joinMapEntries(wholeItem()), // Method reference avoided due to JDK bug
    (k, v) -> entry(k, v));
    // Then
    joined.writeTo(sink);
    DAG dag = p.toDag();
    Edge stage0ToJoin = dag.getInboundEdges("2-way hash-join-joiner").get(0);
    assertTrue("Rebalancing should make the edge distributed", stage0ToJoin.isDistributed());
    assertNull("Didn't rebalance by key, the edge must not be partitioned", stage0ToJoin.getPartitioner());
    execute();
    Function<Entry<Integer, String>, String> formatFn = e -> String.format("(%04d, %s)", e.getKey(), e.getValue());
    assertEquals(streamToString(input.stream().map(i -> tuple2(i, prefix + i)), formatFn), streamToString(sinkStreamOfEntry(), formatFn));
}

Example 8 with Tuple2

use of com.hazelcast.jet.datamodel.Tuple2 in project hazelcast by hazelcast.

the class WriteFilePTest method stressTest.

private void stressTest(boolean graceful, boolean exactlyOnce) throws Exception {
    int numItems = 500;
    Pipeline p = Pipeline.create();
    p.readFrom(SourceBuilder.stream("src", procCtx -> tuple2(new int[1], procCtx.logger())).fillBufferFn((ctx, buf) -> {
        if (ctx.f0()[0] < numItems) {
            buf.add(ctx.f0()[0]++);
            sleepMillis(5);
        }
    }).createSnapshotFn(ctx -> {
        ctx.f1().fine("src vertex saved to snapshot: " + ctx.f0()[0]);
        return ctx.f0()[0];
    }).restoreSnapshotFn((ctx, state) -> {
        ctx.f0()[0] = state.get(0);
        ctx.f1().fine("src vertex restored from snapshot: " + ctx.f0()[0]);
    }).build()).withoutTimestamps().writeTo(Sinks.filesBuilder(directory.toString()).exactlyOnce(exactlyOnce).build()).setLocalParallelism(2);
    JobConfig config = new JobConfig().setProcessingGuarantee(EXACTLY_ONCE).setSnapshotIntervalMillis(50);
    JobProxy job = (JobProxy) instance().getJet().newJob(p, config);
    long endTime = System.nanoTime() + SECONDS.toNanos(60);
    do {
        assertJobStatusEventually(job, RUNNING);
        sleepMillis(100);
        job.restart(graceful);
        try {
            checkFileContents(0, numItems, exactlyOnce, true, false);
            // if content matches, break the loop. Otherwise restart and try again
            break;
        } catch (AssertionError ignored) {
        }
    } while (System.nanoTime() < endTime);
    waitForNextSnapshot(new JobRepository(instance()), job.getId(), 10, true);
    ditchJob(job, instances());
    // when the job is cancelled, there should be no temporary files
    checkFileContents(0, numItems, exactlyOnce, false, false);
}

Example 9 with Tuple2

use of com.hazelcast.jet.datamodel.Tuple2 in project hazelcast by hazelcast.

the class AsyncTransformUsingServiceOrderedP method tryFlushQueue.

/**
 * Drains items from the queue until either:
 * <ul><li>
 *     encountering a non-completed item
 * </li><li>
 *     the outbox gets full
 * </li></ul>
 *
 * @return true if there are no more in-flight items and everything was emitted
 *         to the outbox
 */
boolean tryFlushQueue() {
    // queue. It also doesn't shuffle the stream items.
    for (; ; ) {
        if (!emitFromTraverser(currentTraverser)) {
            return false;
        }
        Object o = queue.peek();
        if (o == null) {
            return true;
        }
        if (o instanceof Watermark) {
            watermarkTraverser.accept((Watermark) o);
            currentTraverser = watermarkTraverser;
            queuedWmCount--;
        } else {
            @SuppressWarnings("unchecked") Tuple2<T, CompletableFuture<IR>> cast = (Tuple2<T, CompletableFuture<IR>>) o;
            T item = cast.f0();
            CompletableFuture<IR> future = cast.f1();
            assert future != null;
            if (!future.isDone()) {
                return false;
            }
            try {
                currentTraverser = mapResultFn.apply(item, future.get());
                if (currentTraverser == null) {
                    currentTraverser = Traversers.empty();
                }
            } catch (Throwable e) {
                throw new JetException("Async operation completed exceptionally: " + e, e);
            }
        }
        queue.remove();
    }
}

Example 10 with Tuple2

use of com.hazelcast.jet.datamodel.Tuple2 in project hazelcast by hazelcast.

the class ExecutionPlanBuilder method getPartitionAssignment.

/**
 * Assign the partitions to their owners. Partitions whose owner isn't in
 * the {@code memberList}, are assigned to one of the members in a
 * round-robin way.
 */
public static Map<MemberInfo, int[]> getPartitionAssignment(NodeEngine nodeEngine, List<MemberInfo> memberList) {
    IPartitionService partitionService = nodeEngine.getPartitionService();
    Map<Address, MemberInfo> membersByAddress = memberList.stream().collect(toMap(MemberInfo::getAddress, identity()));
    final MemberInfo[] partitionOwners = new MemberInfo[partitionService.getPartitionCount()];
    int memberIndex = 0;
    for (int partitionId = 0; partitionId < partitionOwners.length; partitionId++) {
        Address address = partitionService.getPartitionOwnerOrWait(partitionId);
        MemberInfo member = membersByAddress.get(address);
        if (member == null) {
            // if the partition owner isn't in the current memberList, assign to one of the other members in
            // round-robin fashion
            member = memberList.get(memberIndex++ % memberList.size());
        }
        partitionOwners[partitionId] = member;
    }
    return IntStream.range(0, partitionOwners.length).mapToObj(i -> tuple2(partitionOwners[i], i)).collect(Collectors.groupingBy(Tuple2::f0, Collectors.mapping(Tuple2::f1, Collectors.collectingAndThen(Collectors.<Integer>toList(), intList -> intList.stream().mapToInt(Integer::intValue).toArray()))));
}