Examples with Tuple2.tuple2 com.hazelcast.jet.datamodel.Tuple2.tuple2 used on opensource projects

Example 6 with Tuple2.tuple2

use of com.hazelcast.jet.datamodel.Tuple2.tuple2 in project hazelcast by hazelcast.

the class OptUtils method extractKeyConstantExpression.

@SuppressWarnings("checkstyle:AvoidNestedBlocks")
public static RexNode extractKeyConstantExpression(RelOptTable relTable, RexBuilder rexBuilder) {
    HazelcastTable table = relTable.unwrap(HazelcastTable.class);
    RexNode filter = table.getFilter();
    if (filter == null) {
        return null;
    }
    int keyIndex = findKeyIndex(table.getTarget());
    switch(filter.getKind()) {
        // WHERE __key = true, calcite simplifies to just `WHERE __key`
        case INPUT_REF:
            {
                return ((RexInputRef) filter).getIndex() == keyIndex ? rexBuilder.makeLiteral(true) : null;
            }
        // WHERE __key = false, calcite simplifies to `WHERE NOT __key`
        case NOT:
            {
                RexNode operand = ((RexCall) filter).getOperands().get(0);
                return operand.getKind() == SqlKind.INPUT_REF && ((RexInputRef) operand).getIndex() == keyIndex ? rexBuilder.makeLiteral(false) : null;
            }
        // __key = ...
        case EQUALS:
            {
                Tuple2<Integer, RexNode> constantExpressionByIndex = extractConstantExpression((RexCall) filter);
                // noinspection ConstantConditions
                return constantExpressionByIndex != null && constantExpressionByIndex.getKey() == keyIndex ? constantExpressionByIndex.getValue() : null;
            }
        default:
            return null;
    }
}

Example 7 with Tuple2.tuple2

use of com.hazelcast.jet.datamodel.Tuple2.tuple2 in project hazelcast by hazelcast.

the class WindowAggregateTest method aggregateBuilder_withComplexAggrOp.

@Test
public void aggregateBuilder_withComplexAggrOp() {
    // Given
    CoAggregateFixture fx = new CoAggregateFixture();
    // When
    WindowAggregateBuilder1<Integer> b = fx.stage0.aggregateBuilder();
    Tag<Integer> tag0_in = b.tag0();
    Tag<Integer> tag1_in = b.add(fx.newStage());
    CoAggregateOperationBuilder b2 = coAggregateOperationBuilder();
    Tag<Long> tag0 = b2.add(tag0_in, SUMMING);
    Tag<Long> tag1 = b2.add(tag1_in, SUMMING);
    StreamStage<WindowResult<ItemsByTag>> aggregated = b.build(b2.build());
    // Then
    aggregated.writeTo(sink);
    execute();
    assertEquals(fx.expectedString2, streamToString(this.<ItemsByTag>sinkStreamOfWinResult(), wr -> FORMAT_FN_2.apply(wr.end(), tuple2(wr.result().get(tag0), wr.result().get(tag1)))));
}

Example 8 with Tuple2.tuple2

use of com.hazelcast.jet.datamodel.Tuple2.tuple2 in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method when_hashJoinRebalanceMainStage_then_distributedEdge.

@Test
public void when_hashJoinRebalanceMainStage_then_distributedEdge() {
    // Given
    List<Integer> input = sequence(itemCount);
    String prefix = "value-";
    BatchStage<Integer> stage0Rebalanced = batchStageFromList(input).rebalance();
    BatchStage<Entry<Integer, String>> enrichingStage = batchStageFromList(input).map(i -> entry(i, prefix + i));
    // When
    BatchStage<Entry<Integer, String>> joined = stage0Rebalanced.hashJoin(enrichingStage, joinMapEntries(wholeItem()), // Method reference avoided due to JDK bug
    (k, v) -> entry(k, v));
    // Then
    joined.writeTo(sink);
    DAG dag = p.toDag();
    Edge stage0ToJoin = dag.getInboundEdges("2-way hash-join-joiner").get(0);
    assertTrue("Rebalancing should make the edge distributed", stage0ToJoin.isDistributed());
    assertNull("Didn't rebalance by key, the edge must not be partitioned", stage0ToJoin.getPartitioner());
    execute();
    Function<Entry<Integer, String>, String> formatFn = e -> String.format("(%04d, %s)", e.getKey(), e.getValue());
    assertEquals(streamToString(input.stream().map(i -> tuple2(i, prefix + i)), formatFn), streamToString(sinkStreamOfEntry(), formatFn));
}

Example 9 with Tuple2.tuple2

use of com.hazelcast.jet.datamodel.Tuple2.tuple2 in project hazelcast by hazelcast.

the class WriteFilePTest method stressTest.

private void stressTest(boolean graceful, boolean exactlyOnce) throws Exception {
    int numItems = 500;
    Pipeline p = Pipeline.create();
    p.readFrom(SourceBuilder.stream("src", procCtx -> tuple2(new int[1], procCtx.logger())).fillBufferFn((ctx, buf) -> {
        if (ctx.f0()[0] < numItems) {
            buf.add(ctx.f0()[0]++);
            sleepMillis(5);
        }
    }).createSnapshotFn(ctx -> {
        ctx.f1().fine("src vertex saved to snapshot: " + ctx.f0()[0]);
        return ctx.f0()[0];
    }).restoreSnapshotFn((ctx, state) -> {
        ctx.f0()[0] = state.get(0);
        ctx.f1().fine("src vertex restored from snapshot: " + ctx.f0()[0]);
    }).build()).withoutTimestamps().writeTo(Sinks.filesBuilder(directory.toString()).exactlyOnce(exactlyOnce).build()).setLocalParallelism(2);
    JobConfig config = new JobConfig().setProcessingGuarantee(EXACTLY_ONCE).setSnapshotIntervalMillis(50);
    JobProxy job = (JobProxy) instance().getJet().newJob(p, config);
    long endTime = System.nanoTime() + SECONDS.toNanos(60);
    do {
        assertJobStatusEventually(job, RUNNING);
        sleepMillis(100);
        job.restart(graceful);
        try {
            checkFileContents(0, numItems, exactlyOnce, true, false);
            // if content matches, break the loop. Otherwise restart and try again
            break;
        } catch (AssertionError ignored) {
        }
    } while (System.nanoTime() < endTime);
    waitForNextSnapshot(new JobRepository(instance()), job.getId(), 10, true);
    ditchJob(job, instances());
    // when the job is cancelled, there should be no temporary files
    checkFileContents(0, numItems, exactlyOnce, false, false);
}

Example 10 with Tuple2.tuple2

use of com.hazelcast.jet.datamodel.Tuple2.tuple2 in project hazelcast-jet-reference-manual by hazelcast.

the class BuildComputation method s11.

static void s11() {
    // tag::s11[]
    Pipeline p = Pipeline.create();
    // The stream to be enriched: trades
    StreamStage<Trade> trades = p.drawFrom(Sources.mapJournal("trades", mapPutEvents(), mapEventNewValue(), START_FROM_CURRENT));
    // The enriching streams: products, brokers and markets
    BatchStage<Entry<Integer, Product>> prodEntries = p.drawFrom(Sources.map("products"));
    BatchStage<Entry<Integer, Broker>> brokEntries = p.drawFrom(Sources.map("brokers"));
    BatchStage<Entry<Integer, Market>> marketEntries = p.drawFrom(Sources.map("markets"));
    // Obtain a hash-join builder object from the stream to be enriched
    StreamHashJoinBuilder<Trade> builder = trades.hashJoinBuilder();
    // Add enriching streams to the builder
    Tag<Product> productTag = builder.add(prodEntries, joinMapEntries(Trade::productId));
    Tag<Broker> brokerTag = builder.add(brokEntries, joinMapEntries(Trade::brokerId));
    Tag<Market> marketTag = builder.add(marketEntries, joinMapEntries(Trade::marketId));
    // Build the hash join pipeline
    StreamStage<Tuple2<Trade, ItemsByTag>> joined = builder.build(Tuple2::tuple2);
    // end::s11[]
    // tag::s12[]
    StreamStage<String> mapped = joined.map((Tuple2<Trade, ItemsByTag> tuple) -> {
        Trade trade = tuple.f0();
        ItemsByTag ibt = tuple.f1();
        Product product = ibt.get(productTag);
        Broker broker = ibt.get(brokerTag);
        Market market = ibt.get(marketTag);
        return trade + ": " + product + ", " + broker + ", " + market;
    });
// end::s12[]
}