Examples with Tuple2.tuple2 com.hazelcast.jet.datamodel.Tuple2.tuple2 used on opensource projects

Example 1 with Tuple2.tuple2

use of com.hazelcast.jet.datamodel.Tuple2.tuple2 in project hazelcast by hazelcast.

the class MasterJobContext method tryStartJob.

/**
 * Starts the execution of the job if it is not already completed,
 * cancelled or failed.
 * <p>
 * If the job is already cancelled, triggers the job completion procedure.
 * <p>
 * If the job quorum is not satisfied, reschedules the job restart.
 * <p>
 * If there was a membership change and the partition table is not completely
 * fixed yet, reschedules the job restart.
 */
void tryStartJob(Supplier<Long> executionIdSupplier) {
    mc.coordinationService().submitToCoordinatorThread(() -> {
        executionStartTime = System.currentTimeMillis();
        try {
            JobExecutionRecord jobExecRec = mc.jobExecutionRecord();
            jobExecRec.markExecuted();
            Tuple2<DAG, ClassLoader> dagAndClassloader = resolveDagAndCL(executionIdSupplier);
            if (dagAndClassloader == null) {
                return;
            }
            DAG dag = dagAndClassloader.f0();
            assert dag != null;
            ClassLoader classLoader = dagAndClassloader.f1();
            // must call this before rewriteDagWithSnapshotRestore()
            String dotRepresentation = dag.toDotString(defaultParallelism, defaultQueueSize);
            long snapshotId = jobExecRec.snapshotId();
            String snapshotName = mc.jobConfig().getInitialSnapshotName();
            String mapName = snapshotId >= 0 ? jobExecRec.successfulSnapshotDataMapName(mc.jobId()) : snapshotName != null ? EXPORTED_SNAPSHOTS_PREFIX + snapshotName : null;
            if (mapName != null) {
                rewriteDagWithSnapshotRestore(dag, snapshotId, mapName, snapshotName);
            } else {
                logger.info("Didn't find any snapshot to restore for " + mc.jobIdString());
            }
            MembersView membersView = Util.getMembersView(mc.nodeEngine());
            logger.info("Start executing " + mc.jobIdString() + ", execution graph in DOT format:\n" + dotRepresentation + "\nHINT: You can use graphviz or http://viz-js.com to visualize the printed graph.");
            logger.fine("Building execution plan for " + mc.jobIdString());
            Util.doWithClassLoader(classLoader, () -> mc.setExecutionPlanMap(createExecutionPlans(mc.nodeEngine(), membersView.getMembers(), dag, mc.jobId(), mc.executionId(), mc.jobConfig(), jobExecRec.ongoingSnapshotId(), false, mc.jobRecord().getSubject())));
            logger.fine("Built execution plans for " + mc.jobIdString());
            Set<MemberInfo> participants = mc.executionPlanMap().keySet();
            Version coordinatorVersion = mc.nodeEngine().getLocalMember().getVersion().asVersion();
            Function<ExecutionPlan, Operation> operationCtor = plan -> new InitExecutionOperation(mc.jobId(), mc.executionId(), membersView.getVersion(), coordinatorVersion, participants, mc.nodeEngine().getSerializationService().toData(plan), false);
            mc.invokeOnParticipants(operationCtor, this::onInitStepCompleted, null, false);
        } catch (Throwable e) {
            finalizeJob(e);
        }
    });
}

Example 2 with Tuple2.tuple2

use of com.hazelcast.jet.datamodel.Tuple2.tuple2 in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method when_hashJoinBuilderRebalanceMainStage_then_distributedEdge.

@Test
public void when_hashJoinBuilderRebalanceMainStage_then_distributedEdge() {
    // Given
    List<Integer> input = sequence(itemCount);
    String prefix = "value-";
    BatchStage<Integer> stage0Rebalanced = batchStageFromList(input).rebalance();
    BatchStage<Entry<Integer, String>> enrichingStage = batchStageFromList(input).map(i -> entry(i, prefix + i));
    // When
    HashJoinBuilder<Integer> b = stage0Rebalanced.hashJoinBuilder();
    Tag<String> tag1 = b.add(enrichingStage, joinMapEntries(wholeItem()));
    // Method reference avoided due to JDK bug
    BatchStage<Entry<Integer, ItemsByTag>> joined = b.build((k, v) -> entry(k, v));
    // Then
    joined.writeTo(sink);
    DAG dag = p.toDag();
    Edge stage0ToJoin = dag.getInboundEdges("2-way hash-join-joiner").get(0);
    assertTrue("Rebalancing should make the edge distributed", stage0ToJoin.isDistributed());
    assertNull("Didn't rebalance by key, the edge must not be partitioned", stage0ToJoin.getPartitioner());
    execute();
    Function<Entry<Integer, String>, String> formatFn = e -> String.format("(%04d, %s)", e.getKey(), e.getValue());
    assertEquals(streamToString(input.stream().map(i -> tuple2(i, prefix + i)), formatFn), streamToString(this.<Integer, ItemsByTag>sinkStreamOfEntry(), e -> formatFn.apply(entry(e.getKey(), e.getValue().get(tag1)))));
}

Example 3 with Tuple2.tuple2

use of com.hazelcast.jet.datamodel.Tuple2.tuple2 in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method when_hashJoinRebalanceEnrichingStage_then_noEffect.

@Test
public void when_hashJoinRebalanceEnrichingStage_then_noEffect() {
    // Given
    List<Integer> input = sequence(itemCount);
    String prefix = "value-";
    BatchStage<Integer> mainStage = batchStageFromList(input);
    BatchStage<Entry<Integer, String>> enrichingStageRebalanced = batchStageFromList(input).map(i -> entry(i, prefix + i)).rebalance();
    // When
    BatchStage<Entry<Integer, String>> joined = mainStage.hashJoin(enrichingStageRebalanced, joinMapEntries(wholeItem()), Util::entry);
    // Then
    joined.writeTo(sink);
    DAG dag = p.toDag();
    Edge mapToJoin = dag.getInboundEdges("2-way hash-join-collector1").get(0);
    assertTrue("Edge into a hash-join collector vertex must be distributed", mapToJoin.isDistributed());
    assertNull("Didn't rebalance by key, the edge must not be partitioned", mapToJoin.getPartitioner());
    Edge stage0ToJoin = dag.getInboundEdges("2-way hash-join-joiner").get(0);
    assertFalse("Didn't rebalance this stage, why is its edge distributed?", stage0ToJoin.isDistributed());
    assertNull("Didn't rebalance by key, the edge must not be partitioned", stage0ToJoin.getPartitioner());
    execute();
    Function<Entry<Integer, String>, String> formatFn = e -> String.format("(%04d, %s)", e.getKey(), e.getValue());
    assertEquals(streamToString(input.stream().map(i -> tuple2(i, prefix + i)), formatFn), streamToString(sinkStreamOfEntry(), formatFn));
}

Example 4 with Tuple2.tuple2

use of com.hazelcast.jet.datamodel.Tuple2.tuple2 in project hazelcast-jet-reference-manual by hazelcast.

the class BuildComputation method s11.

static void s11() {
    // tag::s11[]
    Pipeline p = Pipeline.create();
    // The stream to be enriched: trades
    StreamStage<Trade> trades = p.drawFrom(Sources.mapJournal("trades", mapPutEvents(), mapEventNewValue(), START_FROM_CURRENT));
    // The enriching streams: products, brokers and markets
    BatchStage<Entry<Integer, Product>> prodEntries = p.drawFrom(Sources.map("products"));
    BatchStage<Entry<Integer, Broker>> brokEntries = p.drawFrom(Sources.map("brokers"));
    BatchStage<Entry<Integer, Market>> marketEntries = p.drawFrom(Sources.map("markets"));
    // Obtain a hash-join builder object from the stream to be enriched
    StreamHashJoinBuilder<Trade> builder = trades.hashJoinBuilder();
    // Add enriching streams to the builder
    Tag<Product> productTag = builder.add(prodEntries, joinMapEntries(Trade::productId));
    Tag<Broker> brokerTag = builder.add(brokEntries, joinMapEntries(Trade::brokerId));
    Tag<Market> marketTag = builder.add(marketEntries, joinMapEntries(Trade::marketId));
    // Build the hash join pipeline
    StreamStage<Tuple2<Trade, ItemsByTag>> joined = builder.build(Tuple2::tuple2);
    // end::s11[]
    // tag::s12[]
    StreamStage<String> mapped = joined.map((Tuple2<Trade, ItemsByTag> tuple) -> {
        Trade trade = tuple.f0();
        ItemsByTag ibt = tuple.f1();
        Product product = ibt.get(productTag);
        Broker broker = ibt.get(brokerTag);
        Market market = ibt.get(marketTag);
        return trade + ": " + product + ", " + broker + ", " + market;
    });
// end::s12[]
}

Example 5 with Tuple2.tuple2

use of com.hazelcast.jet.datamodel.Tuple2.tuple2 in project hazelcast-jet-reference-manual by hazelcast.

the class BuildComputation method s2.

static void s2() {
    // tag::s2[]
    Pipeline p = Pipeline.create();
    StreamStage<Trade> trades = p.drawFrom(Sources.mapJournal("trades", mapPutEvents(), mapEventNewValue(), START_FROM_CURRENT));
    BatchStage<Entry<Integer, Product>> products = p.drawFrom(Sources.<Integer, Product>map("products"));
    StreamStage<Tuple2<Trade, Product>> joined = trades.hashJoin(products, joinMapEntries(Trade::productId), Tuple2::tuple2);
// end::s2[]
}