Examples with DAG com.hazelcast.jet.core.DAG used on opensource projects

Example 36 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast-jet-reference-manual by hazelcast.

the class StockExchangeCoreApi method buildDag.

static DAG buildDag() {
    // tag::s1[]
    DistributedToLongFunction<? super Trade> timestampFn = Trade::timestamp;
    DistributedFunction<? super Trade, ?> keyFn = Trade::productId;
    SlidingWindowPolicy winPolicy = slidingWinPolicy(SLIDING_WINDOW_LENGTH_MILLIS, SLIDE_STEP_MILLIS);
    DAG dag = new DAG();
    Vertex tradeSource = dag.newVertex("trade-source", SourceProcessors.<Trade, Long, Trade>streamMapP(TRADES_MAP_NAME, // <1>
    alwaysTrue(), // <1>
    EventJournalMapEvent::getNewValue, // <2>
    JournalInitialPosition.START_FROM_OLDEST, wmGenParams(// <3>
    timestampFn, // <4>
    limitingLag(SECONDS.toMillis(3)), // <5>
    emitByFrame(winPolicy), // <6>
    SECONDS.toMillis(3))));
    Vertex slidingStage1 = dag.newVertex("sliding-stage-1", Processors.accumulateByFrameP(singletonList(keyFn), singletonList(timestampFn), TimestampKind.EVENT, winPolicy, counting()));
    Vertex slidingStage2 = dag.newVertex("sliding-stage-2", Processors.combineToSlidingWindowP(winPolicy, counting(), TimestampedEntry::new));
    Vertex formatOutput = dag.newVertex("format-output", mapUsingContextP(ContextFactory.withCreateFn(x -> DateTimeFormatter.ofPattern("HH:mm:ss.SSS")), (DateTimeFormatter timeFormat, TimestampedEntry<String, Long> tse) -> String.format("%s %5s %4d", timeFormat.format(Instant.ofEpochMilli(tse.getTimestamp()).atZone(ZoneId.systemDefault())), tse.getKey(), tse.getValue())));
    Vertex sink = dag.newVertex("sink", SinkProcessors.writeFileP(OUTPUT_DIR_NAME));
    tradeSource.localParallelism(1);
    return dag.edge(between(tradeSource, slidingStage1).partitioned(keyFn, HASH_CODE)).edge(between(slidingStage1, slidingStage2).partitioned(entryKey(), HASH_CODE).distributed()).edge(between(slidingStage2, formatOutput).isolated()).edge(between(formatOutput, sink));
// end::s1[]
}

Example 37 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast-jet-reference-manual by hazelcast.

the class S8 method main.

// end::s1[]
static void main() {
    // tag::s2[]
    JetInstance jet = Jet.newJetInstance();
    int upperBound = 10;
    DAG dag = new DAG();
    Vertex generateNumbers = dag.newVertex("generate-numbers", () -> new GenerateNumbersP(upperBound));
    Vertex logInput = dag.newVertex("log-input", DiagnosticProcessors.writeLoggerP(i -> "Received number: " + i));
    dag.edge(Edge.between(generateNumbers, logInput));
    try {
        jet.newJob(dag).join();
    } finally {
        Jet.shutdownAll();
    }
// end::s2[]
}

Example 38 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast-jet-reference-manual by hazelcast.

the class ExpertZoneDag method s0.

// end::s1[]
static void s0() {
    Pipeline pipeline = Pipeline.create();
    // tag::s0[]
    DAG dag = pipeline.toDag();
// end::s0[]
}

Example 39 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.

the class MasterJobContext method tryStartJob.

/**
 * Starts the execution of the job if it is not already completed,
 * cancelled or failed.
 * <p>
 * If the job is already cancelled, triggers the job completion procedure.
 * <p>
 * If the job quorum is not satisfied, reschedules the job restart.
 * <p>
 * If there was a membership change and the partition table is not completely
 * fixed yet, reschedules the job restart.
 */
void tryStartJob(Supplier<Long> executionIdSupplier) {
    mc.coordinationService().submitToCoordinatorThread(() -> {
        executionStartTime = System.currentTimeMillis();
        try {
            JobExecutionRecord jobExecRec = mc.jobExecutionRecord();
            jobExecRec.markExecuted();
            Tuple2<DAG, ClassLoader> dagAndClassloader = resolveDagAndCL(executionIdSupplier);
            if (dagAndClassloader == null) {
                return;
            }
            DAG dag = dagAndClassloader.f0();
            assert dag != null;
            ClassLoader classLoader = dagAndClassloader.f1();
            // must call this before rewriteDagWithSnapshotRestore()
            String dotRepresentation = dag.toDotString(defaultParallelism, defaultQueueSize);
            long snapshotId = jobExecRec.snapshotId();
            String snapshotName = mc.jobConfig().getInitialSnapshotName();
            String mapName = snapshotId >= 0 ? jobExecRec.successfulSnapshotDataMapName(mc.jobId()) : snapshotName != null ? EXPORTED_SNAPSHOTS_PREFIX + snapshotName : null;
            if (mapName != null) {
                rewriteDagWithSnapshotRestore(dag, snapshotId, mapName, snapshotName);
            } else {
                logger.info("Didn't find any snapshot to restore for " + mc.jobIdString());
            }
            MembersView membersView = Util.getMembersView(mc.nodeEngine());
            logger.info("Start executing " + mc.jobIdString() + ", execution graph in DOT format:\n" + dotRepresentation + "\nHINT: You can use graphviz or http://viz-js.com to visualize the printed graph.");
            logger.fine("Building execution plan for " + mc.jobIdString());
            Util.doWithClassLoader(classLoader, () -> mc.setExecutionPlanMap(createExecutionPlans(mc.nodeEngine(), membersView.getMembers(), dag, mc.jobId(), mc.executionId(), mc.jobConfig(), jobExecRec.ongoingSnapshotId(), false, mc.jobRecord().getSubject())));
            logger.fine("Built execution plans for " + mc.jobIdString());
            Set<MemberInfo> participants = mc.executionPlanMap().keySet();
            Version coordinatorVersion = mc.nodeEngine().getLocalMember().getVersion().asVersion();
            Function<ExecutionPlan, Operation> operationCtor = plan -> new InitExecutionOperation(mc.jobId(), mc.executionId(), membersView.getVersion(), coordinatorVersion, participants, mc.nodeEngine().getSerializationService().toData(plan), false);
            mc.invokeOnParticipants(operationCtor, this::onInitStepCompleted, null, false);
        } catch (Throwable e) {
            finalizeJob(e);
        }
    });
}

Example 40 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.

the class Util method copyMapUsingJob.

// used in jet-enterprise
@SuppressWarnings("WeakerAccess")
public static CompletableFuture<Void> copyMapUsingJob(HazelcastInstance instance, int queueSize, String sourceMap, String targetMap) {
    DAG dag = new DAG();
    Vertex source = dag.newVertex("readMap(" + sourceMap + ')', readMapP(sourceMap));
    Vertex sink = dag.newVertex("writeMap(" + targetMap + ')', writeMapP(targetMap));
    dag.edge(between(source, sink).setConfig(new EdgeConfig().setQueueSize(queueSize)));
    JobConfig jobConfig = new JobConfig().setName("copy-" + sourceMap + "-to-" + targetMap);
    return instance.getJet().newJob(dag, jobConfig).getFuture();
}