Examples with Vertex com.hazelcast.jet.core.Vertex used on opensource projects

Example 16 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast-jet by hazelcast.

the class MasterContext method tryStartJob.

/**
 * Starts execution of the job if it is not already completed, cancelled or failed.
 * If the job is already cancelled, the job completion procedure is triggered.
 * If the job quorum is not satisfied, job restart is rescheduled.
 * If there was a membership change and the partition table is not completely
 * fixed yet, job restart is rescheduled.
 */
void tryStartJob(Function<Long, Long> executionIdSupplier) {
    if (!setJobStatusToStarting()) {
        return;
    }
    if (scheduleRestartIfQuorumAbsent() || scheduleRestartIfClusterIsNotSafe()) {
        return;
    }
    DAG dag;
    try {
        dag = deserializeDAG();
    } catch (Exception e) {
        logger.warning("DAG deserialization failed", e);
        finalizeJob(e);
        return;
    }
    // save a copy of the vertex list, because it is going to change
    vertices = new HashSet<>();
    dag.iterator().forEachRemaining(vertices::add);
    executionId = executionIdSupplier.apply(jobId);
    // last started snapshot complete or not complete. The next started snapshot must be greater than this number
    long lastSnapshotId = NO_SNAPSHOT;
    if (isSnapshottingEnabled()) {
        Long snapshotIdToRestore = snapshotRepository.latestCompleteSnapshot(jobId);
        snapshotRepository.deleteAllSnapshotsExceptOne(jobId, snapshotIdToRestore);
        Long lastStartedSnapshot = snapshotRepository.latestStartedSnapshot(jobId);
        if (snapshotIdToRestore != null) {
            logger.info("State of " + jobIdString() + " will be restored from snapshot " + snapshotIdToRestore);
            rewriteDagWithSnapshotRestore(dag, snapshotIdToRestore);
        } else {
            logger.info("No previous snapshot for " + jobIdString() + " found.");
        }
        if (lastStartedSnapshot != null) {
            lastSnapshotId = lastStartedSnapshot;
        }
    }
    MembersView membersView = getMembersView();
    ClassLoader previousCL = swapContextClassLoader(coordinationService.getClassLoader(jobId));
    try {
        int defaultLocalParallelism = getJetInstance(nodeEngine).getConfig().getInstanceConfig().getCooperativeThreadCount();
        logger.info("Start executing " + jobIdString() + ", status " + jobStatus() + "\n" + dag.toString(defaultLocalParallelism));
        logger.fine("Building execution plan for " + jobIdString());
        executionPlanMap = createExecutionPlans(nodeEngine, membersView, dag, getJobConfig(), lastSnapshotId);
    } catch (Exception e) {
        logger.severe("Exception creating execution plan for " + jobIdString(), e);
        finalizeJob(e);
        return;
    } finally {
        Thread.currentThread().setContextClassLoader(previousCL);
    }
    logger.fine("Built execution plans for " + jobIdString());
    Set<MemberInfo> participants = executionPlanMap.keySet();
    Function<ExecutionPlan, Operation> operationCtor = plan -> new InitExecutionOperation(jobId, executionId, membersView.getVersion(), participants, nodeEngine.getSerializationService().toData(plan));
    invoke(operationCtor, this::onInitStepCompleted, null);
}

Example 17 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast-jet by hazelcast.

the class MasterContext method rewriteDagWithSnapshotRestore.

private void rewriteDagWithSnapshotRestore(DAG dag, long snapshotId) {
    logger.info(jobIdString() + ": restoring state from snapshotId=" + snapshotId);
    for (Vertex vertex : dag) {
        // items with keys of type BroadcastKey need to be broadcast to all processors
        DistributedFunction<Entry<Object, Object>, ?> projection = (Entry<Object, Object> e) -> (e.getKey() instanceof BroadcastKey) ? new BroadcastEntry<>(e) : e;
        // We add the vertex even in case when the map is empty: this ensures, that
        // Processor.finishSnapshotRestore() method is always called on all vertices in
        // a job which is restored from a snapshot.
        String mapName = snapshotDataMapName(jobId, snapshotId, vertex.getName());
        Vertex readSnapshotVertex = dag.newVertex("__snapshot_read." + vertex.getName(), readMapP(mapName));
        // We need a separate mapping vertex and can't use readMapP's projectionFn:
        // the projection will cause key/value deserialization on partition thread, which doesn't have job's
        // class loader. If the key/value uses a custom object, it will fail. For example, StreamKafkaP uses
        // TopicPartition as the key or a custom processor can use custom key.
        Vertex mapSnapshotVertex = dag.newVertex("__snapshot_map_." + vertex.getName(), mapP(projection));
        readSnapshotVertex.localParallelism(vertex.getLocalParallelism());
        mapSnapshotVertex.localParallelism(vertex.getLocalParallelism());
        int destOrdinal = dag.getInboundEdges(vertex.getName()).size();
        dag.edge(between(readSnapshotVertex, mapSnapshotVertex).isolated()).edge(new SnapshotRestoreEdge(mapSnapshotVertex, vertex, destOrdinal));
    }
}

Example 18 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast-jet by hazelcast.

the class HazelcastConnectorTest method when_streamCache_withFilterAndProjection.

@Test
public void when_streamCache_withFilterAndProjection() {
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", SourceProcessors.<Integer, Integer, Integer>streamCacheP(streamSourceName, event -> !event.getKey().equals(0), EventJournalCacheEvent::getKey, START_FROM_OLDEST, wmGenParams(i -> i, limitingLag(0), noThrottling(), 10_000)));
    Vertex sink = dag.newVertex("sink", writeListP(streamSinkName));
    dag.edge(between(source, sink));
    Job job = jetInstance.newJob(dag);
    ICacheJet<Integer, Integer> sourceCache = jetInstance.getCacheManager().getCache(streamSourceName);
    range(0, ENTRY_COUNT).forEach(i -> sourceCache.put(i, i));
    assertSizeEventually(ENTRY_COUNT - 1, jetInstance.getList(streamSinkName));
    assertFalse(jetInstance.getList(streamSinkName).contains(0));
    assertTrue(jetInstance.getList(streamSinkName).contains(1));
    job.cancel();
}

Example 19 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast-jet by hazelcast.

the class HazelcastConnectorTest method when_streamMap.

@Test
public void when_streamMap() {
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", streamMapP(streamSourceName, START_FROM_OLDEST, wmGenParams(Entry<Integer, Integer>::getValue, limitingLag(0), noThrottling(), 10_000)));
    Vertex sink = dag.newVertex("sink", writeListP(streamSinkName));
    dag.edge(between(source, sink));
    Job job = jetInstance.newJob(dag);
    IMapJet<Integer, Integer> sourceMap = jetInstance.getMap(streamSourceName);
    range(0, ENTRY_COUNT).forEach(i -> sourceMap.put(i, i));
    assertSizeEventually(ENTRY_COUNT, jetInstance.getList(streamSinkName));
    job.cancel();
}

Example 20 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast-jet by hazelcast.

the class HazelcastConnectorTest method test_defaultFilter_cacheJournal.

@Test
public void test_defaultFilter_cacheJournal() {
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", streamCacheP(streamSourceName, START_FROM_OLDEST, wmGenParams(Entry<Integer, Integer>::getValue, limitingLag(0), noThrottling(), 10_000)));
    Vertex sink = dag.newVertex("sink", writeListP(streamSinkName));
    dag.edge(between(source, sink));
    Job job = jetInstance.newJob(dag);
    ICacheJet<Object, Object> sourceCache = jetInstance.getCacheManager().getCache(streamSourceName);
    // ADDED
    sourceCache.put(1, 1);
    // REMOVED - filtered out
    sourceCache.remove(1);
    // UPDATED
    sourceCache.put(1, 2);
    IListJet<Entry<Integer, Integer>> sinkList = jetInstance.getList(streamSinkName);
    assertTrueEventually(() -> {
        assertEquals(2, sinkList.size());
        Entry<Integer, Integer> e = sinkList.get(0);
        assertEquals(Integer.valueOf(1), e.getKey());
        assertEquals(Integer.valueOf(1), e.getValue());
        e = sinkList.get(1);
        assertEquals(Integer.valueOf(1), e.getKey());
        assertEquals(Integer.valueOf(2), e.getValue());
    }, 10);
    job.cancel();
}