Examples with Vertex com.hazelcast.jet.core.Vertex used on opensource projects

Example 91 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class WordCountTest method testJetTwoPhaseAggregation.

@Test
@Ignore
public void testJetTwoPhaseAggregation() {
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", SourceProcessors.readMapP("words"));
    Vertex mapReduce = dag.newVertex("map-reduce", MapReduceP::new);
    Vertex combineLocal = dag.newVertex("combine-local", CombineP::new);
    Vertex combineGlobal = dag.newVertex("combine-global", CombineP::new);
    Vertex sink = dag.newVertex("sink", SinkProcessors.writeMapP("counts"));
    dag.edge(between(source, mapReduce)).edge(between(mapReduce, combineLocal)).edge(between(combineLocal, combineGlobal).distributed().allToOne("ALL")).edge(between(combineGlobal, sink));
    benchmark("jet", () -> instance.getJet().newJob(dag).join());
    assertCounts((Map<String, Long>) instance.getMap("counts").get("result"));
}

Example 92 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class BackpressureTest method testBackpressure.

@Test
public void testBackpressure() {
    DAG dag = new DAG();
    final int member1Port = hz1.getCluster().getLocalMember().getAddress().getPort();
    final Member member2 = hz2.getCluster().getLocalMember();
    final int ptionOwnedByMember2 = hz1.getPartitionService().getPartitions().stream().filter(p -> p.getOwner().equals(member2)).map(Partition::getPartitionId).findAny().orElseThrow(() -> new RuntimeException("Can't find a partition owned by member " + hz2));
    Vertex source = dag.newVertex("source", ProcessorMetaSupplier.of((Address address) -> ProcessorSupplier.of(address.getPort() == member1Port ? GenerateP::new : noopP())));
    Vertex hiccup = dag.newVertex("hiccup", HiccupP::new);
    Vertex sink = dag.newVertex("sink", SinkProcessors.writeMapP("counts"));
    dag.edge(between(source, hiccup).distributed().partitioned(wholeItem(), (x, y) -> ptionOwnedByMember2)).edge(between(hiccup, sink));
    hz1.getJet().newJob(dag).join();
    assertCounts(hz1.getMap("counts"));
}

Example 93 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class FileSqlConnector method fullScanReader.

@Nonnull
@Override
public Vertex fullScanReader(@Nonnull DAG dag, @Nonnull Table table0, @Nullable Expression<Boolean> predicate, @Nonnull List<Expression<?>> projections, @Nullable FunctionEx<ExpressionEvalContext, EventTimePolicy<JetSqlRow>> eventTimePolicyProvider) {
    if (eventTimePolicyProvider != null) {
        throw QueryException.error("Ordering functions are not supported on top of " + TYPE_NAME + " mappings");
    }
    FileTable table = (FileTable) table0;
    Vertex vStart = dag.newUniqueVertex(table.toString(), table.processorMetaSupplier());
    Vertex vEnd = dag.newUniqueVertex("Project(" + table + ")", SqlProcessors.rowProjector(table.paths(), table.types(), table.queryTargetSupplier(), predicate, projections));
    dag.edge(between(vStart, vEnd).isolated());
    return vEnd;
}

Example 94 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class MasterJobContext method rewriteDagWithSnapshotRestore.

private void rewriteDagWithSnapshotRestore(DAG dag, long snapshotId, String mapName, String snapshotName) {
    IMap<Object, Object> snapshotMap = mc.nodeEngine().getHazelcastInstance().getMap(mapName);
    long resolvedSnapshotId = validateSnapshot(snapshotId, snapshotMap, mc.jobIdString(), snapshotName);
    logger.info(String.format("About to restore the state of %s from snapshot %d, mapName = %s", mc.jobIdString(), resolvedSnapshotId, mapName));
    List<Vertex> originalVertices = new ArrayList<>();
    dag.iterator().forEachRemaining(originalVertices::add);
    Map<String, Integer> vertexToOrdinal = new HashMap<>();
    Vertex readSnapshotVertex = dag.newVertex(SNAPSHOT_VERTEX_PREFIX + "read", readMapP(mapName));
    Vertex explodeVertex = dag.newVertex(SNAPSHOT_VERTEX_PREFIX + "explode", () -> new ExplodeSnapshotP(vertexToOrdinal, resolvedSnapshotId));
    dag.edge(between(readSnapshotVertex, explodeVertex).isolated());
    int index = 0;
    // add the edges
    for (Vertex userVertex : originalVertices) {
        vertexToOrdinal.put(userVertex.getName(), index);
        int destOrdinal = dag.getInboundEdges(userVertex.getName()).size();
        dag.edge(new SnapshotRestoreEdge(explodeVertex, index, userVertex, destOrdinal));
        index++;
    }
}

Example 95 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class LightMasterContext method finalizeJob.

private void finalizeJob(@Nullable Throwable failure) {
    // close ProcessorMetaSuppliers
    for (Vertex vertex : vertices) {
        try {
            vertex.getMetaSupplier().close(failure);
        } catch (Throwable e) {
            logger.severe(jobIdString + " encountered an exception in ProcessorMetaSupplier.complete(), ignoring it", e);
        }
    }
    if (failure == null) {
        jobCompletionFuture.complete(null);
    } else {
        // translate JobTerminateRequestedException(CANCEL_FORCEFUL) to CancellationException
        if (failure instanceof JobTerminateRequestedException && ((JobTerminateRequestedException) failure).mode() == CANCEL_FORCEFUL) {
            CancellationException newFailure = new CancellationException();
            newFailure.initCause(failure);
            failure = newFailure;
        }
        jobCompletionFuture.completeExceptionally(failure);
    }
}