Examples with ProcessorSupplier com.hazelcast.jet.core.ProcessorSupplier used on opensource projects

Example 26 with ProcessorSupplier

use of com.hazelcast.jet.core.ProcessorSupplier in project hazelcast by hazelcast.

the class SourceProcessors method convenientSourceP.

/**
 * Returns a supplier of processors for a source that the user can create
 * using the {@link SourceBuilder}. This variant creates a source that
 * emits items without timestamps.
 *
 * @param createFn function that creates the source's context object
 * @param fillBufferFn function that fills Jet's buffer with items to emit
 * @param createSnapshotFn function that returns a snapshot of the context object's state
 * @param restoreSnapshotFn function that restores the context object's state from a snapshot
 * @param destroyFn function that cleans up the resources held by the context object
 * @param preferredLocalParallelism preferred local parallelism of the source vertex. Special values:
 *                                  {@value Vertex#LOCAL_PARALLELISM_USE_DEFAULT} -> use the cluster's
 *                                  default local parallelism;
 *                                  0 -> create a single processor for the entire cluster (total parallelism = 1)
 * @param isBatch true, if the fillBufferFn will call {@code buffer.close()}, that is whether
 *                the source reads a bounded or unbounded set of data
 *
 * @param <C> type of the source's context object
 * @param <T> type of items the source emits
 * @param <S> type of object saved to state snapshot
 */
@Nonnull
@SuppressWarnings("unchecked")
public static <C, T, S> ProcessorMetaSupplier convenientSourceP(@Nonnull FunctionEx<? super Context, ? extends C> createFn, @Nonnull BiConsumerEx<? super C, ? super SourceBuffer<T>> fillBufferFn, @Nonnull FunctionEx<? super C, ? extends S> createSnapshotFn, @Nonnull BiConsumerEx<? super C, ? super List<S>> restoreSnapshotFn, @Nonnull ConsumerEx<? super C> destroyFn, int preferredLocalParallelism, boolean isBatch, @Nullable Permission permission) {
    checkSerializable(createFn, "createFn");
    checkSerializable(fillBufferFn, "fillBufferFn");
    checkSerializable(destroyFn, "destroyFn");
    checkSerializable(createSnapshotFn, "createSnapshotFn");
    checkSerializable(restoreSnapshotFn, "restoreSnapshotFn");
    checkNotNegative(preferredLocalParallelism + 1, "preferredLocalParallelism must >= -1");
    ProcessorSupplier procSup = ProcessorSupplier.of(() -> new ConvenientSourceP<>(createFn, (BiConsumer<? super C, ? super SourceBufferConsumerSide<?>>) fillBufferFn, createSnapshotFn, restoreSnapshotFn, destroyFn, new SourceBufferImpl.Plain<>(isBatch), null));
    return preferredLocalParallelism != 0 ? ProcessorMetaSupplier.of(preferredLocalParallelism, permission, procSup) : ProcessorMetaSupplier.forceTotalParallelismOne(procSup, permission);
}

Example 27 with ProcessorSupplier

use of com.hazelcast.jet.core.ProcessorSupplier in project hazelcast by hazelcast.

the class SourceProcessors method convenientTimestampedSourceP.

/**
 * Returns a supplier of processors for a source that the user can create
 * using the {@link SourceBuilder}. This variant creates a source that
 * emits timestamped events.
 *
 * @param createFn function that creates the source's context object
 * @param fillBufferFn function that fills Jet's buffer with items to emit
 * @param eventTimePolicy parameters for watermark generation
 * @param createSnapshotFn function that returns a snapshot of the context object's state
 * @param restoreSnapshotFn function that restores the context object's state from a snapshot
 * @param destroyFn function that cleans up the resources held by the context object
 * @param preferredLocalParallelism preferred local parallelism of the source vertex. Special values:
 *                                  {@value Vertex#LOCAL_PARALLELISM_USE_DEFAULT} ->
 *                                  use the cluster's default local parallelism;
 *                                  0 -> create a single processor for the entire cluster (total parallelism = 1)
 *
 * @param <C> type of the context object
 * @param <T> type of items the source emits
 * @param <S> type of the object saved to state snapshot
 */
@Nonnull
@SuppressWarnings("unchecked")
public static <C, T, S> ProcessorMetaSupplier convenientTimestampedSourceP(@Nonnull FunctionEx<? super Context, ? extends C> createFn, @Nonnull BiConsumerEx<? super C, ? super TimestampedSourceBuffer<T>> fillBufferFn, @Nonnull EventTimePolicy<? super T> eventTimePolicy, @Nonnull FunctionEx<? super C, ? extends S> createSnapshotFn, @Nonnull BiConsumerEx<? super C, ? super List<S>> restoreSnapshotFn, @Nonnull ConsumerEx<? super C> destroyFn, int preferredLocalParallelism) {
    checkSerializable(createFn, "createFn");
    checkSerializable(fillBufferFn, "fillBufferFn");
    checkSerializable(destroyFn, "destroyFn");
    checkSerializable(createSnapshotFn, "createSnapshotFn");
    checkSerializable(restoreSnapshotFn, "restoreSnapshotFn");
    checkNotNegative(preferredLocalParallelism + 1, "preferredLocalParallelism must >= -1");
    ProcessorSupplier procSup = ProcessorSupplier.of(() -> new ConvenientSourceP<>(createFn, (BiConsumer<? super C, ? super SourceBufferConsumerSide<?>>) fillBufferFn, createSnapshotFn, restoreSnapshotFn, destroyFn, new SourceBufferImpl.Timestamped<>(), eventTimePolicy));
    return preferredLocalParallelism > 0 ? ProcessorMetaSupplier.of(preferredLocalParallelism, procSup) : ProcessorMetaSupplier.forceTotalParallelismOne(procSup);
}

Example 28 with ProcessorSupplier

use of com.hazelcast.jet.core.ProcessorSupplier in project hazelcast by hazelcast.

the class ExecutionPlanBuilder method createExecutionPlans.

@SuppressWarnings("checkstyle:ParameterNumber")
public static Map<MemberInfo, ExecutionPlan> createExecutionPlans(NodeEngineImpl nodeEngine, List<MemberInfo> memberInfos, DAG dag, long jobId, long executionId, JobConfig jobConfig, long lastSnapshotId, boolean isLightJob, Subject subject) {
    final int defaultParallelism = nodeEngine.getConfig().getJetConfig().getCooperativeThreadCount();
    final Map<MemberInfo, int[]> partitionsByMember = getPartitionAssignment(nodeEngine, memberInfos);
    final Map<Address, int[]> partitionsByAddress = partitionsByMember.entrySet().stream().collect(toMap(en -> en.getKey().getAddress(), Entry::getValue));
    final List<Address> addresses = toList(partitionsByMember.keySet(), MemberInfo::getAddress);
    final int clusterSize = partitionsByMember.size();
    final boolean isJobDistributed = clusterSize > 1;
    final EdgeConfig defaultEdgeConfig = nodeEngine.getConfig().getJetConfig().getDefaultEdgeConfig();
    final Map<MemberInfo, ExecutionPlan> plans = new HashMap<>();
    int memberIndex = 0;
    for (MemberInfo member : partitionsByMember.keySet()) {
        plans.put(member, new ExecutionPlan(partitionsByAddress, jobConfig, lastSnapshotId, memberIndex++, clusterSize, isLightJob, subject));
    }
    final Map<String, Integer> vertexIdMap = assignVertexIds(dag);
    for (Entry<String, Integer> entry : vertexIdMap.entrySet()) {
        final Vertex vertex = dag.getVertex(entry.getKey());
        assert vertex != null;
        final ProcessorMetaSupplier metaSupplier = vertex.getMetaSupplier();
        final int vertexId = entry.getValue();
        // The local parallelism determination here is effective only
        // in jobs submitted as DAG. Otherwise, in jobs submitted as
        // pipeline, we are already doing this determination while
        // converting it to DAG and there is no vertex left with LP=-1.
        final int localParallelism = vertex.determineLocalParallelism(defaultParallelism);
        final int totalParallelism = localParallelism * clusterSize;
        final List<EdgeDef> inbound = toEdgeDefs(dag.getInboundEdges(vertex.getName()), defaultEdgeConfig, e -> vertexIdMap.get(e.getSourceName()), isJobDistributed);
        final List<EdgeDef> outbound = toEdgeDefs(dag.getOutboundEdges(vertex.getName()), defaultEdgeConfig, e -> vertexIdMap.get(e.getDestName()), isJobDistributed);
        String prefix = prefix(jobConfig.getName(), jobId, vertex.getName(), "#PMS");
        ILogger logger = prefixedLogger(nodeEngine.getLogger(metaSupplier.getClass()), prefix);
        JetServiceBackend jetBackend = nodeEngine.getService(JetServiceBackend.SERVICE_NAME);
        JobClassLoaderService jobClassLoaderService = jetBackend.getJobClassLoaderService();
        ClassLoader processorClassLoader = jobClassLoaderService.getClassLoader(jobId);
        try {
            doWithClassLoader(processorClassLoader, () -> metaSupplier.init(new MetaSupplierCtx(nodeEngine, jobId, executionId, jobConfig, logger, vertex.getName(), localParallelism, totalParallelism, clusterSize, isLightJob, partitionsByAddress, subject, processorClassLoader)));
        } catch (Exception e) {
            throw sneakyThrow(e);
        }
        Function<? super Address, ? extends ProcessorSupplier> procSupplierFn = doWithClassLoader(processorClassLoader, () -> metaSupplier.get(addresses));
        for (Entry<MemberInfo, ExecutionPlan> e : plans.entrySet()) {
            final ProcessorSupplier processorSupplier = doWithClassLoader(processorClassLoader, () -> procSupplierFn.apply(e.getKey().getAddress()));
            if (!isLightJob) {
                // We avoid the check for light jobs - the user will get the error anyway, but maybe with less information.
                // And we can recommend the user to use normal job to have more checks.
                checkSerializable(processorSupplier, "ProcessorSupplier in vertex '" + vertex.getName() + '\'');
            }
            final VertexDef vertexDef = new VertexDef(vertexId, vertex.getName(), processorSupplier, localParallelism);
            vertexDef.addInboundEdges(inbound);
            vertexDef.addOutboundEdges(outbound);
            e.getValue().addVertex(vertexDef);
        }
    }
    return plans;
}

Example 29 with ProcessorSupplier

use of com.hazelcast.jet.core.ProcessorSupplier in project hazelcast by hazelcast.

the class CdcSinks method sink.

@Nonnull
private static <K, V> Sink<ChangeRecord> sink(@Nonnull String name, @Nonnull String map, @Nullable ClientConfig clientConfig, @Nonnull FunctionEx<? super ChangeRecord, ? extends K> keyFn, @Nonnull FunctionEx<? super ChangeRecord, ? extends V> valueFn) {
    FunctionEx<? super ChangeRecord, ? extends V> toValueFn = record -> DELETE.equals(record.operation()) ? null : valueFn.apply(record);
    String clientXml = asXmlString(clientConfig);
    ProcessorSupplier supplier = AbstractHazelcastConnectorSupplier.ofMap(clientXml, procFn(name, map, clientXml, keyFn, toValueFn));
    ProcessorMetaSupplier metaSupplier = ProcessorMetaSupplier.of(mapUpdatePermission(clientXml, name), supplier);
    return new SinkImpl<>(name, metaSupplier, DISTRIBUTED_PARTITIONED, keyFn);
}