Examples with Processor com.hazelcast.jet.core.Processor used on opensource projects

Example 11 with Processor

use of com.hazelcast.jet.core.Processor in project hazelcast by hazelcast.

the class UpdateProcessorSupplier method get.

@Nonnull
@Override
public Collection<? extends Processor> get(int count) {
    List<Processor> processors = new ArrayList<>(count);
    for (int i = 0; i < count; i++) {
        String mapName = this.mapName;
        Processor processor = new AsyncTransformUsingServiceBatchedP<>(ServiceFactories.nonSharedService(SecuredFunctions.iMapFn(mapName)), null, MAX_CONCURRENT_OPS, MAX_BATCH_SIZE, (IMap<Object, Object> map, List<JetSqlRow> rows) -> update(rows, map));
        processors.add(processor);
    }
    return processors;
}

Example 12 with Processor

use of com.hazelcast.jet.core.Processor in project hazelcast by hazelcast.

the class ExecutionPlan method initialize.

/**
 * A method called on the members as part of the InitExecutionOperation.
 * Creates tasklets, inboxes/outboxes and connects these to make them ready
 * for a later StartExecutionOperation.
 */
public void initialize(NodeEngineImpl nodeEngine, long jobId, long executionId, @Nonnull SnapshotContext snapshotContext, ConcurrentHashMap<String, File> tempDirectories, InternalSerializationService jobSerializationService) {
    this.nodeEngine = nodeEngine;
    this.jobClassLoaderService = ((JetServiceBackend) nodeEngine.getService(JetServiceBackend.SERVICE_NAME)).getJobClassLoaderService();
    this.executionId = executionId;
    initProcSuppliers(jobId, tempDirectories, jobSerializationService);
    initDag(jobSerializationService);
    this.ptionArrgmt = new PartitionArrangement(partitionAssignment, nodeEngine.getThisAddress());
    Set<Integer> higherPriorityVertices = VertexDef.getHigherPriorityVertices(vertices);
    for (Address destAddr : remoteMembers.get()) {
        Connection conn = getMemberConnection(nodeEngine, destAddr);
        if (conn == null) {
            throw new TopologyChangedException("no connection to job participant: " + destAddr);
        }
        memberConnections.put(destAddr, conn);
    }
    for (VertexDef vertex : vertices) {
        ClassLoader processorClassLoader = isLightJob ? null : jobClassLoaderService.getProcessorClassLoader(jobId, vertex.name());
        Collection<? extends Processor> processors = doWithClassLoader(processorClassLoader, () -> createProcessors(vertex, vertex.localParallelism()));
        String jobPrefix = prefix(jobConfig.getName(), jobId, vertex.name());
        // create StoreSnapshotTasklet and the queues to it
        ConcurrentConveyor<Object> ssConveyor = null;
        if (!isLightJob) {
            // Note that we create the snapshot queues for all non-light jobs, even if they don't have
            // processing guarantee enabled, because in EE one can request a snapshot also for
            // non-snapshotted jobs.
            @SuppressWarnings("unchecked") QueuedPipe<Object>[] snapshotQueues = new QueuedPipe[vertex.localParallelism()];
            Arrays.setAll(snapshotQueues, i -> new OneToOneConcurrentArrayQueue<>(SNAPSHOT_QUEUE_SIZE));
            ssConveyor = ConcurrentConveyor.concurrentConveyor(null, snapshotQueues);
            ILogger storeSnapshotLogger = prefixedLogger(nodeEngine.getLogger(StoreSnapshotTasklet.class), jobPrefix);
            StoreSnapshotTasklet ssTasklet = new StoreSnapshotTasklet(snapshotContext, ConcurrentInboundEdgeStream.create(ssConveyor, 0, 0, true, jobPrefix + "/ssFrom", null), new AsyncSnapshotWriterImpl(nodeEngine, snapshotContext, vertex.name(), memberIndex, memberCount, jobSerializationService), storeSnapshotLogger, vertex.name(), higherPriorityVertices.contains(vertex.vertexId()));
            tasklets.add(ssTasklet);
        }
        int localProcessorIdx = 0;
        for (Processor processor : processors) {
            int globalProcessorIndex = memberIndex * vertex.localParallelism() + localProcessorIdx;
            String processorPrefix = prefix(jobConfig.getName(), jobId, vertex.name(), globalProcessorIndex);
            ILogger logger = prefixedLogger(nodeEngine.getLogger(processor.getClass()), processorPrefix);
            ProcCtx context = new ProcCtx(nodeEngine, jobId, executionId, getJobConfig(), logger, vertex.name(), localProcessorIdx, globalProcessorIndex, isLightJob, partitionAssignment, vertex.localParallelism(), memberIndex, memberCount, tempDirectories, jobSerializationService, subject, processorClassLoader);
            // createOutboundEdgeStreams() populates localConveyorMap and edgeSenderConveyorMap.
            // Also populates instance fields: senderMap, receiverMap, tasklets.
            List<OutboundEdgeStream> outboundStreams = createOutboundEdgeStreams(vertex, localProcessorIdx, jobPrefix, jobSerializationService);
            List<InboundEdgeStream> inboundStreams = createInboundEdgeStreams(vertex, localProcessorIdx, jobPrefix, globalProcessorIndex);
            OutboundCollector snapshotCollector = ssConveyor == null ? null : new ConveyorCollector(ssConveyor, localProcessorIdx, null);
            // vertices which are only used for snapshot restore will not be marked as "source=true" in metrics
            // also do not consider snapshot restore edges for determining source tag
            boolean isSource = vertex.inboundEdges().stream().allMatch(EdgeDef::isSnapshotRestoreEdge) && !vertex.isSnapshotVertex();
            ProcessorTasklet processorTasklet = new ProcessorTasklet(context, nodeEngine.getExecutionService().getExecutor(TASKLET_INIT_CLOSE_EXECUTOR_NAME), jobSerializationService, processor, inboundStreams, outboundStreams, snapshotContext, snapshotCollector, isSource);
            tasklets.add(processorTasklet);
            this.processors.add(processor);
            localProcessorIdx++;
        }
    }
    List<ReceiverTasklet> allReceivers = receiverMap.values().stream().flatMap(o -> o.values().stream()).flatMap(a -> a.values().stream()).collect(toList());
    tasklets.addAll(allReceivers);
}

Example 13 with Processor

use of com.hazelcast.jet.core.Processor in project hazelcast by hazelcast.

the class WriteJdbcP method metaSupplier.

/**
 * Use {@link SinkProcessors#writeJdbcP}.
 */
public static <T> ProcessorMetaSupplier metaSupplier(@Nullable String jdbcUrl, @Nonnull String updateQuery, @Nonnull SupplierEx<? extends CommonDataSource> dataSourceSupplier, @Nonnull BiConsumerEx<? super PreparedStatement, ? super T> bindFn, boolean exactlyOnce, int batchLimit) {
    checkSerializable(dataSourceSupplier, "newConnectionFn");
    checkSerializable(bindFn, "bindFn");
    checkPositive(batchLimit, "batchLimit");
    return ProcessorMetaSupplier.preferLocalParallelismOne(ConnectorPermission.jdbc(jdbcUrl, ACTION_WRITE), new ProcessorSupplier() {

        private transient CommonDataSource dataSource;

        @Override
        public void init(@Nonnull Context context) {
            dataSource = dataSourceSupplier.get();
        }

        @Nonnull
        @Override
        public Collection<? extends Processor> get(int count) {
            return IntStream.range(0, count).mapToObj(i -> new WriteJdbcP<>(updateQuery, dataSource, bindFn, exactlyOnce, batchLimit)).collect(Collectors.toList());
        }

        @Override
        public List<Permission> permissions() {
            return singletonList(ConnectorPermission.jdbc(jdbcUrl, ACTION_WRITE));
        }
    });
}

Example 14 with Processor

use of com.hazelcast.jet.core.Processor in project hazelcast by hazelcast.

the class ProcessorTasklet method init.

@Override
public void init() {
    ManagedContext managedContext = serializationService.getManagedContext();
    if (managedContext != null) {
        Processor toInit = processor instanceof ProcessorWrapper ? ((ProcessorWrapper) processor).getWrapped() : processor;
        Object initialized = null;
        try {
            initialized = managedContext.initialize(toInit);
            toInit = (Processor) initialized;
        } catch (ClassCastException e) {
            throw new IllegalArgumentException(String.format("The initialized object(%s) should be an instance of %s", initialized, Processor.class), e);
        }
        if (processor instanceof ProcessorWrapper) {
            ((ProcessorWrapper) processor).setWrapped(toInit);
        } else {
            processor = toInit;
        }
    }
    try {
        doWithClassLoader(context.classLoader(), () -> processor.init(outbox, context));
    } catch (Exception e) {
        throw sneakyThrow(e);
    }
}

Example 15 with Processor

use of com.hazelcast.jet.core.Processor in project hazelcast by hazelcast.

the class JobMetrics_MiscTest method when_jobSuspended_andMetricsNotStored_then_onlyPeriodicMetricsReturned.

@Test
public void when_jobSuspended_andMetricsNotStored_then_onlyPeriodicMetricsReturned() throws Throwable {
    DAG dag = new DAG();
    Vertex v1 = dag.newVertex("v1", TestProcessors.MockP::new);
    Vertex v2 = dag.newVertex("v2", (SupplierEx<Processor>) TestProcessors.NoOutputSourceP::new);
    dag.edge(between(v1, v2));
    // init
    JobConfig config = new JobConfig().setMetricsEnabled(// enable metric collection
    true).setStoreMetricsAfterJobCompletion(// disable metric saving on completion
    false);
    Job job = hz().getJet().newJob(dag, config);
    // when
    TestProcessors.NoOutputSourceP.executionStarted.await();
    // then
    assertJobStatusEventually(job, JobStatus.RUNNING);
    assertTrueEventually(() -> assertJobHasMetrics(job, false));
    // when
    job.suspend();
    // then
    assertJobStatusEventually(job, SUSPENDED);
    assertTrueEventually(() -> assertEmptyJobMetrics(job, false));
    // when
    job.resume();
    // then
    assertJobStatusEventually(job, RUNNING);
    assertTrueEventually(() -> assertJobHasMetrics(job, false));
    // when
    TestProcessors.NoOutputSourceP.proceedLatch.countDown();
    job.join();
    // then
    assertJobStatusEventually(job, JobStatus.COMPLETED);
    assertEmptyJobMetrics(job, false);
}