Examples with ProcCtx com.hazelcast.jet.impl.execution.init.Contexts.ProcCtx used on opensource projects

Example 6 with ProcCtx

use of com.hazelcast.jet.impl.execution.init.Contexts.ProcCtx in project hazelcast by hazelcast.

the class ReadHadoopNewApiP method init.

@Override
protected void init(@Nonnull Context context) {
    InternalSerializationService serializationService = ((ProcCtx) context).serializationService();
    // we clone the projection of key/value if configured so because some of the
    // record-readers return the same object for `reader.getCurrentKey()`
    // and `reader.getCurrentValue()` which is mutated for each `reader.nextKeyValue()`.
    BiFunctionEx<K, V, R> projectionFn = this.projectionFn;
    if (configuration.getBoolean(COPY_ON_READ, true)) {
        BiFunctionEx<K, V, R> actualProjectionFn = projectionFn;
        projectionFn = (key, value) -> {
            R result = actualProjectionFn.apply(key, value);
            return result == null ? null : serializationService.toObject(serializationService.toData(result));
        };
    }
    traverser = new HadoopFileTraverser<>(configuration, splits, projectionFn);
}

Example 7 with ProcCtx

use of com.hazelcast.jet.impl.execution.init.Contexts.ProcCtx in project hazelcast by hazelcast.

the class ProcessorWrapper method initContext.

protected Context initContext(Context context) {
    // and also other objects could be mocked or null, such as hazelcastInstance())
    if (context instanceof ProcCtx) {
        ProcCtx c = (ProcCtx) context;
        LoggingService loggingService = c.hazelcastInstance().getLoggingService();
        String prefix = prefix(c.jobConfig().getName(), c.jobId(), c.vertexName(), c.globalProcessorIndex());
        ILogger newLogger = prefixedLogger(loggingService.getLogger(wrapped.getClass()), prefix);
        context = new ProcCtx(c.nodeEngine(), c.jobId(), c.executionId(), c.jobConfig(), newLogger, c.vertexName(), c.localProcessorIndex(), c.globalProcessorIndex(), c.isLightJob(), c.partitionAssignment(), c.localParallelism(), c.memberIndex(), c.memberCount(), c.tempDirectories(), c.serializationService(), c.subject(), c.classLoader());
    }
    return context;
}

Example 8 with ProcCtx

use of com.hazelcast.jet.impl.execution.init.Contexts.ProcCtx in project hazelcast by hazelcast.

the class ProcessorTasklet method provideDynamicMetrics.

@Override
public void provideDynamicMetrics(MetricDescriptor descriptor, MetricsCollectionContext mContext) {
    descriptor = descriptor.withTag(MetricTags.VERTEX, this.context.vertexName()).withTag(MetricTags.PROCESSOR_TYPE, this.processor.getClass().getSimpleName()).withTag(MetricTags.PROCESSOR, Integer.toString(this.context.globalProcessorIndex()));
    if (isSource) {
        descriptor = descriptor.withTag(MetricTags.SOURCE, "true");
    }
    if (outstreams.length == 0) {
        descriptor = descriptor.withTag(MetricTags.SINK, "true");
    }
    for (int i = 0; i < instreams.size(); i++) {
        MetricDescriptor descWithOrdinal = descriptor.copy().withTag(MetricTags.ORDINAL, String.valueOf(i));
        mContext.collect(descWithOrdinal, RECEIVED_COUNT, ProbeLevel.INFO, ProbeUnit.COUNT, receivedCounts.get(i));
        mContext.collect(descWithOrdinal, RECEIVED_BATCHES, ProbeLevel.INFO, ProbeUnit.COUNT, receivedBatches.get(i));
    }
    for (int i = 0; i < emittedCounts.length() - (this.context.snapshottingEnabled() ? 0 : 1); i++) {
        String ordinal = i == emittedCounts.length() - 1 ? "snapshot" : String.valueOf(i);
        MetricDescriptor descriptorWithOrdinal = descriptor.copy().withTag(MetricTags.ORDINAL, ordinal);
        mContext.collect(descriptorWithOrdinal, EMITTED_COUNT, ProbeLevel.INFO, ProbeUnit.COUNT, emittedCounts.get(i));
    }
    mContext.collect(descriptor, TOP_OBSERVED_WM, ProbeLevel.INFO, ProbeUnit.MS, watermarkCoalescer.topObservedWm());
    mContext.collect(descriptor, COALESCED_WM, ProbeLevel.INFO, ProbeUnit.MS, watermarkCoalescer.coalescedWm());
    mContext.collect(descriptor, LAST_FORWARDED_WM, ProbeLevel.INFO, ProbeUnit.MS, outbox.lastForwardedWm());
    mContext.collect(descriptor, LAST_FORWARDED_WM_LATENCY, ProbeLevel.INFO, ProbeUnit.MS, lastForwardedWmLatency());
    mContext.collect(descriptor, this);
    // collect static metrics from processor
    mContext.collect(descriptor, this.processor);
    // collect dynamic metrics from processor
    if (processor instanceof DynamicMetricsProvider) {
        ((DynamicMetricsProvider) processor).provideDynamicMetrics(descriptor.copy(), mContext);
    }
    if (context instanceof ProcCtx) {
        ((ProcCtx) context).metricsContext().provideDynamicMetrics(descriptor, mContext);
    }
}

Example 9 with ProcCtx

use of com.hazelcast.jet.impl.execution.init.Contexts.ProcCtx in project hazelcast-jet by hazelcast.

the class ExecutionPlan method initialize.

public void initialize(NodeEngine nodeEngine, long jobId, long executionId, SnapshotContext snapshotContext) {
    this.nodeEngine = nodeEngine;
    this.executionId = executionId;
    initProcSuppliers();
    initDag();
    this.ptionArrgmt = new PartitionArrangement(partitionOwners, nodeEngine.getThisAddress());
    JetInstance instance = getJetInstance(nodeEngine);
    for (VertexDef vertex : vertices) {
        Collection<? extends Processor> processors = createProcessors(vertex, vertex.localParallelism());
        // create StoreSnapshotTasklet and the queues to it
        QueuedPipe<Object>[] snapshotQueues = new QueuedPipe[vertex.localParallelism()];
        Arrays.setAll(snapshotQueues, i -> new OneToOneConcurrentArrayQueue<>(SNAPSHOT_QUEUE_SIZE));
        ConcurrentConveyor<Object> ssConveyor = ConcurrentConveyor.concurrentConveyor(null, snapshotQueues);
        StoreSnapshotTasklet ssTasklet = new StoreSnapshotTasklet(snapshotContext, jobId, new ConcurrentInboundEdgeStream(ssConveyor, 0, 0, lastSnapshotId, true, -1, "ssFrom:" + vertex.name()), nodeEngine, vertex.name(), vertex.isHigherPriorityUpstream());
        tasklets.add(ssTasklet);
        int localProcessorIdx = 0;
        for (Processor p : processors) {
            int globalProcessorIndex = vertex.getProcIdxOffset() + localProcessorIdx;
            String loggerName = createLoggerName(p.getClass().getName(), vertex.name(), globalProcessorIndex);
            ProcCtx context = new ProcCtx(instance, nodeEngine.getSerializationService(), nodeEngine.getLogger(loggerName), vertex.name(), globalProcessorIndex, jobConfig.getProcessingGuarantee(), vertex.localParallelism(), vertex.totalParallelism());
            String probePrefix = String.format("jet.job.%s.%s#%d", idToString(executionId), vertex.name(), localProcessorIdx);
            ((NodeEngineImpl) nodeEngine).getMetricsRegistry().scanAndRegister(p, probePrefix);
            // createOutboundEdgeStreams() populates localConveyorMap and edgeSenderConveyorMap.
            // Also populates instance fields: senderMap, receiverMap, tasklets.
            List<OutboundEdgeStream> outboundStreams = createOutboundEdgeStreams(vertex, localProcessorIdx);
            List<InboundEdgeStream> inboundStreams = createInboundEdgeStreams(vertex, localProcessorIdx);
            OutboundCollector snapshotCollector = new ConveyorCollector(ssConveyor, localProcessorIdx, null);
            ProcessorTasklet processorTasklet = new ProcessorTasklet(context, p, inboundStreams, outboundStreams, snapshotContext, snapshotCollector, jobConfig.getMaxWatermarkRetainMillis());
            tasklets.add(processorTasklet);
            this.processors.add(p);
            localProcessorIdx++;
        }
    }
    List<ReceiverTasklet> allReceivers = receiverMap.values().stream().flatMap(o -> o.values().stream()).flatMap(a -> a.values().stream()).collect(toList());
    tasklets.addAll(allReceivers);
}

Example 10 with ProcCtx

use of com.hazelcast.jet.impl.execution.init.Contexts.ProcCtx in project hazelcast-jet by hazelcast.

the class ProcessorTaskletTest method setUp.

@Before
public void setUp() {
    this.mockInput = IntStream.range(0, MOCK_INPUT_SIZE).boxed().collect(toList());
    this.processor = new PassThroughProcessor();
    this.context = new ProcCtx(null, new DefaultSerializationServiceBuilder().build(), null, null, 0, NONE, 1, 1);
    this.instreams = new ArrayList<>();
    this.outstreams = new ArrayList<>();
}