Examples with JetServiceBackend com.hazelcast.jet.impl.JetServiceBackend used on opensource projects

Example 1 with JetServiceBackend

use of com.hazelcast.jet.impl.JetServiceBackend in project hazelcast by hazelcast.

the class PlanExecutor method execute.

SqlResult execute(ShowStatementPlan plan) {
    Stream<String> rows;
    switch(plan.getShowTarget()) {
        case MAPPINGS:
            rows = catalog.getMappingNames().stream();
            break;
        case VIEWS:
            rows = catalog.getViewNames().stream();
            break;
        case JOBS:
            assert plan.getShowTarget() == ShowStatementTarget.JOBS;
            NodeEngine nodeEngine = getNodeEngine(hazelcastInstance);
            JetServiceBackend jetServiceBackend = nodeEngine.getService(JetServiceBackend.SERVICE_NAME);
            rows = jetServiceBackend.getJobRepository().getJobRecords().stream().map(record -> record.getConfig().getName()).filter(Objects::nonNull);
            break;
        default:
            throw new AssertionError("Unsupported SHOW statement target.");
    }
    SqlRowMetadata metadata = new SqlRowMetadata(singletonList(new SqlColumnMetadata("name", VARCHAR, false)));
    InternalSerializationService serializationService = Util.getSerializationService(hazelcastInstance);
    return new SqlResultImpl(QueryId.create(hazelcastInstance.getLocalEndpoint().getUuid()), new StaticQueryResultProducerImpl(rows.sorted().map(name -> new JetSqlRow(serializationService, new Object[] { name })).iterator()), metadata, false);
}

Example 2 with JetServiceBackend

use of com.hazelcast.jet.impl.JetServiceBackend in project hazelcast by hazelcast.

the class JetInfoCollector method forEachMetric.

@Override
public void forEachMetric(Node node, BiConsumer<PhoneHomeMetrics, String> metricsConsumer) {
    JetConfig jetConfig = node.getNodeEngine().getConfig().getJetConfig();
    boolean isJetEnabled = jetConfig.isEnabled();
    metricsConsumer.accept(PhoneHomeMetrics.JET_ENABLED, String.valueOf(isJetEnabled));
    metricsConsumer.accept(PhoneHomeMetrics.JET_RESOURCE_UPLOAD_ENABLED, String.valueOf(jetConfig.isResourceUploadEnabled()));
    if (isJetEnabled) {
        JetServiceBackend jetServiceBackend = node.getNodeEngine().getService(JetServiceBackend.SERVICE_NAME);
        long jobSubmittedCount = jetServiceBackend.getJobCoordinationService().getJobSubmittedCount();
        metricsConsumer.accept(PhoneHomeMetrics.JET_JOBS_SUBMITTED, String.valueOf(jobSubmittedCount));
    }
}

Example 3 with JetServiceBackend

use of com.hazelcast.jet.impl.JetServiceBackend in project hazelcast by hazelcast.

the class ExecutionPlan method initialize.

/**
 * A method called on the members as part of the InitExecutionOperation.
 * Creates tasklets, inboxes/outboxes and connects these to make them ready
 * for a later StartExecutionOperation.
 */
public void initialize(NodeEngineImpl nodeEngine, long jobId, long executionId, @Nonnull SnapshotContext snapshotContext, ConcurrentHashMap<String, File> tempDirectories, InternalSerializationService jobSerializationService) {
    this.nodeEngine = nodeEngine;
    this.jobClassLoaderService = ((JetServiceBackend) nodeEngine.getService(JetServiceBackend.SERVICE_NAME)).getJobClassLoaderService();
    this.executionId = executionId;
    initProcSuppliers(jobId, tempDirectories, jobSerializationService);
    initDag(jobSerializationService);
    this.ptionArrgmt = new PartitionArrangement(partitionAssignment, nodeEngine.getThisAddress());
    Set<Integer> higherPriorityVertices = VertexDef.getHigherPriorityVertices(vertices);
    for (Address destAddr : remoteMembers.get()) {
        Connection conn = getMemberConnection(nodeEngine, destAddr);
        if (conn == null) {
            throw new TopologyChangedException("no connection to job participant: " + destAddr);
        }
        memberConnections.put(destAddr, conn);
    }
    for (VertexDef vertex : vertices) {
        ClassLoader processorClassLoader = isLightJob ? null : jobClassLoaderService.getProcessorClassLoader(jobId, vertex.name());
        Collection<? extends Processor> processors = doWithClassLoader(processorClassLoader, () -> createProcessors(vertex, vertex.localParallelism()));
        String jobPrefix = prefix(jobConfig.getName(), jobId, vertex.name());
        // create StoreSnapshotTasklet and the queues to it
        ConcurrentConveyor<Object> ssConveyor = null;
        if (!isLightJob) {
            // Note that we create the snapshot queues for all non-light jobs, even if they don't have
            // processing guarantee enabled, because in EE one can request a snapshot also for
            // non-snapshotted jobs.
            @SuppressWarnings("unchecked") QueuedPipe<Object>[] snapshotQueues = new QueuedPipe[vertex.localParallelism()];
            Arrays.setAll(snapshotQueues, i -> new OneToOneConcurrentArrayQueue<>(SNAPSHOT_QUEUE_SIZE));
            ssConveyor = ConcurrentConveyor.concurrentConveyor(null, snapshotQueues);
            ILogger storeSnapshotLogger = prefixedLogger(nodeEngine.getLogger(StoreSnapshotTasklet.class), jobPrefix);
            StoreSnapshotTasklet ssTasklet = new StoreSnapshotTasklet(snapshotContext, ConcurrentInboundEdgeStream.create(ssConveyor, 0, 0, true, jobPrefix + "/ssFrom", null), new AsyncSnapshotWriterImpl(nodeEngine, snapshotContext, vertex.name(), memberIndex, memberCount, jobSerializationService), storeSnapshotLogger, vertex.name(), higherPriorityVertices.contains(vertex.vertexId()));
            tasklets.add(ssTasklet);
        }
        int localProcessorIdx = 0;
        for (Processor processor : processors) {
            int globalProcessorIndex = memberIndex * vertex.localParallelism() + localProcessorIdx;
            String processorPrefix = prefix(jobConfig.getName(), jobId, vertex.name(), globalProcessorIndex);
            ILogger logger = prefixedLogger(nodeEngine.getLogger(processor.getClass()), processorPrefix);
            ProcCtx context = new ProcCtx(nodeEngine, jobId, executionId, getJobConfig(), logger, vertex.name(), localProcessorIdx, globalProcessorIndex, isLightJob, partitionAssignment, vertex.localParallelism(), memberIndex, memberCount, tempDirectories, jobSerializationService, subject, processorClassLoader);
            // createOutboundEdgeStreams() populates localConveyorMap and edgeSenderConveyorMap.
            // Also populates instance fields: senderMap, receiverMap, tasklets.
            List<OutboundEdgeStream> outboundStreams = createOutboundEdgeStreams(vertex, localProcessorIdx, jobPrefix, jobSerializationService);
            List<InboundEdgeStream> inboundStreams = createInboundEdgeStreams(vertex, localProcessorIdx, jobPrefix, globalProcessorIndex);
            OutboundCollector snapshotCollector = ssConveyor == null ? null : new ConveyorCollector(ssConveyor, localProcessorIdx, null);
            // vertices which are only used for snapshot restore will not be marked as "source=true" in metrics
            // also do not consider snapshot restore edges for determining source tag
            boolean isSource = vertex.inboundEdges().stream().allMatch(EdgeDef::isSnapshotRestoreEdge) && !vertex.isSnapshotVertex();
            ProcessorTasklet processorTasklet = new ProcessorTasklet(context, nodeEngine.getExecutionService().getExecutor(TASKLET_INIT_CLOSE_EXECUTOR_NAME), jobSerializationService, processor, inboundStreams, outboundStreams, snapshotContext, snapshotCollector, isSource);
            tasklets.add(processorTasklet);
            this.processors.add(processor);
            localProcessorIdx++;
        }
    }
    List<ReceiverTasklet> allReceivers = receiverMap.values().stream().flatMap(o -> o.values().stream()).flatMap(a -> a.values().stream()).collect(toList());
    tasklets.addAll(allReceivers);
}

Example 4 with JetServiceBackend

use of com.hazelcast.jet.impl.JetServiceBackend in project hazelcast by hazelcast.

the class AsyncOperation method beforeRun.

@Override
public void beforeRun() {
    JetServiceBackend service = getJetServiceBackend();
    service.getLiveOperationRegistry().register(this);
}

Example 5 with JetServiceBackend

use of com.hazelcast.jet.impl.JetServiceBackend in project hazelcast by hazelcast.

the class AsyncOperation method doSendResponse.

private void doSendResponse(Object value) {
    try {
        final JetServiceBackend service = getJetServiceBackend();
        service.getLiveOperationRegistry().deregister(this);
    } finally {
        try {
            sendResponse(value);
        } catch (Exception e) {
            Throwable ex = peel(e);
            if (value instanceof Throwable && ex instanceof HazelcastSerializationException) {
                // Sometimes exceptions are not serializable, for example on
                // https://github.com/hazelcast/hazelcast-jet/issues/1995.
                // When sending exception as a response and the serialization fails,
                // the response will not be sent and the operation will hang.
                // To prevent this from happening, replace the exception with
                // another exception that can be serialized.
                sendResponse(new JetException(stackTraceToString(ex)));
            } else {
                throw e;
            }
        }
    }
}