Examples with Context com.hazelcast.jet.impl.pipeline.PipelineImpl.Context used on opensource projects

Example 1 with Context

use of com.hazelcast.jet.impl.pipeline.PipelineImpl.Context in project hazelcast by hazelcast.

the class JobCoordinationService method submitLightJob.

public CompletableFuture<Void> submitLightJob(long jobId, Data serializedJobDefinition, JobConfig jobConfig, Subject subject) {
    Object jobDefinition = nodeEngine().getSerializationService().toObject(serializedJobDefinition);
    DAG dag;
    if (jobDefinition instanceof DAG) {
        dag = (DAG) jobDefinition;
    } else {
        int coopThreadCount = config.getCooperativeThreadCount();
        dag = ((PipelineImpl) jobDefinition).toDag(new Context() {

            @Override
            public int defaultLocalParallelism() {
                return coopThreadCount;
            }
        });
    }
    // First insert just a marker into the map. This is to prevent initializing the light job if the jobId
    // was submitted twice. This can happen e.g. if the client retries.
    Object oldContext = lightMasterContexts.putIfAbsent(jobId, UNINITIALIZED_LIGHT_JOB_MARKER);
    if (oldContext != null) {
        throw new JetException("duplicate jobId " + idToString(jobId));
    }
    checkPermissions(subject, dag);
    // Initialize and start the job (happens in the constructor). We do this before adding the actual
    // LightMasterContext to the map to avoid possible races of the job initialization and cancellation.
    LightMasterContext mc = new LightMasterContext(nodeEngine, this, dag, jobId, jobConfig, subject);
    oldContext = lightMasterContexts.put(jobId, mc);
    assert oldContext == UNINITIALIZED_LIGHT_JOB_MARKER;
    scheduleJobTimeout(jobId, jobConfig.getTimeoutMillis());
    return mc.getCompletionFuture().whenComplete((r, t) -> {
        Object removed = lightMasterContexts.remove(jobId);
        assert removed instanceof LightMasterContext : "LMC not found: " + removed;
        unscheduleJobTimeout(jobId);
    });
}

Example 2 with Context

use of com.hazelcast.jet.impl.pipeline.PipelineImpl.Context in project hazelcast by hazelcast.

the class JobCoordinationService method submitJob.

public CompletableFuture<Void> submitJob(long jobId, Data serializedJobDefinition, JobConfig jobConfig, Subject subject) {
    CompletableFuture<Void> res = new CompletableFuture<>();
    submitToCoordinatorThread(() -> {
        MasterContext masterContext;
        try {
            assertIsMaster("Cannot submit job " + idToString(jobId) + " to non-master node");
            checkOperationalState();
            // the order of operations is important.
            // first, check if the job is already completed
            JobResult jobResult = jobRepository.getJobResult(jobId);
            if (jobResult != null) {
                logger.fine("Not starting job " + idToString(jobId) + " since already completed with result: " + jobResult);
                return;
            }
            if (!config.isResourceUploadEnabled() && !jobConfig.getResourceConfigs().isEmpty()) {
                throw new JetException(Util.JET_RESOURCE_UPLOAD_DISABLED_MESSAGE);
            }
            int quorumSize = jobConfig.isSplitBrainProtectionEnabled() ? getQuorumSize() : 0;
            Object jobDefinition = deserializeJobDefinition(jobId, jobConfig, serializedJobDefinition);
            DAG dag;
            Data serializedDag;
            if (jobDefinition instanceof PipelineImpl) {
                int coopThreadCount = config.getCooperativeThreadCount();
                dag = ((PipelineImpl) jobDefinition).toDag(new Context() {

                    @Override
                    public int defaultLocalParallelism() {
                        return coopThreadCount;
                    }
                });
                serializedDag = nodeEngine().getSerializationService().toData(dag);
            } else {
                dag = (DAG) jobDefinition;
                serializedDag = serializedJobDefinition;
            }
            checkPermissions(subject, dag);
            Set<String> ownedObservables = ownedObservables(dag);
            JobRecord jobRecord = new JobRecord(nodeEngine.getClusterService().getClusterVersion(), jobId, serializedDag, dagToJson(dag), jobConfig, ownedObservables, subject);
            JobExecutionRecord jobExecutionRecord = new JobExecutionRecord(jobId, quorumSize);
            masterContext = createMasterContext(jobRecord, jobExecutionRecord);
            boolean hasDuplicateJobName;
            synchronized (lock) {
                assertIsMaster("Cannot submit job " + idToString(jobId) + " to non-master node");
                checkOperationalState();
                hasDuplicateJobName = jobConfig.getName() != null && hasActiveJobWithName(jobConfig.getName());
                if (!hasDuplicateJobName) {
                    // just try to initiate the coordination
                    MasterContext prev = masterContexts.putIfAbsent(jobId, masterContext);
                    if (prev != null) {
                        logger.fine("Joining to already existing masterContext " + prev.jobIdString());
                        return;
                    }
                }
            }
            if (hasDuplicateJobName) {
                jobRepository.deleteJob(jobId);
                throw new JobAlreadyExistsException("Another active job with equal name (" + jobConfig.getName() + ") exists: " + idToString(jobId));
            }
            // If job is not currently running, it might be that it is just completed
            if (completeMasterContextIfJobAlreadyCompleted(masterContext)) {
                return;
            }
            // If there is no master context and job result at the same time, it means this is the first submission
            jobSubmitted.inc();
            jobRepository.putNewJobRecord(jobRecord);
            logger.info("Starting job " + idToString(masterContext.jobId()) + " based on submit request");
        } catch (Throwable e) {
            jetServiceBackend.getJobClassLoaderService().tryRemoveClassloadersForJob(jobId, COORDINATOR);
            res.completeExceptionally(e);
            throw e;
        } finally {
            res.complete(null);
        }
        tryStartJob(masterContext);
    });
    return res;
}

Example 3 with Context

use of com.hazelcast.jet.impl.pipeline.PipelineImpl.Context in project hazelcast by hazelcast.

the class Planner method createDag.

@SuppressWarnings("rawtypes")
DAG createDag(Context context) {
    pipeline.makeNamesUnique();
    Map<Transform, List<Transform>> adjacencyMap = pipeline.adjacencyMap();
    validateNoLeakage(adjacencyMap);
    checkTopologicalSort(adjacencyMap.entrySet());
    // Find the greatest common denominator of all frame lengths
    // appearing in the pipeline
    long frameSizeGcd = Util.gcd(adjacencyMap.keySet().stream().map(Transform::preferredWatermarkStride).filter(frameSize -> frameSize > 0).mapToLong(i -> i).toArray());
    if (frameSizeGcd == 0) {
        // even if there are no window aggregations, we want the watermarks for latency debugging
        frameSizeGcd = MAXIMUM_WATERMARK_GAP;
    }
    if (frameSizeGcd > MAXIMUM_WATERMARK_GAP) {
        frameSizeGcd = Util.gcd(frameSizeGcd, MAXIMUM_WATERMARK_GAP);
    }
    LoggingUtil.logFine(LOGGER, "Watermarks in the pipeline will be throttled to %d", frameSizeGcd);
    // Update watermark throttling frame length on all transforms with the determined length
    for (Transform transform : adjacencyMap.keySet()) {
        if (transform instanceof StreamSourceTransform) {
            StreamSourceTransform t = (StreamSourceTransform) transform;
            EventTimePolicy policy = t.getEventTimePolicy();
            if (policy != null) {
                t.setEventTimePolicy(withFrameSize(policy, frameSizeGcd));
            }
        } else if (transform instanceof TimestampTransform) {
            TimestampTransform t = (TimestampTransform) transform;
            t.setEventTimePolicy(withFrameSize(t.getEventTimePolicy(), frameSizeGcd));
        }
    }
    // fuse subsequent map/filter/flatMap transforms into one
    Map<Transform, List<Transform>> originalParents = new HashMap<>();
    List<Transform> transforms = new ArrayList<>(adjacencyMap.keySet());
    for (int i = 0; i < transforms.size(); i++) {
        Transform transform = transforms.get(i);
        List<Transform> chain = findFusableChain(transform, adjacencyMap);
        if (chain == null) {
            continue;
        }
        // remove transforms in the chain and replace the parent with a fused transform
        transforms.removeAll(chain.subList(1, chain.size()));
        Transform fused = fuseFlatMapTransforms(chain);
        transforms.set(i, fused);
        Transform lastInChain = chain.get(chain.size() - 1);
        for (Transform downstream : adjacencyMap.get(lastInChain)) {
            originalParents.put(downstream, new ArrayList<>(downstream.upstream()));
            downstream.upstream().replaceAll(p -> p == lastInChain ? fused : p);
        }
    }
    for (Transform transform : transforms) {
        transform.addToDag(this, context);
    }
    // restore original parents
    for (Entry<Transform, List<Transform>> en : originalParents.entrySet()) {
        List<Transform> upstream = en.getKey().upstream();
        for (int i = 0; i < upstream.size(); i++) {
            en.getKey().upstream().set(i, en.getValue().get(i));
        }
    }
    return dag;
}