Examples with IActivityClusterGraphGenerator org.apache.hyracks.api.job.IActivityClusterGraphGenerator used on opensource projects

Example 1 with IActivityClusterGraphGenerator

use of org.apache.hyracks.api.job.IActivityClusterGraphGenerator in project asterixdb by apache.

the class JobSpecificationActivityClusterGraphGeneratorFactory method createActivityClusterGraphGenerator.

@Override
public IActivityClusterGraphGenerator createActivityClusterGraphGenerator(JobId jobId, final ICCServiceContext ccServiceCtx, Set<JobFlag> jobFlags) throws HyracksException {
    final JobActivityGraphBuilder builder = new JobActivityGraphBuilder(spec, jobFlags);
    PlanUtils.visit(spec, new IConnectorDescriptorVisitor() {

        @Override
        public void visit(IConnectorDescriptor conn) throws HyracksException {
            builder.addConnector(conn);
        }
    });
    PlanUtils.visit(spec, new IOperatorDescriptorVisitor() {

        @Override
        public void visit(IOperatorDescriptor op) {
            op.contributeActivities(builder);
        }
    });
    builder.finish();
    final JobActivityGraph jag = builder.getActivityGraph();
    ActivityClusterGraphBuilder acgb = new ActivityClusterGraphBuilder();
    final ActivityClusterGraph acg = acgb.inferActivityClusters(jobId, jag);
    acg.setFrameSize(spec.getFrameSize());
    acg.setMaxReattempts(spec.getMaxReattempts());
    acg.setJobletEventListenerFactory(spec.getJobletEventListenerFactory());
    acg.setGlobalJobDataFactory(spec.getGlobalJobDataFactory());
    acg.setConnectorPolicyAssignmentPolicy(spec.getConnectorPolicyAssignmentPolicy());
    acg.setUseConnectorPolicyForScheduling(spec.isUseConnectorPolicyForScheduling());
    final Set<Constraint> constraints = new HashSet<>();
    final IConstraintAcceptor acceptor = new IConstraintAcceptor() {

        @Override
        public void addConstraint(Constraint constraint) {
            constraints.add(constraint);
        }
    };
    PlanUtils.visit(spec, new IOperatorDescriptorVisitor() {

        @Override
        public void visit(IOperatorDescriptor op) {
            op.contributeSchedulingConstraints(acceptor, ccServiceCtx);
        }
    });
    PlanUtils.visit(spec, new IConnectorDescriptorVisitor() {

        @Override
        public void visit(IConnectorDescriptor conn) {
            conn.contributeSchedulingConstraints(acceptor, acg.getConnectorMap().get(conn.getConnectorId()), ccServiceCtx);
        }
    });
    constraints.addAll(spec.getUserConstraints());
    return new IActivityClusterGraphGenerator() {

        @Override
        public ActivityClusterGraph initialize() {
            ActivityClusterGraphRewriter rewriter = new ActivityClusterGraphRewriter();
            rewriter.rewrite(acg);
            return acg;
        }

        @Override
        public Set<Constraint> getConstraints() {
            return constraints;
        }
    };
}

Example 2 with IActivityClusterGraphGenerator

use of org.apache.hyracks.api.job.IActivityClusterGraphGenerator in project asterixdb by apache.

the class JobStartWork method doRun.

@Override
protected void doRun() throws Exception {
    IJobManager jobManager = ccs.getJobManager();
    try {
        final CCServiceContext ccServiceCtx = ccs.getContext();
        JobRun run;
        if (!predestributed) {
            //Need to create the ActivityClusterGraph
            IActivityClusterGraphGeneratorFactory acggf = (IActivityClusterGraphGeneratorFactory) DeploymentUtils.deserialize(acggfBytes, deploymentId, ccServiceCtx);
            IActivityClusterGraphGenerator acgg = acggf.createActivityClusterGraphGenerator(jobId, ccServiceCtx, jobFlags);
            run = new JobRun(ccs, deploymentId, jobId, acggf, acgg, jobFlags);
        } else {
            //ActivityClusterGraph has already been distributed
            run = new JobRun(ccs, deploymentId, jobId, ccs.getPreDistributedJobStore().getDistributedJobDescriptor(jobId));
        }
        jobManager.add(run);
        callback.setValue(jobId);
    } catch (Exception e) {
        callback.setException(e);
    }
}

Example 3 with IActivityClusterGraphGenerator

use of org.apache.hyracks.api.job.IActivityClusterGraphGenerator in project asterixdb by apache.

the class DistributeJobWork method doRun.

@Override
protected void doRun() throws Exception {
    try {
        final CCServiceContext ccServiceCtx = ccs.getContext();
        ccs.getPreDistributedJobStore().checkForExistingDistributedJobDescriptor(jobId);
        IActivityClusterGraphGeneratorFactory acggf = (IActivityClusterGraphGeneratorFactory) DeploymentUtils.deserialize(acggfBytes, null, ccServiceCtx);
        IActivityClusterGraphGenerator acgg = acggf.createActivityClusterGraphGenerator(jobId, ccServiceCtx, EnumSet.noneOf(JobFlag.class));
        ActivityClusterGraph acg = acgg.initialize();
        ccs.getPreDistributedJobStore().addDistributedJobDescriptor(jobId, acg, acggf.getJobSpecification(), acgg.getConstraints());
        ccServiceCtx.notifyJobCreation(jobId, acggf.getJobSpecification());
        byte[] acgBytes = JavaSerializationUtils.serialize(acg);
        INodeManager nodeManager = ccs.getNodeManager();
        for (NodeControllerState node : nodeManager.getAllNodeControllerStates()) {
            node.getNodeController().distributeJob(jobId, acgBytes);
        }
        callback.setValue(jobId);
    } catch (Exception e) {
        callback.setException(e);
    }
}