Search in sources :

Example 1 with SchedulableProgramType

use of io.cdap.cdap.api.schedule.SchedulableProgramType in project cdap by caskdata.

the class TimeScheduler method addProgramSchedule.

public void addProgramSchedule(ProgramSchedule schedule) throws AlreadyExistsException, SchedulerException {
    // Verify every trigger does not exist first before adding any of them to Quartz scheduler
    try {
        Map<String, TriggerKey> cronTriggerKeyMap = getCronTriggerKeyMap(schedule);
        for (TriggerKey triggerKey : cronTriggerKeyMap.values()) {
            assertTriggerDoesNotExist(triggerKey);
        }
        ProgramId program = schedule.getProgramId();
        SchedulableProgramType programType = program.getType().getSchedulableType();
        JobDetail job = addJob(program, programType);
        for (Map.Entry<String, TriggerKey> entry : cronTriggerKeyMap.entrySet()) {
            scheduleJob(entry.getValue(), schedule.getName(), entry.getKey(), job);
        }
    } catch (org.quartz.SchedulerException e) {
        throw new SchedulerException(e);
    }
}
Also used : TriggerKey(org.quartz.TriggerKey) JobDetail(org.quartz.JobDetail) SchedulableProgramType(io.cdap.cdap.api.schedule.SchedulableProgramType) ProgramId(io.cdap.cdap.proto.id.ProgramId) HashMap(java.util.HashMap) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap)

Example 2 with SchedulableProgramType

use of io.cdap.cdap.api.schedule.SchedulableProgramType in project cdap by caskdata.

the class TimeScheduler method getScheduledRuntime.

private List<ScheduledRuntime> getScheduledRuntime(ProgramId program, boolean previousRuntimeRequested) throws SchedulerException {
    List<ScheduledRuntime> scheduledRuntimes = new ArrayList<>();
    SchedulableProgramType schedulableType = program.getType().getSchedulableType();
    if (schedulableType == null) {
        throw new IllegalArgumentException("Program " + program + " cannot be scheduled");
    }
    try {
        for (Trigger trigger : scheduler.getTriggersOfJob(jobKeyFor(program, schedulableType))) {
            long time;
            if (previousRuntimeRequested) {
                if (trigger.getPreviousFireTime() == null) {
                    // previous fire time can be null for the triggers which are not yet fired
                    continue;
                }
                time = trigger.getPreviousFireTime().getTime();
            } else {
                // skip the trigger that is not enabled, since it cannot launch program as scheduled
                if (scheduler.getTriggerState(trigger.getKey()) == Trigger.TriggerState.PAUSED) {
                    continue;
                }
                time = trigger.getNextFireTime().getTime();
            }
            ScheduledRuntime runtime = new ScheduledRuntime(trigger.getKey().toString(), time);
            scheduledRuntimes.add(runtime);
        }
    } catch (org.quartz.SchedulerException e) {
        throw new SchedulerException(e);
    }
    return scheduledRuntimes;
}
Also used : SatisfiableTrigger(io.cdap.cdap.internal.app.runtime.schedule.trigger.SatisfiableTrigger) Trigger(org.quartz.Trigger) ProtoTrigger(io.cdap.cdap.proto.ProtoTrigger) AbstractSatisfiableCompositeTrigger(io.cdap.cdap.internal.app.runtime.schedule.trigger.AbstractSatisfiableCompositeTrigger) TimeTrigger(io.cdap.cdap.internal.app.runtime.schedule.trigger.TimeTrigger) ArrayList(java.util.ArrayList) SchedulableProgramType(io.cdap.cdap.api.schedule.SchedulableProgramType) ScheduledRuntime(io.cdap.cdap.proto.ScheduledRuntime)

Example 3 with SchedulableProgramType

use of io.cdap.cdap.api.schedule.SchedulableProgramType in project cdap by caskdata.

the class TimeScheduler method getCronTriggerKeyMap.

/**
 * @return A Map with cron expression as keys and corresponding trigger key as values.
 * Trigger keys are created from program name, programType and scheduleName (and cron expression if the trigger
 * in the schedule is a composite trigger) and TimeScheuler#PAUSED_NEW_TRIGGERS_GROUP
 * if it exists in this group else returns the {@link TriggerKey} prepared with null which gets it with
 * {@link Key#DEFAULT_GROUP}
 * @throws org.quartz.SchedulerException
 */
private Map<String, TriggerKey> getCronTriggerKeyMap(ProgramSchedule schedule) throws org.quartz.SchedulerException {
    ProgramId program = schedule.getProgramId();
    SchedulableProgramType programType = program.getType().getSchedulableType();
    io.cdap.cdap.api.schedule.Trigger trigger = schedule.getTrigger();
    Map<String, TriggerKey> cronTriggerKeyMap = new HashMap<>();
    // Get a set of TimeTrigger if the schedule's trigger is a composite trigger
    if (trigger instanceof AbstractSatisfiableCompositeTrigger) {
        Set<SatisfiableTrigger> triggerSet = ((AbstractSatisfiableCompositeTrigger) trigger).getUnitTriggers().get(ProtoTrigger.Type.TIME);
        if (triggerSet == null) {
            return ImmutableMap.of();
        }
        for (SatisfiableTrigger timeTrigger : triggerSet) {
            String cron = ((TimeTrigger) timeTrigger).getCronExpression();
            String triggerName = AbstractTimeSchedulerService.getTriggerName(program, programType, schedule.getName(), cron);
            cronTriggerKeyMap.put(cron, triggerKeyForName(triggerName));
        }
        return cronTriggerKeyMap;
    }
    // No need to include cron expression in trigger key if the trigger is not composite trigger
    String triggerName = AbstractTimeSchedulerService.scheduleIdFor(program, programType, schedule.getName());
    cronTriggerKeyMap.put(((TimeTrigger) schedule.getTrigger()).getCronExpression(), triggerKeyForName(triggerName));
    return cronTriggerKeyMap;
}
Also used : TimeTrigger(io.cdap.cdap.internal.app.runtime.schedule.trigger.TimeTrigger) HashMap(java.util.HashMap) AbstractSatisfiableCompositeTrigger(io.cdap.cdap.internal.app.runtime.schedule.trigger.AbstractSatisfiableCompositeTrigger) ProgramId(io.cdap.cdap.proto.id.ProgramId) TriggerKey(org.quartz.TriggerKey) SatisfiableTrigger(io.cdap.cdap.internal.app.runtime.schedule.trigger.SatisfiableTrigger) SchedulableProgramType(io.cdap.cdap.api.schedule.SchedulableProgramType)

Example 4 with SchedulableProgramType

use of io.cdap.cdap.api.schedule.SchedulableProgramType in project cdap by caskdata.

the class WorkflowDriver method executeAction.

private void executeAction(WorkflowActionNode node, WorkflowToken token) throws Exception {
    status.put(node.getNodeId(), node);
    CountDownLatch executorTerminateLatch = new CountDownLatch(1);
    ExecutorService executorService = createExecutor(1, executorTerminateLatch, "action-" + node.getNodeId() + "-%d");
    try {
        // Run the action in new thread
        Future<?> future = executorService.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                SchedulableProgramType programType = node.getProgram().getProgramType();
                String programName = node.getProgram().getProgramName();
                String prettyProgramType = ProgramType.valueOf(programType.name()).getPrettyName();
                ProgramWorkflowRunner programWorkflowRunner = workflowProgramRunnerFactory.getProgramWorkflowRunner(programType, token, node.getNodeId(), nodeStates);
                // this should not happen, since null is only passed in from WorkflowDriver, only when calling configure
                if (programWorkflowRunner == null) {
                    throw new UnsupportedOperationException("Operation not allowed.");
                }
                Runnable programRunner = programWorkflowRunner.create(programName);
                LOG.info("Starting {} Program '{}' in workflow", prettyProgramType, programName);
                programRunner.run();
                LOG.info("{} Program '{}' in workflow completed", prettyProgramType, programName);
                return null;
            }
        });
        future.get();
    } catch (Throwable t) {
        Throwables.propagateIfPossible(t, Exception.class);
        throw Throwables.propagate(t);
    } finally {
        executorService.shutdownNow();
        executorTerminateLatch.await();
        status.remove(node.getNodeId());
    }
    workflowStateWriter.setWorkflowToken(workflowRunId, token);
}
Also used : CountDownLatch(java.util.concurrent.CountDownLatch) DatasetManagementException(io.cdap.cdap.api.dataset.DatasetManagementException) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) ExecutorService(java.util.concurrent.ExecutorService) SchedulableProgramType(io.cdap.cdap.api.schedule.SchedulableProgramType) BasicThrowable(io.cdap.cdap.proto.BasicThrowable)

Example 5 with SchedulableProgramType

use of io.cdap.cdap.api.schedule.SchedulableProgramType in project cdap by caskdata.

the class DistributedWorkflowProgramRunner method setupLaunchConfig.

@Override
protected void setupLaunchConfig(ProgramLaunchConfig launchConfig, Program program, ProgramOptions options, CConfiguration cConf, Configuration hConf, File tempDir) throws IOException {
    WorkflowSpecification spec = program.getApplicationSpecification().getWorkflows().get(program.getName());
    List<ClassAcceptor> acceptors = new ArrayList<>();
    acceptors.add(launchConfig.getClassAcceptor());
    // Only interested in MapReduce and Spark nodes.
    // This is because CUSTOM_ACTION types are running inside the driver
    Set<SchedulableProgramType> runnerTypes = EnumSet.of(SchedulableProgramType.MAPREDUCE, SchedulableProgramType.SPARK);
    Iterable<ScheduleProgramInfo> programInfos = spec.getNodeIdMap().values().stream().filter(WorkflowActionNode.class::isInstance).map(WorkflowActionNode.class::cast).map(WorkflowActionNode::getProgram).filter(programInfo -> runnerTypes.contains(programInfo.getProgramType()))::iterator;
    // Can't use Stream.forEach as we want to preserve the IOException being thrown
    for (ScheduleProgramInfo programInfo : programInfos) {
        ProgramType programType = ProgramType.valueOfSchedulableType(programInfo.getProgramType());
        ProgramRunner runner = programRunnerFactory.create(programType);
        try {
            if (runner instanceof DistributedProgramRunner) {
                // Call setupLaunchConfig with the corresponding program.
                // Need to constructs a new ProgramOptions with the scope extracted for the given program
                ProgramId programId = program.getId().getParent().program(programType, programInfo.getProgramName());
                Map<String, String> programUserArgs = RuntimeArguments.extractScope(programId.getType().getScope(), programId.getProgram(), options.getUserArguments().asMap());
                ProgramOptions programOptions = new SimpleProgramOptions(programId, options.getArguments(), new BasicArguments(programUserArgs));
                ((DistributedProgramRunner) runner).setupLaunchConfig(launchConfig, Programs.create(cConf, program, programId, runner), programOptions, cConf, hConf, tempDir);
                acceptors.add(launchConfig.getClassAcceptor());
            }
        } finally {
            if (runner instanceof Closeable) {
                Closeables.closeQuietly((Closeable) runner);
            }
        }
    }
    // Set the class acceptor
    launchConfig.setClassAcceptor(new AndClassAcceptor(acceptors));
    // Find out the default resources requirements based on the programs inside the workflow
    // At least gives the Workflow driver 768 mb of container memory
    Map<String, Resources> runnablesResources = Maps.transformValues(launchConfig.getRunnables(), this::getResources);
    Resources defaultResources = maxResources(new Resources(768), findDriverResources(spec.getNodes(), runnablesResources));
    // Clear and set the runnable for the workflow driver.
    launchConfig.clearRunnables();
    // Extract scoped runtime arguments that only meant for the workflow but not for child nodes
    Map<String, String> runtimeArgs = RuntimeArguments.extractScope("task", "workflow", options.getUserArguments().asMap());
    launchConfig.addRunnable(spec.getName(), new WorkflowTwillRunnable(spec.getName()), 1, runtimeArgs, defaultResources, 0);
}
Also used : URL(java.net.URL) Inject(com.google.inject.Inject) LoggerFactory(org.slf4j.LoggerFactory) ClusterMode(io.cdap.cdap.app.guice.ClusterMode) ProgramRunnerFactory(io.cdap.cdap.app.runtime.ProgramRunnerFactory) Resources(io.cdap.cdap.api.Resources) WorkflowNode(io.cdap.cdap.api.workflow.WorkflowNode) SimpleProgramOptions(io.cdap.cdap.internal.app.runtime.SimpleProgramOptions) Configuration(org.apache.hadoop.conf.Configuration) Map(java.util.Map) Closeables(com.google.common.io.Closeables) ProgramRunner(io.cdap.cdap.app.runtime.ProgramRunner) EnumSet(java.util.EnumSet) TwillController(org.apache.twill.api.TwillController) Collection(java.util.Collection) ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) WorkflowNodeType(io.cdap.cdap.api.workflow.WorkflowNodeType) ResourceSpecification(org.apache.twill.api.ResourceSpecification) Set(java.util.Set) SchedulableProgramType(io.cdap.cdap.api.schedule.SchedulableProgramType) ScheduleProgramInfo(io.cdap.cdap.api.workflow.ScheduleProgramInfo) List(java.util.List) Constants(io.cdap.cdap.common.conf.Constants) Workflow(io.cdap.cdap.api.workflow.Workflow) WorkflowForkNode(io.cdap.cdap.api.workflow.WorkflowForkNode) WorkflowSpecification(io.cdap.cdap.api.workflow.WorkflowSpecification) Program(io.cdap.cdap.app.program.Program) Programs(io.cdap.cdap.app.program.Programs) ProgramType(io.cdap.cdap.proto.ProgramType) ArrayList(java.util.ArrayList) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) ProgramOptions(io.cdap.cdap.app.runtime.ProgramOptions) TwillRunner(org.apache.twill.api.TwillRunner) SystemArguments(io.cdap.cdap.internal.app.runtime.SystemArguments) WorkflowActionNode(io.cdap.cdap.api.workflow.WorkflowActionNode) ClassAcceptor(org.apache.twill.api.ClassAcceptor) ProgramController(io.cdap.cdap.app.runtime.ProgramController) Logger(org.slf4j.Logger) RuntimeArguments(io.cdap.cdap.api.common.RuntimeArguments) ProgramId(io.cdap.cdap.proto.id.ProgramId) Impersonator(io.cdap.cdap.security.impersonation.Impersonator) IOException(java.io.IOException) Maps(com.google.common.collect.Maps) File(java.io.File) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) Closeable(java.io.Closeable) Preconditions(com.google.common.base.Preconditions) WorkflowConditionNode(io.cdap.cdap.api.workflow.WorkflowConditionNode) Collections(java.util.Collections) BasicArguments(io.cdap.cdap.internal.app.runtime.BasicArguments) WorkflowActionNode(io.cdap.cdap.api.workflow.WorkflowActionNode) Closeable(java.io.Closeable) ArrayList(java.util.ArrayList) ClassAcceptor(org.apache.twill.api.ClassAcceptor) ProgramId(io.cdap.cdap.proto.id.ProgramId) SimpleProgramOptions(io.cdap.cdap.internal.app.runtime.SimpleProgramOptions) ProgramOptions(io.cdap.cdap.app.runtime.ProgramOptions) WorkflowSpecification(io.cdap.cdap.api.workflow.WorkflowSpecification) SchedulableProgramType(io.cdap.cdap.api.schedule.SchedulableProgramType) SchedulableProgramType(io.cdap.cdap.api.schedule.SchedulableProgramType) ProgramType(io.cdap.cdap.proto.ProgramType) SimpleProgramOptions(io.cdap.cdap.internal.app.runtime.SimpleProgramOptions) BasicArguments(io.cdap.cdap.internal.app.runtime.BasicArguments) Resources(io.cdap.cdap.api.Resources) ScheduleProgramInfo(io.cdap.cdap.api.workflow.ScheduleProgramInfo) ProgramRunner(io.cdap.cdap.app.runtime.ProgramRunner)

Aggregations

SchedulableProgramType (io.cdap.cdap.api.schedule.SchedulableProgramType)5 ProgramId (io.cdap.cdap.proto.id.ProgramId)3 AbstractSatisfiableCompositeTrigger (io.cdap.cdap.internal.app.runtime.schedule.trigger.AbstractSatisfiableCompositeTrigger)2 SatisfiableTrigger (io.cdap.cdap.internal.app.runtime.schedule.trigger.SatisfiableTrigger)2 TimeTrigger (io.cdap.cdap.internal.app.runtime.schedule.trigger.TimeTrigger)2 IOException (java.io.IOException)2 ArrayList (java.util.ArrayList)2 HashMap (java.util.HashMap)2 TriggerKey (org.quartz.TriggerKey)2 Preconditions (com.google.common.base.Preconditions)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 Maps (com.google.common.collect.Maps)1 Closeables (com.google.common.io.Closeables)1 Inject (com.google.inject.Inject)1 Resources (io.cdap.cdap.api.Resources)1 ApplicationSpecification (io.cdap.cdap.api.app.ApplicationSpecification)1 RuntimeArguments (io.cdap.cdap.api.common.RuntimeArguments)1 DatasetManagementException (io.cdap.cdap.api.dataset.DatasetManagementException)1 ScheduleProgramInfo (io.cdap.cdap.api.workflow.ScheduleProgramInfo)1 Workflow (io.cdap.cdap.api.workflow.Workflow)1