Examples with SchedulableProgramType co.cask.cdap.api.schedule.SchedulableProgramType used on opensource projects

Example 1 with SchedulableProgramType

use of co.cask.cdap.api.schedule.SchedulableProgramType in project cdap by caskdata.

the class DistributedWorkflowProgramRunner method setupLaunchConfig.

@Override
protected void setupLaunchConfig(LaunchConfig launchConfig, Program program, ProgramOptions options, CConfiguration cConf, Configuration hConf, File tempDir) throws IOException {
    WorkflowSpecification spec = program.getApplicationSpecification().getWorkflows().get(program.getName());
    List<ClassAcceptor> acceptors = new ArrayList<>();
    // Only interested in MapReduce and Spark nodes
    Set<SchedulableProgramType> runnerTypes = EnumSet.of(SchedulableProgramType.MAPREDUCE, SchedulableProgramType.SPARK);
    for (WorkflowActionNode node : Iterables.filter(spec.getNodeIdMap().values(), WorkflowActionNode.class)) {
        // For each type, we only need one node to setup the launch context
        ScheduleProgramInfo programInfo = node.getProgram();
        if (!runnerTypes.remove(programInfo.getProgramType())) {
            continue;
        }
        // Find the ProgramRunner of the given type and setup the launch context
        ProgramType programType = ProgramType.valueOfSchedulableType(programInfo.getProgramType());
        ProgramRunner runner = programRunnerFactory.create(programType);
        try {
            if (runner instanceof DistributedProgramRunner) {
                // Call setupLaunchConfig with the corresponding program
                ProgramId programId = program.getId().getParent().program(programType, programInfo.getProgramName());
                ((DistributedProgramRunner) runner).setupLaunchConfig(launchConfig, Programs.create(cConf, program, programId, runner), options, cConf, hConf, tempDir);
                acceptors.add(launchConfig.getClassAcceptor());
            }
        } finally {
            if (runner instanceof Closeable) {
                Closeables.closeQuietly((Closeable) runner);
            }
        }
    }
    // Set the class acceptor
    launchConfig.setClassAcceptor(new AndClassAcceptor(acceptors));
    // Clear and set the runnable for the workflow driver
    launchConfig.clearRunnables();
    Resources resources = findDriverResources(program.getApplicationSpecification().getSpark(), program.getApplicationSpecification().getMapReduce(), spec);
    resources = SystemArguments.getResources(options.getUserArguments(), resources);
    launchConfig.addRunnable(spec.getName(), new WorkflowTwillRunnable(spec.getName()), resources, 1, 0);
}

Example 2 with SchedulableProgramType

use of co.cask.cdap.api.schedule.SchedulableProgramType in project cdap by caskdata.

the class TimeScheduler method addProgramSchedule.

@Override
public void addProgramSchedule(ProgramSchedule schedule) throws AlreadyExistsException, SchedulerException {
    checkInitialized();
    ProgramId program = schedule.getProgramId();
    SchedulableProgramType programType = program.getType().getSchedulableType();
    try {
        assertScheduleDoesNotExist(program, programType, schedule.getName());
    } catch (org.quartz.SchedulerException e) {
        throw new SchedulerException(e);
    }
    JobDetail job = addJob(program, programType);
    scheduleJob(program, programType, schedule.getName(), ((ProtoTrigger.TimeTrigger) schedule.getTrigger()).getCronExpression(), job, schedule.getProperties());
}

Example 3 with SchedulableProgramType

use of co.cask.cdap.api.schedule.SchedulableProgramType in project cdap by caskdata.

the class DistributedWorkflowProgramRunner method findDriverResources.

/**
   * Returns the {@link Resources} requirement for the workflow runnable deduced by Spark
   * or MapReduce driver resources requirement.
   */
private Resources findDriverResources(Map<String, SparkSpecification> sparkSpecs, Map<String, MapReduceSpecification> mrSpecs, WorkflowSpecification spec) {
    // Find the resource requirements from the workflow with 768MB as minimum.
    // It is the largest memory and cores from all Spark and MapReduce programs inside the workflow
    Resources resources = new Resources(768);
    for (WorkflowNode node : spec.getNodeIdMap().values()) {
        if (WorkflowNodeType.ACTION == node.getType()) {
            ScheduleProgramInfo programInfo = ((WorkflowActionNode) node).getProgram();
            SchedulableProgramType programType = programInfo.getProgramType();
            if (programType == SchedulableProgramType.SPARK || programType == SchedulableProgramType.MAPREDUCE) {
                // The program spec shouldn't be null, otherwise the Workflow is not valid
                Resources driverResources;
                if (programType == SchedulableProgramType.SPARK) {
                    driverResources = sparkSpecs.get(programInfo.getProgramName()).getClientResources();
                } else {
                    driverResources = mrSpecs.get(programInfo.getProgramName()).getDriverResources();
                }
                if (driverResources != null) {
                    resources = max(resources, driverResources);
                }
            }
        }
    }
    return resources;
}

Example 4 with SchedulableProgramType

use of co.cask.cdap.api.schedule.SchedulableProgramType in project cdap by caskdata.

the class DatasetBasedStreamSizeScheduleStore method list.

/**
   * @return a list of all the schedules and their states present in the store
   */
public synchronized List<StreamSizeScheduleState> list() throws InterruptedException, TransactionFailureException {
    final List<StreamSizeScheduleState> scheduleStates = Lists.newArrayList();
    factory.createExecutor(ImmutableList.of((TransactionAware) table)).execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            try (Scanner scan = getScannerWithPrefix(table, KEY_PREFIX)) {
                Row row;
                while ((row = scan.next()) != null) {
                    byte[] scheduleBytes = row.get(SCHEDULE_COL);
                    byte[] baseSizeBytes = row.get(BASE_SIZE_COL);
                    byte[] baseTsBytes = row.get(BASE_TS_COL);
                    byte[] lastRunSizeBytes = row.get(LAST_RUN_SIZE_COL);
                    byte[] lastRunTsBytes = row.get(LAST_RUN_TS_COL);
                    byte[] activeBytes = row.get(ACTIVE_COL);
                    byte[] propertyBytes = row.get(PROPERTIES_COL);
                    if (isInvalidRow(row)) {
                        LIMITED_LOG.debug("Stream Sized Schedule entry with Row key {} does not have all columns.", Bytes.toString(row.getRow()));
                        continue;
                    }
                    String rowKey = Bytes.toString(row.getRow());
                    String[] splits = rowKey.split(":");
                    ProgramId program;
                    if (splits.length == 7) {
                        // New Row key for the trigger should be of the form -
                        // streamSizeSchedule:namespace:application:version:type:program:schedule
                        program = new ApplicationId(splits[1], splits[2], splits[3]).program(ProgramType.valueOf(splits[4]), splits[5]);
                    } else if (splits.length == 6) {
                        program = new ApplicationId(splits[1], splits[2]).program(ProgramType.valueOf(splits[3]), splits[4]);
                    } else {
                        continue;
                    }
                    SchedulableProgramType programType = program.getType().getSchedulableType();
                    StreamSizeSchedule schedule = GSON.fromJson(Bytes.toString(scheduleBytes), StreamSizeSchedule.class);
                    long baseSize = Bytes.toLong(baseSizeBytes);
                    long baseTs = Bytes.toLong(baseTsBytes);
                    long lastRunSize = Bytes.toLong(lastRunSizeBytes);
                    long lastRunTs = Bytes.toLong(lastRunTsBytes);
                    boolean active = Bytes.toBoolean(activeBytes);
                    Map<String, String> properties = Maps.newHashMap();
                    if (propertyBytes != null) {
                        properties = GSON.fromJson(Bytes.toString(propertyBytes), STRING_MAP_TYPE);
                    }
                    StreamSizeScheduleState scheduleState = new StreamSizeScheduleState(program, programType, schedule, properties, baseSize, baseTs, lastRunSize, lastRunTs, active);
                    scheduleStates.add(scheduleState);
                    LOG.debug("StreamSizeSchedule found in store: {}", scheduleState);
                }
            }
        }
    });
    return scheduleStates;
}