Examples with ApplicationSpecification co.cask.cdap.api.app.ApplicationSpecification used on opensource projects

Example 46 with ApplicationSpecification

use of co.cask.cdap.api.app.ApplicationSpecification in project cdap by caskdata.

the class ApplicationRegistrationStage method getApplicationSpecificationWithExistingSchedules.

/**
   * uses the existing app spec schedules, however if workflows are deleted in the new app spec,
   * we want to remove the schedules assigned to those deleted workflow from the existing app spec schedules.
   * construct and return a app spec based on this filtered schedules and programs from new app spec.
   * @param input
   * @return {@link ApplicationSpecification} updated spec.
   */
private ApplicationSpecification getApplicationSpecificationWithExistingSchedules(ApplicationWithPrograms input) {
    Map<String, ScheduleSpecification> filteredExistingSchedules = new HashMap<>();
    if (input.getExistingAppSpec() != null) {
        final Set<String> deletedWorkflows = Maps.difference(input.getExistingAppSpec().getWorkflows(), input.getSpecification().getWorkflows()).entriesOnlyOnLeft().keySet();
        // predicate to filter schedules if their workflow is deleted
        Predicate<Map.Entry<String, ScheduleSpecification>> byScheduleProgramStatus = new Predicate<Map.Entry<String, ScheduleSpecification>>() {

            @Override
            public boolean apply(Map.Entry<String, ScheduleSpecification> input) {
                return !deletedWorkflows.contains(input.getValue().getProgram().getProgramName());
            }
        };
        filteredExistingSchedules = Maps.filterEntries(input.getExistingAppSpec().getSchedules(), byScheduleProgramStatus);
    }
    ApplicationSpecification newSpecification = input.getSpecification();
    return new DefaultApplicationSpecification(newSpecification.getName(), newSpecification.getAppVersion(), newSpecification.getDescription(), newSpecification.getConfiguration(), newSpecification.getArtifactId(), newSpecification.getStreams(), newSpecification.getDatasetModules(), newSpecification.getDatasets(), newSpecification.getFlows(), newSpecification.getMapReduce(), newSpecification.getSpark(), newSpecification.getWorkflows(), newSpecification.getServices(), filteredExistingSchedules, newSpecification.getProgramSchedules(), newSpecification.getWorkers(), newSpecification.getPlugins());
}

Example 47 with ApplicationSpecification

use of co.cask.cdap.api.app.ApplicationSpecification in project cdap by caskdata.

the class ProgramLifecycleHttpHandler method getInstances.

/**
   * Returns the number of instances for all program runnables that are passed into the data. The data is an array of
   * Json objects where each object must contain the following three elements: appId, programType, and programId
   * (flow name, service name). Retrieving instances only applies to flows, and user
   * services. For flows, another parameter, "runnableId", must be provided. This corresponds to the
   * flowlet/runnable for which to retrieve the instances.
   * <p>
   * Example input:
   * <pre><code>
   * [{"appId": "App1", "programType": "Service", "programId": "Service1", "runnableId": "Runnable1"},
   *  {"appId": "App1", "programType": "Mapreduce", "programId": "Mapreduce2"},
   *  {"appId": "App2", "programType": "Flow", "programId": "Flow1", "runnableId": "Flowlet1"}]
   * </code></pre>
   * </p><p>
   * The response will be an array of JsonObjects each of which will contain the three input parameters
   * as well as 3 fields:
   * <ul>
   * <li>"provisioned" which maps to the number of instances actually provided for the input runnable;</li>
   * <li>"requested" which maps to the number of instances the user has requested for the input runnable; and</li>
   * <li>"statusCode" which maps to the http status code for the data in that JsonObjects (200, 400, 404).</li>
   * </ul>
   * </p><p>
   * If an error occurs in the input (for the example above, Flowlet1 does not exist), then all JsonObjects for
   * which the parameters have a valid instances will have the provisioned and requested fields status code fields
   * but all JsonObjects for which the parameters are not valid will have an error message and statusCode.
   * </p><p>
   * For example, if there is no Flowlet1 in the above data, then the response could be 200 OK with the following data:
   * </p>
   * <pre><code>
   * [{"appId": "App1", "programType": "Service", "programId": "Service1", "runnableId": "Runnable1",
   *   "statusCode": 200, "provisioned": 2, "requested": 2},
   *  {"appId": "App1", "programType": "Mapreduce", "programId": "Mapreduce2", "statusCode": 400,
   *   "error": "Program type 'Mapreduce' is not a valid program type to get instances"},
   *  {"appId": "App2", "programType": "Flow", "programId": "Flow1", "runnableId": "Flowlet1", "statusCode": 404,
   *   "error": "Program": Flowlet1 not found"}]
   * </code></pre>
   */
@POST
@Path("/instances")
@AuditPolicy(AuditDetail.REQUEST_BODY)
public void getInstances(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId) throws IOException, BadRequestException {
    List<BatchRunnable> runnables = validateAndGetBatchInput(request, BATCH_RUNNABLES_TYPE);
    // cache app specs to perform fewer store lookups
    Map<ApplicationId, ApplicationSpecification> appSpecs = new HashMap<>();
    List<BatchRunnableInstances> output = new ArrayList<>(runnables.size());
    for (BatchRunnable runnable : runnables) {
        // cant get instances for things that are not flows, services, or workers
        if (!canHaveInstances(runnable.getProgramType())) {
            output.add(new BatchRunnableInstances(runnable, HttpResponseStatus.BAD_REQUEST.getCode(), String.format("Program type '%s' is not a valid program type to get instances", runnable.getProgramType().getPrettyName())));
            continue;
        }
        ApplicationId appId = new ApplicationId(namespaceId, runnable.getAppId());
        // populate spec cache if this is the first time we've seen the appid.
        if (!appSpecs.containsKey(appId)) {
            appSpecs.put(appId, store.getApplication(appId));
        }
        ApplicationSpecification spec = appSpecs.get(appId);
        if (spec == null) {
            output.add(new BatchRunnableInstances(runnable, HttpResponseStatus.NOT_FOUND.getCode(), String.format("App: %s not found", appId)));
            continue;
        }
        ProgramId programId = appId.program(runnable.getProgramType(), runnable.getProgramId());
        output.add(getProgramInstances(runnable, spec, programId));
    }
    responder.sendJson(HttpResponseStatus.OK, output);
}

Example 48 with ApplicationSpecification

use of co.cask.cdap.api.app.ApplicationSpecification in project cdap by caskdata.

the class AbstractSchedulerService method getSchedulerForSchedule.

private Scheduler getSchedulerForSchedule(ProgramId program, String scheduleName) throws NotFoundException {
    ApplicationSpecification appSpec = store.getApplication(program.getParent());
    if (appSpec == null) {
        throw new ApplicationNotFoundException(program.getParent());
    }
    Map<String, ScheduleSpecification> schedules = appSpec.getSchedules();
    if (schedules == null || !schedules.containsKey(scheduleName)) {
        throw new ScheduleNotFoundException(program.getParent().schedule(scheduleName));
    }
    ScheduleSpecification scheduleSpec = schedules.get(scheduleName);
    Schedule schedule = scheduleSpec.getSchedule();
    return getScheduler(schedule);
}

Example 49 with ApplicationSpecification

use of co.cask.cdap.api.app.ApplicationSpecification in project cdap by caskdata.

the class DistributedSparkProgramRunner method validateOptions.

@Override
protected void validateOptions(Program program, ProgramOptions options) {
    super.validateOptions(program, options);
    // Extract and verify parameters
    ApplicationSpecification appSpec = program.getApplicationSpecification();
    Preconditions.checkNotNull(appSpec, "Missing application specification for %s", program.getId());
    ProgramType processorType = program.getType();
    Preconditions.checkNotNull(processorType, "Missing processor type for %s", program.getId());
    Preconditions.checkArgument(processorType == ProgramType.SPARK, "Only SPARK process type is supported. Program type is %s for %s", processorType, program.getId());
    SparkSpecification spec = appSpec.getSpark().get(program.getName());
    Preconditions.checkNotNull(spec, "Missing SparkSpecification for %s", program.getId());
}

Example 50 with ApplicationSpecification

use of co.cask.cdap.api.app.ApplicationSpecification in project cdap by caskdata.

the class DistributedSparkProgramRunner method setupLaunchConfig.

@Override
protected void setupLaunchConfig(LaunchConfig launchConfig, Program program, ProgramOptions options, CConfiguration cConf, Configuration hConf, File tempDir) throws IOException {
    // Update the container hConf
    hConf.setBoolean(SparkRuntimeContextConfig.HCONF_ATTR_CLUSTER_MODE, true);
    hConf.set(Constants.Explore.HIVE_METASTORE_TOKEN_SIG, Constants.Explore.HIVE_METASTORE_TOKEN_SERVICE_NAME);
    if (SecurityUtil.isKerberosEnabled(cConf)) {
        // Need to divide the interval by 0.8 because Spark logic has a 0.8 discount on the interval
        // If we don't offset it, it will look for the new credentials too soon
        // Also add 5 seconds to the interval to give master time to push the changes to the Spark client container
        hConf.setLong(SparkRuntimeContextConfig.HCONF_ATTR_CREDENTIALS_UPDATE_INTERVAL_MS, (long) ((secureStoreRenewer.getUpdateInterval() + 5000) / 0.8));
    }
    // Setup the launch config
    ApplicationSpecification appSpec = program.getApplicationSpecification();
    SparkSpecification spec = appSpec.getSpark().get(program.getName());
    Map<String, String> clientArgs = RuntimeArguments.extractScope("task", "client", options.getUserArguments().asMap());
    Resources resources = SystemArguments.getResources(clientArgs, spec.getClientResources());
    // Add runnable. Only one instance for the spark client
    launchConfig.addRunnable(spec.getName(), new SparkTwillRunnable(spec.getName()), 1, clientArgs, spec.getClientResources(), 0);
    // Add extra resources, classpath, dependencies, env and setup ClassAcceptor
    Map<String, LocalizeResource> localizeResources = new HashMap<>();
    Map<String, String> extraEnv = new HashMap<>(SparkPackageUtils.getSparkClientEnv());
    SparkPackageUtils.prepareSparkResources(sparkCompat, locationFactory, tempDir, localizeResources, extraEnv);
    // Add the mapreduce resources and path as well for the InputFormat/OutputFormat classes
    MapReduceContainerHelper.localizeFramework(hConf, localizeResources);
    extraEnv.put(Constants.SPARK_COMPAT_ENV, sparkCompat.getCompat());
    launchConfig.addExtraResources(localizeResources).addExtraDependencies(SparkProgramRuntimeProvider.class).addExtraEnv(extraEnv).addExtraClasspath(MapReduceContainerHelper.addMapReduceClassPath(hConf, new ArrayList<String>())).setClassAcceptor(createBundlerClassAcceptor());
}