Examples with ProgramController co.cask.cdap.app.runtime.ProgramController used on opensource projects

Example 26 with ProgramController

use of co.cask.cdap.app.runtime.ProgramController in project cdap by caskdata.

the class ProgramLifecycleHttpHandler method stopPrograms.

/**
 * Stops all programs that are passed into the data. The data is an array of JSON objects
 * where each object must contain the following three elements: appId, programType, and programId
 * (flow name, service name, etc.).
 * <p>
 * Example input:
 * <pre><code>
 * [{"appId": "App1", "programType": "Service", "programId": "Service1"},
 * {"appId": "App1", "programType": "Mapreduce", "programId": "MapReduce2"},
 * {"appId": "App2", "programType": "Flow", "programId": "Flow1"}]
 * </code></pre>
 * </p><p>
 * The response will be an array of JsonObjects each of which will contain the three input parameters
 * as well as a "statusCode" field which maps to the status code for the data in that JsonObjects.
 * </p><p>
 * If an error occurs in the input (for the example above, App2 does not exist), then all JsonObjects for which the
 * parameters have a valid status will have the status field but all JsonObjects for which the parameters do not have
 * a valid status will have an error message and statusCode.
 * </p><p>
 * For example, if there is no App2 in the data above, then the response would be 200 OK with following possible data:
 * </p>
 * <pre><code>
 * [{"appId": "App1", "programType": "Service", "programId": "Service1", "statusCode": 200},
 * {"appId": "App1", "programType": "Mapreduce", "programId": "Mapreduce2", "statusCode": 200},
 * {"appId":"App2", "programType":"Flow", "programId":"Flow1", "statusCode":404, "error": "App: App2 not found"}]
 * </code></pre>
 */
@POST
@Path("/stop")
@AuditPolicy({ AuditDetail.REQUEST_BODY, AuditDetail.RESPONSE_BODY })
public void stopPrograms(FullHttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId) throws Exception {
    List<BatchProgram> programs = validateAndGetBatchInput(request, BATCH_PROGRAMS_TYPE);
    List<ListenableFuture<BatchProgramResult>> issuedStops = new ArrayList<>(programs.size());
    for (final BatchProgram program : programs) {
        ProgramId programId = new ProgramId(namespaceId, program.getAppId(), program.getProgramType(), program.getProgramId());
        try {
            List<ListenableFuture<ProgramController>> stops = lifecycleService.issueStop(programId, null);
            for (ListenableFuture<ProgramController> stop : stops) {
                ListenableFuture<BatchProgramResult> issuedStop = Futures.transform(stop, new Function<ProgramController, BatchProgramResult>() {

                    @Override
                    public BatchProgramResult apply(ProgramController input) {
                        return new BatchProgramResult(program, HttpResponseStatus.OK.code(), null, input.getRunId().getId());
                    }
                });
                issuedStops.add(issuedStop);
            }
        } catch (NotFoundException e) {
            issuedStops.add(Futures.immediateFuture(new BatchProgramResult(program, HttpResponseStatus.NOT_FOUND.code(), e.getMessage())));
        } catch (BadRequestException e) {
            issuedStops.add(Futures.immediateFuture(new BatchProgramResult(program, HttpResponseStatus.BAD_REQUEST.code(), e.getMessage())));
        }
    }
    List<BatchProgramResult> output = new ArrayList<>(programs.size());
    // need to keep this index in case there is an exception getting the future, since we won't have the program
    // information in that scenario
    int i = 0;
    for (ListenableFuture<BatchProgramResult> issuedStop : issuedStops) {
        try {
            output.add(issuedStop.get());
        } catch (Throwable t) {
            LOG.warn(t.getMessage(), t);
            output.add(new BatchProgramResult(programs.get(i), HttpResponseStatus.INTERNAL_SERVER_ERROR.code(), t.getMessage()));
        }
        i++;
    }
    responder.sendJson(HttpResponseStatus.OK, GSON.toJson(output));
}

Example 27 with ProgramController

use of co.cask.cdap.app.runtime.ProgramController in project cdap by caskdata.

the class AbstractInMemoryProgramRunner method startAll.

/**
 * Starts all instances of a Program component.
 * @param program The program to run
 * @param options options for the program
 * @param numInstances number of component instances to start
 */
protected final ProgramController startAll(Program program, ProgramOptions options, int numInstances) {
    RunId runId = ProgramRunners.getRunId(options);
    Table<String, Integer, ProgramController> components = HashBasedTable.create();
    try {
        for (int instanceId = 0; instanceId < numInstances; instanceId++) {
            ProgramOptions componentOptions = createComponentOptions(instanceId, numInstances, runId, options);
            ProgramController controller = createProgramRunner().run(program, componentOptions);
            components.put(program.getName(), instanceId, controller);
        }
        return new InMemoryProgramController(components, program, options);
    } catch (Throwable t) {
        LOG.error("Failed to start all program instances", t);
        try {
            // Need to stop all started components
            Futures.successfulAsList(Iterables.transform(components.values(), new Function<ProgramController, ListenableFuture<?>>() {

                @Override
                public ListenableFuture<?> apply(ProgramController controller) {
                    return controller.stop();
                }
            })).get();
            throw Throwables.propagate(t);
        } catch (Exception e) {
            LOG.error("Failed to stop all program instances upon startup failure.", e);
            throw Throwables.propagate(e);
        }
    }
}

Example 28 with ProgramController

use of co.cask.cdap.app.runtime.ProgramController in project cdap by caskdata.

the class DefaultPreviewRunner method startPreview.

@Override
public void startPreview(PreviewRequest<?> previewRequest) throws Exception {
    namespaceAdmin.create(new NamespaceMeta.Builder().setName(previewRequest.getProgram().getNamespaceId()).build());
    programId = previewRequest.getProgram();
    AppRequest<?> request = previewRequest.getAppRequest();
    ArtifactSummary artifactSummary = request.getArtifact();
    ApplicationId preview = programId.getParent();
    DataTracerFactoryProvider.setDataTracerFactory(preview, dataTracerFactory);
    String config = request.getConfig() == null ? null : GSON.toJson(request.getConfig());
    try {
        applicationLifecycleService.deployApp(preview.getParent(), preview.getApplication(), preview.getVersion(), artifactSummary, config, NOOP_PROGRAM_TERMINATOR, null, request.canUpdateSchedules());
    } catch (Exception e) {
        this.status = new PreviewStatus(PreviewStatus.Status.DEPLOY_FAILED, new BasicThrowable(e), null, null);
        throw e;
    }
    final PreviewConfig previewConfig = previewRequest.getAppRequest().getPreview();
    ProgramController controller = programLifecycleService.start(programId, previewConfig == null ? Collections.<String, String>emptyMap() : previewConfig.getRuntimeArgs(), false);
    controller.addListener(new AbstractListener() {

        @Override
        public void init(ProgramController.State currentState, @Nullable Throwable cause) {
            setStatus(new PreviewStatus(PreviewStatus.Status.RUNNING, null, System.currentTimeMillis(), null));
            // Only have timer if there is a timeout setting.
            if (previewConfig.getTimeout() != null) {
                timer = new Timer();
                final int timeOutMinutes = previewConfig.getTimeout();
                timer.schedule(new TimerTask() {

                    @Override
                    public void run() {
                        try {
                            LOG.info("Stopping the preview since it has reached running time: {} mins.", timeOutMinutes);
                            stopPreview();
                            killedByTimer = true;
                        } catch (Exception e) {
                            LOG.debug("Error shutting down the preview run with id: {}", programId);
                        }
                    }
                }, timeOutMinutes * 60 * 1000);
            }
        }

        @Override
        public void completed() {
            setStatus(new PreviewStatus(PreviewStatus.Status.COMPLETED, null, status.getStartTime(), System.currentTimeMillis()));
            shutDownUnrequiredServices();
        }

        @Override
        public void killed() {
            if (!killedByTimer) {
                setStatus(new PreviewStatus(PreviewStatus.Status.KILLED, null, status.getStartTime(), System.currentTimeMillis()));
            } else {
                setStatus(new PreviewStatus(PreviewStatus.Status.KILLED_BY_TIMER, null, status.getStartTime(), System.currentTimeMillis()));
            }
            shutDownUnrequiredServices();
        }

        @Override
        public void error(Throwable cause) {
            setStatus(new PreviewStatus(PreviewStatus.Status.RUN_FAILED, new BasicThrowable(cause), status.getStartTime(), System.currentTimeMillis()));
            shutDownUnrequiredServices();
        }
    }, Threads.SAME_THREAD_EXECUTOR);
    runId = controller.getProgramRunId();
}

Example 29 with ProgramController

use of co.cask.cdap.app.runtime.ProgramController in project cdap by caskdata.

the class DistributedProgramRunner method run.

@Override
public final ProgramController run(final Program program, ProgramOptions oldOptions) {
    validateOptions(program, oldOptions);
    final CConfiguration cConf = createContainerCConf(this.cConf);
    final Configuration hConf = createContainerHConf(this.hConf);
    final File tempDir = DirUtils.createTempDir(new File(cConf.get(Constants.CFG_LOCAL_DATA_DIR), cConf.get(Constants.AppFabric.TEMP_DIR)).getAbsoluteFile());
    try {
        final LaunchConfig launchConfig = new LaunchConfig();
        setupLaunchConfig(launchConfig, program, oldOptions, cConf, hConf, tempDir);
        // Add extra localize resources needed by the program runner
        final Map<String, LocalizeResource> localizeResources = new HashMap<>(launchConfig.getExtraResources());
        final ProgramOptions options = addArtifactPluginFiles(oldOptions, localizeResources, DirUtils.createTempDir(tempDir));
        final List<String> additionalClassPaths = new ArrayList<>();
        addContainerJars(cConf, localizeResources, additionalClassPaths);
        prepareHBaseDDLExecutorResources(tempDir, cConf, localizeResources);
        // Save the configuration to files
        final File hConfFile = saveHConf(hConf, new File(tempDir, HADOOP_CONF_FILE_NAME));
        final File cConfFile = saveCConf(cConf, new File(tempDir, CDAP_CONF_FILE_NAME));
        // Localize the program jar
        Location programJarLocation = program.getJarLocation();
        final String programJarName = programJarLocation.getName();
        localizeResources.put(programJarName, new LocalizeResource(program.getJarLocation().toURI(), false));
        // Localize an expanded program jar
        final String expandedProgramJarName = "expanded." + programJarName;
        localizeResources.put(expandedProgramJarName, new LocalizeResource(program.getJarLocation().toURI(), true));
        // Localize the app spec
        localizeResources.put(APP_SPEC_FILE_NAME, new LocalizeResource(saveAppSpec(program, File.createTempFile("appSpec", ".json", tempDir))));
        final URI logbackURI = getLogBackURI(program, tempDir);
        if (logbackURI != null) {
            // Localize the logback xml
            localizeResources.put(LOGBACK_FILE_NAME, new LocalizeResource(logbackURI, false));
        }
        Callable<ProgramController> callable = new Callable<ProgramController>() {

            @Override
            public ProgramController call() throws Exception {
                ProgramTwillApplication twillApplication = new ProgramTwillApplication(program.getId(), launchConfig.getRunnables(), launchConfig.getLaunchOrder(), localizeResources, createEventHandler(cConf, options));
                TwillPreparer twillPreparer = twillRunner.prepare(twillApplication);
                // Add the configuration to container classpath
                twillPreparer.withResources(hConfFile.toURI(), cConfFile.toURI());
                Map<String, String> userArgs = options.getUserArguments().asMap();
                // Setup log level
                twillPreparer.setLogLevels(transformLogLevels(SystemArguments.getLogLevels(userArgs)));
                // Set the configuration for the twill application
                Map<String, String> twillConfigs = new HashMap<>();
                if (DistributedProgramRunner.this instanceof LongRunningDistributedProgramRunner) {
                    twillConfigs.put(Configs.Keys.YARN_ATTEMPT_FAILURES_VALIDITY_INTERVAL, cConf.get(Constants.AppFabric.YARN_ATTEMPT_FAILURES_VALIDITY_INTERVAL));
                }
                // Add the one from the runtime arguments
                twillConfigs.putAll(SystemArguments.getTwillApplicationConfigs(userArgs));
                twillPreparer.withConfiguration(twillConfigs);
                // Setup per runnable configurations
                for (Map.Entry<String, RunnableDefinition> entry : launchConfig.getRunnables().entrySet()) {
                    String runnable = entry.getKey();
                    RunnableDefinition runnableDefinition = entry.getValue();
                    if (runnableDefinition.getMaxRetries() != null) {
                        twillPreparer.withMaxRetries(runnable, runnableDefinition.getMaxRetries());
                    }
                    twillPreparer.setLogLevels(runnable, transformLogLevels(runnableDefinition.getLogLevels()));
                    twillPreparer.withConfiguration(runnable, runnableDefinition.getTwillRunnableConfigs());
                }
                if (options.isDebug()) {
                    twillPreparer.enableDebugging();
                }
                logProgramStart(program, options);
                String serializedOptions = GSON.toJson(options, ProgramOptions.class);
                LOG.info("Starting {} with debugging enabled: {}, programOptions: {}, and logback: {}", program.getId(), options.isDebug(), serializedOptions, logbackURI);
                // Add scheduler queue name if defined
                String schedulerQueueName = options.getArguments().getOption(Constants.AppFabric.APP_SCHEDULER_QUEUE);
                if (schedulerQueueName != null && !schedulerQueueName.isEmpty()) {
                    LOG.info("Setting scheduler queue for app {} as {}", program.getId(), schedulerQueueName);
                    twillPreparer.setSchedulerQueue(schedulerQueueName);
                }
                if (logbackURI != null) {
                    twillPreparer.addJVMOptions("-Dlogback.configurationFile=" + LOGBACK_FILE_NAME);
                }
                String logLevelConf = cConf.get(Constants.COLLECT_APP_CONTAINER_LOG_LEVEL).toUpperCase();
                if ("OFF".equals(logLevelConf)) {
                    twillPreparer.withConfiguration(Collections.singletonMap(Configs.Keys.LOG_COLLECTION_ENABLED, "false"));
                } else {
                    LogEntry.Level logLevel = LogEntry.Level.ERROR;
                    if ("ALL".equals(logLevelConf)) {
                        logLevel = LogEntry.Level.TRACE;
                    } else {
                        try {
                            logLevel = LogEntry.Level.valueOf(logLevelConf.toUpperCase());
                        } catch (Exception e) {
                            LOG.warn("Invalid application container log level {}. Defaulting to ERROR.", logLevelConf);
                        }
                    }
                    twillPreparer.addLogHandler(new LoggerLogHandler(LOG, logLevel));
                }
                // Add secure tokens
                if (User.isHBaseSecurityEnabled(hConf) || UserGroupInformation.isSecurityEnabled()) {
                    twillPreparer.addSecureStore(YarnSecureStore.create(secureStoreRenewer.createCredentials()));
                }
                // Setup the environment for the container logback.xml
                twillPreparer.withEnv(Collections.singletonMap("CDAP_LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR));
                // Add dependencies
                Set<Class<?>> extraDependencies = new HashSet<>(launchConfig.getExtraDependencies());
                extraDependencies.add(HBaseTableUtilFactory.getHBaseTableUtilClass(cConf));
                if (SecureStoreUtils.isKMSBacked(cConf) && SecureStoreUtils.isKMSCapable()) {
                    extraDependencies.add(SecureStoreUtils.getKMSSecureStore());
                }
                twillPreparer.withDependencies(extraDependencies);
                // Add the additional classes to the classpath that comes from the container jar setting
                twillPreparer.withClassPaths(additionalClassPaths);
                twillPreparer.withClassPaths(launchConfig.getExtraClasspath());
                twillPreparer.withEnv(launchConfig.getExtraEnv());
                // The Yarn app classpath goes last
                List<String> yarnAppClassPath = Arrays.asList(hConf.getTrimmedStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH));
                twillPreparer.withApplicationClassPaths(yarnAppClassPath).withClassPaths(yarnAppClassPath).withBundlerClassAcceptor(launchConfig.getClassAcceptor()).withApplicationArguments("--" + RunnableOptions.JAR, programJarName, "--" + RunnableOptions.EXPANDED_JAR, expandedProgramJarName, "--" + RunnableOptions.HADOOP_CONF_FILE, HADOOP_CONF_FILE_NAME, "--" + RunnableOptions.CDAP_CONF_FILE, CDAP_CONF_FILE_NAME, "--" + RunnableOptions.APP_SPEC_FILE, APP_SPEC_FILE_NAME, "--" + RunnableOptions.PROGRAM_OPTIONS, serializedOptions, "--" + RunnableOptions.PROGRAM_ID, GSON.toJson(program.getId())).setClassLoader(MainClassLoader.class.getName());
                // Invoke the before launch hook
                beforeLaunch(program, options);
                TwillController twillController;
                // Change the context classloader to the combine classloader of this ProgramRunner and
                // all the classloaders of the dependencies classes so that Twill can trace classes.
                ClassLoader oldClassLoader = ClassLoaders.setContextClassLoader(new CombineClassLoader(DistributedProgramRunner.this.getClass().getClassLoader(), Iterables.transform(extraDependencies, new Function<Class<?>, ClassLoader>() {

                    @Override
                    public ClassLoader apply(Class<?> input) {
                        return input.getClassLoader();
                    }
                })));
                try {
                    twillController = twillPreparer.start(cConf.getLong(Constants.AppFabric.PROGRAM_MAX_START_SECONDS), TimeUnit.SECONDS);
                } finally {
                    ClassLoaders.setContextClassLoader(oldClassLoader);
                }
                return createProgramController(addCleanupListener(twillController, program, tempDir), new ProgramDescriptor(program.getId(), program.getApplicationSpecification()), ProgramRunners.getRunId(options));
            }
        };
        return impersonator.doAs(program.getId(), callable);
    } catch (Exception e) {
        deleteDirectory(tempDir);
        throw Throwables.propagate(e);
    }
}

Example 30 with ProgramController

use of co.cask.cdap.app.runtime.ProgramController in project cdap by caskdata.

the class InMemoryFlowProgramRunner method run.

@Override
public ProgramController run(Program program, ProgramOptions options) {
    // Extract and verify parameters
    ApplicationSpecification appSpec = program.getApplicationSpecification();
    Preconditions.checkNotNull(appSpec, "Missing application specification.");
    ProgramType processorType = program.getType();
    Preconditions.checkNotNull(processorType, "Missing processor type.");
    Preconditions.checkArgument(processorType == ProgramType.FLOW, "Only FLOW process type is supported.");
    FlowSpecification flowSpec = appSpec.getFlows().get(program.getName());
    Preconditions.checkNotNull(flowSpec, "Missing FlowSpecification for %s", program.getName());
    try {
        // Launch flowlet program runners
        Multimap<String, QueueName> consumerQueues = FlowUtils.configureQueue(program, flowSpec, streamAdmin, queueAdmin, txExecutorFactory);
        final Table<String, Integer, ProgramController> flowlets = createFlowlets(program, options, flowSpec);
        return new FlowProgramController(flowlets, program, options, flowSpec, consumerQueues);
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}