Examples with ProgramDescriptor co.cask.cdap.app.program.ProgramDescriptor used on opensource projects

Example 11 with ProgramDescriptor

use of co.cask.cdap.app.program.ProgramDescriptor in project cdap by caskdata.

the class WorkflowTest method testWorkflow.

@Test(timeout = 120 * 1000L)
public void testWorkflow() throws Exception {
    final ApplicationWithPrograms app = AppFabricTestHelper.deployApplicationWithManager(WorkflowApp.class, TEMP_FOLDER_SUPPLIER);
    final Injector injector = AppFabricTestHelper.getInjector();
    final ProgramDescriptor programDescriptor = Iterators.filter(app.getPrograms().iterator(), input -> input.getProgramId().getType() == ProgramType.WORKFLOW).next();
    String inputPath = createInput();
    String outputPath = new File(tmpFolder.newFolder(), "output").getAbsolutePath();
    BasicArguments userArgs = new BasicArguments(ImmutableMap.of("inputPath", inputPath, "outputPath", outputPath));
    final SettableFuture<String> completion = SettableFuture.create();
    final ProgramController controller = AppFabricTestHelper.submit(app, programDescriptor.getSpecification().getClassName(), userArgs, TEMP_FOLDER_SUPPLIER);
    controller.addListener(new AbstractListener() {

        @Override
        public void init(ProgramController.State currentState, @Nullable Throwable cause) {
            LOG.info("Starting");
            long nowSecs = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
            setStartAndRunning(injector.getInstance(Store.class), controller.getProgramRunId().getParent(), controller.getProgramRunId().getRun(), nowSecs);
        }

        @Override
        public void completed() {
            LOG.info("Completed");
            completion.set("Completed");
        }

        @Override
        public void error(Throwable cause) {
            LOG.info("Error", cause);
            completion.setException(cause);
        }
    }, Threads.SAME_THREAD_EXECUTOR);
    completion.get();
}

Example 12 with ProgramDescriptor

use of co.cask.cdap.app.program.ProgramDescriptor in project cdap by caskdata.

the class ProgramGenerationStage method process.

@Override
public void process(final ApplicationDeployable input) throws Exception {
    List<ProgramDescriptor> programDescriptors = new ArrayList<>();
    final ApplicationSpecification appSpec = input.getSpecification();
    // Now, we iterate through all ProgramSpecification and generate programs
    Iterable<ProgramSpecification> specifications = Iterables.concat(appSpec.getMapReduce().values(), appSpec.getFlows().values(), appSpec.getWorkflows().values(), appSpec.getServices().values(), appSpec.getSpark().values(), appSpec.getWorkers().values());
    for (ProgramSpecification spec : specifications) {
        ProgramType type = ProgramTypes.fromSpecification(spec);
        ProgramId programId = input.getApplicationId().program(type, spec.getName());
        programDescriptors.add(new ProgramDescriptor(programId, appSpec));
    }
    emit(new ApplicationWithPrograms(input, programDescriptors));
}

Example 13 with ProgramDescriptor

use of co.cask.cdap.app.program.ProgramDescriptor in project cdap by caskdata.

the class DistributedProgramRunner method run.

@Override
public final ProgramController run(final Program program, ProgramOptions oldOptions) {
    validateOptions(program, oldOptions);
    final CConfiguration cConf = createContainerCConf(this.cConf);
    final Configuration hConf = createContainerHConf(this.hConf);
    final File tempDir = DirUtils.createTempDir(new File(cConf.get(Constants.CFG_LOCAL_DATA_DIR), cConf.get(Constants.AppFabric.TEMP_DIR)).getAbsoluteFile());
    try {
        final LaunchConfig launchConfig = new LaunchConfig();
        setupLaunchConfig(launchConfig, program, oldOptions, cConf, hConf, tempDir);
        // Add extra localize resources needed by the program runner
        final Map<String, LocalizeResource> localizeResources = new HashMap<>(launchConfig.getExtraResources());
        final ProgramOptions options = addArtifactPluginFiles(oldOptions, localizeResources, DirUtils.createTempDir(tempDir));
        final List<String> additionalClassPaths = new ArrayList<>();
        addContainerJars(cConf, localizeResources, additionalClassPaths);
        prepareHBaseDDLExecutorResources(tempDir, cConf, localizeResources);
        // Save the configuration to files
        final File hConfFile = saveHConf(hConf, new File(tempDir, HADOOP_CONF_FILE_NAME));
        final File cConfFile = saveCConf(cConf, new File(tempDir, CDAP_CONF_FILE_NAME));
        // Localize the program jar
        Location programJarLocation = program.getJarLocation();
        final String programJarName = programJarLocation.getName();
        localizeResources.put(programJarName, new LocalizeResource(program.getJarLocation().toURI(), false));
        // Localize an expanded program jar
        final String expandedProgramJarName = "expanded." + programJarName;
        localizeResources.put(expandedProgramJarName, new LocalizeResource(program.getJarLocation().toURI(), true));
        // Localize the app spec
        localizeResources.put(APP_SPEC_FILE_NAME, new LocalizeResource(saveAppSpec(program, File.createTempFile("appSpec", ".json", tempDir))));
        final URI logbackURI = getLogBackURI(program, tempDir);
        if (logbackURI != null) {
            // Localize the logback xml
            localizeResources.put(LOGBACK_FILE_NAME, new LocalizeResource(logbackURI, false));
        }
        Callable<ProgramController> callable = new Callable<ProgramController>() {

            @Override
            public ProgramController call() throws Exception {
                ProgramTwillApplication twillApplication = new ProgramTwillApplication(program.getId(), launchConfig.getRunnables(), launchConfig.getLaunchOrder(), localizeResources, createEventHandler(cConf, options));
                TwillPreparer twillPreparer = twillRunner.prepare(twillApplication);
                // Add the configuration to container classpath
                twillPreparer.withResources(hConfFile.toURI(), cConfFile.toURI());
                Map<String, String> userArgs = options.getUserArguments().asMap();
                // Setup log level
                twillPreparer.setLogLevels(transformLogLevels(SystemArguments.getLogLevels(userArgs)));
                // Set the configuration for the twill application
                Map<String, String> twillConfigs = new HashMap<>();
                if (DistributedProgramRunner.this instanceof LongRunningDistributedProgramRunner) {
                    twillConfigs.put(Configs.Keys.YARN_ATTEMPT_FAILURES_VALIDITY_INTERVAL, cConf.get(Constants.AppFabric.YARN_ATTEMPT_FAILURES_VALIDITY_INTERVAL));
                }
                // Add the one from the runtime arguments
                twillConfigs.putAll(SystemArguments.getTwillApplicationConfigs(userArgs));
                twillPreparer.withConfiguration(twillConfigs);
                // Setup per runnable configurations
                for (Map.Entry<String, RunnableDefinition> entry : launchConfig.getRunnables().entrySet()) {
                    String runnable = entry.getKey();
                    RunnableDefinition runnableDefinition = entry.getValue();
                    if (runnableDefinition.getMaxRetries() != null) {
                        twillPreparer.withMaxRetries(runnable, runnableDefinition.getMaxRetries());
                    }
                    twillPreparer.setLogLevels(runnable, transformLogLevels(runnableDefinition.getLogLevels()));
                    twillPreparer.withConfiguration(runnable, runnableDefinition.getTwillRunnableConfigs());
                }
                if (options.isDebug()) {
                    twillPreparer.enableDebugging();
                }
                logProgramStart(program, options);
                String serializedOptions = GSON.toJson(options, ProgramOptions.class);
                LOG.info("Starting {} with debugging enabled: {}, programOptions: {}, and logback: {}", program.getId(), options.isDebug(), serializedOptions, logbackURI);
                // Add scheduler queue name if defined
                String schedulerQueueName = options.getArguments().getOption(Constants.AppFabric.APP_SCHEDULER_QUEUE);
                if (schedulerQueueName != null && !schedulerQueueName.isEmpty()) {
                    LOG.info("Setting scheduler queue for app {} as {}", program.getId(), schedulerQueueName);
                    twillPreparer.setSchedulerQueue(schedulerQueueName);
                }
                if (logbackURI != null) {
                    twillPreparer.addJVMOptions("-Dlogback.configurationFile=" + LOGBACK_FILE_NAME);
                }
                String logLevelConf = cConf.get(Constants.COLLECT_APP_CONTAINER_LOG_LEVEL).toUpperCase();
                if ("OFF".equals(logLevelConf)) {
                    twillPreparer.withConfiguration(Collections.singletonMap(Configs.Keys.LOG_COLLECTION_ENABLED, "false"));
                } else {
                    LogEntry.Level logLevel = LogEntry.Level.ERROR;
                    if ("ALL".equals(logLevelConf)) {
                        logLevel = LogEntry.Level.TRACE;
                    } else {
                        try {
                            logLevel = LogEntry.Level.valueOf(logLevelConf.toUpperCase());
                        } catch (Exception e) {
                            LOG.warn("Invalid application container log level {}. Defaulting to ERROR.", logLevelConf);
                        }
                    }
                    twillPreparer.addLogHandler(new LoggerLogHandler(LOG, logLevel));
                }
                // Add secure tokens
                if (User.isHBaseSecurityEnabled(hConf) || UserGroupInformation.isSecurityEnabled()) {
                    twillPreparer.addSecureStore(YarnSecureStore.create(secureStoreRenewer.createCredentials()));
                }
                // Setup the environment for the container logback.xml
                twillPreparer.withEnv(Collections.singletonMap("CDAP_LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR));
                // Add dependencies
                Set<Class<?>> extraDependencies = new HashSet<>(launchConfig.getExtraDependencies());
                extraDependencies.add(HBaseTableUtilFactory.getHBaseTableUtilClass(cConf));
                if (SecureStoreUtils.isKMSBacked(cConf) && SecureStoreUtils.isKMSCapable()) {
                    extraDependencies.add(SecureStoreUtils.getKMSSecureStore());
                }
                twillPreparer.withDependencies(extraDependencies);
                // Add the additional classes to the classpath that comes from the container jar setting
                twillPreparer.withClassPaths(additionalClassPaths);
                twillPreparer.withClassPaths(launchConfig.getExtraClasspath());
                twillPreparer.withEnv(launchConfig.getExtraEnv());
                // The Yarn app classpath goes last
                List<String> yarnAppClassPath = Arrays.asList(hConf.getTrimmedStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH));
                twillPreparer.withApplicationClassPaths(yarnAppClassPath).withClassPaths(yarnAppClassPath).withBundlerClassAcceptor(launchConfig.getClassAcceptor()).withApplicationArguments("--" + RunnableOptions.JAR, programJarName, "--" + RunnableOptions.EXPANDED_JAR, expandedProgramJarName, "--" + RunnableOptions.HADOOP_CONF_FILE, HADOOP_CONF_FILE_NAME, "--" + RunnableOptions.CDAP_CONF_FILE, CDAP_CONF_FILE_NAME, "--" + RunnableOptions.APP_SPEC_FILE, APP_SPEC_FILE_NAME, "--" + RunnableOptions.PROGRAM_OPTIONS, serializedOptions, "--" + RunnableOptions.PROGRAM_ID, GSON.toJson(program.getId())).setClassLoader(MainClassLoader.class.getName());
                // Invoke the before launch hook
                beforeLaunch(program, options);
                TwillController twillController;
                // Change the context classloader to the combine classloader of this ProgramRunner and
                // all the classloaders of the dependencies classes so that Twill can trace classes.
                ClassLoader oldClassLoader = ClassLoaders.setContextClassLoader(new CombineClassLoader(DistributedProgramRunner.this.getClass().getClassLoader(), Iterables.transform(extraDependencies, new Function<Class<?>, ClassLoader>() {

                    @Override
                    public ClassLoader apply(Class<?> input) {
                        return input.getClassLoader();
                    }
                })));
                try {
                    twillController = twillPreparer.start(cConf.getLong(Constants.AppFabric.PROGRAM_MAX_START_SECONDS), TimeUnit.SECONDS);
                } finally {
                    ClassLoaders.setContextClassLoader(oldClassLoader);
                }
                return createProgramController(addCleanupListener(twillController, program, tempDir), new ProgramDescriptor(program.getId(), program.getApplicationSpecification()), ProgramRunners.getRunId(options));
            }
        };
        return impersonator.doAs(program.getId(), callable);
    } catch (Exception e) {
        deleteDirectory(tempDir);
        throw Throwables.propagate(e);
    }
}

Example 14 with ProgramDescriptor

use of co.cask.cdap.app.program.ProgramDescriptor in project cdap by caskdata.

the class SystemMetadataWriterStageTest method createAppWithWorkflow.

@SuppressWarnings("unchecked")
private ApplicationWithPrograms createAppWithWorkflow(ArtifactId artifactId, ApplicationId appId, String workflowName) throws IOException {
    LocationFactory locationFactory = new LocalLocationFactory(TEMP_FOLDER.newFolder());
    AbstractApplication app = new WorkflowAppWithFork();
    ApplicationSpecification appSpec = Specifications.from(app);
    Location workflowJar = AppJarHelper.createDeploymentJar(locationFactory, WorkflowAppWithFork.class);
    ApplicationDeployable appDeployable = new ApplicationDeployable(artifactId, workflowJar, appId, appSpec, null, ApplicationDeployScope.USER);
    return new ApplicationWithPrograms(appDeployable, ImmutableList.of(new ProgramDescriptor(appId.workflow(workflowName), appSpec)));
}

Example 15 with ProgramDescriptor

use of co.cask.cdap.app.program.ProgramDescriptor in project cdap by caskdata.

the class AbstractProgramRuntimeServiceTest method testDeadlock.

@Test(timeout = 5000)
public void testDeadlock() throws IOException, ExecutionException, InterruptedException, TimeoutException {
    // This test is for testing condition in (CDAP-3579)
    // The race condition is if a program finished very fast such that inside the AbstractProgramRuntimeService is
    // still in the run method, it holds the object lock, making the callback from the listener block forever.
    ProgramRunnerFactory runnerFactory = createProgramRunnerFactory();
    final Program program = createDummyProgram();
    final ProgramRuntimeService runtimeService = new AbstractProgramRuntimeService(CConfiguration.create(), runnerFactory, null, new NoOpProgramStateWriter()) {

        @Override
        public ProgramLiveInfo getLiveInfo(ProgramId programId) {
            return new ProgramLiveInfo(programId, "runtime") {
            };
        }

        @Override
        protected Program createProgram(CConfiguration cConf, ProgramRunner programRunner, ProgramDescriptor programDescriptor, ArtifactDetail artifactDetail, File tempDir) throws IOException {
            return program;
        }

        @Override
        protected ArtifactDetail getArtifactDetail(ArtifactId artifactId) throws IOException, ArtifactNotFoundException {
            co.cask.cdap.api.artifact.ArtifactId id = new co.cask.cdap.api.artifact.ArtifactId("dummy", new ArtifactVersion("1.0"), ArtifactScope.USER);
            return new ArtifactDetail(new ArtifactDescriptor(id, Locations.toLocation(TEMP_FOLDER.newFile())), new ArtifactMeta(ArtifactClasses.builder().build()));
        }
    };
    runtimeService.startAndWait();
    try {
        ProgramDescriptor descriptor = new ProgramDescriptor(program.getId(), null, NamespaceId.DEFAULT.artifact("test", "1.0"));
        final ProgramController controller = runtimeService.run(descriptor, new SimpleProgramOptions(program.getId())).getController();
        Tasks.waitFor(ProgramController.State.COMPLETED, new Callable<ProgramController.State>() {

            @Override
            public ProgramController.State call() throws Exception {
                return controller.getState();
            }
        }, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        Tasks.waitFor(true, new Callable<Boolean>() {

            @Override
            public Boolean call() throws Exception {
                return runtimeService.list(ProgramType.WORKER).isEmpty();
            }
        }, 5, TimeUnit.SECONDS, 100, TimeUnit.MICROSECONDS);
    } finally {
        runtimeService.stopAndWait();
    }
}