Examples with ProgramRunner io.cdap.cdap.app.runtime.ProgramRunner used on opensource projects

Example 1 with ProgramRunner

use of io.cdap.cdap.app.runtime.ProgramRunner in project cdap by caskdata.

the class WorkflowTwillRunnable method createModule.

@Override
protected Module createModule(CConfiguration cConf, Configuration hConf, ProgramOptions programOptions, ProgramRunId programRunId) {
    List<Module> modules = new ArrayList<>();
    modules.add(super.createModule(cConf, hConf, programOptions, programRunId));
    if (ProgramRunners.getClusterMode(programOptions) == ClusterMode.ON_PREMISE) {
        modules.add(new DistributedArtifactManagerModule());
    } else {
        modules.add(new AbstractModule() {

            @Override
            protected void configure() {
                bind(PluginFinder.class).to(UnsupportedPluginFinder.class);
            }
        });
    }
    modules.add(new PrivateModule() {

        @Override
        protected void configure() {
            // Bind ProgramRunner for MR, which is used by Workflow.
            // The ProgramRunner for Spark is provided by the DefaultProgramRunnerFactory through the extension mechanism
            MapBinder<ProgramType, ProgramRunner> runnerFactoryBinder = MapBinder.newMapBinder(binder(), ProgramType.class, ProgramRunner.class);
            runnerFactoryBinder.addBinding(ProgramType.MAPREDUCE).to(MapReduceProgramRunner.class);
            // It uses local mode factory because for Workflow we launch the job from the Workflow container directly.
            // The actual execution mode of the job is governed by the framework configuration
            // For mapreduce, it's in the mapred-site.xml
            // for spark, it's in the hConf we shipped from DistributedWorkflowProgramRunner
            bind(ProgramRuntimeProvider.Mode.class).toInstance(ProgramRuntimeProvider.Mode.LOCAL);
            bind(ProgramRunnerFactory.class).to(DefaultProgramRunnerFactory.class).in(Scopes.SINGLETON);
            expose(ProgramRunnerFactory.class);
        }
    });
    return Modules.combine(modules);
}

Example 2 with ProgramRunner

use of io.cdap.cdap.app.runtime.ProgramRunner in project cdap by caskdata.

the class DefaultRuntimeJob method createModules.

/**
 * Returns list of guice modules used to start the program run.
 */
@VisibleForTesting
List<Module> createModules(RuntimeJobEnvironment runtimeJobEnv, CConfiguration cConf, ProgramRunId programRunId, ProgramOptions programOpts) {
    List<Module> modules = new ArrayList<>();
    modules.add(new ConfigModule(cConf));
    RuntimeMonitorType runtimeMonitorType = SystemArguments.getRuntimeMonitorType(cConf, programOpts);
    modules.add(RuntimeMonitors.getRemoteAuthenticatorModule(runtimeMonitorType, programOpts));
    modules.add(new IOModule());
    modules.add(new TMSLogAppenderModule());
    modules.add(new RemoteExecutionDiscoveryModule());
    modules.add(new AuthenticationContextModules().getProgramContainerModule(cConf));
    modules.add(new MetricsClientRuntimeModule().getDistributedModules());
    modules.add(new MessagingServerRuntimeModule().getStandaloneModules());
    modules.add(new AbstractModule() {

        @Override
        protected void configure() {
            bind(ClusterMode.class).toInstance(ClusterMode.ISOLATED);
            bind(UGIProvider.class).to(CurrentUGIProvider.class).in(Scopes.SINGLETON);
            // Bindings from the environment
            bind(TwillRunner.class).annotatedWith(Constants.AppFabric.ProgramRunner.class).toInstance(runtimeJobEnv.getTwillRunner());
            bind(LocationFactory.class).toInstance(runtimeJobEnv.getLocationFactory());
            MapBinder<ProgramType, ProgramRunner> defaultProgramRunnerBinder = MapBinder.newMapBinder(binder(), ProgramType.class, ProgramRunner.class);
            bind(ProgramRuntimeProvider.Mode.class).toInstance(ProgramRuntimeProvider.Mode.DISTRIBUTED);
            bind(ProgramRunnerFactory.class).annotatedWith(Constants.AppFabric.ProgramRunner.class).to(DefaultProgramRunnerFactory.class).in(Scopes.SINGLETON);
            bind(ProgramStateWriter.class).to(MessagingProgramStateWriter.class).in(Scopes.SINGLETON);
            defaultProgramRunnerBinder.addBinding(ProgramType.MAPREDUCE).to(DistributedMapReduceProgramRunner.class);
            defaultProgramRunnerBinder.addBinding(ProgramType.WORKFLOW).to(DistributedWorkflowProgramRunner.class);
            defaultProgramRunnerBinder.addBinding(ProgramType.WORKER).to(DistributedWorkerProgramRunner.class);
            bind(ProgramRunnerFactory.class).to(DefaultProgramRunnerFactory.class).in(Scopes.SINGLETON);
            bind(ProgramRunId.class).toInstance(programRunId);
            bind(RuntimeMonitorType.class).toInstance(runtimeMonitorType);
            install(new FactoryModuleBuilder().implement(Configurator.class, InMemoryConfigurator.class).build(ConfiguratorFactory.class));
            bind(String.class).annotatedWith(Names.named(RemoteIsolatedPluginFinder.ISOLATED_PLUGIN_DIR)).toInstance(programOpts.getArguments().getOption(ProgramOptionConstants.PLUGIN_DIR, DistributedProgramRunner.PLUGIN_DIR));
            bind(PluginFinder.class).to(RemoteIsolatedPluginFinder.class);
            bind(ArtifactRepositoryReader.class).to(RemoteArtifactRepositoryReader.class).in(Scopes.SINGLETON);
            bind(ArtifactRepository.class).to(RemoteArtifactRepository.class);
        }
    });
    return modules;
}

Example 3 with ProgramRunner

use of io.cdap.cdap.app.runtime.ProgramRunner in project cdap by caskdata.

the class DefaultProgramWorkflowRunner method create.

@Override
public Runnable create(String name) {
    ProgramRunner programRunner = programRunnerFactory.create(programType);
    try {
        ProgramId programId = workflowProgram.getId().getParent().program(programType, name);
        Program program = Programs.create(cConf, workflowProgram, programId, programRunner);
        return getProgramRunnable(name, programRunner, program);
    } catch (Exception e) {
        closeProgramRunner(programRunner);
        throw Throwables.propagate(e);
    }
}

Example 4 with ProgramRunner

use of io.cdap.cdap.app.runtime.ProgramRunner in project cdap by caskdata.

the class DistributedWorkflowProgramRunnerTest method setupWorkflowRuntime.

/**
 * Setup the {@link ProgramLaunchConfig} for the given workflow.
 */
private ProgramLaunchConfig setupWorkflowRuntime(String workflowName, Map<String, String> runtimeArgs) throws IOException {
    // Create the distributed workflow program runner
    ProgramRunner programRunner = programRunnerFactory.create(ProgramType.WORKFLOW);
    Assert.assertTrue(programRunner instanceof DistributedWorkflowProgramRunner);
    DistributedWorkflowProgramRunner workflowRunner = (DistributedWorkflowProgramRunner) programRunner;
    // Create the Workflow Program
    Program workflowProgram = createWorkflowProgram(cConf, programRunner, workflowName);
    ProgramLaunchConfig launchConfig = new ProgramLaunchConfig();
    ProgramOptions programOpts = new SimpleProgramOptions(workflowProgram.getId(), new BasicArguments(), new BasicArguments(runtimeArgs));
    // Setup the launching config
    workflowRunner.setupLaunchConfig(launchConfig, workflowProgram, programOpts, cConf, new Configuration(), TEMP_FOLDER.newFolder());
    return launchConfig;
}

Example 5 with ProgramRunner

use of io.cdap.cdap.app.runtime.ProgramRunner in project cdap by caskdata.

the class ArtifactClassLoaderFactory method createClassLoader.

/**
 * Create a classloader that loads classes from a directory where an artifact jar has been expanded, with access to
 * packages that all program type has access to. The classloader created is only for artifact inspection purpose
 * and shouldn't be used for program execution as it doesn't have the proper class filtering for the specific
 * program type for the program being executed.
 *
 * @param unpackDir the directory where the artifact jar has been expanded
 * @return a closeable classloader based off the specified artifact; on closing the returned {@link ClassLoader},
 *         all temporary resources created for the classloader will be removed
 */
CloseableClassLoader createClassLoader(File unpackDir) {
    ProgramRunner programRunner = null;
    try {
        // Try to create a ProgramClassLoader from the Spark runtime system if it is available.
        // It is needed because we don't know what program types that an artifact might have.
        // TODO: CDAP-5613. We shouldn't always expose the Spark classes.
        programRunner = programRunnerFactory.create(ProgramType.SPARK);
    } catch (Exception e) {
        // If Spark is not supported, exception is expected. We'll use the default filter.
        LOG.trace("Spark is not supported. Not using ProgramClassLoader from Spark", e);
    }
    ProgramClassLoader programClassLoader = null;
    if (programRunner instanceof ProgramClassLoaderProvider) {
        programClassLoader = new ProgramClassLoader(cConf, unpackDir, ((ProgramClassLoaderProvider) programRunner).createProgramClassLoaderParent());
    }
    if (programClassLoader == null) {
        programClassLoader = new ProgramClassLoader(cConf, unpackDir, FilterClassLoader.create(getClass().getClassLoader()));
    }
    final ClassLoader finalProgramClassLoader = programClassLoader;
    final ProgramRunner finalProgramRunner = programRunner;
    return new CloseableClassLoader(programClassLoader, () -> {
        Closeables.closeQuietly((Closeable) finalProgramClassLoader);
        if (finalProgramRunner instanceof Closeable) {
            Closeables.closeQuietly((Closeable) finalProgramRunner);
        }
    });
}