Examples with ClusterMode io.cdap.cdap.app.guice.ClusterMode used on opensource projects

Example 6 with ClusterMode

use of io.cdap.cdap.app.guice.ClusterMode in project cdap by caskdata.

the class AbstractProgramRuntimeService method run.

@Override
public final RuntimeInfo run(ProgramDescriptor programDescriptor, ProgramOptions options, RunId runId) {
    ProgramId programId = programDescriptor.getProgramId();
    ProgramRunId programRunId = programId.run(runId);
    ClusterMode clusterMode = ProgramRunners.getClusterMode(options);
    // Creates the ProgramRunner based on the cluster mode
    ProgramRunner runner = (clusterMode == ClusterMode.ON_PREMISE ? programRunnerFactory : Optional.ofNullable(remoteProgramRunnerFactory).orElseThrow(UnsupportedOperationException::new)).create(programId.getType());
    File tempDir = createTempDirectory(programId, runId);
    AtomicReference<Runnable> cleanUpTaskRef = new AtomicReference<>(createCleanupTask(tempDir, runner));
    DelayedProgramController controller = new DelayedProgramController(programRunId);
    RuntimeInfo runtimeInfo = createRuntimeInfo(controller, programId, () -> cleanUpTaskRef.get().run());
    updateRuntimeInfo(runtimeInfo);
    executor.execute(() -> {
        try {
            // Get the artifact details and save it into the program options.
            ArtifactId artifactId = programDescriptor.getArtifactId();
            ArtifactDetail artifactDetail = getArtifactDetail(artifactId);
            ApplicationSpecification appSpec = programDescriptor.getApplicationSpecification();
            ProgramDescriptor newProgramDescriptor = programDescriptor;
            boolean isPreview = Boolean.valueOf(options.getArguments().getOption(ProgramOptionConstants.IS_PREVIEW, "false"));
            // for preview we already have a resolved app spec, so no need to regenerate the app spec again
            if (!isPreview && appSpec != null && ClusterMode.ON_PREMISE.equals(clusterMode)) {
                try {
                    ApplicationSpecification generatedAppSpec = regenerateAppSpec(artifactDetail, programId, artifactId, appSpec, options);
                    appSpec = generatedAppSpec != null ? generatedAppSpec : appSpec;
                    newProgramDescriptor = new ProgramDescriptor(programDescriptor.getProgramId(), appSpec);
                } catch (Exception e) {
                    LOG.warn("Failed to regenerate the app spec for program {}, using the existing app spec", programId);
                }
            }
            ProgramOptions runtimeProgramOptions = updateProgramOptions(artifactId, programId, options, runId, clusterMode, Iterables.getFirst(artifactDetail.getMeta().getClasses().getApps(), null));
            // Take a snapshot of all the plugin artifacts used by the program
            ProgramOptions optionsWithPlugins = createPluginSnapshot(runtimeProgramOptions, programId, tempDir, newProgramDescriptor.getApplicationSpecification());
            // Create and run the program
            Program executableProgram = createProgram(cConf, runner, newProgramDescriptor, artifactDetail, tempDir);
            cleanUpTaskRef.set(createCleanupTask(cleanUpTaskRef.get(), executableProgram));
            controller.setProgramController(runner.run(executableProgram, optionsWithPlugins));
        } catch (Exception e) {
            controller.failed(e);
            programStateWriter.error(programRunId, e);
            LOG.error("Exception while trying to run program", e);
        }
    });
    return runtimeInfo;
}

Example 7 with ClusterMode

use of io.cdap.cdap.app.guice.ClusterMode in project cdap by cdapio.

the class SparkTwillRunnableModuleTest method testSpark.

@Test
public void testSpark() {
    ProgramRunId programRunId = NamespaceId.DEFAULT.app("test").spark("spark").run(RunIds.generate());
    for (ClusterMode mode : ClusterMode.values()) {
        Module module = new SparkTwillRunnable("spark") {

            @Override
            protected ServiceAnnouncer getServiceAnnouncer() {
                return new MockTwillContext();
            }
        }.createModule(CConfiguration.create(), new Configuration(), createProgramOptions(programRunId, mode), programRunId);
        Injector injector = Guice.createInjector(module);
        injector.getInstance(SparkProgramRunner.class);
        injector.getInstance(ExploreClient.class);
        Injector contextInjector = SparkRuntimeContextProvider.createInjector(CConfiguration.create(), new Configuration(), programRunId.getParent(), createProgramOptions(programRunId, mode));
        contextInjector.getInstance(PluginFinder.class);
        contextInjector.getInstance(ExploreClient.class);
    }
}

Example 8 with ClusterMode

use of io.cdap.cdap.app.guice.ClusterMode in project cdap by cdapio.

the class ProgramTwillRunnableModuleTest method testWorkflow.

@Test
public void testWorkflow() {
    ProgramRunId programRunId = NamespaceId.DEFAULT.app("test").workflow("workflow").run(RunIds.generate());
    for (ClusterMode mode : ClusterMode.values()) {
        Module module = new WorkflowTwillRunnable("workflow").createModule(CConfiguration.create(), new Configuration(), createProgramOptions(programRunId, mode), programRunId);
        Injector injector = Guice.createInjector(module);
        injector.getInstance(WorkflowProgramRunner.class);
        // Workflow supports spark, which supports PluginFinder
        injector.getInstance(PluginFinder.class);
        injector.getInstance(ExploreClient.class);
    }
}

Example 9 with ClusterMode

use of io.cdap.cdap.app.guice.ClusterMode in project cdap by cdapio.

the class ProgramTwillRunnableModuleTest method testWorker.

@Test
public void testWorker() {
    ProgramRunId programRunId = NamespaceId.DEFAULT.app("test").worker("worker").run(RunIds.generate());
    for (ClusterMode mode : ClusterMode.values()) {
        Module module = new WorkerTwillRunnable("worker").createModule(CConfiguration.create(), new Configuration(), createProgramOptions(programRunId, mode), programRunId);
        Injector injector = Guice.createInjector(module);
        injector.getInstance(WorkerProgramRunner.class);
        injector.getInstance(ExploreClient.class);
    }
}

Example 10 with ClusterMode

use of io.cdap.cdap.app.guice.ClusterMode in project cdap by cdapio.

the class ProgramTwillRunnableModuleTest method testMapReduce.

@Test
public void testMapReduce() {
    ProgramRunId programRunId = NamespaceId.DEFAULT.app("test").mr("mapreduce").run(RunIds.generate());
    for (ClusterMode mode : ClusterMode.values()) {
        Module module = new MapReduceTwillRunnable("mapreduce").createModule(CConfiguration.create(), new Configuration(), createProgramOptions(programRunId, mode), programRunId);
        Injector injector = Guice.createInjector(module);
        injector.getInstance(MapReduceProgramRunner.class);
        injector.getInstance(ExploreClient.class);
    }
}