Examples with BasicArguments io.cdap.cdap.internal.app.runtime.BasicArguments used on opensource projects

Example 36 with BasicArguments

use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by cdapio.

the class AbstractProgramRuntimeService method createPluginSnapshot.

/**
 * Return the copy of the {@link ProgramOptions} including locations of plugin artifacts in it.
 * @param options the {@link ProgramOptions} in which the locations of plugin artifacts needs to be included
 * @param programId Id of the Program
 * @param tempDir Temporary Directory to create the plugin artifact snapshot
 * @param appSpec program's Application Specification
 * @return the copy of the program options with locations of plugin artifacts included in them
 */
private ProgramOptions createPluginSnapshot(ProgramOptions options, ProgramId programId, File tempDir, @Nullable ApplicationSpecification appSpec) throws Exception {
    // appSpec is null in an unit test
    if (appSpec == null) {
        return options;
    }
    Set<String> files = Sets.newHashSet();
    ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
    builder.putAll(options.getArguments().asMap());
    for (Map.Entry<String, Plugin> pluginEntry : appSpec.getPlugins().entrySet()) {
        Plugin plugin = pluginEntry.getValue();
        File destFile = new File(tempDir, Artifacts.getFileName(plugin.getArtifactId()));
        // Skip if the file has already been copied.
        if (!files.add(destFile.getName())) {
            continue;
        }
        try {
            ArtifactId artifactId = Artifacts.toProtoArtifactId(programId.getNamespaceId(), plugin.getArtifactId());
            copyArtifact(artifactId, noAuthArtifactRepository.getArtifact(Id.Artifact.fromEntityId(artifactId)), destFile);
        } catch (ArtifactNotFoundException e) {
            throw new IllegalArgumentException(String.format("Artifact %s could not be found", plugin.getArtifactId()), e);
        }
    }
    LOG.debug("Plugin artifacts of {} copied to {}", programId, tempDir.getAbsolutePath());
    builder.put(ProgramOptionConstants.PLUGIN_DIR, tempDir.getAbsolutePath());
    return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(builder.build()), options.getUserArguments(), options.isDebug());
}

Example 37 with BasicArguments

use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by cdapio.

the class RemoteExecutionTwillRunnerService method createControllerIfNeeded.

/**
 * Creates a {@link TwillController} based on the given {@link RunRecordDetail} if it should be monitored by this
 * service.
 */
private boolean createControllerIfNeeded(RunRecordDetail runRecordDetail) {
    // Program runs in PENDING and STARTING state will eventually start and controller will be created later.
    if (runRecordDetail.getStatus() != ProgramRunStatus.RUNNING && runRecordDetail.getStatus() != ProgramRunStatus.SUSPENDED) {
        LOG.debug("Skip creating controller for run {} with status {}", runRecordDetail.getProgramRunId(), runRecordDetail.getStatus());
        return false;
    }
    Map<String, String> systemArgs = runRecordDetail.getSystemArgs();
    try {
        ClusterMode clusterMode = ClusterMode.valueOf(systemArgs.getOrDefault(ProgramOptionConstants.CLUSTER_MODE, ClusterMode.ON_PREMISE.name()));
        if (clusterMode != ClusterMode.ISOLATED) {
            LOG.debug("Ignore run {} of non supported cluster mode {}", runRecordDetail.getProgramRunId(), clusterMode);
            return false;
        }
    } catch (IllegalArgumentException e) {
        LOG.warn("Ignore run record with an invalid cluster mode", e);
        return false;
    }
    ProgramOptions programOpts = new SimpleProgramOptions(runRecordDetail.getProgramRunId().getParent(), new BasicArguments(runRecordDetail.getSystemArgs()), new BasicArguments(runRecordDetail.getUserArgs()));
    // Creates a controller via the controller factory.
    // Since there is no startup start needed, the timeout is arbitrarily short
    new ControllerFactory(runRecordDetail.getProgramRunId(), programOpts).create(null, 5, TimeUnit.SECONDS);
    return true;
}

Example 38 with BasicArguments

use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by cdapio.

the class AbstractProgramTwillRunnable method createProgramOptions.

/**
 * Creates a {@link ProgramOptions} by deserializing the given json file.
 */
private ProgramOptions createProgramOptions(File programOptionsFile) throws IOException {
    ProgramOptions original = readJsonFile(programOptionsFile, ProgramOptions.class);
    // Overwrite them with environmental information
    Map<String, String> arguments = new HashMap<>(original.getArguments().asMap());
    arguments.putAll(getExtraSystemArguments());
    // Use the name passed in by the constructor as the program name to construct the ProgramId
    return new SimpleProgramOptions(original.getProgramId(), new BasicArguments(arguments), original.getUserArguments(), original.isDebug());
}

Example 39 with BasicArguments

use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by cdapio.

the class CoreSchedulerServiceTest method testProgramEvents.

@Test
@Category(XSlowTests.class)
public void testProgramEvents() throws Exception {
    // Deploy the app
    deploy(AppWithMultipleSchedules.class, 200);
    CConfiguration cConf = getInjector().getInstance(CConfiguration.class);
    TopicId programEventTopic = NamespaceId.SYSTEM.topic(cConf.get(Constants.AppFabric.PROGRAM_STATUS_RECORD_EVENT_TOPIC));
    ProgramStateWriter programStateWriter = new MessagingProgramStateWriter(cConf, messagingService);
    // These notifications should not trigger the program
    ProgramRunId anotherWorkflowRun = ANOTHER_WORKFLOW.run(RunIds.generate());
    ArtifactId artifactId = ANOTHER_WORKFLOW.getNamespaceId().artifact("test", "1.0").toApiArtifactId();
    ApplicationSpecification appSpec = new DefaultApplicationSpecification(AppWithMultipleSchedules.NAME, ApplicationId.DEFAULT_VERSION, ProjectInfo.getVersion().toString(), "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
    ProgramDescriptor programDescriptor = new ProgramDescriptor(anotherWorkflowRun.getParent(), appSpec);
    BasicArguments systemArgs = new BasicArguments(ImmutableMap.of(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString()));
    ProgramOptions programOptions = new SimpleProgramOptions(anotherWorkflowRun.getParent(), systemArgs, new BasicArguments(), false);
    programStateWriter.start(anotherWorkflowRun, programOptions, null, programDescriptor);
    programStateWriter.running(anotherWorkflowRun, null);
    long lastProcessed = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
    programStateWriter.error(anotherWorkflowRun, null);
    waitUntilProcessed(programEventTopic, lastProcessed);
    ProgramRunId someWorkflowRun = SOME_WORKFLOW.run(RunIds.generate());
    programDescriptor = new ProgramDescriptor(someWorkflowRun.getParent(), appSpec);
    programStateWriter.start(someWorkflowRun, new SimpleProgramOptions(someWorkflowRun.getParent(), systemArgs, new BasicArguments()), null, programDescriptor);
    programStateWriter.running(someWorkflowRun, null);
    lastProcessed = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
    programStateWriter.killed(someWorkflowRun);
    waitUntilProcessed(programEventTopic, lastProcessed);
    Assert.assertEquals(0, getRuns(TRIGGERED_WORKFLOW, ProgramRunStatus.ALL));
    // Enable the schedule
    scheduler.enableSchedule(APP_MULT_ID.schedule(AppWithMultipleSchedules.WORKFLOW_COMPLETED_SCHEDULE));
    // Start a program with user arguments
    startProgram(ANOTHER_WORKFLOW, ImmutableMap.of(AppWithMultipleSchedules.ANOTHER_RUNTIME_ARG_KEY, AppWithMultipleSchedules.ANOTHER_RUNTIME_ARG_VALUE), 200);
    // Wait for a completed run record
    waitForCompleteRuns(1, TRIGGERED_WORKFLOW);
    assertProgramRuns(TRIGGERED_WORKFLOW, ProgramRunStatus.COMPLETED, 1);
    RunRecord run = getProgramRuns(TRIGGERED_WORKFLOW, ProgramRunStatus.COMPLETED).get(0);
    Map<String, List<WorkflowTokenDetail.NodeValueDetail>> tokenData = getWorkflowToken(TRIGGERED_WORKFLOW, run.getPid(), null, null).getTokenData();
    // There should be 2 entries in tokenData
    Assert.assertEquals(2, tokenData.size());
    // The value of TRIGGERED_RUNTIME_ARG_KEY should be ANOTHER_RUNTIME_ARG_VALUE from the triggering workflow
    Assert.assertEquals(AppWithMultipleSchedules.ANOTHER_RUNTIME_ARG_VALUE, tokenData.get(AppWithMultipleSchedules.TRIGGERED_RUNTIME_ARG_KEY).get(0).getValue());
    // The value of TRIGGERED_TOKEN_KEY should be ANOTHER_TOKEN_VALUE from the triggering workflow
    Assert.assertEquals(AppWithMultipleSchedules.ANOTHER_TOKEN_VALUE, tokenData.get(AppWithMultipleSchedules.TRIGGERED_TOKEN_KEY).get(0).getValue());
}

Example 40 with BasicArguments

use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by cdapio.

the class ProgramLifecycleService method start.

/**
 * Starts a Program with the specified argument overrides, skipping cluster lifecycle steps in the run.
 * NOTE: This method should only be called from preview runner.
 *
 * @param programId the {@link ProgramId} to start/stop
 * @param overrides the arguments to override in the program's configured user arguments before starting
 * @param debug {@code true} if the program is to be started in debug mode, {@code false} otherwise
 * @param isPreview true if the program is for preview run, for preview run, the app is already deployed with resolved
 *                  properties, so no need to regenerate app spec again
 * @return {@link ProgramController}
 * @throws ConflictException if the specified program is already running, and if concurrent runs are not allowed
 * @throws NotFoundException if the specified program or the app it belongs to is not found in the specified namespace
 * @throws IOException if there is an error starting the program
 * @throws UnauthorizedException if the logged in user is not authorized to start the program. To start a program,
 *                               a user requires {@link ApplicationPermission#EXECUTE} on the program
 * @throws Exception if there were other exceptions checking if the current user is authorized to start the program
 */
public ProgramController start(ProgramId programId, Map<String, String> overrides, boolean debug, boolean isPreview) throws Exception {
    accessEnforcer.enforce(programId, authenticationContext.getPrincipal(), ApplicationPermission.EXECUTE);
    checkConcurrentExecution(programId);
    Map<String, String> sysArgs = propertiesResolver.getSystemProperties(programId);
    addAppCDAPVersion(programId, sysArgs);
    sysArgs.put(ProgramOptionConstants.SKIP_PROVISIONING, "true");
    sysArgs.put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName());
    sysArgs.put(ProgramOptionConstants.IS_PREVIEW, Boolean.toString(isPreview));
    Map<String, String> userArgs = propertiesResolver.getUserProperties(programId);
    if (overrides != null) {
        userArgs.putAll(overrides);
    }
    authorizePipelineRuntimeImpersonation(userArgs);
    BasicArguments systemArguments = new BasicArguments(sysArgs);
    BasicArguments userArguments = new BasicArguments(userArgs);
    ProgramOptions options = new SimpleProgramOptions(programId, systemArguments, userArguments, debug);
    ProgramDescriptor programDescriptor = store.loadProgram(programId);
    ProgramRunId programRunId = programId.run(RunIds.generate());
    checkCapability(programDescriptor);
    programStateWriter.start(programRunId, options, null, programDescriptor);
    return startInternal(programDescriptor, options, programRunId);
}