Examples with SimpleProgramOptions io.cdap.cdap.internal.app.runtime.SimpleProgramOptions used on opensource projects

Example 6 with SimpleProgramOptions

use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by caskdata.

the class ProgramOptions method fromNotification.

/**
 * Decodes {@link ProgramOptions} from a {@link Notification} object, based on the
 * {@link Notification.Type#PROGRAM_STATUS} type.
 */
static ProgramOptions fromNotification(Notification notification, Gson gson) {
    Map<String, String> properties = notification.getProperties();
    ProgramId programId = gson.fromJson(properties.get(ProgramOptionConstants.PROGRAM_RUN_ID), ProgramRunId.class).getParent();
    String userArgumentsString = properties.get(ProgramOptionConstants.USER_OVERRIDES);
    String systemArgumentsString = properties.get(ProgramOptionConstants.SYSTEM_OVERRIDES);
    String debugString = properties.get(ProgramOptionConstants.DEBUG_ENABLED);
    Type stringStringMap = new TypeToken<Map<String, String>>() {
    }.getType();
    boolean debug = Boolean.parseBoolean(debugString);
    Map<String, String> userArguments = userArgumentsString == null ? Collections.emptyMap() : gson.fromJson(userArgumentsString, stringStringMap);
    Map<String, String> systemArguments = systemArgumentsString == null ? Collections.emptyMap() : gson.fromJson(systemArgumentsString, stringStringMap);
    return new SimpleProgramOptions(programId, new BasicArguments(systemArguments), new BasicArguments(userArguments), debug);
}

Example 7 with SimpleProgramOptions

use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by caskdata.

the class ProvisioningServiceTest method createTaskInfo.

private TaskFields createTaskInfo(ProvisionerInfo provisionerInfo) {
    ProgramRunId programRunId = NamespaceId.DEFAULT.app("app").workflow("wf").run(RunIds.generate());
    Map<String, String> systemArgs = new HashMap<>();
    Map<String, String> userArgs = new HashMap<>();
    Profile profile = new Profile(ProfileId.NATIVE.getProfile(), "label", "desc", provisionerInfo);
    SystemArguments.addProfileArgs(systemArgs, profile);
    systemArgs.put(Constants.APP_CDAP_VERSION, APP_CDAP_VERSION);
    ProgramOptions programOptions = new SimpleProgramOptions(programRunId.getParent(), new BasicArguments(systemArgs), new BasicArguments(userArgs));
    ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
    ApplicationSpecification appSpec = new DefaultApplicationSpecification("name", "1.0.0", APP_CDAP_VERSION, "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
    ProgramDescriptor programDescriptor = new ProgramDescriptor(programRunId.getParent(), appSpec);
    return new TaskFields(programDescriptor, programOptions, programRunId);
}

Example 8 with SimpleProgramOptions

use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by caskdata.

the class ProgramNotificationSubscriberService method handleClusterEvent.

/**
 * Handles a notification related to cluster operations.
 *
 * @param programRunId program run id from the event
 * @param clusterStatus cluster status from the event
 * @param notification the notification to process
 * @param messageIdBytes the unique ID for the notification message
 * @param appMetadataStore the data table to use
 * @param context the table context for performing table operations
 * @return an {@link Optional} of {@link Runnable} to carry a task to execute after handling of this event completed.
 *         See {@link #postProcess()} for details.
 * @throws IOException if failed to read/write to the app metadata store.
 */
private Optional<Runnable> handleClusterEvent(ProgramRunId programRunId, ProgramRunClusterStatus clusterStatus, Notification notification, byte[] messageIdBytes, AppMetadataStore appMetadataStore, StructuredTableContext context) throws IOException {
    Map<String, String> properties = notification.getProperties();
    ProgramOptions programOptions = ProgramOptions.fromNotification(notification, GSON);
    String userId = properties.get(ProgramOptionConstants.USER_ID);
    long endTs = getTimeSeconds(properties, ProgramOptionConstants.CLUSTER_END_TIME);
    ProgramDescriptor programDescriptor = GSON.fromJson(properties.get(ProgramOptionConstants.PROGRAM_DESCRIPTOR), ProgramDescriptor.class);
    switch(clusterStatus) {
        case PROVISIONING:
            appMetadataStore.recordProgramProvisioning(programRunId, programOptions.getUserArguments().asMap(), programOptions.getArguments().asMap(), messageIdBytes, programDescriptor.getArtifactId().toApiArtifactId());
            ProvisionRequest provisionRequest = new ProvisionRequest(programRunId, programOptions, programDescriptor, userId);
            return Optional.of(provisioningService.provision(provisionRequest, context));
        case PROVISIONED:
            Cluster cluster = GSON.fromJson(properties.get(ProgramOptionConstants.CLUSTER), Cluster.class);
            appMetadataStore.recordProgramProvisioned(programRunId, cluster.getNodes().size(), messageIdBytes);
            // Update the ProgramOptions system arguments to include information needed for program execution
            Map<String, String> systemArgs = new HashMap<>(programOptions.getArguments().asMap());
            systemArgs.put(ProgramOptionConstants.USER_ID, properties.get(ProgramOptionConstants.USER_ID));
            systemArgs.put(ProgramOptionConstants.CLUSTER, properties.get(ProgramOptionConstants.CLUSTER));
            systemArgs.put(ProgramOptionConstants.SECURE_KEYS_DIR, properties.get(ProgramOptionConstants.SECURE_KEYS_DIR));
            ProgramOptions newProgramOptions = new SimpleProgramOptions(programOptions.getProgramId(), new BasicArguments(systemArgs), programOptions.getUserArguments());
            // Publish the program STARTING state before starting the program
            programStateWriter.start(programRunId, newProgramOptions, null, programDescriptor);
            // emit provisioning time metric
            long provisioningTime = System.currentTimeMillis() / 1000 - RunIds.getTime(programRunId.getRun(), TimeUnit.SECONDS);
            SystemArguments.getProfileIdFromArgs(programRunId.getNamespaceId(), systemArgs).ifPresent(profileId -> emitProvisioningTimeMetric(programRunId, profileId, programOptions, provisioningTime));
            break;
        case DEPROVISIONING:
            RunRecordDetail recordedMeta = appMetadataStore.recordProgramDeprovisioning(programRunId, messageIdBytes);
            // or an invalid state transition. In both cases, we should not try to deprovision the cluster.
            if (recordedMeta != null) {
                return Optional.of(provisioningService.deprovision(programRunId, context));
            }
            break;
        case DEPROVISIONED:
            appMetadataStore.recordProgramDeprovisioned(programRunId, endTs, messageIdBytes);
            break;
        case ORPHANED:
            appMetadataStore.recordProgramOrphaned(programRunId, endTs, messageIdBytes);
            break;
    }
    return Optional.empty();
}

Example 9 with SimpleProgramOptions

use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by caskdata.

the class ProgramNotificationSubscriberService method doStartUp.

@Override
protected void doStartUp() throws Exception {
    super.doStartUp();
    int batchSize = cConf.getInt(Constants.RuntimeMonitor.INIT_BATCH_SIZE);
    RetryStrategy retryStrategy = RetryStrategies.fromConfiguration(cConf, "system.runtime.monitor.");
    long startTs = System.currentTimeMillis();
    Retries.runWithRetries(() -> store.scanActiveRuns(batchSize, (runRecordDetail) -> {
        if (runRecordDetail.getStartTs() > startTs) {
            return;
        }
        try {
            if (runRecordDetail.getStatus() == ProgramRunStatus.PENDING) {
                runRecordMonitorService.addRequest(runRecordDetail.getProgramRunId());
            } else if (runRecordDetail.getStatus() == ProgramRunStatus.STARTING) {
                runRecordMonitorService.addRequest(runRecordDetail.getProgramRunId());
                // It is unknown what is the state of program runs in STARTING state.
                // A STARTING message is published again to retry STARTING logic.
                ProgramOptions programOptions = new SimpleProgramOptions(runRecordDetail.getProgramRunId().getParent(), new BasicArguments(runRecordDetail.getSystemArgs()), new BasicArguments(runRecordDetail.getUserArgs()));
                LOG.debug("Retrying to start run {}.", runRecordDetail.getProgramRunId());
                programStateWriter.start(runRecordDetail.getProgramRunId(), programOptions, null, this.store.loadProgram(runRecordDetail.getProgramRunId().getParent()));
            }
        } catch (Exception e) {
            ProgramRunId programRunId = runRecordDetail.getProgramRunId();
            LOG.warn("Retrying to start run {} failed. Marking it as failed.", programRunId, e);
            programStateWriter.error(programRunId, e);
        }
    }), retryStrategy, e -> true);
}

Example 10 with SimpleProgramOptions

use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by caskdata.

the class ProgramLifecycleService method createProgramOptions.

@VisibleForTesting
ProgramOptions createProgramOptions(ProgramId programId, Map<String, String> userArgs, Map<String, String> sysArgs, boolean debug) throws NotFoundException, ProfileConflictException {
    ProfileId profileId = SystemArguments.getProfileIdForProgram(programId, userArgs);
    Map<String, String> profileProperties = SystemArguments.getProfileProperties(userArgs);
    Profile profile = profileService.getProfile(profileId, profileProperties);
    if (profile.getStatus() == ProfileStatus.DISABLED) {
        throw new ProfileConflictException(String.format("Profile %s in namespace %s is disabled. It cannot be " + "used to start the program %s", profileId.getProfile(), profileId.getNamespace(), programId.toString()), profileId);
    }
    ProvisionerDetail spec = provisioningService.getProvisionerDetail(profile.getProvisioner().getName());
    if (spec == null) {
        throw new NotFoundException(String.format("Provisioner '%s' not found.", profile.getProvisioner().getName()));
    }
    // get and add any user overrides for profile properties
    Map<String, String> systemArgs = new HashMap<>(sysArgs);
    // add profile properties to the system arguments
    SystemArguments.addProfileArgs(systemArgs, profile);
    // Set the ClusterMode. If it is NATIVE profile, then it is ON_PREMISE, otherwise is ISOLATED
    // This should probably move into the provisioner later once we have a better contract for the
    // provisioner to actually pick what launching mechanism it wants to use.
    systemArgs.put(ProgramOptionConstants.CLUSTER_MODE, (ProfileId.NATIVE.equals(profileId) ? ClusterMode.ON_PREMISE : ClusterMode.ISOLATED).name());
    ProgramSpecification programSpecification = getProgramSpecificationWithoutAuthz(programId);
    if (programSpecification == null) {
        throw new NotFoundException(programId);
    }
    addAppCDAPVersion(programId, systemArgs);
    // put all the plugin requirements if any involved in the run
    systemArgs.put(ProgramOptionConstants.PLUGIN_REQUIREMENTS, GSON.toJson(getPluginRequirements(programSpecification)));
    return new SimpleProgramOptions(programId, new BasicArguments(systemArgs), new BasicArguments(userArgs), debug);
}