Examples with PipelineRun com.epam.pipeline.entity.pipeline.PipelineRun used on opensource projects

Example 11 with PipelineRun

use of com.epam.pipeline.entity.pipeline.PipelineRun in project cloud-pipeline by epam.

the class PipelineRunManager method createRestartRun.

private PipelineRun createRestartRun(final PipelineRun run) {
    PipelineRun restartedRun = new PipelineRun();
    Long runId = pipelineRunDao.createRunId();
    restartedRun.setId(runId);
    restartedRun.setStartDate(DateUtils.now());
    Optional<Pipeline> pipeline = Optional.ofNullable(run.getPipelineId()).map(pipelineId -> pipelineManager.load(pipelineId));
    pipeline.ifPresent(p -> {
        restartedRun.setPipelineName(p.getName());
        restartedRun.setRepository(p.getRepository());
        restartedRun.setPipelineId(p.getId());
        restartedRun.setVersion(run.getVersion());
        restartedRun.setRevisionName(gitManager.getRevisionName(run.getVersion()));
    });
    if (!pipeline.isPresent()) {
        fillMissingPipelineFields(restartedRun);
    }
    restartedRun.setStatus(TaskStatus.RUNNING);
    restartedRun.setCommitStatus(CommitStatus.NOT_COMMITTED);
    restartedRun.setLastChangeCommitTime(DateUtils.now());
    restartedRun.setPodId(getRootPodIDFromPipeline(restartedRun));
    restartedRun.setParams(run.getParams());
    restartedRun.parseParameters();
    restartedRun.setTimeout(run.getTimeout());
    restartedRun.setDockerImage(run.getDockerImage());
    restartedRun.setCmdTemplate(run.getCmdTemplate());
    restartedRun.setNodeCount(run.getNodeCount());
    RunInstance instance = copyInstance(run.getInstance());
    restartedRun.setInstance(instance);
    setRunPrice(instance, restartedRun);
    restartedRun.setSshPassword(PasswordGenerator.generatePassword());
    restartedRun.setOwner(run.getOwner());
    restartedRun.setEntitiesIds(run.getEntitiesIds());
    restartedRun.setConfigurationId(run.getConfigurationId());
    restartedRun.setExecutionPreferences(run.getExecutionPreferences());
    restartedRun.setRunSids(run.getRunSids());
    return restartedRun;
}

Example 12 with PipelineRun

use of com.epam.pipeline.entity.pipeline.PipelineRun in project cloud-pipeline by epam.

the class RunLogManager method loadAllLogsForTask.

@Transactional(propagation = Propagation.SUPPORTS)
public List<RunLog> loadAllLogsForTask(Long runId, String taskName, String parameters) {
    PipelineRun run = pipelineRunManager.loadPipelineRun(runId);
    if (consoleLogTask.equals(taskName)) {
        return getPodLogs(run);
    }
    String taskId = PipelineTask.buildTaskId(taskName, parameters);
    return runLogDao.loadAllLogsForTask(runId, taskId);
}

Example 13 with PipelineRun

use of com.epam.pipeline.entity.pipeline.PipelineRun in project cloud-pipeline by epam.

the class RunLogManager method saveLog.

@Transactional(propagation = Propagation.REQUIRED)
public RunLog saveLog(final RunLog runLog) {
    Assert.notNull(runLog.getRunId(), messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_REQUIRED, "runId", RunLog.class.getSimpleName()));
    Assert.notNull(runLog.getDate(), messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_REQUIRED, "date", RunLog.class.getSimpleName()));
    Assert.notNull(runLog.getStatus(), messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_REQUIRED, "status", RunLog.class.getSimpleName()));
    PipelineRun run = pipelineRunDao.loadPipelineRun(runLog.getRunId());
    Assert.notNull(run, messageHelper.getMessage(MessageConstants.ERROR_PIPELINE_NOT_FOUND, runLog.getRunId()));
    if (!StringUtils.isEmpty(runLog.getLogText())) {
        runLog.setLogText(runLog.getLogText().replaceAll("\\u0000", ""));
    }
    // Check previous status, it may differ from pod status as error may occur during
    // results upload to s3
    TaskStatus statusToSave = runLog.getStatus();
    if (!StringUtils.isEmpty(runLog.getTaskName())) {
        PipelineTask task = self.loadPreviousTaskStatus(run, runLog);
        if (task != null && task.getStatus().isFinal()) {
            statusToSave = task.getStatus();
        }
    }
    // this status
    if (!statusToSave.isFinal() && run.getStatus().isFinal()) {
        statusToSave = run.getStatus();
    }
    runLog.setStatus(statusToSave);
    runLogDao.createRunLog(runLog);
    return runLog;
}

Example 14 with PipelineRun

use of com.epam.pipeline.entity.pipeline.PipelineRun in project cloud-pipeline by epam.

the class DtsRunner method runEntry.

private PipelineRun runEntry(DtsRunConfigurationEntry entry, Long configurationId, List<Long> entitiesIds, PipelineConfiguration configuration) {
    Pipeline pipeline = entry.getPipelineId() == null ? null : pipelineManager.load(entry.getPipelineId());
    PipelineRun run = pipelineRunManager.createPipelineRun(entry.getPipelineVersion(), configuration, pipeline, null, entitiesIds, configurationId);
    run.setConfigName(entry.getConfigName());
    run.setDockerImage(toolManager.getExternalToolName(run.getDockerImage()));
    run.setExecutionPreferences(DtsExecutionPreferences.builder().dtsId(entry.getDtsId()).coresNumber(entry.getCoresNumber()).build());
    Map<SystemParams, String> systemParams = pipelineLauncher.matchCommonParams(run, preferenceManager.getPreference(SystemPreferences.BASE_API_HOST_EXTERNAL), configuration.getGitCredentials());
    systemParams.put(SystemParams.DISTRIBUTION_URL, preferenceManager.getPreference(SystemPreferences.DTS_DISTRIBUTION_URL));
    GitCredentials gitCredentials = configuration.getGitCredentials();
    String gitCloneUrl = gitCredentials == null ? run.getRepository() : gitCredentials.getUrl();
    String pipelineCommand = commandBuilder.build(configuration, systemParams);
    run.setActualCmd(pipelineCommand);
    String launchCmd = preferenceManager.getPreference(SystemPreferences.DTS_LAUNCH_CMD_TEMPLATE);
    String launchScriptUrl = preferenceManager.getPreference(SystemPreferences.DTS_LAUNCH_URL);
    String fullCmd = String.format(launchCmd, launchScriptUrl, launchScriptUrl, gitCloneUrl, run.getRevisionName(), pipelineCommand);
    pipelineRunManager.save(run);
    DtsSubmission submission = buildSubmission(run, configuration, entry.getCoresNumber(), systemParams, fullCmd);
    log.debug("Creating DTS submission");
    try {
        DtsSubmission scheduled = submissionManager.createSubmission(entry.getDtsId(), submission);
        if (scheduled.getState().getStatus() != TaskStatus.RUNNING) {
            return failRun(run, String.format("Submission failed to start: %s", scheduled.getState().getReason()));
        }
        log.debug("Successfully scheduled submission on DTS");
        return run;
    } catch (DtsRequestException e) {
        return failRun(run, e.getMessage());
    }
}

Example 15 with PipelineRun

use of com.epam.pipeline.entity.pipeline.PipelineRun in project cloud-pipeline by epam.

the class CloudPlatformRunner method runConfiguration.

private List<PipelineRun> runConfiguration(Long configurationId, List<RunConfigurationEntry> entries, ResolvedConfiguration resolvedConfigurations) {
    SplitConfig splitConfig = new SplitConfig(entries);
    RunConfigurationEntry mainEntry = splitConfig.getMain();
    List<RunConfigurationEntry> childEntries = splitConfig.getChildEntries();
    boolean isMasterNFSServer = pipelineConfigurationManager.hasNFSParameter(mainEntry.getConfiguration()) || childEntries.stream().noneMatch(entry -> pipelineConfigurationManager.hasNFSParameter(entry.getConfiguration()));
    boolean nfsStarted = isMasterNFSServer;
    PipelineConfiguration mainConfiguration = resolvedConfigurations.getConfiguration(mainEntry.getName());
    List<PipelineConfiguration> childConfigurations = childEntries.stream().map(entry -> resolvedConfigurations.getConfiguration(entry.getName())).collect(Collectors.toList());
    int masterNodeCount = getNodeCount(mainConfiguration.getNodeCount(), 0);
    int totalNodes = childConfigurations.stream().map(PipelineConfiguration::getNodeCount).mapToInt(nodeCount -> getNodeCount(nodeCount, 1)).sum();
    totalNodes += masterNodeCount;
    log.debug("Running total {} nodes", totalNodes + 1);
    mainConfiguration.setNodeCount(totalNodes);
    // create master run
    List<PipelineRun> masterRun = runConfigurationEntry(mainEntry, mainConfiguration, 1, null, isMasterNFSServer, resolvedConfigurations.getAllAssociatedIds(), configurationId);
    List<PipelineRun> launched = new ArrayList<>(masterRun);
    String clusterId = String.valueOf(masterRun.get(0).getId());
    // create master workers
    if (masterNodeCount > 0) {
        mainEntry.getConfiguration().setWorkerCmd(WORKER_CMD_TEMPLATE);
        launched.addAll(runConfigurationEntry(mainEntry, mainConfiguration, masterNodeCount, clusterId, false, resolvedConfigurations.getAllAssociatedIds(), configurationId));
    }
    // create all other workers
    for (int i = 0; i < childConfigurations.size(); i++) {
        PipelineConfiguration childConfig = childConfigurations.get(i);
        boolean startNFS = !nfsStarted && pipelineConfigurationManager.hasNFSParameter(childConfig);
        nfsStarted = nfsStarted || startNFS;
        int copies = getNodeCount(childConfig.getNodeCount(), 1);
        launched.addAll(runConfigurationEntry(childEntries.get(i), childConfig, copies, clusterId, startNFS, resolvedConfigurations.getAllAssociatedIds(), configurationId));
    }
    return launched;
}