Search in sources :

Example 6 with PipeliteException

use of pipelite.exception.PipeliteException in project pipelite by enasequence.

the class ProcessFactory method create.

/**
 * Creates a process.
 *
 * @param processId the process id
 * @param registeredPipeline the registered pipeline
 * @return the process
 * @throws PipeliteException if the new process could not be created
 */
public static Process create(String processId, RegisteredPipeline registeredPipeline) {
    if (processId == null) {
        throw new PipeliteException("Failed to create process. Missing process id.");
    }
    if (registeredPipeline == null) {
        throw new PipeliteException("Failed to create process. Missing registered pipeline.");
    }
    String pipelineName = registeredPipeline.pipelineName();
    if (pipelineName == null) {
        throw new PipeliteException("Failed to create process. Missing pipeline name.");
    }
    try {
        log.atFine().log("Creating %s process %s", pipelineName, processId);
        ProcessBuilder processBuilder = new ProcessBuilder(processId);
        registeredPipeline.configureProcess(processBuilder);
        Process process = processBuilder.build();
        if (process == null) {
            throw new PipeliteException("Failed to create " + pipelineName + " process " + processId + ". Pipeline returned a null process.");
        }
        return process;
    } catch (Exception ex) {
        throw new PipeliteException("Failed to create " + pipelineName + " process " + processId + ". Unexpected exception.", ex);
    }
}
Also used : ProcessBuilder(pipelite.process.builder.ProcessBuilder) PipeliteException(pipelite.exception.PipeliteException) PipeliteException(pipelite.exception.PipeliteException)

Example 7 with PipeliteException

use of pipelite.exception.PipeliteException in project pipelite by enasequence.

the class ConfigureProcessValidator method validate.

/**
 * Validates stage execution graph. Uses 'VALIDATE' as the processId.
 *
 * @param pipeline the schedule or pipeline to validate
 * @return the validation errors
 */
public static List<ValidatorError> validate(RegisteredPipeline pipeline) {
    List<ValidatorError> errors = new ArrayList<>();
    try {
        ProcessBuilder processBuilder = new ProcessBuilder("VALIDATE");
        pipeline.configureProcess(processBuilder);
        processBuilder.build();
    } catch (PipeliteException ex) {
        errors.add(new ValidatorError(pipeline.pipelineName(), ex.getMessage()));
    }
    return errors;
}
Also used : ProcessBuilder(pipelite.process.builder.ProcessBuilder) ArrayList(java.util.ArrayList) PipeliteException(pipelite.exception.PipeliteException)

Example 8 with PipeliteException

use of pipelite.exception.PipeliteException in project pipelite by enasequence.

the class StageRunner method executeStage.

private void executeStage(StageExecutorResultCallback processRunnerResultCallback) {
    StageExecutorResultCallback stageRunnerResultCallback = (executorResult) -> internalErrorHandler.execute(() -> {
        if (executorResult == null) {
            throw new PipeliteException("Missing executor result");
        }
        if (executorResult.isSubmitted()) {
            logContext(log.atFine()).log("Submitted asynchronous stage execution");
            pipeliteServices.stage().saveStage(stage);
        } else if (executorResult.isActive()) {
            if (ZonedDateTime.now().isAfter(timeout)) {
                logContext(log.atSevere()).log("Maximum stage execution time exceeded.");
                // Terminate executor.
                internalErrorHandler.execute(() -> stage.getExecutor().terminate());
                endStageExecution(processRunnerResultCallback, StageExecutorResult.timeoutError());
            } else {
                logContext(log.atFiner()).log("Waiting asynchronous stage execution to complete");
            }
        } else if (executorResult.isSuccess() || executorResult.isError()) {
            endStageExecution(processRunnerResultCallback, executorResult);
        }
    }, (ex) -> endStageExecution(processRunnerResultCallback, StageExecutorResult.internalError(ex)));
    executeStage(processRunnerResultCallback, stageRunnerResultCallback);
}
Also used : ExecutorParameters(pipelite.stage.parameters.ExecutorParameters) PipeliteMetrics(pipelite.metrics.PipeliteMetrics) StageExecutorResult(pipelite.stage.executor.StageExecutorResult) InternalErrorHandler(pipelite.error.InternalErrorHandler) ZonedDateTime(java.time.ZonedDateTime) LogKey(pipelite.log.LogKey) StageExecutor(pipelite.stage.executor.StageExecutor) StageExecutorSerializer(pipelite.stage.executor.StageExecutorSerializer) StageExecutorResultCallback(pipelite.stage.executor.StageExecutorResultCallback) PipeliteException(pipelite.exception.PipeliteException) Process(pipelite.process.Process) AbstractAsyncExecutor(pipelite.executor.AbstractAsyncExecutor) PipeliteServices(pipelite.service.PipeliteServices) Stage(pipelite.stage.Stage) Flogger(lombok.extern.flogger.Flogger) Duration(java.time.Duration) FluentLogger(com.google.common.flogger.FluentLogger) Assert(org.springframework.util.Assert) StageExecutorResultCallback(pipelite.stage.executor.StageExecutorResultCallback) PipeliteException(pipelite.exception.PipeliteException)

Example 9 with PipeliteException

use of pipelite.exception.PipeliteException in project pipelite by enasequence.

the class RegisteredPipelineService method registerPipeline.

public void registerPipeline(RegisteredPipeline registeredPipeline) {
    String pipelineName = registeredPipeline.pipelineName();
    if (pipelineName == null || pipelineName.trim().isEmpty()) {
        throw new PipeliteException("Missing pipeline name");
    }
    if (registeredPipelineMap.containsKey(pipelineName)) {
        throw new PipeliteException("Non-unique pipeline name: " + pipelineName);
    }
    if (registeredPipeline instanceof Schedule) {
        Schedule schedule = (Schedule) registeredPipeline;
        String cron = schedule.configurePipeline().cron();
        if (cron == null) {
            throw new PipeliteException("Missing cron expression for pipeline schedule: " + pipelineName);
        }
        if (!CronUtils.validate(cron)) {
            throw new PipeliteException("Invalid cron expression '" + cron + "' for pipeline schedule: " + pipelineName);
        }
    } else if (registeredPipeline instanceof Pipeline) {
        Pipeline pipeline = (Pipeline) registeredPipeline;
        int pipelineParallelism = pipeline.configurePipeline().pipelineParallelism();
        if (pipelineParallelism < 1) {
            throw new PipeliteException("Invalid pipeline parallelism '" + pipelineParallelism + "' for pipeline: " + pipelineName);
        }
    }
    registeredPipelineMap.put(pipelineName, registeredPipeline);
}
Also used : Schedule(pipelite.Schedule) PipeliteException(pipelite.exception.PipeliteException) RegisteredPipeline(pipelite.RegisteredPipeline) Pipeline(pipelite.Pipeline)

Example 10 with PipeliteException

use of pipelite.exception.PipeliteException in project pipelite by enasequence.

the class RegisteredScheduleService method saveSchedules.

@PostConstruct
public void saveSchedules() {
    registeredPipelineService.getRegisteredPipelines(Schedule.class).forEach(schedule -> {
        String pipelineName = schedule.pipelineName();
        Optional<ScheduleEntity> savedScheduleEntityOpt = scheduleService.getSavedSchedule(pipelineName);
        if (!savedScheduleEntityOpt.isPresent()) {
            createSchedule(schedule);
        } else {
            ScheduleEntity savedScheduleEntity = savedScheduleEntityOpt.get();
            String registeredCron = savedScheduleEntity.getCron();
            String registeredServiceName = savedScheduleEntity.getServiceName();
            String cron = schedule.configurePipeline().cron();
            boolean isCronChanged = !registeredCron.equals(cron);
            boolean isServiceNameChanged = !registeredServiceName.equals(serviceName);
            if (isCronChanged) {
                log.atInfo().log("Cron changed for pipeline schedule: " + schedule.pipelineName());
            }
            if (isServiceNameChanged) {
                log.atInfo().log("Service name changed for pipeline schedule: " + schedule.pipelineName());
            }
            if (isServiceNameChanged && !pipeliteConfiguration.service().isForce()) {
                throw new PipeliteException("Forceful startup not requested. Service name changed for pipeline schedule " + pipelineName + " from " + registeredServiceName + " to " + serviceName);
            }
            if (isServiceNameChanged) {
                log.atWarning().log("Forceful startup requested. Changing service name for pipeline schedule " + pipelineName + " from " + registeredServiceName + " to " + serviceName);
            }
            if (isCronChanged || isServiceNameChanged) {
                log.atInfo().log("Updating pipeline schedule: " + schedule.pipelineName());
                try {
                    savedScheduleEntity.setCron(cron);
                    savedScheduleEntity.setDescription(CronUtils.describe(cron));
                    savedScheduleEntity.setServiceName(serviceName);
                    if (!savedScheduleEntity.isActive()) {
                        savedScheduleEntity.setNextTime(CronUtils.launchTime(savedScheduleEntity.getCron(), savedScheduleEntity.getStartTime()));
                    }
                    scheduleService.saveSchedule(savedScheduleEntity);
                } catch (Exception ex) {
                    throw new PipeliteException("Failed to update pipeline schedule: " + schedule.pipelineName(), ex);
                }
            }
        }
    });
}
Also used : Schedule(pipelite.Schedule) PipeliteException(pipelite.exception.PipeliteException) ScheduleEntity(pipelite.entity.ScheduleEntity) PipeliteException(pipelite.exception.PipeliteException) PostConstruct(javax.annotation.PostConstruct)

Aggregations

PipeliteException (pipelite.exception.PipeliteException)23 StageExecutorResult (pipelite.stage.executor.StageExecutorResult)6 KubernetesClientException (io.fabric8.kubernetes.client.KubernetesClientException)5 DefaultKubernetesClient (io.fabric8.kubernetes.client.DefaultKubernetesClient)4 KubernetesClient (io.fabric8.kubernetes.client.KubernetesClient)4 Pod (io.fabric8.kubernetes.api.model.Pod)2 ByteArrayOutputStream (java.io.ByteArrayOutputStream)2 IOException (java.io.IOException)2 OutputStream (java.io.OutputStream)2 Duration (java.time.Duration)2 ZonedDateTime (java.time.ZonedDateTime)2 TimeoutException (java.util.concurrent.TimeoutException)2 Pipeline (pipelite.Pipeline)2 Schedule (pipelite.Schedule)2 ProcessEntity (pipelite.entity.ProcessEntity)2 ScheduleEntity (pipelite.entity.ScheduleEntity)2 PipeliteMetrics (pipelite.metrics.PipeliteMetrics)2 ProcessBuilder (pipelite.process.builder.ProcessBuilder)2 PipeliteServices (pipelite.service.PipeliteServices)2 AWSBatch (com.amazonaws.services.batch.AWSBatch)1