Examples with RetryUtils io.kestra.core.utils.RetryUtils used on opensource projects

Example 1 with RetryUtils

use of io.kestra.core.utils.RetryUtils in project kestra by kestra-io.

the class DockerScriptRunner method run.

public RunResult run(AbstractBash abstractBash, RunContext runContext, Logger logger, Path workingDirectory, List<String> commandsWithInterpreter, Map<String, String> env, AbstractBash.LogSupplier logSupplier, Map<String, Object> additionalVars) throws Exception {
    DockerClient dockerClient = getDockerClient(abstractBash, runContext, workingDirectory);
    if (abstractBash.getDockerOptions() == null) {
        throw new IllegalArgumentException("Missing required dockerOptions properties");
    }
    String image = runContext.render(abstractBash.getDockerOptions().getImage(), additionalVars);
    NameParser.ReposTag imageParse = NameParser.parseRepositoryTag(image);
    try (CreateContainerCmd container = dockerClient.createContainerCmd(image);
        PullImageCmd pull = dockerClient.pullImageCmd(image);
        PipedInputStream stdOutInputStream = new PipedInputStream();
        PipedOutputStream stdOutOutputStream = new PipedOutputStream(stdOutInputStream);
        PipedInputStream stdErrInputStream = new PipedInputStream();
        PipedOutputStream stdErrOutputStream = new PipedOutputStream(stdErrInputStream)) {
        // properties
        metadata(runContext, container);
        HostConfig hostConfig = new HostConfig();
        if (env != null && env.size() > 0) {
            container.withEnv(env.entrySet().stream().map(throwFunction(r -> runContext.render(r.getKey(), additionalVars) + "=" + runContext.render(r.getValue(), additionalVars))).collect(Collectors.toList()));
        }
        if (workingDirectory != null) {
            container.withWorkingDir(workingDirectory.toFile().getAbsolutePath());
        }
        if (workingDirectory != null) {
            hostConfig.withBinds(new Bind(workingDirectory.toAbsolutePath().toString(), new Volume(workingDirectory.toAbsolutePath().toString()), AccessMode.rw));
        }
        if (abstractBash.getDockerOptions().getUser() != null) {
            container.withUser(runContext.render(abstractBash.getDockerOptions().getUser(), additionalVars));
        }
        if (abstractBash.getDockerOptions().getEntryPoint() != null) {
            container.withEntrypoint(runContext.render(abstractBash.getDockerOptions().getEntryPoint(), additionalVars));
        }
        if (abstractBash.getDockerOptions().getExtraHosts() != null) {
            hostConfig.withExtraHosts(runContext.render(abstractBash.getDockerOptions().getExtraHosts(), additionalVars).toArray(String[]::new));
        }
        container.withHostConfig(hostConfig).withCmd(commandsWithInterpreter).withAttachStderr(true).withAttachStdout(true);
        // pull image
        retryUtils.<Boolean, InternalServerErrorException>of(Exponential.builder().delayFactor(2.0).interval(Duration.ofSeconds(5)).maxInterval(Duration.ofSeconds(120)).maxAttempt(5).build()).run((bool, throwable) -> throwable instanceof InternalServerErrorException || throwable.getCause() instanceof ConnectionClosedException, () -> {
            pull.withTag(!imageParse.tag.equals("") ? imageParse.tag : "latest").exec(new PullImageResultCallback()).awaitCompletion();
            logger.debug("Image pulled [{}:{}]", pull.getRepository(), pull.getTag());
            return true;
        });
        // start container
        CreateContainerResponse exec = container.exec();
        dockerClient.startContainerCmd(exec.getId()).exec();
        logger.debug("Starting command with container id {} [{}]", exec.getId(), String.join(" ", commandsWithInterpreter));
        try {
            // logs
            AbstractLogThread stdOut = logSupplier.call(stdOutInputStream, false);
            AbstractLogThread stdErr = logSupplier.call(stdErrInputStream, true);
            dockerClient.logContainerCmd(exec.getId()).withFollowStream(true).withStdErr(true).withStdOut(true).exec(new ResultCallback.Adapter<Frame>() {

                @SneakyThrows
                @Override
                public void onNext(Frame item) {
                    if (item.getStreamType() == StreamType.STDOUT) {
                        stdOutOutputStream.write(item.getPayload());
                    } else {
                        stdErrOutputStream.write(item.getPayload());
                    }
                }
            });
            WaitContainerResultCallback result = dockerClient.waitContainerCmd(exec.getId()).start();
            Integer exitCode = result.awaitStatusCode();
            stdOutOutputStream.flush();
            stdOutOutputStream.close();
            stdErrOutputStream.flush();
            stdErrOutputStream.close();
            stdOut.join();
            stdErr.join();
            dockerClient.removeContainerCmd(exec.getId()).exec();
            if (exitCode != 0) {
                throw new AbstractBash.BashException(exitCode, stdOut.getLogsCount(), stdErr.getLogsCount());
            } else {
                logger.debug("Command succeed with code " + exitCode);
            }
            return new RunResult(exitCode, stdOut, stdErr);
        } catch (InterruptedException e) {
            logger.warn("Killing process {} for InterruptedException", exec.getId());
            dockerClient.killContainerCmd(exec.getId()).exec();
            dockerClient.removeContainerCmd(exec.getId()).exec();
            throw e;
        }
    }
}