Search in sources :

Example 1 with RetryUtils

use of io.kestra.core.utils.RetryUtils in project kestra by kestra-io.

the class DockerScriptRunner method run.

public RunResult run(AbstractBash abstractBash, RunContext runContext, Logger logger, Path workingDirectory, List<String> commandsWithInterpreter, Map<String, String> env, AbstractBash.LogSupplier logSupplier, Map<String, Object> additionalVars) throws Exception {
    DockerClient dockerClient = getDockerClient(abstractBash, runContext, workingDirectory);
    if (abstractBash.getDockerOptions() == null) {
        throw new IllegalArgumentException("Missing required dockerOptions properties");
    }
    String image = runContext.render(abstractBash.getDockerOptions().getImage(), additionalVars);
    NameParser.ReposTag imageParse = NameParser.parseRepositoryTag(image);
    try (CreateContainerCmd container = dockerClient.createContainerCmd(image);
        PullImageCmd pull = dockerClient.pullImageCmd(image);
        PipedInputStream stdOutInputStream = new PipedInputStream();
        PipedOutputStream stdOutOutputStream = new PipedOutputStream(stdOutInputStream);
        PipedInputStream stdErrInputStream = new PipedInputStream();
        PipedOutputStream stdErrOutputStream = new PipedOutputStream(stdErrInputStream)) {
        // properties
        metadata(runContext, container);
        HostConfig hostConfig = new HostConfig();
        if (env != null && env.size() > 0) {
            container.withEnv(env.entrySet().stream().map(throwFunction(r -> runContext.render(r.getKey(), additionalVars) + "=" + runContext.render(r.getValue(), additionalVars))).collect(Collectors.toList()));
        }
        if (workingDirectory != null) {
            container.withWorkingDir(workingDirectory.toFile().getAbsolutePath());
        }
        if (workingDirectory != null) {
            hostConfig.withBinds(new Bind(workingDirectory.toAbsolutePath().toString(), new Volume(workingDirectory.toAbsolutePath().toString()), AccessMode.rw));
        }
        if (abstractBash.getDockerOptions().getUser() != null) {
            container.withUser(runContext.render(abstractBash.getDockerOptions().getUser(), additionalVars));
        }
        if (abstractBash.getDockerOptions().getEntryPoint() != null) {
            container.withEntrypoint(runContext.render(abstractBash.getDockerOptions().getEntryPoint(), additionalVars));
        }
        if (abstractBash.getDockerOptions().getExtraHosts() != null) {
            hostConfig.withExtraHosts(runContext.render(abstractBash.getDockerOptions().getExtraHosts(), additionalVars).toArray(String[]::new));
        }
        container.withHostConfig(hostConfig).withCmd(commandsWithInterpreter).withAttachStderr(true).withAttachStdout(true);
        // pull image
        retryUtils.<Boolean, InternalServerErrorException>of(Exponential.builder().delayFactor(2.0).interval(Duration.ofSeconds(5)).maxInterval(Duration.ofSeconds(120)).maxAttempt(5).build()).run((bool, throwable) -> throwable instanceof InternalServerErrorException || throwable.getCause() instanceof ConnectionClosedException, () -> {
            pull.withTag(!imageParse.tag.equals("") ? imageParse.tag : "latest").exec(new PullImageResultCallback()).awaitCompletion();
            logger.debug("Image pulled [{}:{}]", pull.getRepository(), pull.getTag());
            return true;
        });
        // start container
        CreateContainerResponse exec = container.exec();
        dockerClient.startContainerCmd(exec.getId()).exec();
        logger.debug("Starting command with container id {} [{}]", exec.getId(), String.join(" ", commandsWithInterpreter));
        try {
            // logs
            AbstractLogThread stdOut = logSupplier.call(stdOutInputStream, false);
            AbstractLogThread stdErr = logSupplier.call(stdErrInputStream, true);
            dockerClient.logContainerCmd(exec.getId()).withFollowStream(true).withStdErr(true).withStdOut(true).exec(new ResultCallback.Adapter<Frame>() {

                @SneakyThrows
                @Override
                public void onNext(Frame item) {
                    if (item.getStreamType() == StreamType.STDOUT) {
                        stdOutOutputStream.write(item.getPayload());
                    } else {
                        stdErrOutputStream.write(item.getPayload());
                    }
                }
            });
            WaitContainerResultCallback result = dockerClient.waitContainerCmd(exec.getId()).start();
            Integer exitCode = result.awaitStatusCode();
            stdOutOutputStream.flush();
            stdOutOutputStream.close();
            stdErrOutputStream.flush();
            stdErrOutputStream.close();
            stdOut.join();
            stdErr.join();
            dockerClient.removeContainerCmd(exec.getId()).exec();
            if (exitCode != 0) {
                throw new AbstractBash.BashException(exitCode, stdOut.getLogsCount(), stdErr.getLogsCount());
            } else {
                logger.debug("Command succeed with code " + exitCode);
            }
            return new RunResult(exitCode, stdOut, stdErr);
        } catch (InterruptedException e) {
            logger.warn("Killing process {} for InterruptedException", exec.getId());
            dockerClient.killContainerCmd(exec.getId()).exec();
            dockerClient.removeContainerCmd(exec.getId()).exec();
            throw e;
        }
    }
}
Also used : RunResult(io.kestra.core.tasks.scripts.RunResult) DockerClientBuilder(com.github.dockerjava.core.DockerClientBuilder) Builder(com.github.dockerjava.httpclient5.ApacheDockerHttpClient.Builder) com.github.dockerjava.api.model(com.github.dockerjava.api.model) Rethrow.throwFunction(io.kestra.core.utils.Rethrow.throwFunction) SneakyThrows(lombok.SneakyThrows) NameParser(com.github.dockerjava.core.NameParser) Exponential(io.kestra.core.models.tasks.retrys.Exponential) IllegalVariableEvaluationException(io.kestra.core.exceptions.IllegalVariableEvaluationException) Slugify(io.kestra.core.utils.Slugify) StringUtils(org.apache.commons.lang3.StringUtils) DockerClient(com.github.dockerjava.api.DockerClient) RetryUtils(io.kestra.core.utils.RetryUtils) ApplicationContext(io.micronaut.context.ApplicationContext) DockerClientConfig(com.github.dockerjava.core.DockerClientConfig) AbstractLogThread(io.kestra.core.tasks.scripts.AbstractLogThread) PipedInputStream(java.io.PipedInputStream) Duration(java.time.Duration) Map(java.util.Map) DefaultDockerClientConfig(com.github.dockerjava.core.DefaultDockerClientConfig) com.github.dockerjava.api.command(com.github.dockerjava.api.command) Path(java.nio.file.Path) InternalServerErrorException(com.github.dockerjava.api.exception.InternalServerErrorException) ApacheDockerHttpClient(com.github.dockerjava.httpclient5.ApacheDockerHttpClient) Logger(org.slf4j.Logger) ImmutableMap(com.google.common.collect.ImmutableMap) Files(java.nio.file.Files) AbstractBash(io.kestra.core.tasks.scripts.AbstractBash) IOException(java.io.IOException) PipedOutputStream(java.io.PipedOutputStream) Collectors(java.util.stream.Collectors) ConnectionClosedException(org.apache.hc.core5.http.ConnectionClosedException) List(java.util.List) RunContext(io.kestra.core.runners.RunContext) ResultCallback(com.github.dockerjava.api.async.ResultCallback) ResultCallback(com.github.dockerjava.api.async.ResultCallback) DockerClient(com.github.dockerjava.api.DockerClient) AbstractLogThread(io.kestra.core.tasks.scripts.AbstractLogThread) ConnectionClosedException(org.apache.hc.core5.http.ConnectionClosedException) SneakyThrows(lombok.SneakyThrows) PipedOutputStream(java.io.PipedOutputStream) RunResult(io.kestra.core.tasks.scripts.RunResult) PipedInputStream(java.io.PipedInputStream) InternalServerErrorException(com.github.dockerjava.api.exception.InternalServerErrorException) NameParser(com.github.dockerjava.core.NameParser)

Aggregations

DockerClient (com.github.dockerjava.api.DockerClient)1 ResultCallback (com.github.dockerjava.api.async.ResultCallback)1 com.github.dockerjava.api.command (com.github.dockerjava.api.command)1 InternalServerErrorException (com.github.dockerjava.api.exception.InternalServerErrorException)1 com.github.dockerjava.api.model (com.github.dockerjava.api.model)1 DefaultDockerClientConfig (com.github.dockerjava.core.DefaultDockerClientConfig)1 DockerClientBuilder (com.github.dockerjava.core.DockerClientBuilder)1 DockerClientConfig (com.github.dockerjava.core.DockerClientConfig)1 NameParser (com.github.dockerjava.core.NameParser)1 ApacheDockerHttpClient (com.github.dockerjava.httpclient5.ApacheDockerHttpClient)1 Builder (com.github.dockerjava.httpclient5.ApacheDockerHttpClient.Builder)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 IllegalVariableEvaluationException (io.kestra.core.exceptions.IllegalVariableEvaluationException)1 Exponential (io.kestra.core.models.tasks.retrys.Exponential)1 RunContext (io.kestra.core.runners.RunContext)1 AbstractBash (io.kestra.core.tasks.scripts.AbstractBash)1 AbstractLogThread (io.kestra.core.tasks.scripts.AbstractLogThread)1 RunResult (io.kestra.core.tasks.scripts.RunResult)1 Rethrow.throwFunction (io.kestra.core.utils.Rethrow.throwFunction)1 RetryUtils (io.kestra.core.utils.RetryUtils)1