Examples with Duration com.datastax.fallout.util.Duration used on opensource projects

Example 6 with Duration

use of com.datastax.fallout.util.Duration in project fallout by datastax.

the class NodeResponse method awaitAsync.

/**
 * An asynchronous wait.
 *
 * The future will complete when isCompleted() returns true
 *
 * Many processes can be waited on via one shared worker thread
 *
 * @see Utils#awaitConditionAsync
 */
public CompletableFuture<Boolean> awaitAsync(WaitOptionsAdjuster adjuster) {
    WaitOptions waitOptions = createWaitOptions();
    adjuster.adjust(waitOptions);
    final Logger logger = waitOptions.logger(this);
    final Pair<Duration, Optional<Duration>> timeouts = waitOptions.effectiveTimeouts();
    final Duration timeout = timeouts.getLeft();
    final Optional<Duration> noOutputTimeout = timeouts.getRight();
    String additionalInfo = noOutputTimeout.map(t -> "no-output-timeout: " + t.toHM() + ", ").orElse("");
    logger.info("Waiting for command ({}hard-timeout: {}): {}", additionalInfo, timeout.toHM(), command);
    if (asyncWaitStarted) {
        String logMsg = "awaitAsync on already awaited NodeResponse: " + command;
        logger.warn(logMsg);
    }
    asyncWaitStarted = true;
    class Context {

        static final int MAX_LAST_LINES = 20;

        final BufferedReader stream;

        final String type;

        final Consumer<String> consumer;

        final EvictingQueue<String> lastLines = EvictingQueue.create(MAX_LAST_LINES);

        CompletableFuture<Boolean> lineSupplier;

        Context(BufferedReader stream, Consumer<String> consumer, String type) {
            this.stream = stream;
            this.consumer = consumer;
            this.type = type;
        }

        void handleLine(String line) {
            lastLines.add(line);
            consumer.accept(line);
        }

        public String appendLastLines(String logMsg) {
            String res = logMsg;
            if (!lastLines.isEmpty()) {
                res += "\n" + type + " (last " + lastLines.size() + " lines):\n";
                for (String line : lastLines) {
                    res += line + "\n";
                }
            }
            return res;
        }
    }
    final Context[] streams;
    try {
        streams = new Context[] { new Context(new BufferedReader(new InputStreamReader(getOutputStream(), StandardCharsets.UTF_8)), waitOptions.stdoutConsumer, "STDOUT"), new Context(new BufferedReader(new InputStreamReader(getErrorStream(), StandardCharsets.UTF_8)), waitOptions.stderrConsumer, "STDERR") };
    } catch (IOException e) {
        logger.error("Error waiting for node response", e);
        return CompletableFuture.completedFuture(false);
    }
    AtomicLong lastOutputLineTime = new AtomicLong(0);
    for (Context ctx : streams) {
        ctx.lineSupplier = CompletableFuture.supplyAsync(() -> {
            try {
                String line;
                while ((line = ctx.stream.readLine()) != null) {
                    String trimmedLine = line.trim();
                    if (!trimmedLine.isEmpty() && !trimmedLine.equals("\n")) {
                        Optional<Logger> outputLineLogger = waitOptions.outputLineLogger(this);
                        if (outputLineLogger.isPresent()) {
                            outputLineLogger.get().info("{}: {}", ctx.type, line);
                        }
                        ctx.handleLine(line);
                    }
                    lastOutputLineTime.set(System.nanoTime());
                }
                return true;
            } catch (Exception e) {
                logger.error("Error waiting for node response output", e);
                return false;
            }
        }, lineSupplierExecutor);
    }
    AtomicBoolean wasNoOutputTimeout = new AtomicBoolean(false);
    List<Utils.TimeoutCheck> timeoutChecks = List.of();
    if (noOutputTimeout.isPresent()) {
        long outputTimeoutNano = noOutputTimeout.get().toNanos();
        timeoutChecks = new ArrayList<>(1);
        timeoutChecks.add(nanoTime -> {
            long lastOutputNano = lastOutputLineTime.get();
            if (// we only timeout processes that have had output before
            lastOutputNano > 0) {
                if (nanoTime >= lastOutputNano + outputTimeoutNano) {
                    wasNoOutputTimeout.set(true);
                    return true;
                }
            }
            return false;
        });
    }
    Utils.AwaitConditionOptions awaitOptions = new Utils.AwaitConditionOptions(logger, this::isCompleted, timeout, timer, waitOptions.checkInterval);
    awaitOptions.addTimeoutChecks(timeoutChecks);
    return Utils.awaitConditionAsync(awaitOptions).thenApplyAsync(completedWithoutTimeout -> {
        completionListeners.forEach(completionListener -> completionListener.accept(this));
        completionListeners.clear();
        String nodeInfo = owner == null ? "" : " on node '" + owner.getId() + "'";
        // command output after the command has been logged as completed.
        if (completedWithoutTimeout && !wasKilled()) {
            // Wait for stream processing to complete
            for (Context ctx : streams) {
                try {
                    Uninterruptibles.getUninterruptibly(ctx.lineSupplier, 1, TimeUnit.MINUTES);
                } catch (ExecutionException e) {
                    throw new CompletionException(e);
                } catch (TimeoutException e) {
                    logger.error("Command{} timed out waiting for output streams to close; this may be due to a zombie process, see FAL-1119", nodeInfo);
                }
            }
        } else {
            // Stop processing output since we will either kill this process due to timeout, or have already
            // killed it.  For killed processes, this is a hack to get around children of the target process
            // keeping the stream open; once FAL-1119 is done, it should no longer be necessary.
            logger.debug("Command{} timed out or killed: cancelling output streams listeners", nodeInfo);
            for (Context ctx : streams) {
                if (!ctx.lineSupplier.isDone()) {
                    ctx.lineSupplier.cancel(true);
                }
            }
        }
        if (!completedWithoutTimeout) {
            // need to kill before potentially fetching fully buffered output from FBNR
            this.kill();
        }
        String outputToLog = waitOptions.outputLogging ? getFormattedCommandOutput() : "";
        if (completedWithoutTimeout) {
            int exitCode = getExitCode();
            if (exitCode == 0) {
                logger.info("Command{} completed with exit code {}: {}{}", nodeInfo, exitCode, command, outputToLog);
            } else {
                String logMsg = "Command{} completed with non-zero exit code {}: {}{}";
                if (waitOptions.exitCodeIsError.test(exitCode)) {
                    String errorMsg = CMD_FAIL_LOG_PREFIX + logMsg;
                    if (outputToLog.isEmpty()) {
                        String cmdOutput = getFormattedCommandOutput();
                        if (cmdOutput.isEmpty()) {
                            for (Context ctx : streams) {
                                outputToLog = ctx.appendLastLines(outputToLog);
                            }
                        } else {
                            // on error we log the full output even when outputLogging was false
                            outputToLog = cmdOutput;
                        }
                    }
                    logger.error(errorMsg, nodeInfo, exitCode, command, outputToLog);
                } else {
                    logger.info(logMsg, nodeInfo, exitCode, command, outputToLog);
                }
            }
        } else {
            if (wasNoOutputTimeout.get()) {
                String errorMsg = CMD_FAIL_LOG_PREFIX + "Command{} timed out due to no output after {}: {}{}";
                logger.error(errorMsg, nodeInfo, waitOptions.noOutputTimeout.get(), command, outputToLog);
            } else {
                String errorMsg = CMD_FAIL_LOG_PREFIX + "Command{} timed out after {}: {}{}";
                logger.error(errorMsg, nodeInfo, timeout, command, outputToLog);
            }
        }
        return completedWithoutTimeout;
    });
}

Example 7 with Duration

use of com.datastax.fallout.util.Duration in project fallout by datastax.

the class RepeatableNodeCommandModule method setup.

@Override
public void setup(Ensemble ensemble, PropertyGroup properties) {
    this.ensemble = ensemble;
    command = getCommand(properties);
    secondaryCommand = commandSecondarySpec.value(properties);
    this.iterationSpec.init(getUnfinishedRunOnceModules(), properties);
    nodeSelector = nodesSpec.createSelector(ensemble, properties);
    timeoutDuration = timeoutSpec.toDuration(properties);
    Duration noOutputTimeout = noOutputTimeoutSpec.value(properties);
    this.noOutputTimeout = noOutputTimeout.value > 0L ? Optional.of(noOutputTimeout) : Optional.empty();
    logger().info(name() + " setup done.");
}

Example 8 with Duration

use of com.datastax.fallout.util.Duration in project fallout by datastax.

the class SleepModule method run.

@Override
public void run(Ensemble ensemble, PropertyGroup properties) {
    emit(Operation.Type.invoke);
    Duration sleepDuration = sleepDurationSpec.value(properties);
    try {
        Thread.sleep(sleepDuration.toMillis());
        emit(Operation.Type.ok);
    } catch (InterruptedException e) {
        emit(Operation.Type.fail);
    }
}

Example 9 with Duration

use of com.datastax.fallout.util.Duration in project fallout by datastax.

the class TimeoutSpec method toDuration.

public Duration toDuration(PropertyGroup propertyGroup) {
    Duration timeoutDuration = timeout.value(propertyGroup);
    if (timeoutDuration != null) {
        return timeoutDuration;
    }
    // deprecated properties
    Long timeValue = timeoutValue.value(propertyGroup);
    TimeUnit timeUnit = timeoutUnit.value(propertyGroup);
    if (timeValue == null && timeUnit != null) {
        timeValue = defaultValue;
    }
    if (timeValue != null && timeUnit == null) {
        timeUnit = defaultUnit;
    }
    if (timeValue != null && timeUnit != null) {
        return new Duration(timeValue, timeUnit);
    }
    // default
    return new Duration(defaultValue, defaultUnit);
}