Examples with SingularityS3UploaderFile com.hubspot.singularity.SingularityS3UploaderFile used on opensource projects

Example 1 with SingularityS3UploaderFile

use of com.hubspot.singularity.SingularityS3UploaderFile in project Singularity by HubSpot.

the class SingularityExecutorTaskLogManager method writeS3MetadataFileForRotatedFiles.

@SuppressFBWarnings
private boolean writeS3MetadataFileForRotatedFiles(boolean finished) {
    final Path serviceLogOutPath = taskDefinition.getServiceLogOutPath();
    final Path serviceLogParent = serviceLogOutPath.getParent();
    final Path logrotateDirectory = serviceLogParent.resolve(configuration.getLogrotateToDirectory());
    List<String> handledLogs = new ArrayList<>();
    int index = 1;
    boolean result = true;
    for (SingularityS3UploaderFile additionalFile : taskDefinition.getExecutorData().getS3UploaderAdditionalFiles()) {
        Path directory = additionalFile.getDirectory().isPresent() ? taskDefinition.getTaskDirectoryPath().resolve(additionalFile.getDirectory().get()) : taskDefinition.getTaskDirectoryPath();
        String fileGlob = additionalFile.getFilename() != null && additionalFile.getFilename().contains("*") ? additionalFile.getFilename() : String.format("%s*.[gb]z*", additionalFile.getFilename());
        result = result && writeS3MetadataFile(additionalFile.getS3UploaderFilenameHint().or(String.format("file%d", index)), directory, fileGlob, additionalFile.getS3UploaderBucket(), additionalFile.getS3UploaderKeyPattern(), finished, additionalFile.getS3StorageClass().or(taskDefinition.getExecutorData().getS3StorageClass()), additionalFile.getApplyS3StorageClassAfterBytes().or(taskDefinition.getExecutorData().getApplyS3StorageClassAfterBytes()), additionalFile.isCheckSubdirectories());
        index++;
        handledLogs.add(additionalFile.getFilename());
    }
    // Allow an additional file to override the upload settings for service.log
    if (!handledLogs.contains(taskDefinition.getServiceLogFileName())) {
        result = result && writeS3MetadataFile("default", logrotateDirectory, String.format("%s*.[gb]z*", taskDefinition.getServiceLogOutPath().getFileName()), Optional.absent(), Optional.absent(), finished, taskDefinition.getExecutorData().getS3StorageClass(), taskDefinition.getExecutorData().getApplyS3StorageClassAfterBytes(), false);
    }
    return result;
}

Example 2 with SingularityS3UploaderFile

use of com.hubspot.singularity.SingularityS3UploaderFile in project Singularity by HubSpot.

the class S3LogResource method getS3PrefixesForDeploy.

private Collection<String> getS3PrefixesForDeploy(S3Configuration s3Configuration, String requestId, String deployId, Optional<Long> startArg, Optional<Long> endArg, String group, SingularityUser user) {
    SingularityDeployHistory deployHistory = getDeployHistory(requestId, deployId, user);
    long start = deployHistory.getDeployMarker().getTimestamp();
    if (startArg.isPresent()) {
        start = Math.max(startArg.get(), start);
    }
    long end = System.currentTimeMillis();
    if (!isCurrentDeploy(requestId, deployId) && deployHistory.getDeployStatistics().isPresent() && deployHistory.getDeployStatistics().get().getLastFinishAt().isPresent()) {
        end = deployHistory.getDeployStatistics().get().getLastFinishAt().get() + TimeUnit.DAYS.toMillis(1);
    }
    if (endArg.isPresent()) {
        end = Math.min(endArg.get(), end);
    }
    Optional<String> tag = Optional.absent();
    if (deployHistory.getDeploy().isPresent() && deployHistory.getDeploy().get().getExecutorData().isPresent()) {
        tag = deployHistory.getDeploy().get().getExecutorData().get().getLoggingTag();
    }
    Collection<String> prefixes = SingularityS3FormatHelper.getS3KeyPrefixes(s3Configuration.getS3KeyFormat(), requestId, deployId, tag, start, end, group);
    for (SingularityS3UploaderFile additionalFile : s3Configuration.getS3UploaderAdditionalFiles()) {
        if (additionalFile.getS3UploaderKeyPattern().isPresent() && !additionalFile.getS3UploaderKeyPattern().get().equals(s3Configuration.getS3KeyFormat())) {
            prefixes.addAll(SingularityS3FormatHelper.getS3KeyPrefixes(additionalFile.getS3UploaderKeyPattern().get(), requestId, deployId, tag, start, end, group));
        }
    }
    LOG.trace("Request {}, deploy {} got S3 prefixes {} for start {}, end {}, tag {}", requestId, deployId, prefixes, start, end, tag);
    return prefixes;
}

Example 3 with SingularityS3UploaderFile

use of com.hubspot.singularity.SingularityS3UploaderFile in project Singularity by HubSpot.

the class S3LogResource method getS3PrefixesForTask.

// Generation of prefixes
private Collection<String> getS3PrefixesForTask(S3Configuration s3Configuration, SingularityTaskId taskId, Optional<Long> startArg, Optional<Long> endArg, String group, SingularityUser user) {
    Optional<SingularityTaskHistory> history = getTaskHistory(taskId, user);
    long start = taskId.getStartedAt();
    if (startArg.isPresent()) {
        start = Math.max(startArg.get(), start);
    }
    long end = start + s3Configuration.getMissingTaskDefaultS3SearchPeriodMillis();
    if (history.isPresent()) {
        SimplifiedTaskState taskState = SingularityTaskHistoryUpdate.getCurrentState(history.get().getTaskUpdates());
        if (taskState == SimplifiedTaskState.DONE) {
            end = Iterables.getLast(history.get().getTaskUpdates()).getTimestamp();
        } else {
            end = System.currentTimeMillis();
        }
    }
    if (endArg.isPresent()) {
        end = Math.min(endArg.get(), end);
    }
    Optional<String> tag = Optional.absent();
    if (history.isPresent() && history.get().getTask().getTaskRequest().getDeploy().getExecutorData().isPresent()) {
        tag = history.get().getTask().getTaskRequest().getDeploy().getExecutorData().get().getLoggingTag();
    }
    Collection<String> prefixes = SingularityS3FormatHelper.getS3KeyPrefixes(s3Configuration.getS3KeyFormat(), taskId, tag, start, end, group);
    for (SingularityS3UploaderFile additionalFile : s3Configuration.getS3UploaderAdditionalFiles()) {
        if (additionalFile.getS3UploaderKeyPattern().isPresent() && !additionalFile.getS3UploaderKeyPattern().get().equals(s3Configuration.getS3KeyFormat())) {
            prefixes.addAll(SingularityS3FormatHelper.getS3KeyPrefixes(additionalFile.getS3UploaderKeyPattern().get(), taskId, tag, start, end, group));
        }
    }
    LOG.trace("Task {} got S3 prefixes {} for start {}, end {}, tag {}", taskId, prefixes, start, end, tag);
    return prefixes;
}

Example 4 with SingularityS3UploaderFile

use of com.hubspot.singularity.SingularityS3UploaderFile in project Singularity by HubSpot.

the class S3LogResource method getBuckets.

private Set<String> getBuckets(String group) {
    Set<String> s3Buckets = new HashSet<>();
    s3Buckets.add(configuration.get().getGroupOverrides().containsKey(group) ? configuration.get().getGroupOverrides().get(group).getS3Bucket() : configuration.get().getS3Bucket());
    s3Buckets.add(configuration.get().getGroupS3SearchConfigs().containsKey(group) ? configuration.get().getGroupS3SearchConfigs().get(group).getS3Bucket() : configuration.get().getS3Bucket());
    for (SingularityS3UploaderFile uploaderFile : configuration.get().getS3UploaderAdditionalFiles()) {
        if (uploaderFile.getS3UploaderBucket().isPresent() && !s3Buckets.contains(uploaderFile.getS3UploaderBucket().get())) {
            s3Buckets.add(uploaderFile.getS3UploaderBucket().get());
        }
    }
    return s3Buckets;
}

Example 5 with SingularityS3UploaderFile

use of com.hubspot.singularity.SingularityS3UploaderFile in project Singularity by HubSpot.

the class S3LogResource method getS3PrefixesForRequest.

private Collection<String> getS3PrefixesForRequest(S3Configuration s3Configuration, String requestId, Optional<Long> startArg, Optional<Long> endArg, String group) {
    Optional<SingularityRequestHistory> firstHistory = requestHistoryHelper.getFirstHistory(requestId);
    checkNotFound(firstHistory.isPresent(), "No request history found for %s", requestId);
    long start = firstHistory.get().getCreatedAt();
    if (startArg.isPresent()) {
        start = Math.max(startArg.get(), start);
    }
    Optional<SingularityRequestHistory> lastHistory = requestHistoryHelper.getLastHistory(requestId);
    long end = System.currentTimeMillis();
    if (lastHistory.isPresent() && (lastHistory.get().getEventType() == RequestHistoryType.DELETED || lastHistory.get().getEventType() == RequestHistoryType.PAUSED)) {
        end = lastHistory.get().getCreatedAt() + TimeUnit.DAYS.toMillis(1);
    }
    if (endArg.isPresent()) {
        end = Math.min(endArg.get(), end);
    }
    Collection<String> prefixes = SingularityS3FormatHelper.getS3KeyPrefixes(s3Configuration.getS3KeyFormat(), requestId, start, end, group);
    for (SingularityS3UploaderFile additionalFile : s3Configuration.getS3UploaderAdditionalFiles()) {
        if (additionalFile.getS3UploaderKeyPattern().isPresent() && !additionalFile.getS3UploaderKeyPattern().get().equals(s3Configuration.getS3KeyFormat())) {
            prefixes.addAll(SingularityS3FormatHelper.getS3KeyPrefixes(additionalFile.getS3UploaderKeyPattern().get(), requestId, start, end, group));
        }
    }
    LOG.trace("Request {} got S3 prefixes {} for start {}, end {}", requestId, prefixes, start, end);
    return prefixes;
}