Examples with SingularityS3Service com.hubspot.singularity.helpers.SingularityS3Service used on opensource projects

Example 1 with SingularityS3Service

use of com.hubspot.singularity.helpers.SingularityS3Service in project Singularity by HubSpot.

the class S3LogResource method getS3LogsWithExecutorService.

// Fetching logs
private List<SingularityS3LogMetadata> getS3LogsWithExecutorService(S3Configuration s3Configuration, ListeningExecutorService executorService, Map<SingularityS3Service, Set<String>> servicesToPrefixes, int totalPrefixCount, final SingularityS3SearchRequest search, final ConcurrentHashMap<String, ContinuationToken> continuationTokens, final boolean paginated) throws InterruptedException, ExecutionException, TimeoutException {
    List<ListenableFuture<List<S3ObjectSummaryHolder>>> futures = Lists.newArrayListWithCapacity(totalPrefixCount);
    final AtomicInteger resultCount = new AtomicInteger();
    for (final Map.Entry<SingularityS3Service, Set<String>> entry : servicesToPrefixes.entrySet()) {
        final String s3Bucket = entry.getKey().getBucket();
        final String group = entry.getKey().getGroup();
        final AmazonS3 s3Client = entry.getKey().getS3Client();
        for (final String s3Prefix : entry.getValue()) {
            final String key = String.format(CONTINUATION_TOKEN_KEY_FORMAT, group, s3Bucket, s3Prefix);
            if (search.getContinuationTokens().containsKey(key) && search.getContinuationTokens().get(key).isLastPage()) {
                LOG.trace("No further content for prefix {} in bucket {}, skipping", s3Prefix, s3Bucket);
                continuationTokens.putIfAbsent(key, search.getContinuationTokens().get(key));
                continue;
            }
            futures.add(executorService.submit(new Callable<List<S3ObjectSummaryHolder>>() {

                @Override
                public List<S3ObjectSummaryHolder> call() throws Exception {
                    ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(s3Bucket).withPrefix(s3Prefix);
                    if (paginated) {
                        Optional<ContinuationToken> token = Optional.absent();
                        if (search.getContinuationTokens().containsKey(key) && !Strings.isNullOrEmpty(search.getContinuationTokens().get(key).getValue())) {
                            request.setContinuationToken(search.getContinuationTokens().get(key).getValue());
                            token = Optional.of(search.getContinuationTokens().get(key));
                        }
                        int targetResultCount = search.getMaxPerPage().or(DEFAULT_TARGET_MAX_RESULTS);
                        request.setMaxKeys(targetResultCount);
                        if (resultCount.get() < targetResultCount) {
                            ListObjectsV2Result result = s3Client.listObjectsV2(request);
                            if (result.getObjectSummaries().isEmpty()) {
                                continuationTokens.putIfAbsent(key, new ContinuationToken(result.getNextContinuationToken(), true));
                                return Collections.emptyList();
                            } else {
                                boolean addToList = incrementIfLessThan(resultCount, result.getObjectSummaries().size(), targetResultCount);
                                if (addToList) {
                                    continuationTokens.putIfAbsent(key, new ContinuationToken(result.getNextContinuationToken(), !result.isTruncated()));
                                    List<S3ObjectSummaryHolder> objectSummaryHolders = new ArrayList<>();
                                    for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
                                        objectSummaryHolders.add(new S3ObjectSummaryHolder(group, objectSummary));
                                    }
                                    return objectSummaryHolders;
                                } else {
                                    continuationTokens.putIfAbsent(key, token.or(new ContinuationToken(null, false)));
                                    return Collections.emptyList();
                                }
                            }
                        } else {
                            continuationTokens.putIfAbsent(key, token.or(new ContinuationToken(null, false)));
                            return Collections.emptyList();
                        }
                    } else {
                        ListObjectsV2Result result = s3Client.listObjectsV2(request);
                        List<S3ObjectSummaryHolder> objectSummaryHolders = new ArrayList<>();
                        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
                            objectSummaryHolders.add(new S3ObjectSummaryHolder(group, objectSummary));
                        }
                        return objectSummaryHolders;
                    }
                }
            }));
        }
    }
    final long start = System.currentTimeMillis();
    List<List<S3ObjectSummaryHolder>> results = Futures.allAsList(futures).get(s3Configuration.getWaitForS3ListSeconds(), TimeUnit.SECONDS);
    List<S3ObjectSummaryHolder> objects = Lists.newArrayListWithExpectedSize(results.size() * 2);
    for (List<S3ObjectSummaryHolder> s3ObjectSummaryHolders : results) {
        for (final S3ObjectSummaryHolder s3ObjectHolder : s3ObjectSummaryHolders) {
            objects.add(s3ObjectHolder);
        }
    }
    LOG.trace("Got {} objects from S3 after {}", objects.size(), JavaUtils.duration(start));
    List<ListenableFuture<SingularityS3LogMetadata>> logFutures = Lists.newArrayListWithCapacity(objects.size());
    final Date expireAt = new Date(System.currentTimeMillis() + s3Configuration.getExpireS3LinksAfterMillis());
    for (final S3ObjectSummaryHolder s3ObjectHolder : objects) {
        final S3ObjectSummary s3Object = s3ObjectHolder.getObjectSummary();
        final AmazonS3 s3Client = s3Services.getServiceByGroupAndBucketOrDefault(s3ObjectHolder.getGroup(), s3Object.getBucketName()).getS3Client();
        logFutures.add(executorService.submit(new Callable<SingularityS3LogMetadata>() {

            @Override
            public SingularityS3LogMetadata call() throws Exception {
                Optional<Long> maybeStartTime = Optional.absent();
                Optional<Long> maybeEndTime = Optional.absent();
                if (!search.isExcludeMetadata()) {
                    GetObjectMetadataRequest metadataRequest = new GetObjectMetadataRequest(s3Object.getBucketName(), s3Object.getKey());
                    Map<String, String> objectMetadata = s3Client.getObjectMetadata(metadataRequest).getUserMetadata();
                    maybeStartTime = getMetadataAsLong(objectMetadata, SingularityS3Log.LOG_START_S3_ATTR);
                    maybeEndTime = getMetadataAsLong(objectMetadata, SingularityS3Log.LOG_END_S3_ATTR);
                }
                if (search.isListOnly()) {
                    return new SingularityS3LogMetadata(s3Object.getKey(), s3Object.getLastModified().getTime(), s3Object.getSize(), maybeStartTime, maybeEndTime);
                } else {
                    GeneratePresignedUrlRequest getUrlRequest = new GeneratePresignedUrlRequest(s3Object.getBucketName(), s3Object.getKey()).withMethod(HttpMethod.GET).withExpiration(expireAt);
                    String getUrl = s3Client.generatePresignedUrl(getUrlRequest).toString();
                    ResponseHeaderOverrides downloadHeaders = new ResponseHeaderOverrides();
                    downloadHeaders.setContentDisposition(CONTENT_DISPOSITION_DOWNLOAD_HEADER);
                    downloadHeaders.setContentEncoding(CONTENT_ENCODING_DOWNLOAD_HEADER);
                    GeneratePresignedUrlRequest downloadUrlRequest = new GeneratePresignedUrlRequest(s3Object.getBucketName(), s3Object.getKey()).withMethod(HttpMethod.GET).withExpiration(expireAt).withResponseHeaders(downloadHeaders);
                    String downloadUrl = s3Client.generatePresignedUrl(downloadUrlRequest).toString();
                    return new SingularityS3Log(getUrl, s3Object.getKey(), s3Object.getLastModified().getTime(), s3Object.getSize(), downloadUrl, maybeStartTime, maybeEndTime);
                }
            }
        }));
    }
    return Futures.allAsList(logFutures).get(s3Configuration.getWaitForS3LinksSeconds(), TimeUnit.SECONDS);
}

Example 2 with SingularityS3Service

use of com.hubspot.singularity.helpers.SingularityS3Service in project Singularity by HubSpot.

the class S3LogResource method getServiceToPrefixes.

private Map<SingularityS3Service, Set<String>> getServiceToPrefixes(SingularityS3SearchRequest search, SingularityUser user) {
    Map<SingularityS3Service, Set<String>> servicesToPrefixes = new HashMap<>();
    if (!search.getTaskIds().isEmpty()) {
        for (String taskId : search.getTaskIds()) {
            SingularityTaskId taskIdObject = getTaskIdObject(taskId);
            String group = getRequestGroupForTask(taskIdObject, user).or(SingularityS3FormatHelper.DEFAULT_GROUP_NAME);
            Set<String> s3Buckets = getBuckets(group);
            Collection<String> prefixes = getS3PrefixesForTask(configuration.get(), taskIdObject, search.getStart(), search.getEnd(), group, user);
            for (String s3Bucket : s3Buckets) {
                SingularityS3Service s3Service = s3Services.getServiceByGroupAndBucketOrDefault(group, s3Bucket);
                if (!servicesToPrefixes.containsKey(s3Service)) {
                    servicesToPrefixes.put(s3Service, new HashSet<String>());
                }
                servicesToPrefixes.get(s3Service).addAll(prefixes);
            }
        }
    }
    if (!search.getRequestsAndDeploys().isEmpty()) {
        for (Map.Entry<String, List<String>> entry : search.getRequestsAndDeploys().entrySet()) {
            String group = getRequestGroup(entry.getKey(), user).or(SingularityS3FormatHelper.DEFAULT_GROUP_NAME);
            Set<String> s3Buckets = getBuckets(group);
            List<String> prefixes = new ArrayList<>();
            if (!entry.getValue().isEmpty()) {
                for (String deployId : entry.getValue()) {
                    prefixes.addAll(getS3PrefixesForDeploy(configuration.get(), entry.getKey(), deployId, search.getStart(), search.getEnd(), group, user));
                }
            } else {
                prefixes.addAll(getS3PrefixesForRequest(configuration.get(), entry.getKey(), search.getStart(), search.getEnd(), group));
            }
            for (String s3Bucket : s3Buckets) {
                SingularityS3Service s3Service = s3Services.getServiceByGroupAndBucketOrDefault(group, s3Bucket);
                if (!servicesToPrefixes.containsKey(s3Service)) {
                    servicesToPrefixes.put(s3Service, new HashSet<String>());
                }
                servicesToPrefixes.get(s3Service).addAll(prefixes);
            }
        }
    }
    // Trim prefixes to search. Less specific prefixes will contain all results of matching + more specific ones
    for (Map.Entry<SingularityS3Service, Set<String>> entry : servicesToPrefixes.entrySet()) {
        Set<String> results = new HashSet<>();
        boolean contains = false;
        for (String prefix : entry.getValue()) {
            for (String unique : results) {
                if (prefix.startsWith(unique) && prefix.length() > unique.length()) {
                    contains = true;
                    break;
                } else if (unique.startsWith(prefix) && unique.length() > prefix.length()) {
                    results.remove(unique);
                    results.add(prefix);
                    contains = true;
                    break;
                }
            }
            if (!contains) {
                results.add(prefix);
            }
        }
        entry.getValue().retainAll(results);
    }
    return servicesToPrefixes;
}

Example 3 with SingularityS3Service

use of com.hubspot.singularity.helpers.SingularityS3Service in project Singularity by HubSpot.

the class S3LogResource method getS3Logs.

private SingularityS3SearchResult getS3Logs(S3Configuration s3Configuration, Map<SingularityS3Service, Set<String>> servicesToPrefixes, final SingularityS3SearchRequest search, final boolean paginated) throws InterruptedException, ExecutionException, TimeoutException {
    int totalPrefixCount = 0;
    for (Map.Entry<SingularityS3Service, Set<String>> entry : servicesToPrefixes.entrySet()) {
        totalPrefixCount += entry.getValue().size();
    }
    if (totalPrefixCount == 0) {
        return SingularityS3SearchResult.empty();
    }
    ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(Math.min(totalPrefixCount, s3Configuration.getMaxS3Threads()), new ThreadFactoryBuilder().setNameFormat("S3LogFetcher-%d").build()));
    try {
        final ConcurrentHashMap<String, ContinuationToken> continuationTokens = new ConcurrentHashMap<>();
        List<SingularityS3LogMetadata> logs = Lists.newArrayList(getS3LogsWithExecutorService(s3Configuration, executorService, servicesToPrefixes, totalPrefixCount, search, continuationTokens, paginated));
        Collections.sort(logs, LOG_COMPARATOR);
        return new SingularityS3SearchResult(continuationTokens, isFinalPageForAllPrefixes(continuationTokens.values()), logs);
    } finally {
        executorService.shutdownNow();
    }
}

Example 4 with SingularityS3Service

use of com.hubspot.singularity.helpers.SingularityS3Service in project Singularity by HubSpot.

the class SingularityMainModule method provideS3Services.

@Provides
@Singleton
public SingularityS3Services provideS3Services(Optional<S3Configuration> config) {
    if (!config.isPresent()) {
        return new SingularityS3Services();
    }
    final ImmutableList.Builder<SingularityS3Service> s3ServiceBuilder = ImmutableList.builder();
    for (Map.Entry<String, S3GroupConfiguration> entry : config.get().getGroupOverrides().entrySet()) {
        s3ServiceBuilder.add(new SingularityS3Service(entry.getKey(), entry.getValue().getS3Bucket(), new AmazonS3Client(new BasicAWSCredentials(entry.getValue().getS3AccessKey(), entry.getValue().getS3SecretKey()))));
    }
    for (Map.Entry<String, S3GroupConfiguration> entry : config.get().getGroupS3SearchConfigs().entrySet()) {
        s3ServiceBuilder.add(new SingularityS3Service(entry.getKey(), entry.getValue().getS3Bucket(), new AmazonS3Client(new BasicAWSCredentials(entry.getValue().getS3AccessKey(), entry.getValue().getS3SecretKey()))));
    }
    SingularityS3Service defaultService = new SingularityS3Service(SingularityS3FormatHelper.DEFAULT_GROUP_NAME, config.get().getS3Bucket(), new AmazonS3Client(new BasicAWSCredentials(config.get().getS3AccessKey(), config.get().getS3SecretKey())));
    return new SingularityS3Services(s3ServiceBuilder.build(), defaultService);
}