use of com.amazonaws.services.s3.model.GeneratePresignedUrlRequest in project Singularity by HubSpot.
the class S3LogResource method getS3LogsWithExecutorService.
// Fetching logs
private List<SingularityS3LogMetadata> getS3LogsWithExecutorService(S3Configuration s3Configuration, ListeningExecutorService executorService, Map<SingularityS3Service, Set<String>> servicesToPrefixes, int totalPrefixCount, final SingularityS3SearchRequest search, final ConcurrentHashMap<String, ContinuationToken> continuationTokens, final boolean paginated) throws InterruptedException, ExecutionException, TimeoutException {
List<ListenableFuture<List<S3ObjectSummaryHolder>>> futures = Lists.newArrayListWithCapacity(totalPrefixCount);
final AtomicInteger resultCount = new AtomicInteger();
for (final Map.Entry<SingularityS3Service, Set<String>> entry : servicesToPrefixes.entrySet()) {
final String s3Bucket = entry.getKey().getBucket();
final String group = entry.getKey().getGroup();
final AmazonS3 s3Client = entry.getKey().getS3Client();
for (final String s3Prefix : entry.getValue()) {
final String key = String.format(CONTINUATION_TOKEN_KEY_FORMAT, group, s3Bucket, s3Prefix);
if (search.getContinuationTokens().containsKey(key) && search.getContinuationTokens().get(key).isLastPage()) {
LOG.trace("No further content for prefix {} in bucket {}, skipping", s3Prefix, s3Bucket);
continuationTokens.putIfAbsent(key, search.getContinuationTokens().get(key));
continue;
}
futures.add(executorService.submit(new Callable<List<S3ObjectSummaryHolder>>() {
@Override
public List<S3ObjectSummaryHolder> call() throws Exception {
ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(s3Bucket).withPrefix(s3Prefix);
if (paginated) {
Optional<ContinuationToken> token = Optional.absent();
if (search.getContinuationTokens().containsKey(key) && !Strings.isNullOrEmpty(search.getContinuationTokens().get(key).getValue())) {
request.setContinuationToken(search.getContinuationTokens().get(key).getValue());
token = Optional.of(search.getContinuationTokens().get(key));
}
int targetResultCount = search.getMaxPerPage().or(DEFAULT_TARGET_MAX_RESULTS);
request.setMaxKeys(targetResultCount);
if (resultCount.get() < targetResultCount) {
ListObjectsV2Result result = s3Client.listObjectsV2(request);
if (result.getObjectSummaries().isEmpty()) {
continuationTokens.putIfAbsent(key, new ContinuationToken(result.getNextContinuationToken(), true));
return Collections.emptyList();
} else {
boolean addToList = incrementIfLessThan(resultCount, result.getObjectSummaries().size(), targetResultCount);
if (addToList) {
continuationTokens.putIfAbsent(key, new ContinuationToken(result.getNextContinuationToken(), !result.isTruncated()));
List<S3ObjectSummaryHolder> objectSummaryHolders = new ArrayList<>();
for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
objectSummaryHolders.add(new S3ObjectSummaryHolder(group, objectSummary));
}
return objectSummaryHolders;
} else {
continuationTokens.putIfAbsent(key, token.or(new ContinuationToken(null, false)));
return Collections.emptyList();
}
}
} else {
continuationTokens.putIfAbsent(key, token.or(new ContinuationToken(null, false)));
return Collections.emptyList();
}
} else {
ListObjectsV2Result result = s3Client.listObjectsV2(request);
List<S3ObjectSummaryHolder> objectSummaryHolders = new ArrayList<>();
for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
objectSummaryHolders.add(new S3ObjectSummaryHolder(group, objectSummary));
}
return objectSummaryHolders;
}
}
}));
}
}
final long start = System.currentTimeMillis();
List<List<S3ObjectSummaryHolder>> results = Futures.allAsList(futures).get(s3Configuration.getWaitForS3ListSeconds(), TimeUnit.SECONDS);
List<S3ObjectSummaryHolder> objects = Lists.newArrayListWithExpectedSize(results.size() * 2);
for (List<S3ObjectSummaryHolder> s3ObjectSummaryHolders : results) {
for (final S3ObjectSummaryHolder s3ObjectHolder : s3ObjectSummaryHolders) {
objects.add(s3ObjectHolder);
}
}
LOG.trace("Got {} objects from S3 after {}", objects.size(), JavaUtils.duration(start));
List<ListenableFuture<SingularityS3LogMetadata>> logFutures = Lists.newArrayListWithCapacity(objects.size());
final Date expireAt = new Date(System.currentTimeMillis() + s3Configuration.getExpireS3LinksAfterMillis());
for (final S3ObjectSummaryHolder s3ObjectHolder : objects) {
final S3ObjectSummary s3Object = s3ObjectHolder.getObjectSummary();
final AmazonS3 s3Client = s3Services.getServiceByGroupAndBucketOrDefault(s3ObjectHolder.getGroup(), s3Object.getBucketName()).getS3Client();
logFutures.add(executorService.submit(new Callable<SingularityS3LogMetadata>() {
@Override
public SingularityS3LogMetadata call() throws Exception {
Optional<Long> maybeStartTime = Optional.absent();
Optional<Long> maybeEndTime = Optional.absent();
if (!search.isExcludeMetadata()) {
GetObjectMetadataRequest metadataRequest = new GetObjectMetadataRequest(s3Object.getBucketName(), s3Object.getKey());
Map<String, String> objectMetadata = s3Client.getObjectMetadata(metadataRequest).getUserMetadata();
maybeStartTime = getMetadataAsLong(objectMetadata, SingularityS3Log.LOG_START_S3_ATTR);
maybeEndTime = getMetadataAsLong(objectMetadata, SingularityS3Log.LOG_END_S3_ATTR);
}
if (search.isListOnly()) {
return new SingularityS3LogMetadata(s3Object.getKey(), s3Object.getLastModified().getTime(), s3Object.getSize(), maybeStartTime, maybeEndTime);
} else {
GeneratePresignedUrlRequest getUrlRequest = new GeneratePresignedUrlRequest(s3Object.getBucketName(), s3Object.getKey()).withMethod(HttpMethod.GET).withExpiration(expireAt);
String getUrl = s3Client.generatePresignedUrl(getUrlRequest).toString();
ResponseHeaderOverrides downloadHeaders = new ResponseHeaderOverrides();
downloadHeaders.setContentDisposition(CONTENT_DISPOSITION_DOWNLOAD_HEADER);
downloadHeaders.setContentEncoding(CONTENT_ENCODING_DOWNLOAD_HEADER);
GeneratePresignedUrlRequest downloadUrlRequest = new GeneratePresignedUrlRequest(s3Object.getBucketName(), s3Object.getKey()).withMethod(HttpMethod.GET).withExpiration(expireAt).withResponseHeaders(downloadHeaders);
String downloadUrl = s3Client.generatePresignedUrl(downloadUrlRequest).toString();
return new SingularityS3Log(getUrl, s3Object.getKey(), s3Object.getLastModified().getTime(), s3Object.getSize(), downloadUrl, maybeStartTime, maybeEndTime);
}
}
}));
}
return Futures.allAsList(logFutures).get(s3Configuration.getWaitForS3LinksSeconds(), TimeUnit.SECONDS);
}
use of com.amazonaws.services.s3.model.GeneratePresignedUrlRequest in project herd by FINRAOS.
the class S3DaoImpl method generateGetObjectPresignedUrl.
@Override
public String generateGetObjectPresignedUrl(String bucketName, String key, Date expiration, S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto) {
GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(bucketName, key, HttpMethod.GET);
generatePresignedUrlRequest.setExpiration(expiration);
AmazonS3Client s3 = getAmazonS3(s3FileTransferRequestParamsDto);
try {
return s3Operations.generatePresignedUrl(generatePresignedUrlRequest, s3).toString();
} finally {
s3.shutdown();
}
}
use of com.amazonaws.services.s3.model.GeneratePresignedUrlRequest in project aws-doc-sdk-examples by awsdocs.
the class GeneratePresignedPutUrl method main.
public static void main(String[] args) {
final String USAGE = "\n" + "To run this example, supply the name of an S3 bucket and a file to\n" + "upload to it.\n" + "\n" + "Ex: GeneratePresignedPutUrl <bucketname> <filename>\n";
if (args.length < 2) {
System.out.println(USAGE);
System.exit(1);
}
String bucket_name = args[0];
String key_name = args[1];
System.out.format("Creating a pre-signed URL for uploading %s to S3 bucket %s...\n", key_name, bucket_name);
final AmazonS3 s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
// Set the pre-signed URL to expire after 12 hours.
java.util.Date expiration = new java.util.Date();
long expirationInMs = expiration.getTime();
expirationInMs += 1000 * 60 * 60 * 12;
expiration.setTime(expirationInMs);
try {
GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(bucket_name, key_name).withMethod(HttpMethod.PUT).withExpiration(expiration);
URL url = s3.generatePresignedUrl(generatePresignedUrlRequest);
// print URL
System.out.println("\n\rGenerated URL: " + url.toString());
// Print curl command to consume URL
System.out.println("\n\rExample command to use URL for file upload: \n\r");
System.out.println("curl --request PUT --upload-file /path/to/" + key_name + " '" + url.toString() + "' -# > /dev/null");
} catch (AmazonServiceException e) {
System.err.println(e.getErrorMessage());
System.exit(1);
}
}
use of com.amazonaws.services.s3.model.GeneratePresignedUrlRequest in project hmftools by hartwigmedical.
the class S3UrlGenerator method generateUrl.
@NotNull
URL generateUrl(@NotNull final String bucketName, @NotNull final String objectKey, final int expirationHours) throws IOException {
try {
LOGGER.info("Generating pre-signed URL for bucket: {}\tobject: {}\texpirationTime: {} hours", bucketName, objectKey, expirationHours);
final long millisNow = System.currentTimeMillis();
final long expirationMillis = millisNow + 1000 * 60 * 60 * expirationHours;
final Date expiration = new java.util.Date(expirationMillis);
final GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(bucketName, objectKey);
generatePresignedUrlRequest.setMethod(HttpMethod.GET);
generatePresignedUrlRequest.setExpiration(expiration);
return s3Client().generatePresignedUrl(generatePresignedUrlRequest);
} catch (AmazonServiceException exception) {
LOGGER.error("Error Message: {}", exception.getMessage());
LOGGER.error("HTTP Code: {}", exception.getStatusCode());
LOGGER.error("Error Code: {}", exception.getErrorCode());
LOGGER.error("Error Type: {}", exception.getErrorType());
LOGGER.error("Request ID: {}", exception.getRequestId());
} catch (AmazonClientException ace) {
LOGGER.error("The client encountered an internal error while trying to communicate with S3");
LOGGER.error("Error Message: {}", ace.getMessage());
}
throw new IOException("Failed to create sbp URL.");
}
use of com.amazonaws.services.s3.model.GeneratePresignedUrlRequest in project dataverse by IQSS.
the class S3AccessIO method generateTemporaryS3Url.
public String generateTemporaryS3Url() throws IOException {
// A. 1 hour by default seems like an OK number. Making it configurable seems like a good idea too. -- L.A.
if (s3 == null) {
throw new IOException("ERROR: s3 not initialised. ");
}
if (dvObject instanceof DataFile) {
key = getMainFileKey();
java.util.Date expiration = new java.util.Date();
long msec = expiration.getTime();
msec += 1000 * getUrlExpirationMinutes();
expiration.setTime(msec);
GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(bucketName, key);
// Default.
generatePresignedUrlRequest.setMethod(HttpMethod.GET);
generatePresignedUrlRequest.setExpiration(expiration);
ResponseHeaderOverrides responseHeaders = new ResponseHeaderOverrides();
// responseHeaders.setContentDisposition("attachment; filename="+this.getDataFile().getDisplayName());
// Encode the file name explicitly specifying the encoding as UTF-8:
// (otherwise S3 may not like non-ASCII characters!)
// Most browsers are happy with just "filename="+URLEncoder.encode(this.getDataFile().getDisplayName(), "UTF-8")
// in the header. But Firefox appears to require that "UTF8" is
// specified explicitly, as below:
responseHeaders.setContentDisposition("attachment; filename*=UTF-8''" + URLEncoder.encode(this.getDataFile().getDisplayName(), "UTF-8"));
// - without it, download will work, but Firefox will leave the special
// characters in the file name encoded. For example, the file name
// will look like "1976%E2%80%932016.txt" instead of "1976–2016.txt",
// where the dash is the "long dash", represented by a 3-byte UTF8
// character "\xE2\x80\x93"
responseHeaders.setContentType(this.getDataFile().getContentType());
generatePresignedUrlRequest.setResponseHeaders(responseHeaders);
URL s = s3.generatePresignedUrl(generatePresignedUrlRequest);
return s.toString();
} else if (dvObject instanceof Dataset) {
throw new IOException("Data Access: GenerateTemporaryS3Url: Invalid DvObject type : Dataset");
} else if (dvObject instanceof Dataverse) {
throw new IOException("Data Access: Invalid DvObject type : Dataverse");
} else {
throw new IOException("Data Access: Invalid DvObject type");
}
}
Aggregations