Examples with AmazonS3Client com.talend.shaded.com.amazonaws.services.s3.AmazonS3Client used on opensource projects

Example 81 with AmazonS3Client

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3Client in project Singularity by HubSpot.

the class S3ArtifactDownloader method downloadThrows.

private void downloadThrows(final S3Artifact s3Artifact, final Path downloadTo) throws Exception {
    log.info("Downloading {}", s3Artifact);
    ClientConfiguration clientConfiguration = new ClientConfiguration().withSocketTimeout(configuration.getS3ChunkDownloadTimeoutMillis());
    if (configuration.isS3UseV2Signing()) {
        clientConfiguration.setSignerOverride("S3SignerType");
    }
    final AmazonS3 s3Client = new AmazonS3Client(getCredentialsForBucket(s3Artifact.getS3Bucket()), clientConfiguration);
    if (configuration.getS3Endpoint().isPresent()) {
        s3Client.setEndpoint(configuration.getS3Endpoint().get());
    }
    long length = 0;
    if (s3Artifact.getFilesize().isPresent()) {
        length = s3Artifact.getFilesize().get();
    } else {
        S3Object details = s3Client.getObject(s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey());
        Preconditions.checkNotNull(details, "Couldn't find object at %s/%s", s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey());
        length = details.getObjectMetadata().getContentLength();
    }
    int numChunks = (int) (length / configuration.getS3ChunkSize());
    if (length % configuration.getS3ChunkSize() > 0) {
        numChunks++;
    }
    final long chunkSize = length / numChunks + (length % numChunks);
    log.info("Downloading {}/{} in {} chunks of {} bytes to {}", s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey(), numChunks, chunkSize, downloadTo);
    final ExecutorService chunkExecutorService = Executors.newFixedThreadPool(numChunks, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("S3ArtifactDownloaderChunkThread-%d").build());
    final List<Future<Path>> futures = Lists.newArrayListWithCapacity(numChunks);
    for (int chunk = 0; chunk < numChunks; chunk++) {
        futures.add(chunkExecutorService.submit(new S3ArtifactChunkDownloader(configuration, log, s3Client, s3Artifact, downloadTo, chunk, chunkSize, length, exceptionNotifier)));
    }
    long remainingMillis = configuration.getS3DownloadTimeoutMillis();
    boolean failed = false;
    for (int chunk = 0; chunk < numChunks; chunk++) {
        final Future<Path> future = futures.get(chunk);
        if (failed) {
            future.cancel(true);
            continue;
        }
        final long start = System.currentTimeMillis();
        if (!handleChunk(s3Artifact, future, downloadTo, chunk, start, remainingMillis)) {
            failed = true;
        }
        remainingMillis -= (System.currentTimeMillis() - start);
    }
    chunkExecutorService.shutdownNow();
    Preconditions.checkState(!failed, "Downloading %s/%s failed", s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey());
}

Example 82 with AmazonS3Client

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3Client in project Singularity by HubSpot.

the class SingularityMainModule method provideS3Services.

@Provides
@Singleton
public SingularityS3Services provideS3Services(Optional<S3Configuration> config) {
    if (!config.isPresent()) {
        return new SingularityS3Services();
    }
    final ImmutableList.Builder<SingularityS3Service> s3ServiceBuilder = ImmutableList.builder();
    for (Map.Entry<String, S3GroupConfiguration> entry : config.get().getGroupOverrides().entrySet()) {
        s3ServiceBuilder.add(new SingularityS3Service(entry.getKey(), entry.getValue().getS3Bucket(), new AmazonS3Client(new BasicAWSCredentials(entry.getValue().getS3AccessKey(), entry.getValue().getS3SecretKey()))));
    }
    for (Map.Entry<String, S3GroupConfiguration> entry : config.get().getGroupS3SearchConfigs().entrySet()) {
        s3ServiceBuilder.add(new SingularityS3Service(entry.getKey(), entry.getValue().getS3Bucket(), new AmazonS3Client(new BasicAWSCredentials(entry.getValue().getS3AccessKey(), entry.getValue().getS3SecretKey()))));
    }
    SingularityS3Service defaultService = new SingularityS3Service(SingularityS3FormatHelper.DEFAULT_GROUP_NAME, config.get().getS3Bucket(), new AmazonS3Client(new BasicAWSCredentials(config.get().getS3AccessKey(), config.get().getS3SecretKey())));
    return new SingularityS3Services(s3ServiceBuilder.build(), defaultService);
}

Example 83 with AmazonS3Client

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3Client in project carina by qaprosoft.

the class AmazonS3Manager method getInstance.

public static AmazonS3Manager getInstance() {
    if (instance == null) {
        synchronized (AmazonS3Manager.class) {
            if (instance == null) {
                instance = new AmazonS3Manager();
                String accessKey = Configuration.get(Parameter.ACCESS_KEY_ID);
                String secretKey = Configuration.get(Parameter.SECRET_KEY);
                System.setProperty("aws.accessKeyId", accessKey);
                System.setProperty("aws.secretKey", secretKey);
                s3client = new AmazonS3Client(new SystemPropertiesCredentialsProvider());
            }
        }
    }
    return instance;
}

Example 84 with AmazonS3Client

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3Client in project herd by FINRAOS.

the class S3DaoImpl method abortMultipartUploads.

@Override
public int abortMultipartUploads(S3FileTransferRequestParamsDto params, Date thresholdDate) {
    // Create an Amazon S3 client.
    AmazonS3Client s3Client = getAmazonS3(params);
    int abortedMultipartUploadsCount = 0;
    try {
        // List upload markers. Null implies initial list request.
        String uploadIdMarker = null;
        String keyMarker = null;
        boolean truncated;
        do {
            // Create the list multipart request, optionally using the last markers.
            ListMultipartUploadsRequest request = new ListMultipartUploadsRequest(params.getS3BucketName());
            request.setUploadIdMarker(uploadIdMarker);
            request.setKeyMarker(keyMarker);
            // Request the multipart upload listing.
            MultipartUploadListing uploadListing = s3Operations.listMultipartUploads(TransferManager.appendSingleObjectUserAgent(request), s3Client);
            for (MultipartUpload upload : uploadListing.getMultipartUploads()) {
                if (upload.getInitiated().compareTo(thresholdDate) < 0) {
                    // Abort the upload.
                    s3Operations.abortMultipartUpload(TransferManager.appendSingleObjectUserAgent(new AbortMultipartUploadRequest(params.getS3BucketName(), upload.getKey(), upload.getUploadId())), s3Client);
                    // Log the information about the aborted multipart upload.
                    LOGGER.info("Aborted S3 multipart upload. s3Key=\"{}\" s3BucketName=\"{}\" s3MultipartUploadInitiatedDate=\"{}\"", upload.getKey(), params.getS3BucketName(), upload.getInitiated());
                    // Increment the counter.
                    abortedMultipartUploadsCount++;
                }
            }
            // Determine whether there are more uploads to list.
            truncated = uploadListing.isTruncated();
            if (truncated) {
                // Record the list markers.
                uploadIdMarker = uploadListing.getNextUploadIdMarker();
                keyMarker = uploadListing.getNextKeyMarker();
            }
        } while (truncated);
    } finally {
        // Shutdown the Amazon S3 client instance to release resources.
        s3Client.shutdown();
    }
    return abortedMultipartUploadsCount;
}

Example 85 with AmazonS3Client

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3Client in project herd by FINRAOS.

the class S3DaoImpl method deleteFileList.

@Override
public void deleteFileList(final S3FileTransferRequestParamsDto params) {
    LOGGER.info("Deleting a list of objects from S3... s3BucketName=\"{}\" s3KeyCount={}", params.getS3BucketName(), params.getFiles().size());
    try {
        // In order to avoid a MalformedXML AWS exception, we send delete request only when we have any keys to delete.
        if (!params.getFiles().isEmpty()) {
            // Create an S3 client.
            AmazonS3Client s3Client = getAmazonS3(params);
            try {
                // Build a list of keys to be deleted.
                List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<>();
                for (File file : params.getFiles()) {
                    keys.add(new DeleteObjectsRequest.KeyVersion(file.getPath().replaceAll("\\\\", "/")));
                }
                // Delete the keys.
                deleteKeyVersions(s3Client, params.getS3BucketName(), keys);
            } finally {
                s3Client.shutdown();
            }
        }
    } catch (Exception e) {
        throw new IllegalStateException(String.format("Failed to delete a list of keys from bucket \"%s\". Reason: %s", params.getS3BucketName(), e.getMessage()), e);
    }
}