Examples with AmazonS3Client com.amazonaws.services.s3.AmazonS3Client used on opensource projects

Example 71 with AmazonS3Client

use of com.amazonaws.services.s3.AmazonS3Client in project apex-malhar by apache.

the class S3RecordReaderModuleAppTest method setup.

@Before
public void setup() throws Exception {
    client = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretKey));
    client.createBucket(testMeta.bucketKey);
    inputDir = testMeta.baseDirectory + File.separator + "input";
    File file1 = new File(inputDir + File.separator + FILE_1);
    File file2 = new File(inputDir + File.separator + FILE_2);
    FileUtils.writeStringToFile(file1, FILE_1_DATA);
    FileUtils.writeStringToFile(file2, FILE_2_DATA);
    client.putObject(new PutObjectRequest(testMeta.bucketKey, "input/" + FILE_1, file1));
    client.putObject(new PutObjectRequest(testMeta.bucketKey, "input/" + FILE_2, file2));
    files = SCHEME + "://" + accessKey + ":" + secretKey + "@" + testMeta.bucketKey + "/input";
}

Example 72 with AmazonS3Client

use of com.amazonaws.services.s3.AmazonS3Client in project apex-malhar by apache.

the class S3BlockReader method setup.

@Override
public void setup(Context.OperatorContext context) {
    super.setup(context);
    s3Client = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretAccessKey));
    ((S3BlockReaderContext) readerContext).setBucketName(bucketName);
    ((S3BlockReaderContext) readerContext).setS3Client(s3Client);
}

Example 73 with AmazonS3Client

use of com.amazonaws.services.s3.AmazonS3Client in project jackrabbit-oak by apache.

the class Utils method openService.

/**
 * Create AmazonS3Client from properties.
 *
 * @param prop properties to configure @link {@link AmazonS3Client}
 * @return {@link AmazonS3Client}
 */
public static AmazonS3Client openService(final Properties prop) {
    String accessKey = prop.getProperty(S3Constants.ACCESS_KEY);
    String secretKey = prop.getProperty(S3Constants.SECRET_KEY);
    AmazonS3Client s3service = null;
    if (StringUtils.isNullOrEmpty(accessKey) || StringUtils.isNullOrEmpty(secretKey)) {
        LOG.info("Configuring Amazon Client from environment");
        s3service = new AmazonS3Client(getClientConfiguration(prop));
    } else {
        LOG.info("Configuring Amazon Client from property file.");
        AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
        s3service = new AmazonS3Client(credentials, getClientConfiguration(prop));
    }
    String region = prop.getProperty(S3Constants.S3_REGION);
    String endpoint = null;
    String propEndPoint = prop.getProperty(S3Constants.S3_END_POINT);
    if ((propEndPoint != null) & !"".equals(propEndPoint)) {
        endpoint = propEndPoint;
    } else {
        if (StringUtils.isNullOrEmpty(region)) {
            com.amazonaws.regions.Region s3Region = Regions.getCurrentRegion();
            if (s3Region != null) {
                region = s3Region.getName();
            } else {
                throw new AmazonClientException("parameter [" + S3Constants.S3_REGION + "] not configured and cannot be derived from environment");
            }
        }
        if (DEFAULT_AWS_BUCKET_REGION.equals(region)) {
            endpoint = S3 + DOT + AWSDOTCOM;
        } else if (Region.EU_Ireland.toString().equals(region)) {
            endpoint = "s3-eu-west-1" + DOT + AWSDOTCOM;
        } else {
            endpoint = S3 + DASH + region + DOT + AWSDOTCOM;
        }
    }
    /*
         * setting endpoint to remove latency of redirection. If endpoint is
         * not set, invocation first goes us standard region, which
         * redirects it to correct location.
         */
    s3service.setEndpoint(endpoint);
    LOG.info("S3 service endpoint [{}] ", endpoint);
    return s3service;
}

Example 74 with AmazonS3Client

use of com.amazonaws.services.s3.AmazonS3Client in project jackrabbit-oak by apache.

the class DataStoreUtils method deleteBucket.

public static void deleteBucket(String bucket, Map<String, ?> map, Date date) throws Exception {
    log.info("cleaning bucket [" + bucket + "]");
    Properties props = new Properties();
    props.putAll(map);
    AmazonS3Client s3service = Utils.openService(props);
    TransferManager tmx = new TransferManager(s3service);
    if (s3service.doesBucketExist(bucket)) {
        for (int i = 0; i < 4; i++) {
            tmx.abortMultipartUploads(bucket, date);
            ObjectListing prevObjectListing = s3service.listObjects(bucket);
            while (prevObjectListing != null) {
                List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
                for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
                    deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
                }
                if (deleteList.size() > 0) {
                    DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
                    delObjsReq.setKeys(deleteList);
                    s3service.deleteObjects(delObjsReq);
                }
                if (!prevObjectListing.isTruncated())
                    break;
                prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
            }
        }
        s3service.deleteBucket(bucket);
        log.info("bucket [ " + bucket + "] cleaned");
    } else {
        log.info("bucket [" + bucket + "] doesn't exists");
    }
    tmx.shutdownNow();
    s3service.shutdown();
}

Example 75 with AmazonS3Client

use of com.amazonaws.services.s3.AmazonS3Client in project Singularity by HubSpot.

the class S3ArtifactDownloader method downloadThrows.

private void downloadThrows(final S3Artifact s3Artifact, final Path downloadTo) throws Exception {
    log.info("Downloading {}", s3Artifact);
    ClientConfiguration clientConfiguration = new ClientConfiguration().withSocketTimeout(configuration.getS3ChunkDownloadTimeoutMillis());
    if (configuration.isS3UseV2Signing()) {
        clientConfiguration.setSignerOverride("S3SignerType");
    }
    final AmazonS3 s3Client = new AmazonS3Client(getCredentialsForBucket(s3Artifact.getS3Bucket()), clientConfiguration);
    if (configuration.getS3Endpoint().isPresent()) {
        s3Client.setEndpoint(configuration.getS3Endpoint().get());
    }
    long length = 0;
    if (s3Artifact.getFilesize().isPresent()) {
        length = s3Artifact.getFilesize().get();
    } else {
        S3Object details = s3Client.getObject(s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey());
        Preconditions.checkNotNull(details, "Couldn't find object at %s/%s", s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey());
        length = details.getObjectMetadata().getContentLength();
    }
    int numChunks = (int) (length / configuration.getS3ChunkSize());
    if (length % configuration.getS3ChunkSize() > 0) {
        numChunks++;
    }
    final long chunkSize = length / numChunks + (length % numChunks);
    log.info("Downloading {}/{} in {} chunks of {} bytes to {}", s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey(), numChunks, chunkSize, downloadTo);
    final ExecutorService chunkExecutorService = Executors.newFixedThreadPool(numChunks, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("S3ArtifactDownloaderChunkThread-%d").build());
    final List<Future<Path>> futures = Lists.newArrayListWithCapacity(numChunks);
    for (int chunk = 0; chunk < numChunks; chunk++) {
        futures.add(chunkExecutorService.submit(new S3ArtifactChunkDownloader(configuration, log, s3Client, s3Artifact, downloadTo, chunk, chunkSize, length, exceptionNotifier)));
    }
    long remainingMillis = configuration.getS3DownloadTimeoutMillis();
    boolean failed = false;
    for (int chunk = 0; chunk < numChunks; chunk++) {
        final Future<Path> future = futures.get(chunk);
        if (failed) {
            future.cancel(true);
            continue;
        }
        final long start = System.currentTimeMillis();
        if (!handleChunk(s3Artifact, future, downloadTo, chunk, start, remainingMillis)) {
            failed = true;
        }
        remainingMillis -= (System.currentTimeMillis() - start);
    }
    chunkExecutorService.shutdownNow();
    Preconditions.checkState(!failed, "Downloading %s/%s failed", s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey());
}