Examples with Bucket com.amazonaws.services.s3.model.Bucket used on opensource projects

Example 91 with Bucket

use of com.amazonaws.services.s3.model.Bucket in project deeplearning4j by deeplearning4j.

the class S3Downloader method paginate.

/**
     * Paginates through a bucket's keys invoking the listener 
     * at each key
     * @param bucket the bucket to iterate
     * @param listener the listener
     */
public void paginate(String bucket, BucketKeyListener listener) {
    AmazonS3 s3 = getClient();
    ObjectListing list = s3.listObjects(bucket);
    for (S3ObjectSummary summary : list.getObjectSummaries()) {
        if (listener != null)
            listener.onKey(s3, bucket, summary.getKey());
    }
    while (list.isTruncated()) {
        list = s3.listNextBatchOfObjects(list);
        for (S3ObjectSummary summary : list.getObjectSummaries()) {
            if (listener != null)
                listener.onKey(s3, bucket, summary.getKey());
        }
    }
}

Example 92 with Bucket

use of com.amazonaws.services.s3.model.Bucket in project deeplearning4j by deeplearning4j.

the class S3Downloader method listObjects.

/**
     * Simple way of retrieving the listings for a bucket
     * @param bucket the bucket to retrieve listings for
     * @return the object listing for this bucket
     */
public ObjectListing listObjects(String bucket) {
    AmazonS3 s3 = getClient();
    ObjectListing list = s3.listObjects(bucket);
    return list;
}

Example 93 with Bucket

use of com.amazonaws.services.s3.model.Bucket in project presto by prestodb.

the class TestPrestoS3FileSystem method testReadRetryCounters.

@SuppressWarnings({ "ResultOfMethodCallIgnored", "OverlyStrongTypeCast", "ConstantConditions" })
@Test
public void testReadRetryCounters() throws Exception {
    try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
        int maxRetries = 2;
        MockAmazonS3 s3 = new MockAmazonS3();
        s3.setGetObjectHttpErrorCode(SC_INTERNAL_SERVER_ERROR);
        Configuration configuration = new Configuration();
        configuration.set(S3_MAX_BACKOFF_TIME, "1ms");
        configuration.set(S3_MAX_RETRY_TIME, "5s");
        configuration.setInt(S3_MAX_CLIENT_RETRIES, maxRetries);
        fs.initialize(new URI("s3n://test-bucket/"), configuration);
        fs.setS3Client(s3);
        try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) {
            inputStream.read();
        } catch (Throwable expected) {
            assertInstanceOf(expected, AmazonS3Exception.class);
            assertEquals(((AmazonS3Exception) expected).getStatusCode(), SC_INTERNAL_SERVER_ERROR);
            assertEquals(PrestoS3FileSystem.getFileSystemStats().getReadRetries().getTotalCount(), maxRetries);
            assertEquals(PrestoS3FileSystem.getFileSystemStats().getGetObjectRetries().getTotalCount(), (maxRetries + 1L) * maxRetries);
        }
    }
}

Example 94 with Bucket

use of com.amazonaws.services.s3.model.Bucket in project heron by twitter.

the class S3Uploader method initialize.

@Override
public void initialize(Config config) {
    bucket = S3Context.bucket(config);
    String accessKey = S3Context.accessKey(config);
    String accessSecret = S3Context.secretKey(config);
    String awsProfile = S3Context.awsProfile(config);
    String proxy = S3Context.proxyUri(config);
    String endpoint = S3Context.uri(config);
    String customRegion = S3Context.region(config);
    AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard();
    if (Strings.isNullOrEmpty(bucket)) {
        throw new RuntimeException("Missing heron.uploader.s3.bucket config value");
    }
    // by not specifying a CredentialsProvider.
    if (!Strings.isNullOrEmpty(accessKey) || !Strings.isNullOrEmpty(accessSecret)) {
        if (!Strings.isNullOrEmpty(awsProfile)) {
            throw new RuntimeException("Please provide access_key/secret_key " + "or aws_profile, not both.");
        }
        if (Strings.isNullOrEmpty(accessKey)) {
            throw new RuntimeException("Missing heron.uploader.s3.access_key config value");
        }
        if (Strings.isNullOrEmpty(accessSecret)) {
            throw new RuntimeException("Missing heron.uploader.s3.secret_key config value");
        }
        builder.setCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, accessSecret)));
    } else if (!Strings.isNullOrEmpty(awsProfile)) {
        builder.setCredentials(new ProfileCredentialsProvider(awsProfile));
    }
    if (!Strings.isNullOrEmpty(proxy)) {
        URI proxyUri;
        try {
            proxyUri = new URI(proxy);
        } catch (URISyntaxException e) {
            throw new RuntimeException("Invalid heron.uploader.s3.proxy_uri config value: " + proxy, e);
        }
        ClientConfiguration clientCfg = new ClientConfiguration();
        clientCfg.withProtocol(Protocol.HTTPS).withProxyHost(proxyUri.getHost()).withProxyPort(proxyUri.getPort());
        if (!Strings.isNullOrEmpty(proxyUri.getUserInfo())) {
            String[] info = proxyUri.getUserInfo().split(":", 2);
            clientCfg.setProxyUsername(info[0]);
            if (info.length > 1) {
                clientCfg.setProxyPassword(info[1]);
            }
        }
        builder.setClientConfiguration(clientCfg);
    }
    s3Client = builder.withRegion(customRegion).withPathStyleAccessEnabled(true).withChunkedEncodingDisabled(true).withPayloadSigningEnabled(true).build();
    if (!Strings.isNullOrEmpty(endpoint)) {
        s3Client.setEndpoint(endpoint);
    }
    final String topologyName = Context.topologyName(config);
    final String topologyPackageLocation = Context.topologyPackageFile(config);
    pathPrefix = S3Context.pathPrefix(config);
    packageFileHandler = new File(topologyPackageLocation);
    // The path the packaged topology will be uploaded to
    remoteFilePath = generateS3Path(pathPrefix, topologyName, packageFileHandler.getName());
    // Generate the location of the backup file incase we need to revert the deploy
    previousVersionFilePath = generateS3Path(pathPrefix, topologyName, "previous_" + packageFileHandler.getName());
}

Example 95 with Bucket

use of com.amazonaws.services.s3.model.Bucket in project ice by Netflix.

the class AwsUtils method listAllObjects.

/**
     * List all object summary with given prefix in the s3 bucket.
     * @param bucket
     * @param prefix
     * @return
     */
public static List<S3ObjectSummary> listAllObjects(String bucket, String prefix, String accountId, String assumeRole, String externalId) {
    AmazonS3Client s3Client = AwsUtils.s3Client;
    try {
        ListObjectsRequest request = new ListObjectsRequest().withBucketName(bucket).withPrefix(prefix);
        List<S3ObjectSummary> result = Lists.newLinkedList();
        if (!StringUtils.isEmpty(accountId) && !StringUtils.isEmpty(assumeRole)) {
            Credentials assumedCredentials = getAssumedCredentials(accountId, assumeRole, externalId);
            s3Client = new AmazonS3Client(new BasicSessionCredentials(assumedCredentials.getAccessKeyId(), assumedCredentials.getSecretAccessKey(), assumedCredentials.getSessionToken()), clientConfig);
        }
        ObjectListing page = null;
        do {
            if (page != null)
                request.setMarker(page.getNextMarker());
            page = s3Client.listObjects(request);
            result.addAll(page.getObjectSummaries());
        } while (page.isTruncated());
        return result;
    } finally {
        if (s3Client != AwsUtils.s3Client)
            s3Client.shutdown();
    }
}