Examples with AmazonS3 com.amazonaws.services.s3.AmazonS3 used on opensource projects

Example 16 with AmazonS3

use of com.amazonaws.services.s3.AmazonS3 in project h2o-3 by h2oai.

the class PersistS3 method importFiles.

public void importFiles(String path, String pattern, ArrayList<String> files, ArrayList<String> keys, ArrayList<String> fails, ArrayList<String> dels) {
    Log.info("ImportS3 processing (" + path + ")");
    // List of processed files
    AmazonS3 s3 = getClient();
    String[] parts = decodePath(path);
    ObjectListing currentList = s3.listObjects(parts[0], parts[1]);
    processListing(currentList, files, fails, true);
    while (currentList.isTruncated()) {
        currentList = s3.listNextBatchOfObjects(currentList);
        processListing(currentList, files, fails, true);
    }
    keys.addAll(files);
// write barrier was here : DKV.write_barrier();
}

Example 17 with AmazonS3

use of com.amazonaws.services.s3.AmazonS3 in project h2o-3 by h2oai.

the class PersistS3 method uriToKey.

@Override
public Key uriToKey(URI uri) throws IOException {
    AmazonS3 s3 = getClient();
    // Decompose URI into bucket, key
    String[] parts = decodePath(uri.toString());
    try {
        ObjectMetadata om = s3.getObjectMetadata(parts[0], parts[1]);
        // Voila: create S3 specific key pointing to the file
        return S3FileVec.make(encodePath(parts[0], parts[1]), om.getContentLength());
    } catch (AmazonServiceException e) {
        if (e.getErrorCode().contains("404")) {
            throw new IOException(e);
        } else {
            Log.err("AWS failed for " + Arrays.toString(parts) + ": " + e.getMessage());
            throw e;
        }
    }
}

Example 18 with AmazonS3

use of com.amazonaws.services.s3.AmazonS3 in project ORCID-Source by ORCID.

the class S3Utils method createBuckets.

//Create S3 buckets for a given prefix
public static void createBuckets(String bucketPrefix, String accessKey, String secretKey) {
    AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
    AmazonS3 s3 = new AmazonS3Client(credentials);
    String api12JsonPrefix = bucketPrefix + "-api-1-2-json-";
    String api12XMLPrefix = bucketPrefix + "-api-1-2-xml-";
    String api20JsonPrefix = bucketPrefix + "-api-2-0-json-";
    String api20XMLPrefix = bucketPrefix + "-api-2-0-xml-";
    for (int i = 0; i <= 10; i++) {
        char lastCharacter = (i == 10 ? 'x' : Character.forDigit(i, 10));
        if (!s3.doesBucketExist(api12JsonPrefix + lastCharacter)) {
            s3.createBucket((api12JsonPrefix + lastCharacter), Region.EU_Ireland);
        }
        if (!s3.doesBucketExist(api12XMLPrefix + lastCharacter)) {
            s3.createBucket((api12XMLPrefix + lastCharacter), Region.EU_Ireland);
        }
        if (!s3.doesBucketExist(api20JsonPrefix + lastCharacter)) {
            s3.createBucket((api20JsonPrefix + lastCharacter), Region.EU_Ireland);
        }
        if (!s3.doesBucketExist(api20XMLPrefix + lastCharacter)) {
            s3.createBucket((api20XMLPrefix + lastCharacter), Region.EU_Ireland);
        }
    }
}

Example 19 with AmazonS3

use of com.amazonaws.services.s3.AmazonS3 in project ignite by apache.

the class S3CheckpointSpiSelfTest method afterSpiStopped.

/**
     * @throws Exception If error.
     */
@Override
protected void afterSpiStopped() throws Exception {
    AWSCredentials cred = new BasicAWSCredentials(IgniteS3TestSuite.getAccessKey(), IgniteS3TestSuite.getSecretKey());
    AmazonS3 s3 = new AmazonS3Client(cred);
    String bucketName = S3CheckpointSpi.BUCKET_NAME_PREFIX + "unit-test-bucket";
    try {
        ObjectListing list = s3.listObjects(bucketName);
        while (true) {
            for (S3ObjectSummary sum : list.getObjectSummaries()) s3.deleteObject(bucketName, sum.getKey());
            if (list.isTruncated())
                list = s3.listNextBatchOfObjects(list);
            else
                break;
        }
    } catch (AmazonClientException e) {
        throw new IgniteSpiException("Failed to read checkpoint bucket: " + bucketName, e);
    }
}

Example 20 with AmazonS3

use of com.amazonaws.services.s3.AmazonS3 in project jackrabbit by apache.

the class Utils method deleteBucket.

/**
     * Delete S3 bucket. This method first deletes all objects from bucket and
     * then delete empty bucket.
     * 
     * @param bucketName the bucket name.
     */
public static void deleteBucket(final String bucketName) throws IOException {
    Properties prop = readConfig(DEFAULT_CONFIG_FILE);
    AmazonS3 s3service = openService(prop);
    ObjectListing prevObjectListing = s3service.listObjects(bucketName);
    while (true) {
        for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
            s3service.deleteObject(bucketName, s3ObjSumm.getKey());
        }
        if (!prevObjectListing.isTruncated()) {
            break;
        }
        prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
    }
    s3service.deleteBucket(bucketName);
}