Examples with AmazonS3 com.talend.shaded.com.amazonaws.services.s3.AmazonS3 used on opensource projects

Example 11 with AmazonS3

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3 in project jackrabbit by apache.

the class Utils method deleteBucket.

/**
     * Delete S3 bucket. This method first deletes all objects from bucket and
     * then delete empty bucket.
     * 
     * @param bucketName the bucket name.
     */
public static void deleteBucket(final String bucketName) throws IOException {
    Properties prop = readConfig(DEFAULT_CONFIG_FILE);
    AmazonS3 s3service = openService(prop);
    ObjectListing prevObjectListing = s3service.listObjects(bucketName);
    while (true) {
        for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
            s3service.deleteObject(bucketName, s3ObjSumm.getKey());
        }
        if (!prevObjectListing.isTruncated()) {
            break;
        }
        prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
    }
    s3service.deleteBucket(bucketName);
}

Example 12 with AmazonS3

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3 in project h2o-3 by h2oai.

the class PersistS3 method importFiles.

public void importFiles(String path, String pattern, ArrayList<String> files, ArrayList<String> keys, ArrayList<String> fails, ArrayList<String> dels) {
    Log.info("ImportS3 processing (" + path + ")");
    // List of processed files
    AmazonS3 s3 = getClient();
    String[] parts = decodePath(path);
    ObjectListing currentList = s3.listObjects(parts[0], parts[1]);
    processListing(currentList, files, fails, true);
    while (currentList.isTruncated()) {
        currentList = s3.listNextBatchOfObjects(currentList);
        processListing(currentList, files, fails, true);
    }
    keys.addAll(files);
// write barrier was here : DKV.write_barrier();
}

Example 13 with AmazonS3

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3 in project h2o-3 by h2oai.

the class PersistS3 method uriToKey.

@Override
public Key uriToKey(URI uri) throws IOException {
    AmazonS3 s3 = getClient();
    // Decompose URI into bucket, key
    String[] parts = decodePath(uri.toString());
    try {
        ObjectMetadata om = s3.getObjectMetadata(parts[0], parts[1]);
        // Voila: create S3 specific key pointing to the file
        return S3FileVec.make(encodePath(parts[0], parts[1]), om.getContentLength());
    } catch (AmazonServiceException e) {
        if (e.getErrorCode().contains("404")) {
            throw new IOException(e);
        } else {
            Log.err("AWS failed for " + Arrays.toString(parts) + ": " + e.getMessage());
            throw e;
        }
    }
}

Example 14 with AmazonS3

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3 in project h2o-2 by h2oai.

the class ImportFiles2 method serveS3.

protected void serveS3() {
    Futures fs = new Futures();
    assert path.startsWith("s3://");
    path = path.substring(5);
    int bend = path.indexOf('/');
    if (bend == -1)
        bend = path.length();
    String bucket = path.substring(0, bend);
    String prefix = bend < path.length() ? path.substring(bend + 1) : "";
    AmazonS3 s3 = PersistS3.getClient();
    if (!s3.doesBucketExist(bucket))
        throw new IllegalArgumentException("S3 Bucket " + bucket + " not found!");
    ;
    ArrayList<String> succ = new ArrayList<String>();
    ArrayList<String> fail = new ArrayList<String>();
    ObjectListing currentList = s3.listObjects(bucket, prefix);
    while (true) {
        for (S3ObjectSummary obj : currentList.getObjectSummaries()) try {
            succ.add(S3FileVec.make(obj, fs).toString());
        } catch (Throwable e) {
            fail.add(obj.getKey());
            Log.err("Failed to loadfile from S3: path = " + obj.getKey() + ", error = " + e.getClass().getName() + ", msg = " + e.getMessage());
        }
        if (currentList.isTruncated())
            currentList = s3.listNextBatchOfObjects(currentList);
        else
            break;
    }
    keys = succ.toArray(new String[succ.size()]);
    files = keys;
    fails = fail.toArray(new String[fail.size()]);
    this.prefix = getCommonPrefix(keys);
}

Example 15 with AmazonS3

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3 in project h2o-2 by h2oai.

the class ImportS3 method serve.

@Override
protected Response serve() {
    String bucket = _bucket.value();
    Log.info("ImportS3 processing (" + bucket + ")");
    JsonObject json = new JsonObject();
    JsonArray succ = new JsonArray();
    JsonArray fail = new JsonArray();
    AmazonS3 s3 = PersistS3.getClient();
    ObjectListing currentList = s3.listObjects(bucket);
    processListing(currentList, succ, fail);
    while (currentList.isTruncated()) {
        currentList = s3.listNextBatchOfObjects(currentList);
        processListing(currentList, succ, fail);
    }
    json.add(NUM_SUCCEEDED, new JsonPrimitive(succ.size()));
    json.add(SUCCEEDED, succ);
    json.add(NUM_FAILED, new JsonPrimitive(fail.size()));
    json.add(FAILED, fail);
    DKV.write_barrier();
    Response r = Response.done(json);
    r.setBuilder(SUCCEEDED + "." + KEY, new KeyCellBuilder());
    return r;
}