Examples with ObjectListing com.amazonaws.services.s3.model.ObjectListing used on opensource projects

Example 6 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project jackrabbit-oak by apache.

the class S3Backend method deleteAllMetadataRecords.

public void deleteAllMetadataRecords(String prefix) {
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(addMetaKeyPrefix(prefix));
        ObjectListing metaList = s3service.listObjects(listObjectsRequest);
        List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
        for (S3ObjectSummary s3ObjSumm : metaList.getObjectSummaries()) {
            deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
        }
        if (deleteList.size() > 0) {
            DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
            delObjsReq.setKeys(deleteList);
            DeleteObjectsResult dobjs = s3service.deleteObjects(delObjsReq);
        }
    } finally {
        if (contextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
}

Example 7 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project jackrabbit-oak by apache.

the class S3Backend method deleteAllMetadataRecords.

@Override
public void deleteAllMetadataRecords(String prefix) {
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(addMetaKeyPrefix(prefix));
        ObjectListing metaList = s3service.listObjects(listObjectsRequest);
        List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
        for (S3ObjectSummary s3ObjSumm : metaList.getObjectSummaries()) {
            deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
        }
        if (deleteList.size() > 0) {
            DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
            delObjsReq.setKeys(deleteList);
            s3service.deleteObjects(delObjsReq);
        }
    } finally {
        if (contextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
}

Example 8 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project presto by prestodb.

the class PrestoS3FileSystem method listPrefix.

private Iterator<LocatedFileStatus> listPrefix(Path path) {
    String key = keyFromPath(path);
    if (!key.isEmpty()) {
        key += PATH_SEPARATOR;
    }
    ListObjectsRequest request = new ListObjectsRequest().withBucketName(uri.getHost()).withPrefix(key).withDelimiter(PATH_SEPARATOR);
    STATS.newListObjectsCall();
    Iterator<ObjectListing> listings = new AbstractSequentialIterator<ObjectListing>(s3.listObjects(request)) {

        @Override
        protected ObjectListing computeNext(ObjectListing previous) {
            if (!previous.isTruncated()) {
                return null;
            }
            return s3.listNextBatchOfObjects(previous);
        }
    };
    return Iterators.concat(Iterators.transform(listings, this::statusFromListing));
}

Example 9 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project h2o-3 by h2oai.

the class PersistS3 method importFiles.

public void importFiles(String path, String pattern, ArrayList<String> files, ArrayList<String> keys, ArrayList<String> fails, ArrayList<String> dels) {
    Log.info("ImportS3 processing (" + path + ")");
    // List of processed files
    AmazonS3 s3 = getClient();
    String[] parts = decodePath(path);
    ObjectListing currentList = s3.listObjects(parts[0], parts[1]);
    processListing(currentList, files, fails, true);
    while (currentList.isTruncated()) {
        currentList = s3.listNextBatchOfObjects(currentList);
        processListing(currentList, files, fails, true);
    }
    keys.addAll(files);
// write barrier was here : DKV.write_barrier();
}

Example 10 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project h2o-2 by h2oai.

the class ImportFiles2 method serveS3.

protected void serveS3() {
    Futures fs = new Futures();
    assert path.startsWith("s3://");
    path = path.substring(5);
    int bend = path.indexOf('/');
    if (bend == -1)
        bend = path.length();
    String bucket = path.substring(0, bend);
    String prefix = bend < path.length() ? path.substring(bend + 1) : "";
    AmazonS3 s3 = PersistS3.getClient();
    if (!s3.doesBucketExist(bucket))
        throw new IllegalArgumentException("S3 Bucket " + bucket + " not found!");
    ;
    ArrayList<String> succ = new ArrayList<String>();
    ArrayList<String> fail = new ArrayList<String>();
    ObjectListing currentList = s3.listObjects(bucket, prefix);
    while (true) {
        for (S3ObjectSummary obj : currentList.getObjectSummaries()) try {
            succ.add(S3FileVec.make(obj, fs).toString());
        } catch (Throwable e) {
            fail.add(obj.getKey());
            Log.err("Failed to loadfile from S3: path = " + obj.getKey() + ", error = " + e.getClass().getName() + ", msg = " + e.getMessage());
        }
        if (currentList.isTruncated())
            currentList = s3.listNextBatchOfObjects(currentList);
        else
            break;
    }
    keys = succ.toArray(new String[succ.size()]);
    files = keys;
    fails = fail.toArray(new String[fail.size()]);
    this.prefix = getCommonPrefix(keys);
}