Search in sources :

Example 71 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project jackrabbit-oak by apache.

the class S3DataStoreUtils method deleteBucket.

public static void deleteBucket(String bucket, Date date) throws Exception {
    log.info("cleaning bucket [" + bucket + "]");
    Properties props = getS3Config();
    AmazonS3Client s3service = Utils.openService(props);
    TransferManager tmx = new TransferManager(s3service);
    if (s3service.doesBucketExist(bucket)) {
        for (int i = 0; i < 4; i++) {
            tmx.abortMultipartUploads(bucket, date);
            ObjectListing prevObjectListing = s3service.listObjects(bucket);
            while (prevObjectListing != null) {
                List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
                for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
                    deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
                }
                if (deleteList.size() > 0) {
                    DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
                    delObjsReq.setKeys(deleteList);
                    s3service.deleteObjects(delObjsReq);
                }
                if (!prevObjectListing.isTruncated())
                    break;
                prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
            }
        }
        s3service.deleteBucket(bucket);
        log.info("bucket [ " + bucket + "] cleaned");
    } else {
        log.info("bucket [" + bucket + "] doesn't exists");
    }
    tmx.shutdownNow();
    s3service.shutdown();
}
Also used : TransferManager(com.amazonaws.services.s3.transfer.TransferManager) AmazonS3Client(com.amazonaws.services.s3.AmazonS3Client) ArrayList(java.util.ArrayList) ObjectListing(com.amazonaws.services.s3.model.ObjectListing) S3ObjectSummary(com.amazonaws.services.s3.model.S3ObjectSummary) Properties(java.util.Properties) DeleteObjectsRequest(com.amazonaws.services.s3.model.DeleteObjectsRequest)

Example 72 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project jackrabbit-oak by apache.

the class S3Backend method getAllMetadataRecords.

public List<DataRecord> getAllMetadataRecords(String prefix) {
    List<DataRecord> metadataList = new ArrayList<DataRecord>();
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(addMetaKeyPrefix(prefix));
        ObjectListing prevObjectListing = s3service.listObjects(listObjectsRequest);
        for (final S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
            metadataList.add(new S3DataRecord(s3service, bucket, stripMetaKeyPrefix(s3ObjSumm.getKey()), s3ObjSumm.getLastModified().getTime(), s3ObjSumm.getSize(), true));
        }
    } finally {
        if (contextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
    return metadataList;
}
Also used : ListObjectsRequest(com.amazonaws.services.s3.model.ListObjectsRequest) ArrayList(java.util.ArrayList) ObjectListing(com.amazonaws.services.s3.model.ObjectListing) S3ObjectSummary(com.amazonaws.services.s3.model.S3ObjectSummary) DataRecord(org.apache.jackrabbit.core.data.DataRecord)

Example 73 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project jackrabbit-oak by apache.

the class S3Backend method renameKeys.

/**
     * This method rename object keys in S3 concurrently. The number of
     * concurrent threads is defined by 'maxConnections' property in
     * aws.properties. As S3 doesn't have "move" command, this method simulate
     * move as copy object object to new key and then delete older key.
     */
private void renameKeys() throws DataStoreException {
    long startTime = System.currentTimeMillis();
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    long count = 0;
    try {
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        ObjectListing prevObjectListing = s3service.listObjects(bucket);
        List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
        int nThreads = Integer.parseInt(properties.getProperty("maxConnections"));
        ExecutorService executor = Executors.newFixedThreadPool(nThreads, new NamedThreadFactory("s3-object-rename-worker"));
        boolean taskAdded = false;
        while (true) {
            for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
                executor.execute(new KeyRenameThread(s3ObjSumm.getKey()));
                taskAdded = true;
                count++;
                // delete the object if it follows old key name format
                if (s3ObjSumm.getKey().startsWith(KEY_PREFIX)) {
                    deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
                }
            }
            if (!prevObjectListing.isTruncated())
                break;
            prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
        }
        // This will make the executor accept no new threads
        // and finish all existing threads in the queue
        executor.shutdown();
        try {
            // Wait until all threads are finish
            while (taskAdded && !executor.awaitTermination(10, TimeUnit.SECONDS)) {
                LOG.info("Rename S3 keys tasks timedout. Waiting again");
            }
        } catch (InterruptedException ie) {
        }
        LOG.info("Renamed [{}] keys, time taken [{}]sec", count, ((System.currentTimeMillis() - startTime) / 1000));
        // Delete older keys.
        if (deleteList.size() > 0) {
            DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
            int batchSize = 500, startIndex = 0, size = deleteList.size();
            int endIndex = batchSize < size ? batchSize : size;
            while (endIndex <= size) {
                delObjsReq.setKeys(Collections.unmodifiableList(deleteList.subList(startIndex, endIndex)));
                DeleteObjectsResult dobjs = s3service.deleteObjects(delObjsReq);
                LOG.info("Records[{}] deleted in datastore from index [{}] to [{}]", new Object[] { dobjs.getDeletedObjects().size(), startIndex, (endIndex - 1) });
                if (endIndex == size) {
                    break;
                } else {
                    startIndex = endIndex;
                    endIndex = (startIndex + batchSize) < size ? (startIndex + batchSize) : size;
                }
            }
        }
    } finally {
        if (contextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
}
Also used : NamedThreadFactory(org.apache.jackrabbit.core.data.util.NamedThreadFactory) ArrayList(java.util.ArrayList) ObjectListing(com.amazonaws.services.s3.model.ObjectListing) S3ObjectSummary(com.amazonaws.services.s3.model.S3ObjectSummary) DeleteObjectsResult(com.amazonaws.services.s3.model.DeleteObjectsResult) DeleteObjectsRequest(com.amazonaws.services.s3.model.DeleteObjectsRequest) ExecutorService(java.util.concurrent.ExecutorService)

Example 74 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project hadoop by apache.

the class S3AFileSystem method getFileStatus.

/**
   * Return a file status object that represents the path.
   * @param f The path we want information from
   * @return a FileStatus object
   * @throws java.io.FileNotFoundException when the path does not exist;
   * @throws IOException on other problems.
   */
public S3AFileStatus getFileStatus(final Path f) throws IOException {
    incrementStatistic(INVOCATION_GET_FILE_STATUS);
    final Path path = qualify(f);
    String key = pathToKey(path);
    LOG.debug("Getting path status for {}  ({})", path, key);
    if (!key.isEmpty()) {
        try {
            ObjectMetadata meta = getObjectMetadata(key);
            if (objectRepresentsDirectory(key, meta.getContentLength())) {
                LOG.debug("Found exact file: fake directory");
                return new S3AFileStatus(true, path, username);
            } else {
                LOG.debug("Found exact file: normal file");
                return new S3AFileStatus(meta.getContentLength(), dateToLong(meta.getLastModified()), path, getDefaultBlockSize(path), username);
            }
        } catch (AmazonServiceException e) {
            if (e.getStatusCode() != 404) {
                throw translateException("getFileStatus", path, e);
            }
        } catch (AmazonClientException e) {
            throw translateException("getFileStatus", path, e);
        }
        // Necessary?
        if (!key.endsWith("/")) {
            String newKey = key + "/";
            try {
                ObjectMetadata meta = getObjectMetadata(newKey);
                if (objectRepresentsDirectory(newKey, meta.getContentLength())) {
                    LOG.debug("Found file (with /): fake directory");
                    return new S3AFileStatus(true, path, username);
                } else {
                    LOG.warn("Found file (with /): real file? should not happen: {}", key);
                    return new S3AFileStatus(meta.getContentLength(), dateToLong(meta.getLastModified()), path, getDefaultBlockSize(path), username);
                }
            } catch (AmazonServiceException e) {
                if (e.getStatusCode() != 404) {
                    throw translateException("getFileStatus", newKey, e);
                }
            } catch (AmazonClientException e) {
                throw translateException("getFileStatus", newKey, e);
            }
        }
    }
    try {
        key = maybeAddTrailingSlash(key);
        ListObjectsRequest request = new ListObjectsRequest();
        request.setBucketName(bucket);
        request.setPrefix(key);
        request.setDelimiter("/");
        request.setMaxKeys(1);
        ObjectListing objects = listObjects(request);
        if (!objects.getCommonPrefixes().isEmpty() || !objects.getObjectSummaries().isEmpty()) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Found path as directory (with /): {}/{}", objects.getCommonPrefixes().size(), objects.getObjectSummaries().size());
                for (S3ObjectSummary summary : objects.getObjectSummaries()) {
                    LOG.debug("Summary: {} {}", summary.getKey(), summary.getSize());
                }
                for (String prefix : objects.getCommonPrefixes()) {
                    LOG.debug("Prefix: {}", prefix);
                }
            }
            return new S3AFileStatus(false, path, username);
        } else if (key.isEmpty()) {
            LOG.debug("Found root directory");
            return new S3AFileStatus(true, path, username);
        }
    } catch (AmazonServiceException e) {
        if (e.getStatusCode() != 404) {
            throw translateException("getFileStatus", key, e);
        }
    } catch (AmazonClientException e) {
        throw translateException("getFileStatus", key, e);
    }
    LOG.debug("Not Found: {}", path);
    throw new FileNotFoundException("No such file or directory: " + path);
}
Also used : Path(org.apache.hadoop.fs.Path) ListObjectsRequest(com.amazonaws.services.s3.model.ListObjectsRequest) AmazonClientException(com.amazonaws.AmazonClientException) AmazonServiceException(com.amazonaws.AmazonServiceException) FileNotFoundException(java.io.FileNotFoundException) ObjectListing(com.amazonaws.services.s3.model.ObjectListing) S3ObjectSummary(com.amazonaws.services.s3.model.S3ObjectSummary) ObjectMetadata(com.amazonaws.services.s3.model.ObjectMetadata)

Example 75 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project hadoop by apache.

the class TestS3AGetFileStatus method testNotFound.

@Test
public void testNotFound() throws Exception {
    Path path = new Path("/dir");
    String key = path.toUri().getPath().substring(1);
    when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key)))).thenThrow(NOT_FOUND);
    when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key + "/")))).thenThrow(NOT_FOUND);
    ObjectListing objects = mock(ObjectListing.class);
    when(objects.getCommonPrefixes()).thenReturn(Collections.<String>emptyList());
    when(objects.getObjectSummaries()).thenReturn(Collections.<S3ObjectSummary>emptyList());
    when(s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objects);
    exception.expect(FileNotFoundException.class);
    fs.getFileStatus(path);
}
Also used : Path(org.apache.hadoop.fs.Path) ListObjectsRequest(com.amazonaws.services.s3.model.ListObjectsRequest) ObjectListing(com.amazonaws.services.s3.model.ObjectListing) Test(org.junit.Test)

Aggregations

ObjectListing (com.amazonaws.services.s3.model.ObjectListing)104 S3ObjectSummary (com.amazonaws.services.s3.model.S3ObjectSummary)81 ListObjectsRequest (com.amazonaws.services.s3.model.ListObjectsRequest)55 ArrayList (java.util.ArrayList)44 AmazonS3 (com.amazonaws.services.s3.AmazonS3)22 AmazonClientException (com.amazonaws.AmazonClientException)17 DeleteObjectsRequest (com.amazonaws.services.s3.model.DeleteObjectsRequest)16 IOException (java.io.IOException)14 AmazonS3Client (com.amazonaws.services.s3.AmazonS3Client)12 Date (java.util.Date)12 Test (org.junit.Test)11 Test (org.testng.annotations.Test)11 AmazonServiceException (com.amazonaws.AmazonServiceException)10 HashMap (java.util.HashMap)10 Path (org.apache.hadoop.fs.Path)9 AmazonS3Exception (com.amazonaws.services.s3.model.AmazonS3Exception)8 DeleteObjectsResult (com.amazonaws.services.s3.model.DeleteObjectsResult)7 S3Object (com.amazonaws.services.s3.model.S3Object)7 Properties (java.util.Properties)6 FileStatus (org.apache.hadoop.fs.FileStatus)6