Examples with ObjectListing com.amazonaws.services.s3.model.ObjectListing used on opensource projects

Example 51 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project YCSB by brianfrankcooper.

the class S3Client method scanFromStorage.

/**
 * Perform an emulation of a database scan operation on a S3 bucket.
 *
 * @param bucket
 *            The name of the bucket
 * @param startkey
 *            The file key of the first file to read.
 * @param recordcount
 *            The number of files to read
 * @param fields
 *            The list of fields to read, or null for all of them
 * @param result
 *            A Vector of HashMaps, where each HashMap is a set field/value
 *            pairs for one file
 */
protected Status scanFromStorage(String bucket, String startkey, int recordcount, Vector<HashMap<String, ByteIterator>> result, SSECustomerKey ssecLocal) {
    int counter = 0;
    ObjectListing listing = s3Client.listObjects(bucket);
    List<S3ObjectSummary> summaries = listing.getObjectSummaries();
    List<String> keyList = new ArrayList();
    int startkeyNumber = 0;
    int numberOfIteration = 0;
    // getting the list of files in the bucket
    while (listing.isTruncated()) {
        listing = s3Client.listNextBatchOfObjects(listing);
        summaries.addAll(listing.getObjectSummaries());
    }
    for (S3ObjectSummary summary : summaries) {
        String summaryKey = summary.getKey();
        keyList.add(summaryKey);
    }
    // Sorting the list of files in Alphabetical order
    // sorting the list
    Collections.sort(keyList);
    // Getting the position of the startingfile for the scan
    for (String key : keyList) {
        if (key.equals(startkey)) {
            startkeyNumber = counter;
        } else {
            counter = counter + 1;
        }
    }
    // if not using the total number of Files
    if (recordcount < keyList.size()) {
        numberOfIteration = recordcount;
    } else {
        numberOfIteration = keyList.size();
    }
    // of the Files or Till the recordcount number
    for (int i = startkeyNumber; i < numberOfIteration; i++) {
        HashMap<String, ByteIterator> resultTemp = new HashMap<String, ByteIterator>();
        readFromStorage(bucket, keyList.get(i), resultTemp, ssecLocal);
        result.add(resultTemp);
    }
    return Status.OK;
}

Example 52 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project exhibitor by soabase.

the class S3PseudoLock method getFileNames.

@Override
protected List<String> getFileNames(String lockPrefix) throws Exception {
    ListObjectsRequest request = new ListObjectsRequest();
    request.setBucketName(bucket);
    request.setPrefix(lockPrefix);
    ObjectListing objectListing = client.listObjects(request);
    return Lists.transform(objectListing.getObjectSummaries(), new Function<S3ObjectSummary, String>() {

        @Override
        public String apply(S3ObjectSummary summary) {
            return summary.getKey();
        }
    });
}

Example 53 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project exhibitor by soabase.

the class TestS3BackupProviderBase method testGetAvailableBackupKeys.

@Test
public void testGetAvailableBackupKeys() throws Exception {
    ObjectListing listing = new ObjectListing() {

        @Override
        public List<S3ObjectSummary> getObjectSummaries() {
            List<S3ObjectSummary> list = Lists.newArrayList();
            S3ObjectSummary summary = new S3ObjectSummary();
            summary.setKey("exhibitor-backup" + S3BackupProvider.SEPARATOR + "one" + S3BackupProvider.SEPARATOR + "1234");
            list.add(summary);
            summary = new S3ObjectSummary();
            summary.setKey("exhibitor-backup" + S3BackupProvider.SEPARATOR + "two" + S3BackupProvider.SEPARATOR + "1234");
            list.add(summary);
            summary = new S3ObjectSummary();
            summary.setKey("exhibitor-backup" + S3BackupProvider.SEPARATOR + "three" + S3BackupProvider.SEPARATOR + "1234");
            list.add(summary);
            return list;
        }
    };
    MockS3Client s3Client = new MockS3Client(null, listing);
    S3BackupProvider provider = new S3BackupProvider(new MockS3ClientFactory(s3Client), new PropertyBasedS3Credential(new Properties()), new PropertyBasedS3ClientConfig(new Properties()), null);
    List<BackupMetaData> backups = provider.getAvailableBackups(null, Maps.<String, String>newHashMap());
    List<String> backupNames = Lists.transform(backups, new Function<BackupMetaData, String>() {

        @Override
        public String apply(BackupMetaData metaData) {
            return metaData.getName();
        }
    });
    Assert.assertEquals(backupNames, Arrays.asList("one", "two", "three"));
}

Example 54 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project crate by crate.

the class S3FileReadingCollectorTest method createBatchIterator.

private BatchIterator<Row> createBatchIterator(Collection<String> fileUris, String compression, final S3ObjectInputStream s3InputStream, boolean collectSourceUriFailure) {
    InputFactory.Context<LineCollectorExpression<?>> ctx = inputFactory.ctxForRefs(txnCtx, FileLineReferenceResolver::getImplementation);
    List<Input<?>> inputs = new ArrayList<>(2);
    Reference raw = createReference(SourceLineExpression.COLUMN_NAME, DataTypes.STRING);
    inputs.add(ctx.add(raw));
    if (collectSourceUriFailure) {
        Reference sourceUriFailure = createReference(SourceUriFailureExpression.COLUMN_NAME, DataTypes.STRING);
        // noinspection unchecked
        sourceUriFailureInput = (Input<String>) ctx.add(sourceUriFailure);
        inputs.add(sourceUriFailureInput);
    }
    return FileReadingIterator.newInstance(fileUris, inputs, ctx.expressions(), compression, Map.of(S3FileInputFactory.NAME, new FileInputFactory() {

        @Override
        public FileInput create(URI uri, Settings withClauseOptions) throws IOException {
            return new S3FileInput(new S3ClientHelper() {

                @Override
                protected AmazonS3 initClient(String accessKey, String secretKey, String endpoint, String protocol) throws IOException {
                    AmazonS3 client = mock(AmazonS3Client.class);
                    ObjectListing objectListing = mock(ObjectListing.class);
                    S3ObjectSummary summary = mock(S3ObjectSummary.class);
                    S3Object s3Object = mock(S3Object.class);
                    when(client.listObjects(anyString(), anyString())).thenReturn(objectListing);
                    when(objectListing.getObjectSummaries()).thenReturn(Collections.singletonList(summary));
                    when(summary.getKey()).thenReturn("foo");
                    when(client.getObject("fakebucket", "foo")).thenReturn(s3Object);
                    when(s3Object.getObjectContent()).thenReturn(s3InputStream);
                    when(client.listNextBatchOfObjects(any(ObjectListing.class))).thenReturn(objectListing);
                    when(objectListing.isTruncated()).thenReturn(false);
                    return client;
                }
            }, uri, "https");
        }
    }), false, 1, 0, CopyFromParserProperties.DEFAULT, FileUriCollectPhase.InputFormat.JSON, Settings.EMPTY);
}

Example 55 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project crate by crate.

the class S3FileInput method expandUri.

@Override
public List<URI> expandUri() throws IOException {
    if (isGlobbed() == false) {
        return List.of(normalizedS3URI.uri());
    }
    if (client == null) {
        client = clientBuilder.client(preGlobUri, protocolSetting);
    }
    List<URI> uris = new ArrayList<>();
    ObjectListing list = client.listObjects(preGlobUri.bucket(), preGlobUri.key());
    addKeyUris(uris, list);
    while (list.isTruncated()) {
        list = client.listNextBatchOfObjects(list);
        addKeyUris(uris, list);
    }
    return uris;
}