Examples with AbstractDataStorageItem com.epam.pipeline.entity.datastorage.AbstractDataStorageItem used on opensource projects

Example 6 with AbstractDataStorageItem

use of com.epam.pipeline.entity.datastorage.AbstractDataStorageItem in project cloud-pipeline by epam.

the class S3Helper method listFiles.

private DataStorageListing listFiles(AmazonS3 client, String bucket, String requestPath, Integer pageSize, String marker) {
    ListObjectsV2Request req = new ListObjectsV2Request();
    req.setBucketName(bucket);
    req.setPrefix(requestPath);
    req.setDelimiter(S3Constants.DELIMITER);
    if (pageSize != null) {
        req.setMaxKeys(pageSize);
    }
    if (StringUtils.hasValue(marker)) {
        req.setStartAfter(marker);
    }
    ListObjectsV2Result listing;
    List<AbstractDataStorageItem> items = new ArrayList<>();
    String previous = null;
    do {
        listing = client.listObjectsV2(req);
        for (String name : listing.getCommonPrefixes()) {
            previous = getPreviousKey(previous, name);
            items.add(parseFolder(requestPath, name));
        }
        for (S3ObjectSummary s3ObjectSummary : listing.getObjectSummaries()) {
            DataStorageFile file = AbstractS3ObjectWrapper.getWrapper(s3ObjectSummary).convertToStorageFile(requestPath);
            if (file != null) {
                previous = getPreviousKey(previous, s3ObjectSummary.getKey());
                items.add(file);
            }
        }
        req.setContinuationToken(listing.getNextContinuationToken());
    } while (listing.isTruncated() && (pageSize == null || items.size() < pageSize));
    String returnToken = listing.isTruncated() ? previous : null;
    return new DataStorageListing(returnToken, items);
}