Examples with IndexRequestContainer com.epam.pipeline.elasticsearchagent.service.impl.IndexRequestContainer used on opensource projects

Example 1 with IndexRequestContainer

use of com.epam.pipeline.elasticsearchagent.service.impl.IndexRequestContainer in project cloud-pipeline by epam.

the class S3Helper method listFiles.

private void listFiles(AmazonS3 client, String bucket) {
    ListObjectsV2Request req = new ListObjectsV2Request();
    req.setBucketName(bucket);
    req.setPrefix("");
    ListObjectsV2Result listing;
    try (IndexRequestContainer walker = new IndexRequestContainer(bulkRequestCreator, bulkInsertSize)) {
        do {
            listing = client.listObjectsV2(req);
            for (S3ObjectSummary s3ObjectSummary : listing.getObjectSummaries()) {
                DataStorageFile file = convertToStorageFile(s3ObjectSummary);
                if (file != null) {
                    if (enableTags) {
                        file.setTags(listObjectTags(client, bucket, s3ObjectSummary.getKey()));
                    }
                    walker.add(createIndexRequest(file));
                }
            }
            req.setContinuationToken(listing.getNextContinuationToken());
        } while (listing.isTruncated());
    }
}