Examples with S3ExportConfiguration com.amazonaws.services.qldb.model.S3ExportConfiguration used on opensource projects

Example 1 with S3ExportConfiguration

use of com.amazonaws.services.qldb.model.S3ExportConfiguration in project amazon-qldb-dmv-sample-java by aws-samples.

the class JournalS3ExportReader method readExport.

/**
 * Read the S3 export within a {@link JournalBlock}.
 *
 * @param describeJournalS3ExportResult
 *              The result from the QLDB database describing a journal export.
 * @param amazonS3
 *              The low level S3 client.
 * @return a list of {@link JournalBlock}.
 */
public static List<JournalBlock> readExport(final DescribeJournalS3ExportResult describeJournalS3ExportResult, final AmazonS3 amazonS3) {
    S3ExportConfiguration exportConfiguration = describeJournalS3ExportResult.getExportDescription().getS3ExportConfiguration();
    ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request().withBucketName(exportConfiguration.getBucket()).withPrefix(exportConfiguration.getPrefix());
    ListObjectsV2Result listObjectsV2Result = amazonS3.listObjectsV2(listObjectsRequest);
    log.info("Found the following objects for list from s3: ");
    listObjectsV2Result.getObjectSummaries().forEach(s3ObjectSummary -> log.info(s3ObjectSummary.getKey()));
    // Validate initial manifest file was written.
    String expectedManifestKey = exportConfiguration.getPrefix() + describeJournalS3ExportResult.getExportDescription().getExportId() + ".started" + ".manifest";
    String initialManifestKey = listObjectsV2Result.getObjectSummaries().stream().filter(s3ObjectSummary -> s3ObjectSummary.getKey().equalsIgnoreCase(expectedManifestKey)).map(S3ObjectSummary::getKey).findFirst().orElseThrow(() -> new IllegalStateException("Initial manifest not found."));
    log.info("Found the initial manifest with key " + initialManifestKey);
    // Find the final manifest file, it should contain the exportId in it.
    String completedManifestFileKey = listObjectsV2Result.getObjectSummaries().stream().filter(s3ObjectSummary -> s3ObjectSummary.getKey().endsWith("completed.manifest") && (s3ObjectSummary.getKey().contains(describeJournalS3ExportResult.getExportDescription().getExportId()))).map(S3ObjectSummary::getKey).findFirst().orElseThrow(() -> new IllegalStateException("Completed manifest not found."));
    log.info("Found the completed manifest with key " + completedManifestFileKey);
    // Read manifest file to find data file keys.
    S3Object completedManifestObject = amazonS3.getObject(exportConfiguration.getBucket(), completedManifestFileKey);
    List<String> dataFileKeys = getDataFileKeysFromManifest(completedManifestObject);
    log.info("Found the following keys in the manifest files: " + dataFileKeys);
    List<JournalBlock> journalBlocks = new ArrayList<>();
    for (String key : dataFileKeys) {
        log.info("Reading file with S3 key " + key + " from bucket: " + exportConfiguration.getBucket());
        S3Object s3Object = amazonS3.getObject(exportConfiguration.getBucket(), key);
        List<JournalBlock> blocks = getJournalBlocks(s3Object);
        compareKeyWithContentRange(key, blocks.get(0), blocks.get(blocks.size() - 1));
        journalBlocks.addAll(blocks);
    }
    return journalBlocks;
}

Example 2 with S3ExportConfiguration

use of com.amazonaws.services.qldb.model.S3ExportConfiguration in project amazon-qldb-dmv-sample-java by aws-samples.

the class ExportJournal method createExport.

/**
 * Request QLDB to export the contents of the journal for the given time
 * period and s3 configuration. Before calling this function the S3 bucket
 * should be created, see {@link #createS3BucketIfNotExists(String, AmazonS3)}.
 *
 * @param name
 *              Name of the ledger.
 * @param startTime
 *              Time from when the journal contents should be exported.
 * @param endTime
 *              Time until which the journal contents should be exported.
 * @param bucket
 *              S3 bucket to write the data to.
 * @param prefix
 *              S3 prefix to be prefixed to the files written.
 * @param s3EncryptionConfiguration
 *              Encryption configuration for S3.
 * @param roleArn
 *              The IAM role ARN to be used when exporting the journal.
 * @return {@link ExportJournalToS3Result} from QLDB.
 */
public static ExportJournalToS3Result createExport(String name, Date startTime, Date endTime, String bucket, String prefix, S3EncryptionConfiguration s3EncryptionConfiguration, String roleArn) {
    log.info("Let's create a journal export for ledger with name: {}...", name);
    S3ExportConfiguration s3ExportConfiguration = new S3ExportConfiguration().withBucket(bucket).withPrefix(prefix).withEncryptionConfiguration(s3EncryptionConfiguration);
    ExportJournalToS3Request request = new ExportJournalToS3Request().withName(name).withInclusiveStartTime(startTime).withExclusiveEndTime(endTime).withS3ExportConfiguration(s3ExportConfiguration).withRoleArn(roleArn);
    try {
        ExportJournalToS3Result result = client.exportJournalToS3(request);
        log.info("Requested QLDB to export contents of the journal.");
        return result;
    } catch (InvalidParameterException ipe) {
        log.error("The eventually consistent behavior of the IAM service may cause this export" + " to fail its first attempts, please retry.");
        throw ipe;
    }
}