Examples with SyntheticDataFile gov.cms.bfd.pipeline.ccw.rif.extract.synthetic.SyntheticDataFixer2.SyntheticDataFile used on opensource projects

Example 1 with SyntheticDataFile

use of gov.cms.bfd.pipeline.ccw.rif.extract.synthetic.SyntheticDataFixer2.SyntheticDataFile in project beneficiary-fhir-data by CMSgov.

the class SyntheticDataUploader2 method uploadSyntheticData.

/**
 * @param s3Client the {@link AmazonS3} client to use
 * @param s3KeyPrefix the S3 key prefix to upload all objects under/into
 * @param syntheticDataPathGrabber the {@link Function} that returns the {@link Path} to upload
 *     from, for a given {@link SyntheticDataFile}
 * @throws MalformedURLException Any {@link MalformedURLException}s encountered will be bubbled
 *     up.
 */
private static void uploadSyntheticData(AmazonS3 s3Client, String s3KeyPrefix, Function<SyntheticDataFile, Path> syntheticDataPathGrabber) throws MalformedURLException {
    Bucket bucket = new Bucket(TestDataSetLocation.S3_BUCKET_TEST_DATA);
    // Build a DataSetManifest for the data to be uploaded.
    List<DataSetManifestEntry> manifestEntries = new LinkedList<>();
    for (SyntheticDataFile syntheticDataFile : SyntheticDataFile.values()) manifestEntries.add(new DataSetManifestEntry(syntheticDataPathGrabber.apply(syntheticDataFile).getFileName().toString(), syntheticDataFile.getRifFile().getFileType()));
    DataSetManifest manifest = new DataSetManifest(Instant.now(), 0, manifestEntries);
    // Upload the manifest and every file in it.
    PutObjectRequest manifestRequest = DataSetTestUtilities.createPutRequest(bucket, s3KeyPrefix, manifest);
    manifestRequest.setCannedAcl(CannedAccessControlList.PublicRead);
    s3Client.putObject(manifestRequest);
    LOGGER.info("Uploaded: manifest");
    for (SyntheticDataFile syntheticDataFile : SyntheticDataFile.values()) {
        DataSetManifestEntry manifestEntry = manifest.getEntries().stream().filter(e -> e.getName().equals(syntheticDataPathGrabber.apply(syntheticDataFile).getFileName().toString())).findFirst().get();
        PutObjectRequest fileRequest = DataSetTestUtilities.createPutRequest(bucket, s3KeyPrefix, manifest, manifestEntry, syntheticDataPathGrabber.apply(syntheticDataFile).toUri().toURL());
        fileRequest.setCannedAcl(CannedAccessControlList.PublicRead);
        s3Client.putObject(fileRequest);
        LOGGER.info("Uploaded: {}", syntheticDataFile.name());
    }
}

Example 2 with SyntheticDataFile

use of gov.cms.bfd.pipeline.ccw.rif.extract.synthetic.SyntheticDataFixer2.SyntheticDataFile in project beneficiary-fhir-data by CMSgov.

the class SyntheticDataUploader3 method uploadSyntheticData.

/**
 * @param s3Client the {@link AmazonS3} client to use
 * @param s3KeyPrefix the S3 key prefix to upload all objects under/into
 * @param syntheticDataPathGrabber the {@link Function} that returns the {@link Path} to upload
 *     from, for a given {@link SyntheticDataFile}
 * @throws MalformedURLException Any {@link MalformedURLException}s encountered will be bubbled
 *     up.
 */
private static void uploadSyntheticData(AmazonS3 s3Client, String s3KeyPrefix, Function<SyntheticDataFile, Path> syntheticDataPathGrabber) throws MalformedURLException {
    Bucket bucket = new Bucket(TestDataSetLocation.S3_BUCKET_TEST_DATA);
    // Build a DataSetManifest for the data to be uploaded.
    List<DataSetManifestEntry> manifestEntries = new LinkedList<>();
    for (SyntheticDataFile syntheticDataFile : SyntheticDataFile.values()) manifestEntries.add(new DataSetManifestEntry(syntheticDataPathGrabber.apply(syntheticDataFile).getFileName().toString(), syntheticDataFile.getRifFile().getFileType()));
    DataSetManifest manifest = new DataSetManifest(Instant.now(), 0, manifestEntries);
    // Upload the manifest and every file in it.
    PutObjectRequest manifestRequest = DataSetTestUtilities.createPutRequest(bucket, s3KeyPrefix, manifest);
    manifestRequest.setCannedAcl(CannedAccessControlList.PublicRead);
    s3Client.putObject(manifestRequest);
    LOGGER.info("Uploaded: manifest");
    for (SyntheticDataFile syntheticDataFile : SyntheticDataFile.values()) {
        DataSetManifestEntry manifestEntry = manifest.getEntries().stream().filter(e -> e.getName().equals(syntheticDataPathGrabber.apply(syntheticDataFile).getFileName().toString())).findFirst().get();
        PutObjectRequest fileRequest = DataSetTestUtilities.createPutRequest(bucket, s3KeyPrefix, manifest, manifestEntry, syntheticDataPathGrabber.apply(syntheticDataFile).toUri().toURL());
        fileRequest.setCannedAcl(CannedAccessControlList.PublicRead);
        s3Client.putObject(fileRequest);
        LOGGER.info("Uploaded: {}", syntheticDataFile.name());
    }
}