Examples with RifFilesProcessor gov.cms.bfd.pipeline.ccw.rif.extract.RifFilesProcessor used on opensource projects

Example 1 with RifFilesProcessor

use of gov.cms.bfd.pipeline.ccw.rif.extract.RifFilesProcessor in project beneficiary-fhir-data by CMSgov.

the class RifLoaderIT method loadSample.

/**
 * Runs {@link RifLoader} against the specified {@link StaticRifResourceGroup}.
 *
 * @param sampleName a human-friendly name that will be logged to identify the data load being
 *     kicked off here
 * @param options the {@link LoadAppOptions} to use
 * @param rifFilesEvent the {@link RifFilesEvent} to load
 * @return the number of RIF records that were loaded (as reported by the {@link RifLoader})
 */
private int loadSample(String sampleName, LoadAppOptions options, RifFilesEvent rifFilesEvent) {
    LOGGER.info("Loading RIF files: '{}'...", sampleName);
    // Create the processors that will handle each stage of the pipeline.
    RifFilesProcessor processor = new RifFilesProcessor();
    RifLoader loader = new RifLoader(options, PipelineTestUtils.get().getPipelineApplicationState());
    // Link up the pipeline and run it.
    LOGGER.info("Loading RIF records...");
    AtomicInteger failureCount = new AtomicInteger(0);
    AtomicInteger loadCount = new AtomicInteger(0);
    for (RifFileEvent rifFileEvent : rifFilesEvent.getFileEvents()) {
        RifFileRecords rifFileRecords = processor.produceRecords(rifFileEvent);
        loader.process(rifFileRecords, error -> {
            failureCount.incrementAndGet();
            LOGGER.warn("Record(s) failed to load.", error);
        }, result -> {
            loadCount.incrementAndGet();
        });
        Slf4jReporter.forRegistry(rifFileEvent.getEventMetrics()).outputTo(LOGGER).build().report();
    }
    LOGGER.info("Loaded RIF files: '{}', record count: '{}'.", sampleName, loadCount.get());
    Slf4jReporter.forRegistry(PipelineTestUtils.get().getPipelineApplicationState().getMetrics()).outputTo(LOGGER).build().report();
    // Verify that the expected number of records were run successfully.
    assertEquals(0, failureCount.get(), "Load errors encountered.");
    return loadCount.get();
}

Example 2 with RifFilesProcessor

use of gov.cms.bfd.pipeline.ccw.rif.extract.RifFilesProcessor in project beneficiary-fhir-data by CMSgov.

the class PipelineApplication method createCcwRifLoadJob.

/**
 * @param loadOptions the {@link CcwRifLoadOptions} to use
 * @param appState the {@link PipelineApplicationState} to use
 * @return a {@link CcwRifLoadJob} instance for the application to use
 */
private static PipelineJob<?> createCcwRifLoadJob(CcwRifLoadOptions loadOptions, PipelineApplicationState appState) {
    /*
     * Create the services that will be used to handle each stage in the extract, transform, and
     * load process.
     */
    S3TaskManager s3TaskManager = new S3TaskManager(appState.getMetrics(), loadOptions.getExtractionOptions());
    RifFilesProcessor rifProcessor = new RifFilesProcessor();
    RifLoader rifLoader = new RifLoader(loadOptions.getLoadOptions(), appState);
    /*
     * Create the DataSetMonitorListener that will glue those stages together and run them all for
     * each data set that is found.
     */
    DataSetMonitorListener dataSetMonitorListener = new DefaultDataSetMonitorListener(appState.getMetrics(), PipelineApplication::handleUncaughtException, rifProcessor, rifLoader);
    CcwRifLoadJob ccwRifLoadJob = new CcwRifLoadJob(appState.getMetrics(), loadOptions.getExtractionOptions(), s3TaskManager, dataSetMonitorListener);
    return ccwRifLoadJob;
}

Example 3 with RifFilesProcessor

use of gov.cms.bfd.pipeline.ccw.rif.extract.RifFilesProcessor in project beneficiary-fhir-data by CMSgov.

the class LoadedFilterManagerIT method loadData.

/**
 * @param sampleResources the sample RIF resources to load
 */
private static void loadData(DataSource dataSource, List<StaticRifResource> sampleResources) {
    LoadAppOptions loadOptions = CcwRifLoadTestUtils.getLoadOptions();
    RifFilesEvent rifFilesEvent = new RifFilesEvent(Instant.now(), sampleResources.stream().map(StaticRifResource::toRifFile).collect(Collectors.toList()));
    // Create the processors that will handle each stage of the pipeline.
    RifFilesProcessor processor = new RifFilesProcessor();
    RifLoader loader = new RifLoader(loadOptions, PipelineTestUtils.get().getPipelineApplicationState());
    // Link up the pipeline and run it.
    for (RifFileEvent rifFileEvent : rifFilesEvent.getFileEvents()) {
        RifFileRecords rifFileRecords = processor.produceRecords(rifFileEvent);
        loader.process(rifFileRecords, error -> {
        }, result -> {
        });
    }
}

Example 4 with RifFilesProcessor

use of gov.cms.bfd.pipeline.ccw.rif.extract.RifFilesProcessor in project beneficiary-fhir-data by CMSgov.

the class ServerTestUtils method loadData.

/**
 * @param sampleResources the sample RIF resources to load
 * @return the {@link List} of RIF records that were loaded (e.g. {@link Beneficiary}s, etc.)
 */
public List<Object> loadData(List<StaticRifResource> sampleResources) {
    LoadAppOptions loadOptions = CcwRifLoadTestUtils.getLoadOptions();
    RifFilesEvent rifFilesEvent = new RifFilesEvent(Instant.now(), sampleResources.stream().map(r -> r.toRifFile()).collect(Collectors.toList()));
    // Create the processors that will handle each stage of the pipeline.
    RifFilesProcessor processor = new RifFilesProcessor();
    // Link up the pipeline and run it.
    RifLoader loader = new RifLoader(loadOptions, PipelineTestUtils.get().getPipelineApplicationState());
    LOGGER.info("Loading RIF records...");
    List<Object> recordsLoaded = new ArrayList<>();
    for (RifFileEvent rifFileEvent : rifFilesEvent.getFileEvents()) {
        RifFileRecords rifFileRecords = processor.produceRecords(rifFileEvent);
        loader.process(rifFileRecords, error -> {
            LOGGER.warn("Record(s) failed to load.", error);
        }, result -> {
            recordsLoaded.add(result.getRifRecordEvent().getRecord());
        });
    }
    LOGGER.info("Loaded RIF records: '{}'.", recordsLoaded.size());
    return recordsLoaded;
}

Example 5 with RifFilesProcessor

use of gov.cms.bfd.pipeline.ccw.rif.extract.RifFilesProcessor in project beneficiary-fhir-data by CMSgov.

the class ServerTestUtils method parseData.

/**
 * @param sampleResources the sample RIF resources to parse
 * @return the {@link List} of RIF records that were parsed (e.g. {@link Beneficiary}s, etc.)
 */
public static List<Object> parseData(List<StaticRifResource> sampleResources) {
    RifFilesEvent rifFilesEvent = new RifFilesEvent(Instant.now(), sampleResources.stream().map(r -> r.toRifFile()).collect(Collectors.toList()));
    RifFilesProcessor processor = new RifFilesProcessor();
    List<Object> recordsParsed = new ArrayList<>();
    for (RifFileEvent rifFileEvent : rifFilesEvent.getFileEvents()) {
        RifFileRecords rifFileRecords = processor.produceRecords(rifFileEvent);
        rifFileRecords.getRecords().map(r -> r.getRecord()).forEach(r -> recordsParsed.add(r));
    }
    return recordsParsed;
}