Examples with RifFileRecords gov.cms.bfd.model.rif.RifFileRecords used on opensource projects

Example 1 with RifFileRecords

use of gov.cms.bfd.model.rif.RifFileRecords in project beneficiary-fhir-data by CMSgov.

the class RifLoaderIT method loadSample.

/**
 * Runs {@link RifLoader} against the specified {@link StaticRifResourceGroup}.
 *
 * @param sampleName a human-friendly name that will be logged to identify the data load being
 *     kicked off here
 * @param options the {@link LoadAppOptions} to use
 * @param rifFilesEvent the {@link RifFilesEvent} to load
 * @return the number of RIF records that were loaded (as reported by the {@link RifLoader})
 */
private int loadSample(String sampleName, LoadAppOptions options, RifFilesEvent rifFilesEvent) {
    LOGGER.info("Loading RIF files: '{}'...", sampleName);
    // Create the processors that will handle each stage of the pipeline.
    RifFilesProcessor processor = new RifFilesProcessor();
    RifLoader loader = new RifLoader(options, PipelineTestUtils.get().getPipelineApplicationState());
    // Link up the pipeline and run it.
    LOGGER.info("Loading RIF records...");
    AtomicInteger failureCount = new AtomicInteger(0);
    AtomicInteger loadCount = new AtomicInteger(0);
    for (RifFileEvent rifFileEvent : rifFilesEvent.getFileEvents()) {
        RifFileRecords rifFileRecords = processor.produceRecords(rifFileEvent);
        loader.process(rifFileRecords, error -> {
            failureCount.incrementAndGet();
            LOGGER.warn("Record(s) failed to load.", error);
        }, result -> {
            loadCount.incrementAndGet();
        });
        Slf4jReporter.forRegistry(rifFileEvent.getEventMetrics()).outputTo(LOGGER).build().report();
    }
    LOGGER.info("Loaded RIF files: '{}', record count: '{}'.", sampleName, loadCount.get());
    Slf4jReporter.forRegistry(PipelineTestUtils.get().getPipelineApplicationState().getMetrics()).outputTo(LOGGER).build().report();
    // Verify that the expected number of records were run successfully.
    assertEquals(0, failureCount.get(), "Load errors encountered.");
    return loadCount.get();
}

Example 2 with RifFileRecords

use of gov.cms.bfd.model.rif.RifFileRecords in project beneficiary-fhir-data by CMSgov.

the class RifFilesProcessor method produceRecords.

/**
 * @param rifFileEvent the {@link RifFileEvent} that is being processed
 * @return a {@link RifFileRecords} with the {@link RifRecordEvent}s produced from the specified
 *     {@link RifFileEvent}
 */
public RifFileRecords produceRecords(RifFileEvent rifFileEvent) {
    RifFile file = rifFileEvent.getFile();
    /*
     * Approach used here to parse CSV as a Java 8 Stream is courtesy of
     * https://rumianom.pl/rumianom/entry/apache-commons-csv-with-java.
     */
    CSVParser parser = RifParsingUtils.createCsvParser(file);
    boolean isGrouped;
    BiFunction<RifFileEvent, List<CSVRecord>, RifRecordEvent<?>> recordParser;
    if (file.getFileType() == RifFileType.BENEFICIARY) {
        isGrouped = false;
        recordParser = RifFilesProcessor::buildBeneficiaryEvent;
    } else if (file.getFileType() == RifFileType.BENEFICIARY_HISTORY) {
        isGrouped = false;
        recordParser = RifFilesProcessor::buildBeneficiaryHistoryEvent;
    } else if (file.getFileType() == RifFileType.MEDICARE_BENEFICIARY_ID_HISTORY) {
        isGrouped = false;
        recordParser = RifFilesProcessor::buildMedicareBeneficiaryIdHistoryEvent;
    } else if (file.getFileType() == RifFileType.PDE) {
        isGrouped = false;
        recordParser = RifFilesProcessor::buildPartDEvent;
    } else if (file.getFileType() == RifFileType.CARRIER) {
        isGrouped = true;
        recordParser = RifFilesProcessor::buildCarrierClaimEvent;
    } else if (file.getFileType() == RifFileType.INPATIENT) {
        isGrouped = true;
        recordParser = RifFilesProcessor::buildInpatientClaimEvent;
    } else if (file.getFileType() == RifFileType.OUTPATIENT) {
        isGrouped = true;
        recordParser = RifFilesProcessor::buildOutpatientClaimEvent;
    } else if (file.getFileType() == RifFileType.SNF) {
        isGrouped = true;
        recordParser = RifFilesProcessor::buildSNFClaimEvent;
    } else if (file.getFileType() == RifFileType.HOSPICE) {
        isGrouped = true;
        recordParser = RifFilesProcessor::buildHospiceClaimEvent;
    } else if (file.getFileType() == RifFileType.HHA) {
        isGrouped = true;
        recordParser = RifFilesProcessor::buildHHAClaimEvent;
    } else if (file.getFileType() == RifFileType.DME) {
        isGrouped = true;
        recordParser = RifFilesProcessor::buildDMEClaimEvent;
    } else {
        throw new UnsupportedRifFileTypeException("Unsupported file type:" + file.getFileType());
    }
    /*
     * Use the CSVParser to drive a Stream of grouped CSVRecords
     * (specifically, group by claim ID/lines).
     */
    CsvRecordGrouper grouper = new ColumnValueCsvRecordGrouper(isGrouped ? file.getFileType().getIdColumn() : null);
    Iterator<List<CSVRecord>> csvIterator = new CsvRecordGroupingIterator(parser, grouper);
    Spliterator<List<CSVRecord>> spliterator = Spliterators.spliteratorUnknownSize(csvIterator, Spliterator.ORDERED | Spliterator.NONNULL);
    Stream<List<CSVRecord>> csvRecordStream = StreamSupport.stream(spliterator, false).onClose(() -> {
        try {
            /*
                     * This will also close the Reader and InputStream that the
                     * CSVParser was consuming.
                     */
            parser.close();
        } catch (IOException e) {
            LOGGER.warn("Unable to close CSVParser", e);
        }
    });
    /* Map each record group to a single RifRecordEvent. */
    Stream<RifRecordEvent<?>> rifRecordStream = csvRecordStream.map(csvRecordGroup -> {
        try {
            Timer.Context parsingTimer = rifFileEvent.getEventMetrics().timer(MetricRegistry.name(getClass().getSimpleName(), "recordParsing")).time();
            RifRecordEvent<?> recordEvent = recordParser.apply(rifFileEvent, csvRecordGroup);
            parsingTimer.close();
            return recordEvent;
        } catch (InvalidRifValueException e) {
            LOGGER.warn("Parse error encountered near line number '{}'.", csvRecordGroup.get(0).getRecordNumber());
            throw new InvalidRifValueException(e);
        }
    });
    return new RifFileRecords(rifFileEvent, rifRecordStream);
}

Example 3 with RifFileRecords

use of gov.cms.bfd.model.rif.RifFileRecords in project beneficiary-fhir-data by CMSgov.

the class RifFilesProcessorTest method processBeneficiaryHistoryRecord_SAMPLE_A.

/**
 * Ensures that {@link RifFilesProcessor} can correctly handle {@link
 * StaticRifResource#SAMPLE_A_BENEFICIARY_HISTORY}.
 */
@Test
public void processBeneficiaryHistoryRecord_SAMPLE_A() {
    RifFilesEvent filesEvent = new RifFilesEvent(Instant.now(), StaticRifResource.SAMPLE_A_BENEFICIARY_HISTORY.toRifFile());
    RifFilesProcessor processor = new RifFilesProcessor();
    RifFileRecords rifFileRecords = processor.produceRecords(filesEvent.getFileEvents().get(0));
    List<RifRecordEvent<?>> rifEventsList = rifFileRecords.getRecords().collect(Collectors.toList());
    assertEquals(StaticRifResource.SAMPLE_A_BENEFICIARY_HISTORY.getRecordCount(), rifEventsList.size());
    RifRecordEvent<?> rifRecordEvent0 = rifEventsList.get(0);
    assertEquals(StaticRifResource.SAMPLE_A_BENEFICIARY_HISTORY.getRifFileType(), rifRecordEvent0.getFileEvent().getFile().getFileType());
    assertNotNull(rifRecordEvent0.getRecord());
    assertTrue(rifRecordEvent0.getRecord() instanceof BeneficiaryHistory);
    BeneficiaryHistory beneficiaryHistory0 = (BeneficiaryHistory) rifRecordEvent0.getRecord();
    assertEquals(beneficiaryHistory0.getBeneficiaryId(), rifRecordEvent0.getBeneficiaryId());
    assertEquals(RecordAction.INSERT, rifRecordEvent0.getRecordAction());
    assertEquals("567834", beneficiaryHistory0.getBeneficiaryId());
    assertEquals(LocalDate.of(1979, Month.MARCH, 17), beneficiaryHistory0.getBirthDate());
    assertEquals(('2'), beneficiaryHistory0.getSex());
    assertEquals("543217066Z", beneficiaryHistory0.getHicn());
    assertEquals(Optional.of("3456689"), beneficiaryHistory0.getMedicareBeneficiaryId());
    assertEquals(LocalDate.of(1990, Month.MARCH, 17), beneficiaryHistory0.getMbiEffectiveDate().get());
    assertEquals(LocalDate.of(1995, Month.MARCH, 17), beneficiaryHistory0.getMbiObsoleteDate().get());
    /*
     * We should expect and be able to cope with BENEFICIARY_HISTORY records that
     * are exact duplicates.
     */
    for (RifRecordEvent<?> rifRecordEvent : new RifRecordEvent<?>[] { rifEventsList.get(1), rifEventsList.get(2) }) {
        assertEquals(StaticRifResource.SAMPLE_A_BENEFICIARY_HISTORY.getRifFileType(), rifRecordEvent.getFileEvent().getFile().getFileType());
        assertNotNull(rifRecordEvent.getRecord());
        assertTrue(rifRecordEvent.getRecord() instanceof BeneficiaryHistory);
        BeneficiaryHistory beneficiaryHistory = (BeneficiaryHistory) rifRecordEvent.getRecord();
        assertEquals(RecordAction.INSERT, rifRecordEvent.getRecordAction());
        assertEquals("567834", beneficiaryHistory.getBeneficiaryId());
        assertEquals(LocalDate.of(1980, Month.MARCH, 17), beneficiaryHistory.getBirthDate());
        assertEquals(('1'), beneficiaryHistory.getSex());
        assertEquals("543217066T", beneficiaryHistory.getHicn());
        assertEquals(Optional.of("3456789"), beneficiaryHistory.getMedicareBeneficiaryId());
        assertEquals(LocalDate.of(1990, Month.MARCH, 17), beneficiaryHistory0.getMbiEffectiveDate().get());
        assertEquals(LocalDate.of(1995, Month.MARCH, 17), beneficiaryHistory0.getMbiObsoleteDate().get());
    }
}

Example 4 with RifFileRecords

use of gov.cms.bfd.model.rif.RifFileRecords in project beneficiary-fhir-data by CMSgov.

the class RifFilesProcessorTest method process1MedicareBeneficiaryIdHistoryRecord.

/**
 * Ensures that {@link RifFilesProcessor} can correctly handle {@link
 * StaticRifResource#SAMPLE_A_MEDICARE_BENEFICIARY_ID_HISTORY}.
 */
@Test
public void process1MedicareBeneficiaryIdHistoryRecord() {
    RifFilesEvent filesEvent = new RifFilesEvent(Instant.now(), StaticRifResource.SAMPLE_A_MEDICARE_BENEFICIARY_ID_HISTORY.toRifFile());
    RifFilesProcessor processor = new RifFilesProcessor();
    RifFileRecords rifFileRecords = processor.produceRecords(filesEvent.getFileEvents().get(0));
    List<RifRecordEvent<?>> rifEventsList = rifFileRecords.getRecords().collect(Collectors.toList());
    assertEquals(StaticRifResource.SAMPLE_A_MEDICARE_BENEFICIARY_ID_HISTORY.getRecordCount(), rifEventsList.size());
    RifRecordEvent<?> rifRecordEvent0 = rifEventsList.get(0);
    assertEquals(StaticRifResource.SAMPLE_A_MEDICARE_BENEFICIARY_ID_HISTORY.getRifFileType(), rifRecordEvent0.getFileEvent().getFile().getFileType());
    assertNotNull(rifRecordEvent0.getRecord());
    assertTrue(rifRecordEvent0.getRecord() instanceof MedicareBeneficiaryIdHistory);
    MedicareBeneficiaryIdHistory medicareBeneficiaryIdHistory = (MedicareBeneficiaryIdHistory) rifRecordEvent0.getRecord();
    assertEquals("567834", medicareBeneficiaryIdHistory.getBeneficiaryId().get());
    assertEquals(LocalDate.of(2011, Month.APRIL, 16), medicareBeneficiaryIdHistory.getMbiEffectiveDate().get());
    assertEquals("9AB2WW3GR44", medicareBeneficiaryIdHistory.getMedicareBeneficiaryId().get());
}

Example 5 with RifFileRecords

use of gov.cms.bfd.model.rif.RifFileRecords in project beneficiary-fhir-data by CMSgov.

the class RifFilesProcessorTest method process1BeneRecord.

/**
 * Ensures that {@link RifFilesProcessor} can correctly handle {@link
 * StaticRifResource#SAMPLE_A_BENES}.
 */
@Test
public void process1BeneRecord() {
    RifFilesEvent filesEvent = new RifFilesEvent(Instant.now(), StaticRifResource.SAMPLE_A_BENES.toRifFile());
    RifFilesProcessor processor = new RifFilesProcessor();
    RifFileRecords rifFileRecords = processor.produceRecords(filesEvent.getFileEvents().get(0));
    List<RifRecordEvent<?>> rifEventsList = rifFileRecords.getRecords().collect(Collectors.toList());
    assertEquals(StaticRifResource.SAMPLE_A_BENES.getRecordCount(), rifEventsList.size());
    RifRecordEvent<?> rifRecordEvent = rifEventsList.get(0);
    assertEquals(StaticRifResource.SAMPLE_A_BENES.getRifFileType(), rifRecordEvent.getFileEvent().getFile().getFileType());
    assertNotNull(rifRecordEvent.getRecord());
    assertTrue(rifRecordEvent.getRecord() instanceof Beneficiary);
    Beneficiary beneRow = (Beneficiary) rifRecordEvent.getRecord();
    assertEquals(beneRow.getBeneficiaryId(), rifRecordEvent.getBeneficiaryId());
    assertEquals(RecordAction.INSERT, rifRecordEvent.getRecordAction());
    assertEquals("567834", beneRow.getBeneficiaryId());
    assertEquals("MO", beneRow.getStateCode());
    assertEquals("123", beneRow.getCountyCode());
    assertEquals("12345", beneRow.getPostalCode());
    assertEquals(LocalDate.of(1981, Month.MARCH, 17), beneRow.getBirthDate());
    assertEquals(('1'), beneRow.getSex());
    assertEquals(new Character('1'), beneRow.getRace().get());
    assertEquals(new Character('1'), beneRow.getEntitlementCodeOriginal().get());
    assertEquals(new Character('1'), beneRow.getEntitlementCodeCurrent().get());
    assertEquals(new Character('N'), beneRow.getEndStageRenalDiseaseCode().get());
    assertEquals(new String("20"), beneRow.getMedicareEnrollmentStatusCode().get());
    assertEquals(new Character('0'), beneRow.getPartBTerminationCode().get());
    assertEquals(new Character('0'), beneRow.getPartBTerminationCode().get());
    assertEquals("543217066U", beneRow.getHicnUnhashed().orElse(null));
    assertEquals("Doe", beneRow.getNameSurname());
    assertEquals("John", beneRow.getNameGiven());
    assertEquals(new Character('A'), beneRow.getNameMiddleInitial().get());
    assertEquals("3456789", beneRow.getMedicareBeneficiaryId().get());
    assertEquals(LocalDate.of(1981, Month.MARCH, 17), beneRow.getBeneficiaryDateOfDeath().get());
    assertEquals(LocalDate.of(1963, Month.OCTOBER, 3), beneRow.getMedicareCoverageStartDate().get());
    assertEquals(new Character('1'), beneRow.getHmoIndicatorAprInd().get());
    assertEquals(new BigDecimal(5), beneRow.getPartDMonthsCount().get());
    assertEquals("00", beneRow.getPartDLowIncomeCostShareGroupFebCode().get());
    assertEquals(new Character('N'), beneRow.getPartDRetireeDrugSubsidyDecInd().get());
    assertEquals("204 SOUTH ST", beneRow.getDerivedMailingAddress1().get());
    assertEquals("7560 123TH ST", beneRow.getDerivedMailingAddress2().get());
    assertEquals("SURREY", beneRow.getDerivedMailingAddress3().get());
    assertEquals("DAEJEON SI 34867", beneRow.getDerivedMailingAddress4().get());
    assertEquals("COLOMBIA", beneRow.getDerivedMailingAddress5().get());
    assertEquals("SURREY", beneRow.getDerivedMailingAddress6().get());
    assertEquals("PODUNK", beneRow.getDerivedCityName().get());
    assertEquals("IA", beneRow.getDerivedStateCode().get());
    assertEquals("123456789", beneRow.getDerivedZipCode().get());
    assertEquals(LocalDate.of(2020, Month.JULY, 30), beneRow.getMbiEffectiveDate().get());
    assertEquals(new BigDecimal("1"), beneRow.getBeneLinkKey().get());
}