Search in sources :

Example 11 with RifRecordEvent

use of gov.cms.bfd.model.rif.RifRecordEvent in project beneficiary-fhir-data by CMSgov.

the class RifLoader method process.

/**
 * Consumes the input {@link Stream} of {@link RifRecordEvent}s, pushing each {@link
 * RifRecordEvent}'s record to the database, and passing the result for each of those bundles to
 * the specified error handler and result handler, as appropriate.
 *
 * <p>This is a <a href=
 * "https://docs.oracle.com/javase/8/docs/api/java/util/stream/package-summary.html#StreamOps">
 * terminal operation</a>.
 *
 * @param dataToLoad the FHIR {@link RifRecordEvent}s to be loaded
 * @param errorHandler the {@link Consumer} to pass each error that occurs to (possibly one error
 *     per {@link RifRecordEvent}, if every input element fails to load), which will be run on the
 *     caller's thread
 * @param resultHandler the {@link Consumer} to pass each the {@link RifRecordLoadResult} for each
 *     of the successfully-processed input {@link RifRecordEvent}s, which will be run on the
 *     caller's thread
 */
public void process(RifFileRecords dataToLoad, Consumer<Throwable> errorHandler, Consumer<RifRecordLoadResult> resultHandler) {
    BlockingThreadPoolExecutor loadExecutor = createLoadExecutor(options);
    MetricRegistry fileEventMetrics = dataToLoad.getSourceEvent().getEventMetrics();
    Timer.Context timerDataSetFile = appState.getMetrics().timer(MetricRegistry.name(getClass().getSimpleName(), "dataSet", "file", "processed")).time();
    LOGGER.info("Processing '{}'...", dataToLoad);
    dataToLoad.getSourceEvent().getEventMetrics().register(MetricRegistry.name(getClass().getSimpleName(), "loadExecutorService", "queueSize"), new Gauge<Integer>() {

        /**
         * @see com.codahale.metrics.Gauge#getValue()
         */
        @Override
        public Integer getValue() {
            return loadExecutor.getQueue().size();
        }
    });
    dataToLoad.getSourceEvent().getEventMetrics().register(MetricRegistry.name(getClass().getSimpleName(), "loadExecutorService", "activeBatches"), new Gauge<Integer>() {

        /**
         * @see com.codahale.metrics.Gauge#getValue()
         */
        @Override
        public Integer getValue() {
            return loadExecutor.getActiveCount();
        }
    });
    // Trim the LoadedFiles & LoadedBatches table
    trimLoadedFiles(errorHandler);
    // Insert a LoadedFiles entry
    final long loadedFileId = insertLoadedFile(dataToLoad.getSourceEvent(), errorHandler);
    if (loadedFileId < 0) {
        // Something went wrong, the error handler was called.
        return;
    }
    try (PostgreSqlCopyInserter postgresBatch = new PostgreSqlCopyInserter(appState.getEntityManagerFactory(), fileEventMetrics)) {
        // Define the Consumer that will handle each batch.
        Consumer<List<RifRecordEvent<?>>> batchProcessor = recordsBatch -> {
            /*
             * Submit the RifRecordEvent for asynchronous processing. Note
             * that, due to the ExecutorService's configuration (see in
             * constructor), this will block if too many tasks are already
             * pending. That's desirable behavior, as it prevents
             * OutOfMemoryErrors.
             */
            processAsync(loadExecutor, recordsBatch, loadedFileId, postgresBatch, resultHandler, errorHandler);
        };
        // Collect records into batches and submit each to batchProcessor.
        if (RECORD_BATCH_SIZE > 1)
            BatchSpliterator.batches(dataToLoad.getRecords(), RECORD_BATCH_SIZE).forEach(batchProcessor);
        else
            dataToLoad.getRecords().map(record -> {
                List<RifRecordEvent<?>> ittyBittyBatch = new LinkedList<>();
                ittyBittyBatch.add(record);
                return ittyBittyBatch;
            }).forEach(batchProcessor);
        // Wait for all submitted batches to complete.
        try {
            loadExecutor.shutdown();
            boolean terminatedSuccessfully = loadExecutor.awaitTermination(72, TimeUnit.HOURS);
            if (!terminatedSuccessfully)
                throw new IllegalStateException(String.format("%s failed to complete processing the records in time: '%s'.", this.getClass().getSimpleName(), dataToLoad));
        } catch (InterruptedException e) {
            // Interrupts should not be used on this thread, so go boom.
            throw new RuntimeException(e);
        }
        // Submit the queued PostgreSQL COPY operations, if any.
        if (!postgresBatch.isEmpty()) {
            postgresBatch.submit();
        }
    }
    LOGGER.info("Processed '{}'.", dataToLoad);
    timerDataSetFile.stop();
    logRecordCounts();
}
Also used : Arrays(java.util.Arrays) Connection(java.sql.Connection) RifFileType(gov.cms.bfd.model.rif.RifFileType) CSVRecord(org.apache.commons.csv.CSVRecord) LoggerFactory(org.slf4j.LoggerFactory) LoadedBatchBuilder(gov.cms.bfd.model.rif.LoadedBatchBuilder) IdHasher(gov.cms.bfd.pipeline.sharedutils.IdHasher) BigDecimal(java.math.BigDecimal) CSVFormat(org.apache.commons.csv.CSVFormat) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) CarrierClaimLine(gov.cms.bfd.model.rif.CarrierClaimLine) Map(java.util.Map) CriteriaBuilder(javax.persistence.criteria.CriteriaBuilder) JoinType(javax.persistence.criteria.JoinType) BaseConnection(org.postgresql.core.BaseConnection) BeneficiaryHistory(gov.cms.bfd.model.rif.BeneficiaryHistory) PipelineApplicationState(gov.cms.bfd.pipeline.sharedutils.PipelineApplicationState) HikariProxyConnection(com.zaxxer.hikari.pool.HikariProxyConnection) Entity(javax.persistence.Entity) CriteriaQuery(javax.persistence.criteria.CriteriaQuery) RifFileRecords(gov.cms.bfd.model.rif.RifFileRecords) LoadAction(gov.cms.bfd.pipeline.ccw.rif.load.RifRecordLoadResult.LoadAction) Instant(java.time.Instant) CarrierClaimCsvWriter(gov.cms.bfd.model.rif.CarrierClaimCsvWriter) Collectors(java.util.stream.Collectors) BeneficiaryMonthly(gov.cms.bfd.model.rif.BeneficiaryMonthly) RifFilesEvent(gov.cms.bfd.model.rif.RifFilesEvent) UncheckedIOException(java.io.UncheckedIOException) Objects(java.util.Objects) Beneficiary(gov.cms.bfd.model.rif.Beneficiary) LoadedBatch(gov.cms.bfd.model.rif.LoadedBatch) List(java.util.List) Stream(java.util.stream.Stream) SkipReasonCode(gov.cms.bfd.model.rif.SkippedRifRecord.SkipReasonCode) EntityManagerFactory(javax.persistence.EntityManagerFactory) CarrierClaim(gov.cms.bfd.model.rif.CarrierClaim) LocalDate(java.time.LocalDate) LoadedFile(gov.cms.bfd.model.rif.LoadedFile) Entry(java.util.Map.Entry) Timer(com.codahale.metrics.Timer) Optional(java.util.Optional) Gauge(com.codahale.metrics.Gauge) RifParsingUtils(gov.cms.bfd.model.rif.parse.RifParsingUtils) RifRecordEvent(gov.cms.bfd.model.rif.RifRecordEvent) CSVPrinter(org.apache.commons.csv.CSVPrinter) SkippedRifRecord(gov.cms.bfd.model.rif.SkippedRifRecord) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Session(org.hibernate.Session) BeneficiaryCsvWriter(gov.cms.bfd.model.rif.BeneficiaryCsvWriter) Beneficiary_(gov.cms.bfd.model.rif.Beneficiary_) ArrayList(java.util.ArrayList) SQLException(java.sql.SQLException) Table(javax.persistence.Table) CopyManager(org.postgresql.copy.CopyManager) LinkedList(java.util.LinkedList) Root(javax.persistence.criteria.Root) Period(java.time.Period) RifFileEvent(gov.cms.bfd.model.rif.RifFileEvent) MetricRegistry(com.codahale.metrics.MetricRegistry) Logger(org.slf4j.Logger) RecordAction(gov.cms.bfd.model.rif.RecordAction) BadCodeMonkeyException(gov.cms.bfd.sharedutils.exceptions.BadCodeMonkeyException) FileWriter(java.io.FileWriter) RifRecordBase(gov.cms.bfd.model.rif.RifRecordBase) IOException(java.io.IOException) EntityManager(javax.persistence.EntityManager) File(java.io.File) Work(org.hibernate.jdbc.Work) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) EntityTransaction(javax.persistence.EntityTransaction) FileReader(java.io.FileReader) Comparator(java.util.Comparator) MetricRegistry(com.codahale.metrics.MetricRegistry) RifRecordEvent(gov.cms.bfd.model.rif.RifRecordEvent) LinkedList(java.util.LinkedList) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Timer(com.codahale.metrics.Timer) List(java.util.List) ArrayList(java.util.ArrayList) LinkedList(java.util.LinkedList)

Example 12 with RifRecordEvent

use of gov.cms.bfd.model.rif.RifRecordEvent in project beneficiary-fhir-data by CMSgov.

the class RifLoader method updateBeneficiaryMonthly.

/**
 * Ensures that a {@link BeneficiaryMonthly} record is created or updated for the specified {@link
 * Beneficiary}, if that {@link Beneficiary} already exists and is just being updated.
 *
 * @param newBeneficiaryRecord the {@link Beneficiary} record being processed
 * @param oldBeneficiaryRecord the previous/current version of the {@link Beneficiary} (as it
 *     exists in the database before applying the specified {@link RifRecordEvent})
 */
private static void updateBeneficiaryMonthly(Beneficiary newBeneficiaryRecord, Optional<Beneficiary> oldBeneficiaryRecord) {
    if (newBeneficiaryRecord.getBeneEnrollmentReferenceYear().isPresent()) {
        int year = newBeneficiaryRecord.getBeneEnrollmentReferenceYear().get().intValue();
        List<BeneficiaryMonthly> currentYearBeneficiaryMonthly = new ArrayList<BeneficiaryMonthly>();
        BeneficiaryMonthly beneficiaryMonthly = getBeneficiaryMonthly(newBeneficiaryRecord, LocalDate.of(year, 1, 1), newBeneficiaryRecord.getEntitlementBuyInJanInd(), newBeneficiaryRecord.getFipsStateCntyJanCode(), newBeneficiaryRecord.getHmoIndicatorJanInd(), newBeneficiaryRecord.getMedicaidDualEligibilityJanCode(), newBeneficiaryRecord.getMedicareStatusJanCode(), newBeneficiaryRecord.getPartCContractNumberJanId(), newBeneficiaryRecord.getPartCPbpNumberJanId(), newBeneficiaryRecord.getPartCPlanTypeJanCode(), newBeneficiaryRecord.getPartDContractNumberJanId(), newBeneficiaryRecord.getPartDLowIncomeCostShareGroupJanCode(), newBeneficiaryRecord.getPartDPbpNumberJanId(), newBeneficiaryRecord.getPartDRetireeDrugSubsidyJanInd(), newBeneficiaryRecord.getPartDSegmentNumberJanId());
        if (beneficiaryMonthly != null) {
            currentYearBeneficiaryMonthly.add(beneficiaryMonthly);
        }
        beneficiaryMonthly = getBeneficiaryMonthly(newBeneficiaryRecord, LocalDate.of(year, 2, 1), newBeneficiaryRecord.getEntitlementBuyInFebInd(), newBeneficiaryRecord.getFipsStateCntyFebCode(), newBeneficiaryRecord.getHmoIndicatorFebInd(), newBeneficiaryRecord.getMedicaidDualEligibilityFebCode(), newBeneficiaryRecord.getMedicareStatusFebCode(), newBeneficiaryRecord.getPartCContractNumberFebId(), newBeneficiaryRecord.getPartCPbpNumberFebId(), newBeneficiaryRecord.getPartCPlanTypeFebCode(), newBeneficiaryRecord.getPartDContractNumberFebId(), newBeneficiaryRecord.getPartDLowIncomeCostShareGroupFebCode(), newBeneficiaryRecord.getPartDPbpNumberFebId(), newBeneficiaryRecord.getPartDRetireeDrugSubsidyFebInd(), newBeneficiaryRecord.getPartDSegmentNumberFebId());
        if (beneficiaryMonthly != null) {
            currentYearBeneficiaryMonthly.add(beneficiaryMonthly);
        }
        beneficiaryMonthly = getBeneficiaryMonthly(newBeneficiaryRecord, LocalDate.of(year, 3, 1), newBeneficiaryRecord.getEntitlementBuyInMarInd(), newBeneficiaryRecord.getFipsStateCntyMarCode(), newBeneficiaryRecord.getHmoIndicatorMarInd(), newBeneficiaryRecord.getMedicaidDualEligibilityMarCode(), newBeneficiaryRecord.getMedicareStatusMarCode(), newBeneficiaryRecord.getPartCContractNumberMarId(), newBeneficiaryRecord.getPartCPbpNumberMarId(), newBeneficiaryRecord.getPartCPlanTypeMarCode(), newBeneficiaryRecord.getPartDContractNumberMarId(), newBeneficiaryRecord.getPartDLowIncomeCostShareGroupMarCode(), newBeneficiaryRecord.getPartDPbpNumberMarId(), newBeneficiaryRecord.getPartDRetireeDrugSubsidyMarInd(), newBeneficiaryRecord.getPartDSegmentNumberMarId());
        if (beneficiaryMonthly != null) {
            currentYearBeneficiaryMonthly.add(beneficiaryMonthly);
        }
        beneficiaryMonthly = getBeneficiaryMonthly(newBeneficiaryRecord, LocalDate.of(year, 4, 1), newBeneficiaryRecord.getEntitlementBuyInAprInd(), newBeneficiaryRecord.getFipsStateCntyAprCode(), newBeneficiaryRecord.getHmoIndicatorAprInd(), newBeneficiaryRecord.getMedicaidDualEligibilityAprCode(), newBeneficiaryRecord.getMedicareStatusAprCode(), newBeneficiaryRecord.getPartCContractNumberAprId(), newBeneficiaryRecord.getPartCPbpNumberAprId(), newBeneficiaryRecord.getPartCPlanTypeAprCode(), newBeneficiaryRecord.getPartDContractNumberAprId(), newBeneficiaryRecord.getPartDLowIncomeCostShareGroupAprCode(), newBeneficiaryRecord.getPartDPbpNumberAprId(), newBeneficiaryRecord.getPartDRetireeDrugSubsidyAprInd(), newBeneficiaryRecord.getPartDSegmentNumberAprId());
        if (beneficiaryMonthly != null) {
            currentYearBeneficiaryMonthly.add(beneficiaryMonthly);
        }
        beneficiaryMonthly = getBeneficiaryMonthly(newBeneficiaryRecord, LocalDate.of(year, 5, 1), newBeneficiaryRecord.getEntitlementBuyInMayInd(), newBeneficiaryRecord.getFipsStateCntyMayCode(), newBeneficiaryRecord.getHmoIndicatorMayInd(), newBeneficiaryRecord.getMedicaidDualEligibilityMayCode(), newBeneficiaryRecord.getMedicareStatusMayCode(), newBeneficiaryRecord.getPartCContractNumberMayId(), newBeneficiaryRecord.getPartCPbpNumberMayId(), newBeneficiaryRecord.getPartCPlanTypeMayCode(), newBeneficiaryRecord.getPartDContractNumberMayId(), newBeneficiaryRecord.getPartDLowIncomeCostShareGroupMayCode(), newBeneficiaryRecord.getPartDPbpNumberMayId(), newBeneficiaryRecord.getPartDRetireeDrugSubsidyMayInd(), newBeneficiaryRecord.getPartDSegmentNumberMayId());
        if (beneficiaryMonthly != null) {
            currentYearBeneficiaryMonthly.add(beneficiaryMonthly);
        }
        beneficiaryMonthly = getBeneficiaryMonthly(newBeneficiaryRecord, LocalDate.of(year, 6, 1), newBeneficiaryRecord.getEntitlementBuyInJunInd(), newBeneficiaryRecord.getFipsStateCntyJunCode(), newBeneficiaryRecord.getHmoIndicatorJunInd(), newBeneficiaryRecord.getMedicaidDualEligibilityJunCode(), newBeneficiaryRecord.getMedicareStatusJunCode(), newBeneficiaryRecord.getPartCContractNumberJunId(), newBeneficiaryRecord.getPartCPbpNumberJunId(), newBeneficiaryRecord.getPartCPlanTypeJunCode(), newBeneficiaryRecord.getPartDContractNumberJunId(), newBeneficiaryRecord.getPartDLowIncomeCostShareGroupJunCode(), newBeneficiaryRecord.getPartDPbpNumberJunId(), newBeneficiaryRecord.getPartDRetireeDrugSubsidyJunInd(), newBeneficiaryRecord.getPartDSegmentNumberJunId());
        if (beneficiaryMonthly != null) {
            currentYearBeneficiaryMonthly.add(beneficiaryMonthly);
        }
        beneficiaryMonthly = getBeneficiaryMonthly(newBeneficiaryRecord, LocalDate.of(year, 7, 1), newBeneficiaryRecord.getEntitlementBuyInJulInd(), newBeneficiaryRecord.getFipsStateCntyJulCode(), newBeneficiaryRecord.getHmoIndicatorJulInd(), newBeneficiaryRecord.getMedicaidDualEligibilityJulCode(), newBeneficiaryRecord.getMedicareStatusJulCode(), newBeneficiaryRecord.getPartCContractNumberJulId(), newBeneficiaryRecord.getPartCPbpNumberJulId(), newBeneficiaryRecord.getPartCPlanTypeJulCode(), newBeneficiaryRecord.getPartDContractNumberJulId(), newBeneficiaryRecord.getPartDLowIncomeCostShareGroupJulCode(), newBeneficiaryRecord.getPartDPbpNumberJulId(), newBeneficiaryRecord.getPartDRetireeDrugSubsidyJulInd(), newBeneficiaryRecord.getPartDSegmentNumberJulId());
        if (beneficiaryMonthly != null) {
            currentYearBeneficiaryMonthly.add(beneficiaryMonthly);
        }
        beneficiaryMonthly = getBeneficiaryMonthly(newBeneficiaryRecord, LocalDate.of(year, 8, 1), newBeneficiaryRecord.getEntitlementBuyInAugInd(), newBeneficiaryRecord.getFipsStateCntyAugCode(), newBeneficiaryRecord.getHmoIndicatorAugInd(), newBeneficiaryRecord.getMedicaidDualEligibilityAugCode(), newBeneficiaryRecord.getMedicareStatusAugCode(), newBeneficiaryRecord.getPartCContractNumberAugId(), newBeneficiaryRecord.getPartCPbpNumberAugId(), newBeneficiaryRecord.getPartCPlanTypeAugCode(), newBeneficiaryRecord.getPartDContractNumberAugId(), newBeneficiaryRecord.getPartDLowIncomeCostShareGroupAugCode(), newBeneficiaryRecord.getPartDPbpNumberAugId(), newBeneficiaryRecord.getPartDRetireeDrugSubsidyAugInd(), newBeneficiaryRecord.getPartDSegmentNumberAugId());
        if (beneficiaryMonthly != null) {
            currentYearBeneficiaryMonthly.add(beneficiaryMonthly);
        }
        beneficiaryMonthly = getBeneficiaryMonthly(newBeneficiaryRecord, LocalDate.of(year, 9, 1), newBeneficiaryRecord.getEntitlementBuyInSeptInd(), newBeneficiaryRecord.getFipsStateCntySeptCode(), newBeneficiaryRecord.getHmoIndicatorSeptInd(), newBeneficiaryRecord.getMedicaidDualEligibilitySeptCode(), newBeneficiaryRecord.getMedicareStatusSeptCode(), newBeneficiaryRecord.getPartCContractNumberSeptId(), newBeneficiaryRecord.getPartCPbpNumberSeptId(), newBeneficiaryRecord.getPartCPlanTypeSeptCode(), newBeneficiaryRecord.getPartDContractNumberSeptId(), newBeneficiaryRecord.getPartDLowIncomeCostShareGroupSeptCode(), newBeneficiaryRecord.getPartDPbpNumberSeptId(), newBeneficiaryRecord.getPartDRetireeDrugSubsidySeptInd(), newBeneficiaryRecord.getPartDSegmentNumberSeptId());
        if (beneficiaryMonthly != null) {
            currentYearBeneficiaryMonthly.add(beneficiaryMonthly);
        }
        beneficiaryMonthly = getBeneficiaryMonthly(newBeneficiaryRecord, LocalDate.of(year, 10, 1), newBeneficiaryRecord.getEntitlementBuyInOctInd(), newBeneficiaryRecord.getFipsStateCntyOctCode(), newBeneficiaryRecord.getHmoIndicatorOctInd(), newBeneficiaryRecord.getMedicaidDualEligibilityOctCode(), newBeneficiaryRecord.getMedicareStatusOctCode(), newBeneficiaryRecord.getPartCContractNumberOctId(), newBeneficiaryRecord.getPartCPbpNumberOctId(), newBeneficiaryRecord.getPartCPlanTypeOctCode(), newBeneficiaryRecord.getPartDContractNumberOctId(), newBeneficiaryRecord.getPartDLowIncomeCostShareGroupOctCode(), newBeneficiaryRecord.getPartDPbpNumberOctId(), newBeneficiaryRecord.getPartDRetireeDrugSubsidyOctInd(), newBeneficiaryRecord.getPartDSegmentNumberOctId());
        if (beneficiaryMonthly != null) {
            currentYearBeneficiaryMonthly.add(beneficiaryMonthly);
        }
        beneficiaryMonthly = getBeneficiaryMonthly(newBeneficiaryRecord, LocalDate.of(year, 11, 1), newBeneficiaryRecord.getEntitlementBuyInNovInd(), newBeneficiaryRecord.getFipsStateCntyNovCode(), newBeneficiaryRecord.getHmoIndicatorNovInd(), newBeneficiaryRecord.getMedicaidDualEligibilityNovCode(), newBeneficiaryRecord.getMedicareStatusNovCode(), newBeneficiaryRecord.getPartCContractNumberNovId(), newBeneficiaryRecord.getPartCPbpNumberNovId(), newBeneficiaryRecord.getPartCPlanTypeNovCode(), newBeneficiaryRecord.getPartDContractNumberNovId(), newBeneficiaryRecord.getPartDLowIncomeCostShareGroupNovCode(), newBeneficiaryRecord.getPartDPbpNumberNovId(), newBeneficiaryRecord.getPartDRetireeDrugSubsidyNovInd(), newBeneficiaryRecord.getPartDSegmentNumberNovId());
        if (beneficiaryMonthly != null) {
            currentYearBeneficiaryMonthly.add(beneficiaryMonthly);
        }
        beneficiaryMonthly = getBeneficiaryMonthly(newBeneficiaryRecord, LocalDate.of(year, 12, 1), newBeneficiaryRecord.getEntitlementBuyInDecInd(), newBeneficiaryRecord.getFipsStateCntyDecCode(), newBeneficiaryRecord.getHmoIndicatorDecInd(), newBeneficiaryRecord.getMedicaidDualEligibilityDecCode(), newBeneficiaryRecord.getMedicareStatusDecCode(), newBeneficiaryRecord.getPartCContractNumberDecId(), newBeneficiaryRecord.getPartCPbpNumberDecId(), newBeneficiaryRecord.getPartCPlanTypeDecCode(), newBeneficiaryRecord.getPartDContractNumberDecId(), newBeneficiaryRecord.getPartDLowIncomeCostShareGroupDecCode(), newBeneficiaryRecord.getPartDPbpNumberDecId(), newBeneficiaryRecord.getPartDRetireeDrugSubsidyDecInd(), newBeneficiaryRecord.getPartDSegmentNumberDecId());
        if (beneficiaryMonthly != null) {
            currentYearBeneficiaryMonthly.add(beneficiaryMonthly);
        }
        if (currentYearBeneficiaryMonthly.size() > 0) {
            List<BeneficiaryMonthly> currentBeneficiaryMonthlyWithUpdates;
            if (oldBeneficiaryRecord.isPresent() && oldBeneficiaryRecord.get().getBeneficiaryMonthlys().size() > 0) {
                currentBeneficiaryMonthlyWithUpdates = oldBeneficiaryRecord.get().getBeneficiaryMonthlys();
                List<BeneficiaryMonthly> currentYearBeneficiaryMonthlyPrevious = oldBeneficiaryRecord.get().getBeneficiaryMonthlys().stream().filter(e -> year == e.getYearMonth().getYear()).collect(Collectors.toList());
                for (BeneficiaryMonthly previousEnrollment : currentYearBeneficiaryMonthlyPrevious) {
                    currentBeneficiaryMonthlyWithUpdates.remove(previousEnrollment);
                }
            } else {
                currentBeneficiaryMonthlyWithUpdates = new LinkedList<BeneficiaryMonthly>();
            }
            currentBeneficiaryMonthlyWithUpdates.addAll(currentYearBeneficiaryMonthly);
            newBeneficiaryRecord.setBeneficiaryMonthlys(currentBeneficiaryMonthlyWithUpdates);
        }
    }
}
Also used : Arrays(java.util.Arrays) Connection(java.sql.Connection) RifFileType(gov.cms.bfd.model.rif.RifFileType) CSVRecord(org.apache.commons.csv.CSVRecord) LoggerFactory(org.slf4j.LoggerFactory) LoadedBatchBuilder(gov.cms.bfd.model.rif.LoadedBatchBuilder) IdHasher(gov.cms.bfd.pipeline.sharedutils.IdHasher) BigDecimal(java.math.BigDecimal) CSVFormat(org.apache.commons.csv.CSVFormat) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) CarrierClaimLine(gov.cms.bfd.model.rif.CarrierClaimLine) Map(java.util.Map) CriteriaBuilder(javax.persistence.criteria.CriteriaBuilder) JoinType(javax.persistence.criteria.JoinType) BaseConnection(org.postgresql.core.BaseConnection) BeneficiaryHistory(gov.cms.bfd.model.rif.BeneficiaryHistory) PipelineApplicationState(gov.cms.bfd.pipeline.sharedutils.PipelineApplicationState) HikariProxyConnection(com.zaxxer.hikari.pool.HikariProxyConnection) Entity(javax.persistence.Entity) CriteriaQuery(javax.persistence.criteria.CriteriaQuery) RifFileRecords(gov.cms.bfd.model.rif.RifFileRecords) LoadAction(gov.cms.bfd.pipeline.ccw.rif.load.RifRecordLoadResult.LoadAction) Instant(java.time.Instant) CarrierClaimCsvWriter(gov.cms.bfd.model.rif.CarrierClaimCsvWriter) Collectors(java.util.stream.Collectors) BeneficiaryMonthly(gov.cms.bfd.model.rif.BeneficiaryMonthly) RifFilesEvent(gov.cms.bfd.model.rif.RifFilesEvent) UncheckedIOException(java.io.UncheckedIOException) Objects(java.util.Objects) Beneficiary(gov.cms.bfd.model.rif.Beneficiary) LoadedBatch(gov.cms.bfd.model.rif.LoadedBatch) List(java.util.List) Stream(java.util.stream.Stream) SkipReasonCode(gov.cms.bfd.model.rif.SkippedRifRecord.SkipReasonCode) EntityManagerFactory(javax.persistence.EntityManagerFactory) CarrierClaim(gov.cms.bfd.model.rif.CarrierClaim) LocalDate(java.time.LocalDate) LoadedFile(gov.cms.bfd.model.rif.LoadedFile) Entry(java.util.Map.Entry) Timer(com.codahale.metrics.Timer) Optional(java.util.Optional) Gauge(com.codahale.metrics.Gauge) RifParsingUtils(gov.cms.bfd.model.rif.parse.RifParsingUtils) RifRecordEvent(gov.cms.bfd.model.rif.RifRecordEvent) CSVPrinter(org.apache.commons.csv.CSVPrinter) SkippedRifRecord(gov.cms.bfd.model.rif.SkippedRifRecord) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Session(org.hibernate.Session) BeneficiaryCsvWriter(gov.cms.bfd.model.rif.BeneficiaryCsvWriter) Beneficiary_(gov.cms.bfd.model.rif.Beneficiary_) ArrayList(java.util.ArrayList) SQLException(java.sql.SQLException) Table(javax.persistence.Table) CopyManager(org.postgresql.copy.CopyManager) LinkedList(java.util.LinkedList) Root(javax.persistence.criteria.Root) Period(java.time.Period) RifFileEvent(gov.cms.bfd.model.rif.RifFileEvent) MetricRegistry(com.codahale.metrics.MetricRegistry) Logger(org.slf4j.Logger) RecordAction(gov.cms.bfd.model.rif.RecordAction) BadCodeMonkeyException(gov.cms.bfd.sharedutils.exceptions.BadCodeMonkeyException) FileWriter(java.io.FileWriter) RifRecordBase(gov.cms.bfd.model.rif.RifRecordBase) IOException(java.io.IOException) EntityManager(javax.persistence.EntityManager) File(java.io.File) Work(org.hibernate.jdbc.Work) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) EntityTransaction(javax.persistence.EntityTransaction) FileReader(java.io.FileReader) Comparator(java.util.Comparator) ArrayList(java.util.ArrayList) BeneficiaryMonthly(gov.cms.bfd.model.rif.BeneficiaryMonthly)

Example 13 with RifRecordEvent

use of gov.cms.bfd.model.rif.RifRecordEvent in project beneficiary-fhir-data by CMSgov.

the class RifFilesProcessorTest method process1HHAClaimRecord.

/**
 * Ensures that {@link RifFilesProcessor} can correctly handle {@link
 * StaticRifResource#SAMPLE_A_HHA}.
 */
@Test
public void process1HHAClaimRecord() {
    RifFilesEvent filesEvent = new RifFilesEvent(Instant.now(), StaticRifResource.SAMPLE_A_HHA.toRifFile());
    RifFilesProcessor processor = new RifFilesProcessor();
    RifFileRecords rifFileRecords = processor.produceRecords(filesEvent.getFileEvents().get(0));
    List<RifRecordEvent<?>> rifEventsList = rifFileRecords.getRecords().collect(Collectors.toList());
    assertEquals(StaticRifResource.SAMPLE_A_HHA.getRecordCount(), rifEventsList.size());
    RifRecordEvent<?> rifRecordEvent = rifEventsList.get(0);
    assertEquals(StaticRifResource.SAMPLE_A_HHA.getRifFileType(), rifRecordEvent.getFileEvent().getFile().getFileType());
    assertNotNull(rifRecordEvent.getRecord());
    assertTrue(rifRecordEvent.getRecord() instanceof HHAClaim);
    // Verify the claim header.
    HHAClaim claimGroup = (HHAClaim) rifRecordEvent.getRecord();
    assertEquals(RecordAction.INSERT, rifRecordEvent.getRecordAction());
    assertEquals(claimGroup.getBeneficiaryId(), rifRecordEvent.getBeneficiaryId());
    assertEquals("567834", claimGroup.getBeneficiaryId());
    assertEquals("2925555555", claimGroup.getClaimId());
    assertEquals(new BigDecimal(900), claimGroup.getClaimGroupId());
    assertEquals('W', claimGroup.getNearLineRecordIdCode());
    assertEquals("10", claimGroup.getClaimTypeCode());
    assertEquals('2', claimGroup.getClaimServiceClassificationTypeCode());
    assertEquals(LocalDate.of(2015, 6, 23), claimGroup.getDateFrom());
    assertEquals(LocalDate.of(2015, 6, 23), claimGroup.getDateThrough());
    assertEquals("45645", claimGroup.getProviderNumber());
    assertEquals("P", claimGroup.getClaimNonPaymentReasonCode().get());
    assertEquals(new BigDecimal("188.00"), claimGroup.getPaymentAmount());
    assertEquals(new BigDecimal("11.00"), claimGroup.getPrimaryPayerPaidAmount());
    assertEquals("UT", claimGroup.getProviderStateCode());
    assertEquals("1811111111", claimGroup.getOrganizationNpi().get());
    assertEquals("2222222222", claimGroup.getAttendingPhysicianNpi().get());
    assertEquals("30", claimGroup.getPatientDischargeStatusCode());
    assertEquals(new BigDecimal("199.99"), claimGroup.getTotalChargeAmount());
    assertEquals("H5555", claimGroup.getDiagnosisPrincipalCode().get());
    assertEquals('9', claimGroup.getDiagnosisPrincipalCodeVersion().get().charValue());
    assertEquals('L', claimGroup.getClaimLUPACode().get().charValue());
    assertEquals('1', claimGroup.getClaimReferralCode().get().charValue());
    assertEquals(new BigDecimal(3), claimGroup.getTotalVisitCount());
    assertEquals(LocalDate.of(2015, 6, 23), claimGroup.getCareStartDate().get());
    // assertEquals("308683096577486", claimGroup.getFiDocumentClaimControlNumber().get());
    // assertEquals("10493204767560565", claimGroup.getFiOriginalClaimControlNumber().get());
    assertEquals(1, claimGroup.getLines().size());
    // Verify one of the claim lines.
    HHAClaimLine claimLine = claimGroup.getLines().get(0);
    assertEquals(new BigDecimal(1), claimLine.getLineNumber());
    assertEquals("0023", claimLine.getRevenueCenterCode());
    assertEquals(new BigDecimal("26.00"), claimGroup.getLines().get(0).getPaymentAmount());
    assertEquals(new BigDecimal("24.00"), claimGroup.getLines().get(0).getNonCoveredChargeAmount());
    assertEquals(new BigDecimal("25.00"), claimGroup.getLines().get(0).getTotalChargeAmount());
    assertEquals("345345345", claimLine.getRevenueCenterRenderingPhysicianNPI().get());
}
Also used : RifRecordEvent(gov.cms.bfd.model.rif.RifRecordEvent) HHAClaim(gov.cms.bfd.model.rif.HHAClaim) RifFileRecords(gov.cms.bfd.model.rif.RifFileRecords) HHAClaimLine(gov.cms.bfd.model.rif.HHAClaimLine) RifFilesEvent(gov.cms.bfd.model.rif.RifFilesEvent) BigDecimal(java.math.BigDecimal) Test(org.junit.jupiter.api.Test)

Example 14 with RifRecordEvent

use of gov.cms.bfd.model.rif.RifRecordEvent in project beneficiary-fhir-data by CMSgov.

the class RifFilesProcessorTest method process1DMEClaimRecord.

/**
 * Ensures that {@link RifFilesProcessor} can correctly handle {@link
 * StaticRifResource#SAMPLE_A_DME}.
 */
@Test
public void process1DMEClaimRecord() {
    RifFilesEvent filesEvent = new RifFilesEvent(Instant.now(), StaticRifResource.SAMPLE_A_DME.toRifFile());
    RifFilesProcessor processor = new RifFilesProcessor();
    RifFileRecords rifFileRecords = processor.produceRecords(filesEvent.getFileEvents().get(0));
    List<RifRecordEvent<?>> rifEventsList = rifFileRecords.getRecords().collect(Collectors.toList());
    assertEquals(StaticRifResource.SAMPLE_A_DME.getRecordCount(), rifEventsList.size());
    RifRecordEvent<?> rifRecordEvent = rifEventsList.get(0);
    assertEquals(StaticRifResource.SAMPLE_A_DME.getRifFileType(), rifRecordEvent.getFileEvent().getFile().getFileType());
    assertNotNull(rifRecordEvent.getRecord());
    assertTrue(rifRecordEvent.getRecord() instanceof DMEClaim);
    // Verify the claim header.
    DMEClaim claimGroup = (DMEClaim) rifRecordEvent.getRecord();
    assertEquals(RecordAction.INSERT, rifRecordEvent.getRecordAction());
    assertEquals(claimGroup.getBeneficiaryId(), rifRecordEvent.getBeneficiaryId());
    assertEquals("567834", claimGroup.getBeneficiaryId());
    assertEquals("2188888888", claimGroup.getClaimId());
    assertEquals(new BigDecimal(900), claimGroup.getClaimGroupId());
    assertEquals('M', claimGroup.getNearLineRecordIdCode());
    assertEquals("82", claimGroup.getClaimTypeCode());
    assertEquals(LocalDate.of(2014, 02, 03), claimGroup.getDateFrom());
    assertEquals(LocalDate.of(2014, 02, 03), claimGroup.getDateThrough());
    assertEquals(LocalDate.of(2014, 02, 14), claimGroup.getWeeklyProcessDate());
    assertEquals('1', claimGroup.getClaimEntryCode());
    assertEquals("01", claimGroup.getClaimDispositionCode());
    assertEquals("99999", claimGroup.getCarrierNumber());
    assertEquals("1", claimGroup.getPaymentDenialCode());
    assertEquals(new BigDecimal("777.75"), claimGroup.getPaymentAmount());
    assertEquals(new BigDecimal("0"), claimGroup.getPrimaryPayerPaidAmount());
    assertEquals('A', claimGroup.getProviderAssignmentIndicator());
    assertEquals(new BigDecimal("666.75"), claimGroup.getProviderPaymentAmount());
    assertEquals(new BigDecimal("666.66"), claimGroup.getBeneficiaryPaymentAmount());
    assertEquals(new BigDecimal("1752.75"), claimGroup.getSubmittedChargeAmount());
    assertEquals(new BigDecimal("754.79"), claimGroup.getAllowedChargeAmount());
    assertEquals(new BigDecimal("777.00"), claimGroup.getBeneficiaryPartBDeductAmount());
    assertEquals('3', claimGroup.getHcpcsYearCode().get().charValue());
    assertEquals("R5555", claimGroup.getDiagnosis1Code().get());
    assertEquals('0', claimGroup.getDiagnosis1CodeVersion().get().charValue());
    assertEquals("1306849450", claimGroup.getReferringPhysicianNpi().get());
    assertEquals("0", claimGroup.getClinicalTrialNumber().get());
    assertEquals(1, claimGroup.getLines().size());
    assertEquals("74655592568216", claimGroup.getClaimCarrierControlNumber().get());
    // Verify one of the claim lines.
    DMEClaimLine claimLine = claimGroup.getLines().get(0);
    assertEquals(new BigDecimal(1), claimLine.getLineNumber());
    assertEquals("9994931888", claimLine.getProviderTaxNumber());
    assertEquals("A5", claimLine.getProviderSpecialityCode().get());
    assertEquals('1', claimLine.getProviderParticipatingIndCode().get().charValue());
    assertEquals(new BigDecimal("60"), claimLine.getServiceCount());
    assertEquals('P', claimLine.getCmsServiceTypeCode());
    assertEquals("12", claimLine.getPlaceOfServiceCode());
    assertEquals(LocalDate.of(2014, 02, 03), claimLine.getFirstExpenseDate().get());
    assertEquals(LocalDate.of(2014, 02, 03), claimLine.getLastExpenseDate().get());
    assertEquals("345", claimLine.getHcpcsCode().get());
    assertFalse(claimLine.getHcpcsSecondModifierCode().isPresent());
    assertFalse(claimLine.getHcpcsThirdModifierCode().isPresent());
    assertFalse(claimLine.getHcpcsFourthModifierCode().isPresent());
    assertEquals("D9Z", claimLine.getBetosCode().get());
    assertEquals(new BigDecimal("123.45"), claimLine.getPaymentAmount());
    assertEquals(new BigDecimal("11.00"), claimLine.getBeneficiaryPaymentAmount());
    assertEquals(new BigDecimal("120.00"), claimLine.getProviderPaymentAmount());
    assertEquals(new BigDecimal("18.00"), claimLine.getBeneficiaryPartBDeductAmount());
    assertTrue(claimLine.getPrimaryPayerCode().isPresent());
    assertEquals(new BigDecimal("11.00"), claimLine.getPrimaryPayerPaidAmount());
    assertEquals(new BigDecimal("20.20"), claimLine.getCoinsuranceAmount());
    assertEquals(new BigDecimal("20.29"), claimLine.getPrimaryPayerAllowedChargeAmount());
    assertEquals(new BigDecimal("130.45"), claimLine.getSubmittedChargeAmount());
    assertEquals(new BigDecimal("129.45"), claimLine.getAllowedChargeAmount());
    assertEquals("A", claimLine.getProcessingIndicatorCode().get());
    assertEquals('0', claimLine.getPaymentCode().get().charValue());
    assertEquals('0', claimLine.getServiceDeductibleCode().get().charValue());
    assertEquals(new BigDecimal("82.29"), claimLine.getPurchasePriceAmount());
    assertEquals("1244444444", claimLine.getProviderNPI().get());
    assertEquals("AL", claimLine.getPricingStateCode().get());
    assertEquals("MO", claimLine.getProviderStateCode());
    assertEquals(new Character('3'), claimLine.getSupplierTypeCode().get());
    assertEquals(new BigDecimal("0.00"), claimLine.getScreenSavingsAmount().get());
    assertEquals(new BigDecimal("60"), claimLine.getMtusCount());
    assertEquals('3', claimLine.getMtusCode().get().charValue());
    assertEquals(new BigDecimal("44.4"), claimLine.getHctHgbTestResult());
    assertEquals("R2", claimLine.getHctHgbTestTypeCode().get());
    assertEquals("500904610", claimLine.getNationalDrugCode().get());
}
Also used : DMEClaim(gov.cms.bfd.model.rif.DMEClaim) RifRecordEvent(gov.cms.bfd.model.rif.RifRecordEvent) RifFileRecords(gov.cms.bfd.model.rif.RifFileRecords) RifFilesEvent(gov.cms.bfd.model.rif.RifFilesEvent) BigDecimal(java.math.BigDecimal) DMEClaimLine(gov.cms.bfd.model.rif.DMEClaimLine) Test(org.junit.jupiter.api.Test)

Example 15 with RifRecordEvent

use of gov.cms.bfd.model.rif.RifRecordEvent in project beneficiary-fhir-data by CMSgov.

the class RifFilesProcessorTest method process1PDERecord.

/**
 * Ensures that {@link RifFilesProcessor} can correctly handle {@link
 * StaticRifResource#SAMPLE_A_PDE}.
 */
@Test
public void process1PDERecord() {
    RifFilesEvent filesEvent = new RifFilesEvent(Instant.now(), StaticRifResource.SAMPLE_A_PDE.toRifFile());
    RifFilesProcessor processor = new RifFilesProcessor();
    RifFileRecords rifFileRecords = processor.produceRecords(filesEvent.getFileEvents().get(0));
    List<RifRecordEvent<?>> rifEventsList = rifFileRecords.getRecords().collect(Collectors.toList());
    assertEquals(StaticRifResource.SAMPLE_A_PDE.getRecordCount(), rifEventsList.size());
    RifRecordEvent<?> rifRecordEvent = rifEventsList.get(0);
    assertEquals(StaticRifResource.SAMPLE_A_PDE.getRifFileType(), rifRecordEvent.getFileEvent().getFile().getFileType());
    assertNotNull(rifRecordEvent.getRecord());
    assertTrue(rifRecordEvent.getRecord() instanceof PartDEvent);
    PartDEvent pdeRow = (PartDEvent) rifRecordEvent.getRecord();
    assertEquals(RecordAction.INSERT, rifRecordEvent.getRecordAction());
    assertEquals(pdeRow.getBeneficiaryId(), rifRecordEvent.getBeneficiaryId());
    assertEquals("89", pdeRow.getEventId());
    assertEquals(new BigDecimal(900), pdeRow.getClaimGroupId());
    assertEquals("567834", pdeRow.getBeneficiaryId());
    assertEquals(LocalDate.of(2015, Month.MAY, 12), pdeRow.getPrescriptionFillDate());
    assertEquals(LocalDate.of(2015, Month.MAY, 27), pdeRow.getPaymentDate().get());
    assertEquals("01", pdeRow.getServiceProviderIdQualiferCode());
    assertEquals("1023011079", pdeRow.getServiceProviderId());
    assertEquals("01", pdeRow.getPrescriberIdQualifierCode());
    assertEquals("1750384806", pdeRow.getPrescriberId());
    assertEquals(new BigDecimal(799999), pdeRow.getPrescriptionReferenceNumber());
    assertEquals("500904610", pdeRow.getNationalDrugCode());
    assertEquals("H9999", pdeRow.getPlanContractId());
    assertEquals("020", pdeRow.getPlanBenefitPackageId());
    assertEquals(1, pdeRow.getCompoundCode());
    assertEquals('0', pdeRow.getDispenseAsWrittenProductSelectionCode());
    assertEquals(new BigDecimal("60"), pdeRow.getQuantityDispensed());
    assertEquals(new BigDecimal(30), pdeRow.getDaysSupply());
    assertEquals(new BigDecimal(3), pdeRow.getFillNumber());
    assertEquals(new Character('P'), pdeRow.getDispensingStatusCode().get());
    assertEquals('C', pdeRow.getDrugCoverageStatusCode());
    assertEquals(new Character('A'), pdeRow.getAdjustmentDeletionCode().get());
    assertEquals(new Character('X'), pdeRow.getNonstandardFormatCode().get());
    assertEquals(new Character('M'), pdeRow.getPricingExceptionCode().get());
    assertEquals(new Character('C'), pdeRow.getCatastrophicCoverageCode().get());
    assertEquals(new BigDecimal("995.34"), pdeRow.getGrossCostBelowOutOfPocketThreshold());
    assertEquals(new BigDecimal("15.25"), pdeRow.getGrossCostAboveOutOfPocketThreshold());
    assertEquals(new BigDecimal("235.85"), pdeRow.getPatientPaidAmount());
    assertEquals(new BigDecimal("17.30"), pdeRow.getOtherTrueOutOfPocketPaidAmount());
    assertEquals(new BigDecimal("122.23"), pdeRow.getLowIncomeSubsidyPaidAmount());
    assertEquals(new BigDecimal("42.42"), pdeRow.getPatientLiabilityReductionOtherPaidAmount());
    assertEquals(new BigDecimal("126.99"), pdeRow.getPartDPlanCoveredPaidAmount());
    assertEquals(new BigDecimal("17.98"), pdeRow.getPartDPlanNonCoveredPaidAmount());
    assertEquals(new BigDecimal("550.00"), pdeRow.getTotalPrescriptionCost());
    assertEquals(new Character('3'), pdeRow.getPrescriptionOriginationCode().get());
    assertEquals(new BigDecimal("317.22"), pdeRow.getGapDiscountAmount());
    assertEquals(new Character('G'), pdeRow.getBrandGenericCode().get());
    assertEquals("01", pdeRow.getPharmacyTypeCode());
    assertEquals("02", pdeRow.getPatientResidenceCode());
    assertEquals("08", pdeRow.getSubmissionClarificationCode().get());
}
Also used : RifRecordEvent(gov.cms.bfd.model.rif.RifRecordEvent) RifFileRecords(gov.cms.bfd.model.rif.RifFileRecords) RifFilesEvent(gov.cms.bfd.model.rif.RifFilesEvent) PartDEvent(gov.cms.bfd.model.rif.PartDEvent) BigDecimal(java.math.BigDecimal) Test(org.junit.jupiter.api.Test)

Aggregations

RifRecordEvent (gov.cms.bfd.model.rif.RifRecordEvent)27 RifFileRecords (gov.cms.bfd.model.rif.RifFileRecords)16 RifFilesEvent (gov.cms.bfd.model.rif.RifFilesEvent)15 CSVRecord (org.apache.commons.csv.CSVRecord)14 RecordAction (gov.cms.bfd.model.rif.RecordAction)13 Test (org.junit.jupiter.api.Test)13 BigDecimal (java.math.BigDecimal)11 Beneficiary (gov.cms.bfd.model.rif.Beneficiary)6 BadCodeMonkeyException (gov.cms.bfd.sharedutils.exceptions.BadCodeMonkeyException)6 BeneficiaryHistory (gov.cms.bfd.model.rif.BeneficiaryHistory)5 CarrierClaim (gov.cms.bfd.model.rif.CarrierClaim)5 CarrierClaimLine (gov.cms.bfd.model.rif.CarrierClaimLine)4 RifFileEvent (gov.cms.bfd.model.rif.RifFileEvent)4 IOException (java.io.IOException)4 List (java.util.List)4 BeneficiaryMonthly (gov.cms.bfd.model.rif.BeneficiaryMonthly)3 LoadedBatch (gov.cms.bfd.model.rif.LoadedBatch)3 LoadedFile (gov.cms.bfd.model.rif.LoadedFile)3 RifFileType (gov.cms.bfd.model.rif.RifFileType)3 SkippedRifRecord (gov.cms.bfd.model.rif.SkippedRifRecord)3