Examples with RifRecordBase gov.cms.bfd.model.rif.RifRecordBase used on opensource projects

Example 1 with RifRecordBase

use of gov.cms.bfd.model.rif.RifRecordBase in project beneficiary-fhir-data by CMSgov.

the class RifLoader method process.

/**
 * @param recordsBatch the {@link RifRecordEvent}s to process
 * @param loadedFileBuilder the builder for the {@LoadedFile} associated with this batch
 * @param postgresBatch the {@link PostgreSqlCopyInserter} for the current set of {@link
 *     RifFilesEvent}s being processed
 * @return the {@link RifRecordLoadResult}s that model the results of the operation
 */
private List<RifRecordLoadResult> process(List<RifRecordEvent<?>> recordsBatch, long loadedFileId, PostgreSqlCopyInserter postgresBatch) {
    RifFileEvent fileEvent = recordsBatch.get(0).getFileEvent();
    MetricRegistry fileEventMetrics = fileEvent.getEventMetrics();
    RifFileType rifFileType = fileEvent.getFile().getFileType();
    if (rifFileType == RifFileType.BENEFICIARY_HISTORY) {
        for (RifRecordEvent<?> rifRecordEvent : recordsBatch) {
            hashBeneficiaryHistoryHicn(rifRecordEvent);
            hashBeneficiaryHistoryMbi(rifRecordEvent);
        }
    }
    // Only one of each failure/success Timer.Contexts will be applied.
    Timer.Context timerBatchSuccess = appState.getMetrics().timer(MetricRegistry.name(getClass().getSimpleName(), "recordBatches")).time();
    Timer.Context timerBatchTypeSuccess = fileEventMetrics.timer(MetricRegistry.name(getClass().getSimpleName(), "recordBatches", rifFileType.name())).time();
    Timer.Context timerBundleFailure = appState.getMetrics().timer(MetricRegistry.name(getClass().getSimpleName(), "recordBatches", "failed")).time();
    EntityManager entityManager = null;
    EntityTransaction txn = null;
    // TODO: refactor the following to be less of an indented mess
    try {
        entityManager = appState.getEntityManagerFactory().createEntityManager();
        txn = entityManager.getTransaction();
        txn.begin();
        List<RifRecordLoadResult> loadResults = new ArrayList<>(recordsBatch.size());
        /*
       * Dev Note: All timestamps of records in the batch and the LoadedBatch must be the same for data consistency.
       * The timestamp from the LoadedBatchBuilder is used.
       */
        LoadedBatchBuilder loadedBatchBuilder = new LoadedBatchBuilder(loadedFileId, recordsBatch.size());
        for (RifRecordEvent<?> rifRecordEvent : recordsBatch) {
            RecordAction recordAction = rifRecordEvent.getRecordAction();
            RifRecordBase record = rifRecordEvent.getRecord();
            LOGGER.trace("Loading '{}' record.", rifFileType);
            // Set lastUpdated to the same value for the whole batch
            record.setLastUpdated(Optional.of(loadedBatchBuilder.getTimestamp()));
            // Associate the beneficiary with this file loaded
            loadedBatchBuilder.associateBeneficiary(rifRecordEvent.getBeneficiaryId());
            LoadStrategy strategy = selectStrategy(recordAction);
            LoadAction loadAction;
            if (strategy == LoadStrategy.INSERT_IDEMPOTENT) {
                // Check to see if record already exists.
                Timer.Context timerIdempotencyQuery = fileEventMetrics.timer(MetricRegistry.name(getClass().getSimpleName(), "idempotencyQueries")).time();
                Object recordId = appState.getEntityManagerFactory().getPersistenceUnitUtil().getIdentifier(record);
                Objects.requireNonNull(recordId);
                Object recordInDb = entityManager.find(record.getClass(), recordId);
                timerIdempotencyQuery.close();
                // Log if we have a non-2022 enrollment year INSERT
                if (isBackdatedBene(rifRecordEvent)) {
                    Beneficiary bene = (Beneficiary) rifRecordEvent.getRecord();
                    LOGGER.info("Inserted beneficiary with non-2022 enrollment year (beneficiaryId={})", bene.getBeneficiaryId());
                }
                if (recordInDb == null) {
                    loadAction = LoadAction.INSERTED;
                    tweakIfBeneficiary(entityManager, loadedBatchBuilder, rifRecordEvent);
                    entityManager.persist(record);
                // FIXME Object recordInDbAfterUpdate = entityManager.find(record.getClass(), recordId);
                } else {
                    loadAction = LoadAction.DID_NOTHING;
                }
            } else if (strategy == LoadStrategy.INSERT_UPDATE_NON_IDEMPOTENT) {
                if (rifRecordEvent.getRecordAction().equals(RecordAction.INSERT)) {
                    loadAction = LoadAction.INSERTED;
                    // Log if we have a non-2022 enrollment year INSERT
                    if (isBackdatedBene(rifRecordEvent)) {
                        Beneficiary bene = (Beneficiary) rifRecordEvent.getRecord();
                        LOGGER.info("Inserted beneficiary with non-2022 enrollment year (beneficiaryId={})", bene.getBeneficiaryId());
                    }
                    tweakIfBeneficiary(entityManager, loadedBatchBuilder, rifRecordEvent);
                    entityManager.persist(record);
                } else if (rifRecordEvent.getRecordAction().equals(RecordAction.UPDATE)) {
                    loadAction = LoadAction.UPDATED;
                    // Skip this record if the year is not 2022 and its an update.
                    if (isBackdatedBene(rifRecordEvent)) {
                        /*
               * Serialize the record's CSV data back to actual RIF/CSV, as that's how we'll store
               * it in the DB.
               */
                        StringBuffer rifData = new StringBuffer();
                        try (CSVPrinter csvPrinter = new CSVPrinter(rifData, RifParsingUtils.CSV_FORMAT)) {
                            for (CSVRecord csvRow : rifRecordEvent.getRawCsvRecords()) {
                                csvPrinter.printRecord(csvRow);
                            }
                        }
                        // Save the skipped record to the DB.
                        SkippedRifRecord skippedRifRecord = new SkippedRifRecord(rifRecordEvent.getFileEvent().getParentFilesEvent().getTimestamp(), SkipReasonCode.DELAYED_BACKDATED_ENROLLMENT_BFD_1566, rifRecordEvent.getFileEvent().getFile().getFileType().name(), rifRecordEvent.getRecordAction(), ((Beneficiary) record).getBeneficiaryId(), rifData.toString());
                        entityManager.persist(skippedRifRecord);
                        LOGGER.info("Skipped RIF record, due to '{}'.", skippedRifRecord.getSkipReason());
                    } else {
                        tweakIfBeneficiary(entityManager, loadedBatchBuilder, rifRecordEvent);
                        entityManager.merge(record);
                    }
                } else {
                    throw new BadCodeMonkeyException(String.format("Unhandled %s: '%s'.", RecordAction.class, rifRecordEvent.getRecordAction()));
                }
            } else
                throw new BadCodeMonkeyException();
            LOGGER.trace("Loaded '{}' record.", rifFileType);
            fileEventMetrics.meter(MetricRegistry.name(getClass().getSimpleName(), "records", loadAction.name())).mark(1);
            loadResults.add(new RifRecordLoadResult(rifRecordEvent, loadAction));
        }
        LoadedBatch loadedBatch = loadedBatchBuilder.build();
        entityManager.persist(loadedBatch);
        txn.commit();
        // Update the metrics now that things have been pushed.
        timerBatchSuccess.stop();
        timerBatchTypeSuccess.stop();
        return loadResults;
    } catch (Throwable t) {
        timerBundleFailure.stop();
        fileEventMetrics.meter(MetricRegistry.name(getClass().getSimpleName(), "recordBatches", "failed")).mark(1);
        LOGGER.warn("Failed to load '{}' record.", rifFileType, t);
        throw new RifLoadFailure(recordsBatch, t);
    } finally {
        /*
       * Some errors (e.g. HSQL constraint violations) seem to cause the
       * rollback to fail. Extra error handling is needed here, too, to
       * ensure that the failing data is captured.
       */
        try {
            if (txn != null && txn.isActive())
                txn.rollback();
        } catch (Throwable t) {
            timerBundleFailure.stop();
            fileEventMetrics.meter(MetricRegistry.name(getClass().getSimpleName(), "recordBatches", "failed")).mark(1);
            LOGGER.warn("Failed to load '{}' record.", rifFileType, t);
            throw new RifLoadFailure(recordsBatch, t);
        }
        if (entityManager != null)
            entityManager.close();
    }
}

Example 2 with RifRecordBase

use of gov.cms.bfd.model.rif.RifRecordBase in project beneficiary-fhir-data by CMSgov.

the class RifLoader method tweakIfBeneficiary.

/**
 * Applies various "tweaks" to the {@link Beneficiary} (if any) in the specified {@link
 * RifRecordEvent}:
 *
 * <ul>
 *   <li>Hashes any MBIs or HICNs in it.
 *   <li>Updates its {@link Beneficiary#getBeneficiaryMonthlys()} records, as needed.
 *   <li>Adds a {@link BeneficiaryHistory} record for previous the {@link Beneficiary}, as needed.
 * </ul>
 *
 * @param entityManager the {@link EntityManager} to use
 * @param loadedBatchBuilder the {@link LoadedBatchBuilder} to use
 * @param rifRecordEvent the {@link RifRecordEvent} to handle the {@link Beneficiary} (if any) for
 */
private void tweakIfBeneficiary(EntityManager entityManager, LoadedBatchBuilder loadedBatchBuilder, RifRecordEvent<?> rifRecordEvent) {
    RifRecordBase record = rifRecordEvent.getRecord();
    // Nothing to do here unless it's a Beneficiary record.
    if (!(record instanceof Beneficiary)) {
        return;
    }
    Beneficiary newBeneficiaryRecord = (Beneficiary) record;
    Optional<Beneficiary> oldBeneficiaryRecord = Optional.empty();
    /*
     * Grab the the previous/current version of the Beneficiary (if any, as it exists in the
     * database before applying the specified RifRecordEvent).
     */
    if (rifRecordEvent.getRecordAction() == RecordAction.UPDATE) {
        /*
       * FIXME We need to enforce a new invariant on the incoming RIF files: no repeats of same
       * record/PK in same RIF file allowed. Otherwise, we're running the risk of data race bugs and
       * out-of-order application due to the asynchronous nature of this processing.
       */
        CriteriaBuilder builder = entityManager.getCriteriaBuilder();
        CriteriaQuery<Beneficiary> criteria = builder.createQuery(Beneficiary.class);
        Root<Beneficiary> root = criteria.from(Beneficiary.class);
        root.fetch(Beneficiary_.beneficiaryMonthlys, JoinType.LEFT);
        criteria.select(root);
        criteria.where(builder.equal(root.get(Beneficiary_.beneficiaryId), newBeneficiaryRecord.getBeneficiaryId()));
        oldBeneficiaryRecord = Optional.ofNullable(entityManager.createQuery(criteria).getSingleResult());
    }
    /*
     * Ensure that hashes are added for the secrets in bene records (HICNs and MBIs). When the
     * secret value hasn't changed (for UPDATE records), we copy the old hash over, to avoid
     * unnecessary hashing. Otherwise, we can waste hours recomputing hashes that don't need
     * recomputing.
     */
    if (oldBeneficiaryRecord.isPresent() && Objects.equals(newBeneficiaryRecord.getHicnUnhashed(), oldBeneficiaryRecord.get().getHicnUnhashed())) {
        newBeneficiaryRecord.setHicn(oldBeneficiaryRecord.get().getHicn());
    } else {
        hashBeneficiaryHicn(rifRecordEvent);
    }
    if (oldBeneficiaryRecord.isPresent() && Objects.equals(newBeneficiaryRecord.getMedicareBeneficiaryId(), oldBeneficiaryRecord.get().getMedicareBeneficiaryId())) {
        newBeneficiaryRecord.setMbiHash(oldBeneficiaryRecord.get().getMbiHash());
    } else {
        hashBeneficiaryMbi(rifRecordEvent);
    }
    if (rifRecordEvent.getRecordAction() == RecordAction.UPDATE) {
        /*
       * When beneficiaries are updated, we need to be careful to capture their current/previous
       * state as a BeneficiaryHistory record. (Note: this has to be done AFTER the secret hashes
       * have been updated, as per above.
       */
        updateBeneficaryHistory(entityManager, newBeneficiaryRecord, oldBeneficiaryRecord, loadedBatchBuilder.getTimestamp());
    }
    updateBeneficiaryMonthly(newBeneficiaryRecord, oldBeneficiaryRecord);
}

Example 3 with RifRecordBase

use of gov.cms.bfd.model.rif.RifRecordBase in project beneficiary-fhir-data by CMSgov.

the class SamhsaMatcherR4FromClaimTransformerV2Test method getClaim.

/**
 * Generates the Claim object to be used in a test.
 *
 * @param type the type
 * @return the claim to be used for the test, should match the input type
 */
public static RifRecordBase getClaim(Class<? extends RifRecordBase> type) {
    List<Object> parsedRecords = ServerTestUtils.parseData(Arrays.asList(StaticRifResourceGroup.SAMPLE_A.getResources()));
    RifRecordBase claim = parsedRecords.stream().filter(type::isInstance).map(type::cast).findFirst().orElse(null);
    if (claim != null) {
        claim.setLastUpdated(Instant.now());
    } else {
        throw new IllegalStateException("Test setup issue, did not find expected InpatientClaim in sample record.");
    }
    return claim;
}

Example 4 with RifRecordBase

use of gov.cms.bfd.model.rif.RifRecordBase in project beneficiary-fhir-data by CMSgov.

the class SamhsaMatcherFromClaimTransformerTest method getClaim.

/**
 * Generates the Claim object to be used in a test.
 *
 * @param type the type
 * @return the claim to be used for the test, should match the input type
 */
public static RifRecordBase getClaim(Class<? extends RifRecordBase> type) {
    List<Object> parsedRecords = ServerTestUtils.parseData(Arrays.asList(StaticRifResourceGroup.SAMPLE_A.getResources()));
    RifRecordBase claim = parsedRecords.stream().filter(type::isInstance).map(type::cast).findFirst().orElse(null);
    if (claim != null) {
        claim.setLastUpdated(Instant.now());
    } else {
        throw new IllegalStateException("Test setup issue, did not find expected InpatientClaim in sample record.");
    }
    return claim;
}