Examples with LoadedFile gov.cms.bfd.model.rif.LoadedFile used on opensource projects

Example 1 with LoadedFile

use of gov.cms.bfd.model.rif.LoadedFile in project beneficiary-fhir-data by CMSgov.

the class LoadedFilterManager method refreshFilters.

/**
 * Called periodically to build and refresh the filters list from the entityManager.
 *
 * <p>The {@link #lastBatchCreated} and {@link #firstBatchCreated} fields are updated by this
 * call.
 */
@Scheduled(fixedDelay = 1000, initialDelay = 2000)
public void refreshFilters() {
    /*
     * Dev note: the pipeline has a process to trim the files list. Nevertheless, building a set of
     * bloom filters may take a while. This method is expected to be called on it's own thread by
     * the the Spring framework. In addition, it doesn't lock this object until the end of the
     * process, so this filter building process can happen without interfering with serving. Also,
     * this refresh time will be proportional to the number of files which have been loaded in the
     * past refresh period. If no files have been loaded, this refresh should take less than a
     * millisecond.
     */
    try {
        // If new batches are present, then build new filters for the affected files
        final Instant currentLastBatchCreated = fetchLastLoadedBatchCreated().orElse(BEFORE_LAST_UPDATED_FEATURE);
        if (this.lastBatchCreated == null || this.lastBatchCreated.isBefore(currentLastBatchCreated)) {
            LOGGER.info("Refreshing LoadedFile filters with new filters from {} to {}", lastBatchCreated, currentLastBatchCreated);
            List<LoadedTuple> loadedTuples = fetchLoadedTuples(this.lastBatchCreated);
            List<LoadedFileFilter> newFilters = updateFilters(this.filters, loadedTuples, this::fetchLoadedBatches);
            // If batches been trimmed, then remove filters which are no longer present
            final Instant currentFirstBatchUpdate = fetchFirstLoadedBatchCreated().orElse(BEFORE_LAST_UPDATED_FEATURE);
            if (this.firstBatchCreated == null || this.firstBatchCreated.isBefore(currentFirstBatchUpdate)) {
                LOGGER.info("Trimmed LoadedFile filters before {}", currentFirstBatchUpdate);
                List<LoadedFile> loadedFiles = fetchLoadedFiles();
                newFilters = trimFilters(newFilters, loadedFiles);
            }
            LOGGER.info("Updating timestamps. currentFirstBatchUpdate={} currentLastBatchCreated={}", currentFirstBatchUpdate, currentLastBatchCreated);
            set(newFilters, currentFirstBatchUpdate, currentLastBatchCreated);
        }
    } catch (Throwable ex) {
        LOGGER.error("Error found refreshing LoadedFile filters", ex);
    }
}

Example 2 with LoadedFile

use of gov.cms.bfd.model.rif.LoadedFile in project beneficiary-fhir-data by CMSgov.

the class RifLoaderIT method trimLoadedFiles.

@Test
public void trimLoadedFiles() {
    PipelineTestUtils.get().doTestWithDb((dataSource, entityManager) -> {
        // Setup a loaded file with an old date
        loadSample(Arrays.asList(StaticRifResourceGroup.SAMPLE_A.getResources()));
        final List<LoadedFile> loadedFiles = PipelineTestUtils.get().findLoadedFiles(entityManager);
        final EntityTransaction txn = entityManager.getTransaction();
        txn.begin();
        LoadedFile oldFile = loadedFiles.get(loadedFiles.size() - 1);
        oldFile.setCreated(Instant.now().minus(101, ChronoUnit.DAYS));
        txn.commit();
        // Look at the files now
        final List<LoadedFile> beforeFiles = PipelineTestUtils.get().findLoadedFiles(entityManager);
        final Instant oldDate = Instant.now().minus(99, ChronoUnit.DAYS);
        assertTrue(beforeFiles.stream().anyMatch(file -> file.getCreated().isBefore(oldDate)), "Expect to have old files");
        // Load another set that will cause the old file to be trimmed
        loadSample(Arrays.asList(StaticRifResourceGroup.SAMPLE_U.getResources()));
        // Verify that old file was trimmed
        final List<LoadedFile> afterFiles = PipelineTestUtils.get().findLoadedFiles(entityManager);
        assertFalse(afterFiles.stream().anyMatch(file -> file.getCreated().isBefore(oldDate)), "Expect to not have old files");
    });
}

Example 3 with LoadedFile

use of gov.cms.bfd.model.rif.LoadedFile in project beneficiary-fhir-data by CMSgov.

the class RifLoader method insertLoadedFile.

/**
 * Insert the LoadedFile into the database
 *
 * @param fileEvent to base this new LoadedFile
 * @param errorHandler to call if something bad happens
 * @return the loadedFileId of the new LoadedFile record
 */
private long insertLoadedFile(RifFileEvent fileEvent, Consumer<Throwable> errorHandler) {
    if (fileEvent == null || fileEvent.getFile().getFileType() == null) {
        throw new IllegalArgumentException();
    }
    final LoadedFile loadedFile = new LoadedFile();
    loadedFile.setRifType(fileEvent.getFile().getFileType().toString());
    loadedFile.setCreated(Instant.now());
    try {
        EntityManager em = appState.getEntityManagerFactory().createEntityManager();
        EntityTransaction txn = null;
        try {
            // Insert the passed in loaded file
            txn = em.getTransaction();
            txn.begin();
            em.persist(loadedFile);
            txn.commit();
            LOGGER.info("Inserting LoadedFile {} of type {} created at {}", loadedFile.getLoadedFileId(), loadedFile.getRifType(), loadedFile.getCreated());
            return loadedFile.getLoadedFileId();
        } finally {
            if (em != null && em.isOpen()) {
                if (txn != null && txn.isActive()) {
                    txn.rollback();
                }
                em.close();
            }
        }
    } catch (Exception ex) {
        errorHandler.accept(ex);
        return -1;
    }
}

Example 4 with LoadedFile

use of gov.cms.bfd.model.rif.LoadedFile in project beneficiary-fhir-data by CMSgov.

the class LoadedFilterManager method fetchLoadedTuples.

/**
 * Fetch the tuple of (loadedFileId, LoadedFile.created, max(LoadedBatch.created))
 *
 * @param after limits the query to include batches created after this timestamp
 * @return tuples that meet the after criteria or an empty list
 */
private List<LoadedTuple> fetchLoadedTuples(Instant after) {
    final CriteriaBuilder cb = entityManager.getCriteriaBuilder();
    CriteriaQuery<LoadedTuple> query = cb.createQuery(LoadedTuple.class);
    final Root<LoadedFile> f = query.from(LoadedFile.class);
    Join<LoadedFile, LoadedBatch> b = f.join("batches");
    query = query.select(cb.construct(LoadedTuple.class, f.get("loadedFileId"), f.get("created"), cb.max(b.get("created"))));
    if (after != null) {
        query = query.where(cb.greaterThan(b.get("created"), after));
    }
    query = query.groupBy(f.get("loadedFileId"), f.get("created")).orderBy(cb.desc(f.get("created")));
    return entityManager.createQuery(query).getResultList();
}

Example 5 with LoadedFile

use of gov.cms.bfd.model.rif.LoadedFile in project beneficiary-fhir-data by CMSgov.

the class RifLoaderIT method buildSyntheticLoadedFiles.

@Disabled
@Test
public void buildSyntheticLoadedFiles() {
    PipelineTestUtils.get().doTestWithDb((dataSource, entityManager) -> {
        loadSample(Arrays.asList(StaticRifResourceGroup.SYNTHETIC_DATA.getResources()));
        // Verify that a loaded files exsits
        final List<LoadedFile> loadedFiles = PipelineTestUtils.get().findLoadedFiles(entityManager);
        assertTrue(loadedFiles.size() > 0, "Expected to have at least one file");
        final LoadedFile file = loadedFiles.get(0);
        final List<LoadedBatch> batches = loadBatches(entityManager, file.getLoadedFileId());
        assertTrue(batches.size() > 0);
    });
}