Examples with ContractDTO gov.cms.ab2d.common.dto.ContractDTO used on opensource projects

Example 16 with ContractDTO

use of gov.cms.ab2d.common.dto.ContractDTO in project ab2d by CMSgov.

the class CoverageDriverImpl method isCoverageAvailable.

/**
 * Determine whether database contains all necessary enrollment for a contract and that no updates to that
 * enrollment are currently occurring.
 *
 * Steps
 *      - Lock coverage so no other workers can modify coverage while this check is occurring
 *      - Create any {@link CoveragePeriod}s that are currently missing
 *      - Check the following to determine whether a job can run (return false if any are not met)
 *          - Look for whether months have failed to update during earlier attempts
 *          - Look for coverage periods that have never been successfully searched and queue them
 *          - Look for coverage periods currently being updated
 *
 * @param job job to check for coverage
 * @throws CoverageDriverException if enrollment state violates assumed preconditions or database lock cannot be retrieved
 * @throws InterruptedException if trying to lock the table is interrupted
 */
@Trace(metricName = "EnrollmentIsAvailable", dispatcher = true)
@Override
public boolean isCoverageAvailable(Job job, ContractDTO contract) throws InterruptedException {
    String contractNumber = job.getContractNumber();
    assert contractNumber.equals(contract.getContractNumber());
    Lock coverageLock = coverageLockWrapper.getCoverageLock();
    // Track whether locked or not to prevent an illegal monitor exception
    boolean locked = false;
    try {
        locked = coverageLock.tryLock(MINUTE, TimeUnit.MINUTES);
        if (!locked) {
            log.warn("Could not retrieve lock after timeout of {} minute(s)." + " Cannot confirm coverage metadata is available", MINUTE);
            return false;
        }
        // Check whether a coverage period is missing for this contract.
        // If so then create those coverage periods.
        discoverCoveragePeriods(mapping.map(contract));
        log.info("queueing never searched coverage metadata periods for {}", contractNumber);
        /*
             * If any relevant coverage period has never been pulled from BFD successfully then automatically fail the
             * search
             */
        List<CoveragePeriod> neverSearched = coverageService.coveragePeriodNeverSearchedSuccessfully().stream().filter(period -> Objects.equals(contract.getContractNumber(), period.getContractNumber())).toList();
        if (!neverSearched.isEmpty()) {
            // Check that we've not submitted and failed these jobs
            neverSearched.forEach(period -> checkCoveragePeriodValidity(job, period));
            // Add all never searched coverage periods to the queue for processing
            neverSearched.forEach(period -> coverageProcessor.queueCoveragePeriod(period, false));
            return false;
        }
        log.info("checking whether any coverage metadata is currently being updated for {}", contractNumber);
        /*
             * If coverage periods are submitted, in progress or null then ignore for the moment.
             *
             * There will always be at least one coverage period returned.
             */
        List<CoveragePeriod> periods = coverageService.findAssociatedCoveragePeriods(contract.getContractNumber());
        if (periods.isEmpty()) {
            log.error("There are no existing coverage periods for this job so no metadata exists");
            throw new CoverageDriverException("There are no existing coverage periods for this job so no ");
        }
        return periods.stream().map(CoveragePeriod::getStatus).noneMatch(status -> status == null || status == CoverageJobStatus.IN_PROGRESS || status == CoverageJobStatus.SUBMITTED);
    } catch (InterruptedException interruptedException) {
        log.error("Interrupted attempting to retrieve lock. Cannot confirm coverage metadata is available");
        throw interruptedException;
    } finally {
        if (locked) {
            coverageLock.unlock();
        }
    }
}

Example 17 with ContractDTO

use of gov.cms.ab2d.common.dto.ContractDTO in project ab2d by CMSgov.

the class CoverageProcessorImpl method startJob.

/**
 * Attempt to start a coverage search of BFD
 * @param mapping a mapping job
 */
@Override
public boolean startJob(CoverageMapping mapping) {
    synchronized (inProgressMappings) {
        if (inShutdown.get()) {
            log.warn("cannot start job because service has been shutdown");
            return false;
        }
        ContractDTO contract = contractWorkerClient.getContractByContractNumber(mapping.getContractNumber());
        if (contract == null) {
            log.warn("cannot grab contract using contract number {}", mapping.getContractNumber());
            return false;
        }
        log.info("starting search for {} during {}-{}", mapping.getContractNumber(), mapping.getPeriod().getMonth(), mapping.getPeriod().getYear());
        // Currently, we are using the STU3 version to get patient mappings
        CoverageMappingCallable callable = new CoverageMappingCallable(STU3, mapping, bfdClient, contractCoverageMapping.map(contract).getCorrectedYear(mapping.getPeriod().getYear()));
        executor.submit(callable);
        inProgressMappings.add(callable);
        return true;
    }
}

Example 18 with ContractDTO

use of gov.cms.ab2d.common.dto.ContractDTO in project ab2d by CMSgov.

the class ContractProcessorImpl method process.

/**
 * Process the contract - execute an entire {@link Job} from start to finish. Under the hood beneficiaries are
 * loaded from our database, queued for a thread pool to process, and the results processed and written out.
 * <p>
 * Under the hood all of this is done by paging through beneficiaries. We queue {@link #eobJobPatientQueuePageSize}
 * beneficiaries at a time and then check to see what requests have finished before attempting
 * to queue more beneficiaries.
 * <p>
 * Steps
 * - Calculate number of beneficiaries expected to process for the job
 * - Open a stream to the file system to write out results
 * - Page through enrollment queueing requests for each patient, and processing results
 * - Handle remaining requests waiting to be finished
 * - Report all files generated by running the job
 * - Cleanup stream
 * <p>
 * Periodically run at various points in this method
 * - Update the progress tracker with number of beneficiaries and eobs processed
 * - Are too many failures occurring in requests and do we need to shut down?
 * - Has the job been cancelled externally?
 * - Are too many requests waiting to run and do we have to wait for some requests to be processed?
 * - Log progress to database or console
 *
 * @return - the job output records containing the file information
 */
public List<JobOutput> process(Job job) {
    var contractNumber = job.getContractNumber();
    log.info("Beginning to process contract {}", keyValue(CONTRACT_LOG, contractNumber));
    // noinspection OptionalGetWithoutIsPresent
    ContractDTO contract = contractWorkerClient.getContractByContractNumber(contractNumber);
    int numBenes = coverageDriver.numberOfBeneficiariesToProcess(job, contract);
    jobChannelService.sendUpdate(job.getJobUuid(), JobMeasure.PATIENTS_EXPECTED, numBenes);
    log.info("Contract [{}] has [{}] Patients", contractNumber, numBenes);
    // Create the aggregator
    AggregatorCallable aggregator = new AggregatorCallable(searchConfig.getEfsMount(), job.getJobUuid(), contractNumber, ndjsonRollOver, searchConfig.getStreamingDir(), searchConfig.getFinishedDir(), searchConfig.getMultiplier());
    List<JobOutput> jobOutputs = new ArrayList<>();
    try {
        // Let the aggregator create all the necessary directories
        JobHelper.workerSetUpJobDirectories(job.getJobUuid(), searchConfig.getEfsMount(), searchConfig.getStreamingDir(), searchConfig.getFinishedDir());
        // Create the aggregator thread
        Future<Integer> aggregatorFuture = aggregatorThreadPool.submit(aggregator);
        ContractData contractData = new ContractData(contract, job);
        contractData.addAggregatorHandle(aggregatorFuture);
        // Iterate through pages of beneficiary data
        loadEobRequests(contractData);
        // Wait for remaining work to finish before cleaning up after the job
        // This should be at most eobJobPatientQueueMaxSize requests
        processRemainingRequests(contractData);
        log.info("Finished writing {} EOBs for contract {}", jobProgressService.getStatus(job.getJobUuid()).getEobsProcessedCount(), contractNumber);
        // Mark the job as finished for the aggregator (all file data has been written out)
        JobHelper.workerFinishJob(searchConfig.getEfsMount() + "/" + job.getJobUuid() + "/" + searchConfig.getStreamingDir());
        // Wait for the aggregator to finish
        while (!isDone(aggregatorFuture, job.getJobUuid(), true)) {
            Thread.sleep(1000);
        }
        // Retrieve all the job output info
        jobOutputs.addAll(getOutputs(job.getJobUuid(), DATA));
        jobOutputs.addAll(getOutputs(job.getJobUuid(), ERROR));
        log.info("Number of outputs: " + jobOutputs.size());
    } catch (InterruptedException | IOException ex) {
        log.error("interrupted while processing job for contract");
    }
    return jobOutputs;
}

Example 19 with ContractDTO

use of gov.cms.ab2d.common.dto.ContractDTO in project ab2d by CMSgov.

the class ContractProcessorImpl method loadEobRequests.

/**
 * Load beneficiaries and create an EOB request for each patient. Patients are loaded a page at a time. The page size is
 * configurable. At the end of loading all requests, the number of requests loaded is compared to the expected
 * number and the job immediately failed if not equal.
 * <p>
 * Steps:
 * - load a page of beneficiaries from the database
 * - create a request per patient and queue each request
 * - update the progress tracker with the number of patients added
 * - check if the job has been cancelled
 * - process any requests to BFD that are complete
 *
 * @param contractData job requests record and object for storing in motion requests
 * @throws InterruptedException if job is shut down during a busy wait for space in the queue
 */
private void loadEobRequests(ContractData contractData) throws InterruptedException {
    String jobUuid = contractData.getJob().getJobUuid();
    ContractDTO contract = contractData.getContract();
    // Handle first page of beneficiaries and then enter loop
    CoveragePagingResult current = coverageDriver.pageCoverage(new CoveragePagingRequest(eobJobPatientQueuePageSize, null, mapping.map(contract), contractData.getJob().getCreatedAt()));
    loadRequestBatch(contractData, current, searchConfig.getNumberBenesPerBatch());
    jobChannelService.sendUpdate(jobUuid, JobMeasure.PATIENT_REQUEST_QUEUED, current.size());
    // noinspection WhileLoopReplaceableByForEach
    while (current.getNextRequest().isPresent()) {
        if (eobClaimRequestsQueue.size(jobUuid) > eobJobPatientQueueMaxSize) {
            // Wait for queue to empty out some before adding more
            // noinspection BusyWait
            Thread.sleep(1000);
            continue;
        }
        // Queue a batch of patients
        current = coverageDriver.pageCoverage(current.getNextRequest().get());
        loadRequestBatch(contractData, current, searchConfig.getNumberBenesPerBatch());
        jobChannelService.sendUpdate(jobUuid, JobMeasure.PATIENT_REQUEST_QUEUED, current.size());
        processFinishedRequests(contractData);
    }
    // Verify that the number of benes requested matches the number expected from the database and fail
    // immediately if the two do not match
    ProgressTracker progressTracker = jobProgressService.getStatus(jobUuid);
    int totalQueued = progressTracker.getPatientRequestQueuedCount();
    int totalExpected = progressTracker.getPatientsExpected();
    if (totalQueued != totalExpected) {
        throw new ContractProcessingException("expected " + totalExpected + " patients from database but retrieved " + totalQueued);
    }
}

Example 20 with ContractDTO

use of gov.cms.ab2d.common.dto.ContractDTO in project ab2d by CMSgov.

the class JobPreProcessorImpl method preprocess.

@Override
@Transactional(propagation = Propagation.REQUIRES_NEW, isolation = Isolation.SERIALIZABLE)
public Job preprocess(String jobUuid) {
    Job job = jobRepository.findByJobUuid(jobUuid);
    if (job == null) {
        log.error("Job was not found");
        throw new IllegalArgumentException("Job " + jobUuid + " was not found");
    }
    // validate status is SUBMITTED
    if (!SUBMITTED.equals(job.getStatus())) {
        final String errMsg = String.format("Job %s is not in %s status", jobUuid, SUBMITTED);
        log.error("Job is not in submitted status");
        throw new IllegalArgumentException(errMsg);
    }
    ContractDTO contract = contractWorkerClient.getContractByContractNumber(job.getContractNumber());
    if (contract == null) {
        throw new IllegalArgumentException("A job must always have a contract.");
    }
    Optional<OffsetDateTime> sinceValue = Optional.ofNullable(job.getSince());
    if (sinceValue.isPresent()) {
        // If the user provided a 'since' value
        job.setSinceSource(SinceSource.USER);
        jobRepository.save(job);
    } else if (job.getFhirVersion().supportDefaultSince() && !contract.hasDateIssue()) {
        // If the user did not, but this version supports a default 'since', populate it
        job = updateSinceTime(job, contract);
        jobRepository.save(job);
    }
    try {
        if (!coverageDriver.isCoverageAvailable(job, contract)) {
            log.info("coverage metadata is not up to date so job will not be started");
            return job;
        }
        eventLogger.logAndAlert(EventUtils.getJobChangeEvent(job, IN_PROGRESS, EOB_JOB_STARTED + " for " + contract.getContractNumber() + " in progress"), PUBLIC_LIST);
        job.setStatus(IN_PROGRESS);
        job.setStatusMessage(null);
        job = jobRepository.save(job);
    } catch (CoverageDriverException coverageDriverException) {
        eventLogger.logAndAlert(EventUtils.getJobChangeEvent(job, FAILED, EOB_JOB_COVERAGE_ISSUE + " Job for " + contract.getContractNumber() + " in progress"), PUBLIC_LIST);
        job.setStatus(FAILED);
        job.setStatusMessage("could not pull coverage information for contract");
        job = jobRepository.save(job);
    } catch (InterruptedException ie) {
        throw new RuntimeException("could not determine whether coverage metadata was up to date", ie);
    }
    return job;
}