Examples with NullPipelineJobArguments gov.cms.bfd.pipeline.sharedutils.NullPipelineJobArguments used on opensource projects

Example 1 with NullPipelineJobArguments

use of gov.cms.bfd.pipeline.sharedutils.NullPipelineJobArguments in project beneficiary-fhir-data by CMSgov.

the class PipelineManagerIT method runInterruptibleJobsThenStop.

/**
 * Verifies that {@link PipelineManager#stop()} works, as expected.
 *
 * @throws Exception Any unhandled {@link Exception}s will cause this test case to fail.
 */
@Test
public void runInterruptibleJobsThenStop() throws Exception {
    // Create the pipeline and have it run a mock job.
    PipelineJobRecordStore jobRecordStore = new PipelineJobRecordStore(PipelineTestUtils.get().getPipelineApplicationState().getMetrics());
    try (PipelineManager pipelineManager = new PipelineManager(PipelineTestUtils.get().getPipelineApplicationState().getMetrics(), jobRecordStore)) {
        MockJob mockJob = new MockJob(Optional.of(new PipelineJobSchedule(1, ChronoUnit.MILLIS)), () -> {
            // Add an artificial delay that we'll be able to measure.
            Thread.sleep(500);
            return PipelineJobOutcome.WORK_DONE;
        });
        pipelineManager.registerJob(mockJob);
        jobRecordStore.submitPendingJob(MockJob.JOB_TYPE, null);
        // Wait until the mock job has started.
        Awaitility.await().atMost(1, TimeUnit.SECONDS).until(() -> jobRecordStore.getJobRecords().stream().filter(j -> MockJob.JOB_TYPE.equals(j.getJobType()) && j.isStarted()).findAny().isPresent());
        // Stop the pipeline and then make sure that the job was actually interrupted.
        pipelineManager.stop();
        PipelineJobRecord<NullPipelineJobArguments> mockJobRecord = jobRecordStore.findMostRecent(MockJob.JOB_TYPE).get();
        assertTrue(mockJobRecord.getCanceledTime().isPresent());
        assertTrue(mockJobRecord.getDuration().get().toMillis() < 500);
    }
}

Example 2 with NullPipelineJobArguments

use of gov.cms.bfd.pipeline.sharedutils.NullPipelineJobArguments in project beneficiary-fhir-data by CMSgov.

the class PipelineApplication method main.

/**
 * This method is the one that will get called when users launch the application from the command
 * line.
 *
 * @param args (should be empty, as this application accepts configuration via environment
 *     variables)
 * @throws Exception any unhandled checked {@link Exception}s that are encountered will cause the
 *     application to halt
 */
public static void main(String[] args) throws Exception {
    LOGGER.info("Application starting up!");
    configureUnexpectedExceptionHandlers();
    AppConfiguration appConfig = null;
    try {
        appConfig = AppConfiguration.readConfigFromEnvironmentVariables();
        LOGGER.info("Application configured: '{}'", appConfig);
    } catch (AppConfigurationException e) {
        System.err.println(e.getMessage());
        LOGGER.warn("Invalid app configuration.", e);
        System.exit(EXIT_CODE_BAD_CONFIG);
    }
    MetricRegistry appMetrics = new MetricRegistry();
    appMetrics.registerAll(new MemoryUsageGaugeSet());
    appMetrics.registerAll(new GarbageCollectorMetricSet());
    Slf4jReporter appMetricsReporter = Slf4jReporter.forRegistry(appMetrics).outputTo(LOGGER).build();
    MetricOptions metricOptions = appConfig.getMetricOptions();
    if (metricOptions.getNewRelicMetricKey().isPresent()) {
        SenderConfiguration configuration = SenderConfiguration.builder(metricOptions.getNewRelicMetricHost().orElse(null), metricOptions.getNewRelicMetricPath().orElse(null)).httpPoster(new OkHttpPoster()).apiKey(metricOptions.getNewRelicMetricKey().orElse(null)).build();
        MetricBatchSender metricBatchSender = MetricBatchSender.create(configuration);
        Attributes commonAttributes = new Attributes().put("host", metricOptions.getHostname().orElse("unknown")).put("appName", metricOptions.getNewRelicAppName().orElse(null));
        NewRelicReporter newRelicReporter = NewRelicReporter.build(appMetrics, metricBatchSender).commonAttributes(commonAttributes).build();
        newRelicReporter.start(metricOptions.getNewRelicMetricPeriod().orElse(15), TimeUnit.SECONDS);
    }
    appMetricsReporter.start(1, TimeUnit.HOURS);
    /*
     * Create the PipelineManager that will be responsible for running and managing the various
     * jobs.
     */
    PipelineJobRecordStore jobRecordStore = new PipelineJobRecordStore(appMetrics);
    PipelineManager pipelineManager = new PipelineManager(appMetrics, jobRecordStore);
    registerShutdownHook(appMetrics, pipelineManager);
    LOGGER.info("Job processing started.");
    // Create a pooled data source for use by the DatabaseSchemaUpdateJob.
    final HikariDataSource pooledDataSource = PipelineApplicationState.createPooledDataSource(appConfig.getDatabaseOptions(), appMetrics);
    /*
     * Register and wait for the database schema job to run, so that we don't have to worry about
     * declaring it as a dependency (since it is for pretty much everything right now).
     */
    pipelineManager.registerJob(new DatabaseSchemaUpdateJob(pooledDataSource));
    PipelineJobRecord<NullPipelineJobArguments> dbSchemaJobRecord = jobRecordStore.submitPendingJob(DatabaseSchemaUpdateJob.JOB_TYPE, null);
    try {
        jobRecordStore.waitForJobs(dbSchemaJobRecord);
    } catch (InterruptedException e) {
        pooledDataSource.close();
        throw new InterruptedException();
    }
    /*
     * Create and register the other jobs.
     */
    if (appConfig.getCcwRifLoadOptions().isPresent()) {
        // Create an application state that reuses the existing pooled data source with the ccw/rif
        // persistence unit.
        final PipelineApplicationState appState = new PipelineApplicationState(appMetrics, pooledDataSource, PipelineApplicationState.PERSISTENCE_UNIT_NAME, Clock.systemUTC());
        pipelineManager.registerJob(createCcwRifLoadJob(appConfig.getCcwRifLoadOptions().get(), appState));
        LOGGER.info("Registered CcwRifLoadJob.");
    } else {
        LOGGER.warn("CcwRifLoadJob is disabled in app configuration.");
    }
    if (appConfig.getRdaLoadOptions().isPresent()) {
        LOGGER.info("RDA API jobs are enabled in app configuration.");
        // Create an application state that reuses the existing pooled data source with the rda
        // persistence unit.
        final PipelineApplicationState rdaAppState = new PipelineApplicationState(appMetrics, pooledDataSource, PipelineApplicationState.RDA_PERSISTENCE_UNIT_NAME, Clock.systemUTC());
        final RdaLoadOptions rdaLoadOptions = appConfig.getRdaLoadOptions().get();
        final Optional<RdaServerJob> mockServerJob = rdaLoadOptions.createRdaServerJob();
        if (mockServerJob.isPresent()) {
            pipelineManager.registerJob(mockServerJob.get());
            LOGGER.warn("Registered RdaServerJob.");
        } else {
            LOGGER.info("Skipping RdaServerJob registration - not enabled in app configuration.");
        }
        pipelineManager.registerJob(rdaLoadOptions.createFissClaimsLoadJob(rdaAppState));
        LOGGER.info("Registered RdaFissClaimLoadJob.");
        pipelineManager.registerJob(rdaLoadOptions.createMcsClaimsLoadJob(rdaAppState));
        LOGGER.info("Registered RdaMcsClaimLoadJob.");
    } else {
        LOGGER.info("RDA API jobs are not enabled in app configuration.");
    }
/*
     * At this point, we're done here with the main thread. From now on, the PipelineManager's
     * executor service should be the only non-daemon thread running (and whatever it kicks off).
     * Once/if that thread stops, the application will run all registered shutdown hooks and Wait
     * for the PipelineManager to stop running jobs, and then check to see if we should exit
     * normally with 0 or abnormally with a non-0 because a job failed.
     */
}

Example 3 with NullPipelineJobArguments

use of gov.cms.bfd.pipeline.sharedutils.NullPipelineJobArguments in project beneficiary-fhir-data by CMSgov.

the class SchedulerJob method call.

/**
 * @see gov.cms.bfd.pipeline.sharedutils.PipelineJob#call()
 */
@Override
public PipelineJobOutcome call() throws Exception {
    boolean scheduledAJob = false;
    while (true) {
        try (Timer.Context timer = appMetrics.timer(MetricRegistry.name(getClass().getSimpleName(), "call", "iteration")).time()) {
            Instant now = Instant.now();
            Set<PipelineJob<NullPipelineJobArguments>> scheduledJobs = pipelineManager.getScheduledJobs();
            for (PipelineJob<NullPipelineJobArguments> scheduledJob : scheduledJobs) {
                PipelineJobSchedule jobSchedule = scheduledJob.getSchedule().get();
                Optional<PipelineJobRecord<NullPipelineJobArguments>> mostRecentExecution = jobRecordsStore.findMostRecent(scheduledJob.getType());
                /* Calculate whether or not we should trigger an execution of the next job. */
                boolean shouldTriggerJob;
                if (!mostRecentExecution.isPresent()) {
                    // If the job has never run, we'll always trigger it now, regardless of schedule.
                    shouldTriggerJob = true;
                } else {
                    if (!mostRecentExecution.get().isCompleted()) {
                        // If the job's still pending or running, don't double-trigger it.
                        shouldTriggerJob = false;
                    } else {
                        if (mostRecentExecution.get().isCompletedSuccessfully()) {
                            // If the job's not running, check to see if it's time to trigger it again.
                            // Note: This calculation is based on completion time, not submission or start time.
                            Instant nextExecution = mostRecentExecution.get().getStartedTime().get().plus(jobSchedule.getRepeatDelay(), jobSchedule.getRepeatDelayUnit());
                            shouldTriggerJob = now.equals(nextExecution) || now.isAfter(nextExecution);
                        } else {
                            // We don't re-run failed jobs.
                            shouldTriggerJob = false;
                        }
                    }
                }
                // If we shouldn't trigger this job, move on to the next.
                if (!shouldTriggerJob) {
                    continue;
                }
                // Trigger the job (for future execution, when VolunteerJob picks it up)!
                jobRecordsStore.submitPendingJob(scheduledJob.getType(), null);
            }
        }
        try {
            Thread.sleep(SCHEDULER_TICK_MILLIS);
        } catch (InterruptedException e) {
            /*
         * Jobs are only interrupted/cancelled as part of application shutdown, so when encountered,
         * we'll break out of our scheduling loop and close up shop here.
         */
            break;
        }
    }
    /*
     * Did we schedule at least one job? If we ever move to an autoscaled version of this
     * application, it will be important to ensure that we "collude" with the PipelineJobRecordStore
     * to ignore this PipelineJobOutcome and ensure that the record doesn't get marked as completed,
     * even when the application shuts down. (If that happened, then scheduled triggers would stop
     * firing.)
     */
    return scheduledAJob ? PipelineJobOutcome.WORK_DONE : PipelineJobOutcome.NOTHING_TO_DO;
}

Example 4 with NullPipelineJobArguments

use of gov.cms.bfd.pipeline.sharedutils.NullPipelineJobArguments in project beneficiary-fhir-data by CMSgov.

the class PipelineManagerIT method runWayTooManyJobsThenStop.

/**
 * Verifies that {@link PipelineManager} can performantly handle large numbers of job executions,
 * as expected. Note that "performantly" is relative to what we need: fast enough to run a few
 * jobs every second of the day. Mostly, though, this test case is used as a good way to inspect
 * and evaluate the various metrics that the application collects.
 *
 * <p>This is intentionally left ignored most of the time, so as to not slow down our builds. It
 * should only be run if/when someone is looking into performance issues.
 *
 * @throws Exception Any unhandled {@link Exception}s will cause this test case to fail.
 */
@Test
@Disabled
public void runWayTooManyJobsThenStop() throws Exception {
    // Let's speed things up a bit, so we can run more iterations, faster.
    SchedulerJob.SCHEDULER_TICK_MILLIS = 1;
    VolunteerJob.VOLUNTEER_TICK_MILLIS = 1;
    MetricRegistry appMetrics = new MetricRegistry();
    Slf4jReporter.forRegistry(appMetrics).outputTo(LOGGER).build().start(30, TimeUnit.SECONDS);
    // Create the pipeline.
    PipelineJobRecordStore jobRecordStore = new PipelineJobRecordStore(appMetrics);
    try (PipelineManager pipelineManager = new PipelineManager(appMetrics, jobRecordStore)) {
        // Register a mock unscheduled job.
        MockJob mockUnscheduledJob = new MockJob(Optional.empty(), () -> {
            return PipelineJobOutcome.WORK_DONE;
        });
        pipelineManager.registerJob(mockUnscheduledJob);
        // Register a second scheduled job.
        MockJob mockScheduledJob = new MockJob(Optional.of(new PipelineJobSchedule(1, ChronoUnit.MILLIS)), () -> {
            return PipelineJobOutcome.WORK_DONE;
        }) {

            /*
             * Very hacky, but here we're extending MockJob with an anonymous class that has a
             * different getType() value.
             */
            /**
             * @see gov.cms.bfd.pipeline.app.PipelineManagerIT.MockJob#getType()
             */
            @Override
            public PipelineJobType<NullPipelineJobArguments> getType() {
                return new PipelineJobType<>(this);
            }
        };
        pipelineManager.registerJob(mockScheduledJob);
        /*
       * Submit way too many executions of the unscheduled job. The number here corresponds to how
       * many executions you'd get if it was run once a second, every second of the day.
       */
        for (int i = 0; i < 24 * 60 * 60; i++) {
            jobRecordStore.submitPendingJob(MockJob.JOB_TYPE, null);
        }
        /*
       * Wait until all of the jobs have completed, with a large timeout. Don't worry: it only takes
       * about 500 seconds on my system.
       */
        Awaitility.await().atMost(20, TimeUnit.MINUTES).until(() -> jobRecordStore.getJobRecords().stream().filter(j -> MockJob.JOB_TYPE.equals(j.getJobType()) && !j.isCompleted()).findAny().isPresent() == false);
        // Stop the pipeline.
        pipelineManager.stop();
        // Verify that all jobs completed successfully.
        Set<PipelineJobRecord<?>> unsuccessfulJobs = jobRecordStore.getJobRecords().stream().filter(j -> MockJob.JOB_TYPE.equals(j.getJobType()) && !j.isCompletedSuccessfully()).collect(Collectors.toSet());
        assertEquals(0, unsuccessfulJobs.size());
        // Ensure that the final metrics get logged.
        Slf4jReporter.forRegistry(appMetrics).outputTo(LOGGER).build().report();
    } finally {
        configureTimers();
    }
}