use of gov.cms.bfd.pipeline.ccw.rif.load.RifRecordLoadResult in project beneficiary-fhir-data by CMSgov.
the class DefaultDataSetMonitorListener method dataAvailable.
/**
* @see
* gov.cms.bfd.pipeline.ccw.rif.extract.s3.DataSetMonitorListener#dataAvailable(gov.cms.bfd.model.rif.RifFilesEvent)
*/
@Override
public void dataAvailable(RifFilesEvent rifFilesEvent) {
Timer.Context timerDataSet = appMetrics.timer(MetricRegistry.name(PipelineApplication.class.getSimpleName(), "dataSet", "processed")).time();
Consumer<Throwable> errorHandler = error -> {
/*
* This will be called on the same thread used to run each
* RifLoader task (probably a background one). This is not
* the right place to do any error _recovery_ (that'd have
* to be inside RifLoader itself), but it is likely the
* right place to decide when/if a failure is "bad enough"
* that the rest of processing should be stopped. Right now
* we stop that way for _any_ failure, but we probably want
* to be more discriminating than that.
*/
errorOccurred(error);
};
Consumer<RifRecordLoadResult> resultHandler = result -> {
/*
* Don't really *need* to do anything here. The RifLoader
* already records metrics for each data set.
*/
};
/*
* Each ETL stage produces a stream that will be handed off to
* and processed by the next stage.
*/
for (RifFileEvent rifFileEvent : rifFilesEvent.getFileEvents()) {
Slf4jReporter dataSetFileMetricsReporter = Slf4jReporter.forRegistry(rifFileEvent.getEventMetrics()).outputTo(LOGGER).build();
dataSetFileMetricsReporter.start(2, TimeUnit.MINUTES);
RifFileRecords rifFileRecords = rifProcessor.produceRecords(rifFileEvent);
rifLoader.process(rifFileRecords, errorHandler, resultHandler);
dataSetFileMetricsReporter.stop();
dataSetFileMetricsReporter.report();
}
timerDataSet.stop();
}
Aggregations