Examples with DataSampler gov.cms.bfd.pipeline.bridge.util.DataSampler used on opensource projects

Example 1 with DataSampler

use of gov.cms.bfd.pipeline.bridge.util.DataSampler in project beneficiary-fhir-data by CMSgov.

the class RDABridge method run.

/**
 * Reads all relevant source files, executing task logic for each claim found.
 *
 * @param config The configurations to use when generating the RDA data.
 * @throws IOException If there was an issue accessing any of the files.
 */
public void run(ConfigLoader config) throws IOException {
    WrappedCounter fissSequence = new WrappedCounter(config.intValue(AppConfig.Fields.fissSeqStart));
    WrappedCounter mcsSequence = new WrappedCounter(config.intValue(AppConfig.Fields.mcsSeqStart));
    if (fissSequence.get() < 1 || mcsSequence.get() < 1) {
        throw new IllegalArgumentException("Sequences must start at 1 or higher.");
    }
    Path inputDirectory = Paths.get(config.stringValue(AppConfig.Fields.inputDirPath));
    Map<String, BeneficiaryData> mbiMap = parseMbiNumbers(inputDirectory.resolve(config.stringValue(AppConfig.Fields.mbiSource)));
    Path outputDirectory = Paths.get(config.stringOption(AppConfig.Fields.outputDirPath).orElse("output"));
    // ResultOfMethodCallIgnored - Don't need to know if it had to be created.
    // noinspection ResultOfMethodCallIgnored
    outputDirectory.toFile().mkdir();
    String fissOutputFile = config.stringOption(AppConfig.Fields.fissOutputFile).orElse("FISS.ndjson");
    String mcsOutputFile = config.stringOption(AppConfig.Fields.mcsOutputFile).orElse("MCS.ndjson");
    Path fissOutputPath = outputDirectory.resolve(fissOutputFile);
    Path mcsOutputPath = outputDirectory.resolve(mcsOutputFile);
    String fissOutputType = FilenameUtils.getExtension(fissOutputPath.getFileName().toString());
    String mcsOutputType = FilenameUtils.getExtension(mcsOutputPath.getFileName().toString());
    if (!sinkMap.containsKey(fissOutputType)) {
        throw new IllegalArgumentException("Unsupported fiss output file type '" + fissOutputType + "'");
    } else if (!sinkMap.containsKey(mcsOutputType)) {
        throw new IllegalArgumentException("Unsupported mcs output file type '" + mcsOutputType + "'");
    } else {
        final int FISS_ID = 0;
        final int MCS_ID = 1;
        // Grab given ratios for fiss/mcs attribution output
        float fissRatio = config.floatOption(AppConfig.Fields.attributionFissRatio).orElse(1.0f);
        // Convert ratio to proportion
        float fissProportion = 1.0f - (1.0f / (1.0f + fissRatio));
        float mcsProportion = 1.0f - fissProportion;
        DataSampler<String> mbiSampler = DataSampler.<String>builder().maxValues(config.intOption(AppConfig.Fields.attributionSetSize).orElse(10_000)).registerSampleSet(FISS_ID, fissProportion).registerSampleSet(MCS_ID, mcsProportion).build();
        try (Sink<MessageOrBuilder> fissSink = sinkMap.get(fissOutputType).apply(new SinkArguments(fissOutputPath, fissSequence));
            Sink<MessageOrBuilder> mcsSink = sinkMap.get(mcsOutputType).apply(new SinkArguments(mcsOutputPath, mcsSequence))) {
            // Sorting the files so tests are more deterministic
            List<String> fissSources = config.stringValues(AppConfig.Fields.fissSources);
            Collections.sort(fissSources);
            for (String fissSource : fissSources) {
                executeTransformation(SourceType.FISS, inputDirectory, fissSource, fissSequence, mbiMap, fissSink, mbiSampler, FISS_ID);
            }
            // Sorting the files so tests are more deterministic
            List<String> mcsSources = config.stringValues(AppConfig.Fields.mcsSources);
            Collections.sort(mcsSources);
            for (String mcsSource : mcsSources) {
                executeTransformation(SourceType.MCS, inputDirectory, mcsSource, mcsSequence, mbiMap, mcsSink, mbiSampler, MCS_ID);
            }
        }
        if (config.booleanValue(AppConfig.Fields.buildAttributionSet, false)) {
            String templateFileName = config.stringOption(AppConfig.Fields.attributionTemplateFile).orElse(DEFAULT_INPUT_FILE);
            String outputFileName = config.stringOption(AppConfig.Fields.attributionScriptFile).orElse(DEFAULT_OUTPUT_FILE);
            AttributionBuilder builder = new AttributionBuilder(templateFileName, outputFileName);
            builder.run(mbiSampler);
        }
    }
}