Examples with LoadAppOptions gov.cms.bfd.pipeline.ccw.rif.load.LoadAppOptions used on opensource projects

Example 1 with LoadAppOptions

use of gov.cms.bfd.pipeline.ccw.rif.load.LoadAppOptions in project beneficiary-fhir-data by CMSgov.

the class LoadedFilterManagerIT method loadData.

/**
 * @param sampleResources the sample RIF resources to load
 */
private static void loadData(DataSource dataSource, List<StaticRifResource> sampleResources) {
    LoadAppOptions loadOptions = CcwRifLoadTestUtils.getLoadOptions();
    RifFilesEvent rifFilesEvent = new RifFilesEvent(Instant.now(), sampleResources.stream().map(StaticRifResource::toRifFile).collect(Collectors.toList()));
    // Create the processors that will handle each stage of the pipeline.
    RifFilesProcessor processor = new RifFilesProcessor();
    RifLoader loader = new RifLoader(loadOptions, PipelineTestUtils.get().getPipelineApplicationState());
    // Link up the pipeline and run it.
    for (RifFileEvent rifFileEvent : rifFilesEvent.getFileEvents()) {
        RifFileRecords rifFileRecords = processor.produceRecords(rifFileEvent);
        loader.process(rifFileRecords, error -> {
        }, result -> {
        });
    }
}

Example 2 with LoadAppOptions

use of gov.cms.bfd.pipeline.ccw.rif.load.LoadAppOptions in project beneficiary-fhir-data by CMSgov.

the class ServerTestUtils method loadData.

/**
 * @param sampleResources the sample RIF resources to load
 * @return the {@link List} of RIF records that were loaded (e.g. {@link Beneficiary}s, etc.)
 */
public List<Object> loadData(List<StaticRifResource> sampleResources) {
    LoadAppOptions loadOptions = CcwRifLoadTestUtils.getLoadOptions();
    RifFilesEvent rifFilesEvent = new RifFilesEvent(Instant.now(), sampleResources.stream().map(r -> r.toRifFile()).collect(Collectors.toList()));
    // Create the processors that will handle each stage of the pipeline.
    RifFilesProcessor processor = new RifFilesProcessor();
    // Link up the pipeline and run it.
    RifLoader loader = new RifLoader(loadOptions, PipelineTestUtils.get().getPipelineApplicationState());
    LOGGER.info("Loading RIF records...");
    List<Object> recordsLoaded = new ArrayList<>();
    for (RifFileEvent rifFileEvent : rifFilesEvent.getFileEvents()) {
        RifFileRecords rifFileRecords = processor.produceRecords(rifFileEvent);
        loader.process(rifFileRecords, error -> {
            LOGGER.warn("Record(s) failed to load.", error);
        }, result -> {
            recordsLoaded.add(result.getRifRecordEvent().getRecord());
        });
    }
    LOGGER.info("Loaded RIF records: '{}'.", recordsLoaded.size());
    return recordsLoaded;
}

Example 3 with LoadAppOptions

use of gov.cms.bfd.pipeline.ccw.rif.load.LoadAppOptions in project beneficiary-fhir-data by CMSgov.

the class AppConfiguration method readConfigFromEnvironmentVariables.

/**
 * Per <code>/dev/design-decisions-readme.md</code>, this application accepts its configuration
 * via environment variables. Read those in, and build an {@link AppConfiguration} instance from
 * them.
 *
 * <p>As a convenience, this method will also verify that AWS credentials were provided, such that
 * {@link DefaultAWSCredentialsProviderChain} can load them. If not, an {@link
 * AppConfigurationException} will be thrown.
 *
 * @return the {@link AppConfiguration} instance represented by the configuration provided to this
 *     application via the environment variables
 * @throws AppConfigurationException An {@link AppConfigurationException} will be thrown if the
 *     configuration passed to the application are incomplete or incorrect.
 */
static AppConfiguration readConfigFromEnvironmentVariables() {
    int hicnHashIterations = readEnvIntPositiveRequired(ENV_VAR_KEY_HICN_HASH_ITERATIONS);
    byte[] hicnHashPepper = readEnvBytesRequired(ENV_VAR_KEY_HICN_HASH_PEPPER);
    int hicnHashCacheSize = readEnvIntOptional(ENV_VAR_KEY_HICN_HASH_CACHE_SIZE).orElse(DEFAULT_HICN_HASH_CACHE_SIZE);
    String databaseUrl = readEnvStringRequired(ENV_VAR_KEY_DATABASE_URL);
    String databaseUsername = readEnvStringRequired(ENV_VAR_KEY_DATABASE_USERNAME);
    String databasePassword = readEnvStringRequired(ENV_VAR_KEY_DATABASE_PASSWORD);
    int loaderThreads = readEnvIntPositiveRequired(ENV_VAR_KEY_LOADER_THREADS);
    boolean idempotencyRequired = readEnvBooleanRequired(ENV_VAR_KEY_IDEMPOTENCY_REQUIRED);
    boolean filteringNonNullAndNon2022Benes = readEnvBooleanOptional(ENV_VAR_KEY_RIF_FILTERING_NON_NULL_AND_NON_2022_BENES).orElse(DEFAULT_RIF_FILTERING_NON_NULL_AND_NON_2022_BENES);
    Optional<String> newRelicMetricKey = readEnvStringOptional(ENV_VAR_NEW_RELIC_METRIC_KEY);
    Optional<String> newRelicAppName = readEnvStringOptional(ENV_VAR_NEW_RELIC_APP_NAME);
    Optional<String> newRelicMetricHost = readEnvStringOptional(ENV_VAR_NEW_RELIC_METRIC_HOST);
    Optional<String> newRelicMetricPath = readEnvStringOptional(ENV_VAR_NEW_RELIC_METRIC_PATH);
    Optional<Integer> newRelicMetricPeriod = readEnvIntOptional(ENV_VAR_NEW_RELIC_METRIC_PERIOD);
    /*
     * Note: For CcwRifLoadJob, databaseMaxPoolSize needs to be double the number of loader threads
     * when idempotent loads are being used. Apparently, the queries need a separate Connection?
     */
    Optional<Integer> databaseMaxPoolSize = readEnvIntOptional(ENV_VAR_KEY_DATABASE_MAX_POOL_SIZE);
    if (databaseMaxPoolSize.isPresent() && databaseMaxPoolSize.get() < 1)
        throw new AppConfigurationException(String.format("Invalid value for configuration environment variable '%s': '%s'", ENV_VAR_KEY_DATABASE_MAX_POOL_SIZE, databaseMaxPoolSize));
    if (!databaseMaxPoolSize.isPresent())
        databaseMaxPoolSize = Optional.of(loaderThreads * 2);
    Optional<String> hostname;
    try {
        hostname = Optional.of(InetAddress.getLocalHost().getHostName());
    } catch (UnknownHostException e) {
        hostname = Optional.empty();
    }
    MetricOptions metricOptions = new MetricOptions(newRelicMetricKey, newRelicAppName, newRelicMetricHost, newRelicMetricPath, newRelicMetricPeriod, hostname);
    DatabaseOptions databaseOptions = new DatabaseOptions(databaseUrl, databaseUsername, databasePassword, databaseMaxPoolSize.get());
    LoadAppOptions loadOptions = new LoadAppOptions(IdHasher.Config.builder().hashIterations(hicnHashIterations).hashPepper(hicnHashPepper).cacheSize(hicnHashCacheSize).build(), loaderThreads, idempotencyRequired, filteringNonNullAndNon2022Benes);
    CcwRifLoadOptions ccwRifLoadOptions = readCcwRifLoadOptionsFromEnvironmentVariables(loadOptions);
    RdaLoadOptions rdaLoadOptions = readRdaLoadOptionsFromEnvironmentVariables(loadOptions.getIdHasherConfig());
    return new AppConfiguration(metricOptions, databaseOptions, ccwRifLoadOptions, rdaLoadOptions);
}