Examples with Config com.typesafe.config.Config used on opensource projects

Example 46 with Config

use of com.typesafe.config.Config in project incubator-gobblin by apache.

the class InMemoryValueInspector method getOwnConfigs.

/**
 * {@inheritDoc}.
 *
 * <p>
 *   If present in the cache, return the cached {@link com.typesafe.config.Config} for given input
 *   Otherwise, simply delegate the functionality to the internal {ConfigStoreValueInspector} and store the value into cache
 * </p>
 */
@Override
public Map<ConfigKeyPath, Config> getOwnConfigs(Collection<ConfigKeyPath> configKeys) {
    Collection<ConfigKeyPath> configKeysNotInCache = new ArrayList<>();
    Map<ConfigKeyPath, Config> result = new HashMap<>();
    for (ConfigKeyPath configKey : configKeys) {
        Config cachedValue = this.ownConfigCache.getIfPresent(configKey);
        if (cachedValue == null) {
            configKeysNotInCache.add(configKey);
        } else {
            result.put(configKey, cachedValue);
        }
    }
    // for ConfigKeyPath which are not in cache
    if (configKeysNotInCache.size() > 0) {
        Map<ConfigKeyPath, Config> configsFromFallBack = this.valueFallback.getOwnConfigs(configKeysNotInCache);
        this.ownConfigCache.putAll(configsFromFallBack);
        result.putAll(configsFromFallBack);
    }
    return result;
}

Example 47 with Config

use of com.typesafe.config.Config in project incubator-gobblin by apache.

the class AvroToRestJsonEntryConverter method convertRecord.

/**
 * Use resource key(Optional) and rest json entry as a template and fill in template using Avro as a reference.
 * e.g:
 *  Rest JSON entry HOCON template:
 *    AccountId=${sf_account_id},Member_Id__c=${member_id}
 *  Avro:
 *    {"sf_account_id":{"string":"0016000000UiCYHAA3"},"member_id":{"long":296458833}}
 *
 *  Converted Json:
 *    {"AccountId":"0016000000UiCYHAA3","Member_Id__c":296458833}
 *
 *  As it's template based approach, it can produce nested JSON structure even Avro is flat (or vice versa).
 *
 * e.g:
 *  Rest resource template:
 *    /sobject/account/memberId/${member_id}
 *  Avro:
 *    {"sf_account_id":{"string":"0016000000UiCYHAA3"},"member_id":{"long":296458833}}
 *  Converted resource:
 *    /sobject/account/memberId/296458833
 *
 *  Converted resource will be used to form end point.
 *    http://www.server.com:9090/sobject/account/memberId/296458833
 *
 * {@inheritDoc}
 * @see org.apache.gobblin.converter.Converter#convertRecord(java.lang.Object, java.lang.Object, org.apache.gobblin.configuration.WorkUnitState)
 */
@Override
public Iterable<RestEntry<JsonObject>> convertRecord(Void outputSchema, GenericRecord inputRecord, WorkUnitState workUnit) throws DataConversionException {
    Config srcConfig = ConfigFactory.parseString(inputRecord.toString(), ConfigParseOptions.defaults().setSyntax(ConfigSyntax.JSON));
    String resourceKey = workUnit.getProp(CONVERTER_AVRO_REST_ENTRY_RESOURCE_KEY, "");
    if (!StringUtils.isEmpty(resourceKey)) {
        final String dummyKey = "DUMMY";
        Config tmpConfig = ConfigFactory.parseString(dummyKey + "=" + resourceKey).resolveWith(srcConfig);
        resourceKey = tmpConfig.getString(dummyKey);
    }
    String hoconInput = workUnit.getProp(CONVERTER_AVRO_REST_JSON_ENTRY_TEMPLATE);
    if (StringUtils.isEmpty(hoconInput)) {
        return new SingleRecordIterable<>(new RestEntry<>(resourceKey, parser.parse(inputRecord.toString()).getAsJsonObject()));
    }
    Config destConfig = ConfigFactory.parseString(hoconInput).resolveWith(srcConfig);
    JsonObject json = parser.parse(destConfig.root().render(ConfigRenderOptions.concise())).getAsJsonObject();
    return new SingleRecordIterable<>(new RestEntry<>(resourceKey, json));
}

Example 48 with Config

use of com.typesafe.config.Config in project incubator-gobblin by apache.

the class DefaultHttpClientConfigurator method stateToConfig.

protected Config stateToConfig(State state) {
    String proxyUrlKey = getPrefixedPropertyName(PROXY_URL_KEY);
    String proxyPortKey = getPrefixedPropertyName(PROXY_PORT_KEY);
    String proxyHostportKey = getPrefixedPropertyName(PROXY_HOSTPORT_KEY);
    Config cfg = ConfigFactory.empty();
    if (state.contains(proxyUrlKey)) {
        cfg = cfg.withValue(PROXY_URL_KEY, ConfigValueFactory.fromAnyRef(state.getProp(proxyUrlKey)));
    }
    if (state.contains(proxyPortKey)) {
        cfg = cfg.withValue(PROXY_PORT_KEY, ConfigValueFactory.fromAnyRef(state.getPropAsInt(proxyPortKey)));
    }
    if (state.contains(proxyHostportKey)) {
        cfg = cfg.withValue(PROXY_HOSTPORT_KEY, ConfigValueFactory.fromAnyRef(state.getProp(proxyHostportKey)));
    }
    return cfg;
}

Example 49 with Config

use of com.typesafe.config.Config in project incubator-gobblin by apache.

the class ReplicationConfiguration method buildFromConfig.

public static ReplicationConfiguration buildFromConfig(Config input) throws InstantiationException, IllegalAccessException, ClassNotFoundException {
    Preconditions.checkArgument(input != null, "can not build ReplicationConfig from null");
    Config config = input.resolve();
    return new Builder().withReplicationMetaData(ReplicationMetaData.buildMetaData(config)).withReplicationCopyMode(ReplicationCopyMode.getReplicationCopyMode(config)).withSelectionConfig(config.getConfig("gobblin.selected.policy")).withReplicationSource(config).withReplicationReplica(config).withDefaultDataFlowTopologyConfig_PullMode(config).withDefaultDataFlowTopologyConfig_PushMode(config).withDataFlowTopologyConfig(config).withCopyRouteGenerator(config).withDeleteTarget(config).build();
}

Example 50 with Config

use of com.typesafe.config.Config in project incubator-gobblin by apache.

the class ReplicationDataValidPathPicker method getValidPaths.

@SuppressWarnings("unchecked")
public static Collection<Path> getValidPaths(HadoopFsEndPoint hadoopFsEndPoint) throws IOException {
    Config selectionConfig = hadoopFsEndPoint.getSelectionConfig();
    FileSystemDataset tmpDataset = new HadoopFsEndPointDataset(hadoopFsEndPoint);
    FileSystem theFs = FileSystem.get(hadoopFsEndPoint.getFsURI(), new Configuration());
    /**
     * Use {@link FileSystemDatasetVersion} as
     * {@link DateTimeDatasetVersionFinder} / {@link GlobModTimeDatasetVersionFinder} use {@link TimestampedDatasetVersion}
     * {@link SingleVersionFinder} uses {@link FileStatusDatasetVersion}
     */
    VersionFinder<FileSystemDatasetVersion> finder;
    try {
        finder = (VersionFinder<FileSystemDatasetVersion>) ConstructorUtils.invokeConstructor(Class.forName(selectionConfig.getString(FINDER_CLASS)), theFs, selectionConfig);
    } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException e) {
        throw new IllegalArgumentException(e);
    }
    List<FileSystemDatasetVersion> versions = Ordering.natural().reverse().sortedCopy(finder.findDatasetVersions(tmpDataset));
    VersionSelectionPolicy<FileSystemDatasetVersion> selector;
    try {
        selector = (VersionSelectionPolicy<FileSystemDatasetVersion>) ConstructorUtils.invokeConstructor(Class.forName(selectionConfig.getString(POLICY_CLASS)), selectionConfig);
    } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException e) {
        throw new IllegalArgumentException(e);
    }
    Collection<FileSystemDatasetVersion> versionsSelected = selector.listSelectedVersions(versions);
    List<Path> result = new ArrayList<Path>();
    for (FileSystemDatasetVersion t : versionsSelected) {
        // get the first element out
        result.add(t.getPaths().iterator().next());
    }
    return result;
}