Examples with ConfigKeyPath org.apache.gobblin.config.store.api.ConfigKeyPath used on opensource projects

Example 26 with ConfigKeyPath

use of org.apache.gobblin.config.store.api.ConfigKeyPath in project incubator-gobblin by apache.

the class TestInMemoryTopology method mockupConfigValues.

private void mockupConfigValues() {
    // mock up the configuration values for root
    Map<String, String> rootMap = new HashMap<>();
    rootMap.put("keyInRoot", "valueInRoot");
    when(mockConfigStore.getOwnConfig(SingleLinkedListConfigKeyPath.ROOT, version)).thenReturn(ConfigFactory.parseMap(rootMap));
    Collection<ConfigKeyPath> currentLevel = mockConfigStore.getChildren(SingleLinkedListConfigKeyPath.ROOT, version);
    while (!currentLevel.isEmpty()) {
        Collection<ConfigKeyPath> nextLevel = new ArrayList<ConfigKeyPath>();
        for (ConfigKeyPath p : currentLevel) {
            mockupConfigValueForKey(p);
            nextLevel.addAll(mockConfigStore.getChildren(p, version));
        }
        currentLevel = nextLevel;
    }
}

Example 27 with ConfigKeyPath

use of org.apache.gobblin.config.store.api.ConfigKeyPath in project incubator-gobblin by apache.

the class SimpleHadoopFilesystemConfigStore method getOwnConfig.

/**
 * Retrieves the {@link Config} for the given {@link ConfigKeyPath} by reading the {@link #MAIN_CONF_FILE_NAME}
 * associated with the dataset specified by the given {@link ConfigKeyPath}. If the {@link Path} described by the
 * {@link ConfigKeyPath} does not exist then an empty {@link Config} is returned.
 *
 * @param  configKey      the config key path whose properties are needed.
 * @param  version        the configuration version in the configuration store.
 *
 * @return a {@link Config} for the given configKey.
 *
 * @throws VersionDoesNotExistException if the version specified cannot be found in the {@link ConfigStore}.
 */
@Override
public Config getOwnConfig(ConfigKeyPath configKey, String version) throws VersionDoesNotExistException {
    Preconditions.checkNotNull(configKey, "configKey cannot be null!");
    Preconditions.checkArgument(!Strings.isNullOrEmpty(version), "version cannot be null or empty!");
    Path datasetDir = getDatasetDirForKey(configKey, version);
    Path mainConfFile = new Path(datasetDir, MAIN_CONF_FILE_NAME);
    try {
        if (!this.fs.exists(mainConfFile)) {
            return ConfigFactory.empty();
        }
        FileStatus configFileStatus = this.fs.getFileStatus(mainConfFile);
        if (!configFileStatus.isDirectory()) {
            try (InputStream mainConfInputStream = this.fs.open(configFileStatus.getPath())) {
                return ConfigFactory.parseReader(new InputStreamReader(mainConfInputStream, Charsets.UTF_8));
            }
        }
        return ConfigFactory.empty();
    } catch (IOException e) {
        throw new RuntimeException(String.format("Error while getting config for configKey: \"%s\"", configKey), e);
    }
}

Example 28 with ConfigKeyPath

use of org.apache.gobblin.config.store.api.ConfigKeyPath in project incubator-gobblin by apache.

the class SimpleHadoopFilesystemConfigStore method getOwnImports.

/**
 * Retrieves all the {@link ConfigKeyPath}s that are imported by the given {@link ConfigKeyPath}. This method does this
 * by reading the {@link #INCLUDES_CONF_FILE_NAME} file associated with the dataset specified by the given
 * {@link ConfigKeyPath}. If the {@link Path} described by the {@link ConfigKeyPath} does not exist, then an empty
 * {@link List} is returned.
 *
 * @param  configKey      the config key path whose tags are needed
 * @param  version        the configuration version in the configuration store.
 *
 * @return a {@link List} of {@link ConfigKeyPath}s where each entry is a {@link ConfigKeyPath} imported by the dataset
 * specified by the configKey.
 *
 * @throws VersionDoesNotExistException if the version specified cannot be found in the {@link ConfigStore}.
 */
public List<ConfigKeyPath> getOwnImports(ConfigKeyPath configKey, String version, Optional<Config> runtimeConfig) throws VersionDoesNotExistException {
    Preconditions.checkNotNull(configKey, "configKey cannot be null!");
    Preconditions.checkArgument(!Strings.isNullOrEmpty(version), "version cannot be null or empty!");
    List<ConfigKeyPath> configKeyPaths = new ArrayList<>();
    Path datasetDir = getDatasetDirForKey(configKey, version);
    Path includesFile = new Path(datasetDir, INCLUDES_CONF_FILE_NAME);
    try {
        if (!this.fs.exists(includesFile)) {
            return configKeyPaths;
        }
        FileStatus includesFileStatus = this.fs.getFileStatus(includesFile);
        if (!includesFileStatus.isDirectory()) {
            try (InputStream includesConfInStream = this.fs.open(includesFileStatus.getPath())) {
                configKeyPaths.addAll(getResolvedConfigKeyPaths(includesConfInStream, runtimeConfig));
            }
        }
    } catch (IOException e) {
        throw new RuntimeException(String.format("Error while getting config for configKey: \"%s\"", configKey), e);
    }
    return configKeyPaths;
}

Example 29 with ConfigKeyPath

use of org.apache.gobblin.config.store.api.ConfigKeyPath in project incubator-gobblin by apache.

the class ConfigClientUtils method buildUriInClientFormat.

/**
 * Build the URI based on the {@link ConfigStore} or input cnofigKeyURI
 *
 * @param configKeyPath : relative path to the input config store cs
 * @param returnURIWithAuthority  : return the URI with input config store's authority and absolute path
 * @param cs            : the config store of the input configKeyURI
 * @return              : return the URI of the same format with the input configKeyURI
 *
 * for example, configKeyPath is /tags/retention,
 * with returnURIWithAuthority as false, return "etl-hdfs:///tags/retention
 * with returnURIWithAuthority as true , then return
 * etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest/tags/retention
 */
public static URI buildUriInClientFormat(ConfigKeyPath configKeyPath, ConfigStore cs, boolean returnURIWithAuthority) {
    try {
        if (!returnURIWithAuthority) {
            return new URI(cs.getStoreURI().getScheme(), null, configKeyPath.getAbsolutePathString(), null, null);
        }
        URI storeRoot = cs.getStoreURI();
        // will cause the Path creation failure if not handled here
        if (configKeyPath.isRootPath()) {
            return storeRoot;
        }
        // remote the first "/";
        Path absPath = new Path(storeRoot.getPath(), configKeyPath.getAbsolutePathString().substring(1));
        return new URI(storeRoot.getScheme(), storeRoot.getAuthority(), absPath.toString(), null, null);
    } catch (URISyntaxException e) {
        // should not come here
        throw new RuntimeException("Can not build URI based on " + configKeyPath);
    }
}

Example 30 with ConfigKeyPath

use of org.apache.gobblin.config.store.api.ConfigKeyPath in project incubator-gobblin by apache.

the class TestConfigClientUtils method testIsAncestorOrSame.

@Test
public void testIsAncestorOrSame() throws Exception {
    // Path ancestor = new Path("/");
    // Path descendant = new Path("/");
    URI ancestor = new URI("etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest/");
    URI descendant = new URI("etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest/datasets/a1/a2");
    Assert.assertTrue(ConfigClientUtils.isAncestorOrSame(descendant, ancestor));
    // ends with "/"
    descendant = new URI("etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest/datasets/a1/a2/");
    Assert.assertTrue(ConfigClientUtils.isAncestorOrSame(descendant, ancestor));
    // wrong authority
    descendant = new URI("etl-hdfs://ltx1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest/datasets/a1/a2");
    Assert.assertTrue(!ConfigClientUtils.isAncestorOrSame(descendant, ancestor));
    // wrong path
    descendant = new URI("etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/sahil/HdfsBasedConfigTest/datasets/a1/a2");
    Assert.assertTrue(!ConfigClientUtils.isAncestorOrSame(descendant, ancestor));
    ConfigKeyPath data = SingleLinkedListConfigKeyPath.ROOT.createChild("data");
    ConfigKeyPath data2 = SingleLinkedListConfigKeyPath.ROOT.createChild("data2");
    ConfigKeyPath identity = SingleLinkedListConfigKeyPath.ROOT.createChild("data").createChild("databases").createChild("Identity");
    Assert.assertTrue(ConfigClientUtils.isAncestorOrSame(identity, data));
    Assert.assertTrue(ConfigClientUtils.isAncestorOrSame(identity, SingleLinkedListConfigKeyPath.ROOT));
    Assert.assertTrue(!ConfigClientUtils.isAncestorOrSame(identity, data2));
}