Examples with ConfigKeyPath org.apache.gobblin.config.store.api.ConfigKeyPath used on opensource projects

Example 6 with ConfigKeyPath

use of org.apache.gobblin.config.store.api.ConfigKeyPath in project incubator-gobblin by apache.

the class TestCircularDependency method addConfigStoreImports.

private static void addConfigStoreImports(ConfigStore mockup, String version, ConfigKeyPath self, ConfigKeyPath... configKeyPaths) {
    List<ConfigKeyPath> ownImports = new ArrayList<ConfigKeyPath>();
    for (ConfigKeyPath p : configKeyPaths) {
        ownImports.add(p);
    }
    when(mockup.getOwnImports(self, version)).thenReturn(ownImports);
}

Example 7 with ConfigKeyPath

use of org.apache.gobblin.config.store.api.ConfigKeyPath in project incubator-gobblin by apache.

the class SimpleHdfsConfigStoreTest method testGetOwnImports.

@Test
public void testGetOwnImports() throws IOException, URISyntaxException, ConfigStoreCreationException {
    String datasetName = "dataset-test-get-own-imports";
    String tagKey1 = "/path/to/tag1";
    String tagKey2 = "/path/to/tag2";
    Path datasetPath = new Path(CONFIG_DIR_PATH, datasetName);
    try {
        this.fs.mkdirs(datasetPath);
        BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(this.fs.create(new Path(datasetPath, "includes.conf")), Charsets.UTF_8));
        writer.write(tagKey1);
        writer.newLine();
        writer.write(tagKey2);
        writer.close();
        ConfigKeyPath datasetConfigKey = SingleLinkedListConfigKeyPath.ROOT.createChild(datasetName);
        List<ConfigKeyPath> imports = this._simpleHadoopFilesystemConfigStore.getOwnImports(datasetConfigKey, VERSION);
        Assert.assertEquals(imports.size(), 2);
        Assert.assertEquals(imports.get(0).getAbsolutePathString(), tagKey2);
        Assert.assertEquals(imports.get(1).getAbsolutePathString(), tagKey1);
    } finally {
        if (this.fs.exists(datasetPath)) {
            this.fs.delete(datasetPath, true);
        }
    }
}

Example 8 with ConfigKeyPath

use of org.apache.gobblin.config.store.api.ConfigKeyPath in project incubator-gobblin by apache.

the class SimpleHdfsConfigStoreTest method testGetOwnImportsWithRuntimeConfigResolution.

@Test
public void testGetOwnImportsWithRuntimeConfigResolution() throws IOException, URISyntaxException, ConfigStoreCreationException {
    String datasetName = "dataset-test-get-own-imports-resolution";
    Path datasetPath = new Path(CONFIG_DIR_PATH, datasetName);
    Properties prop = new Properties();
    prop.put(TAG_NAME_RUNTIME_PROP_KEY, TAG_NAME_RUNTIME_PROP_VALUE);
    Optional<Config> runtimeConfig = Optional.fromNullable(ConfigUtils.propertiesToConfig(prop));
    try {
        this.fs.mkdirs(datasetPath);
        BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(this.fs.create(new Path(datasetPath, "includes.conf")), Charsets.UTF_8));
        writer.write("/path/to/${?" + TAG_NAME_RUNTIME_PROP_KEY + "}");
        writer.close();
        ConfigKeyPath datasetConfigKey = SingleLinkedListConfigKeyPath.ROOT.createChild(datasetName);
        List<ConfigKeyPath> imports = this._simpleHadoopFilesystemConfigStore.getOwnImports(datasetConfigKey, VERSION, runtimeConfig);
        Assert.assertEquals(imports.size(), 1);
        Assert.assertEquals(imports.get(0).getAbsolutePathString(), "/path/to/" + TAG_NAME_RUNTIME_PROP_VALUE);
    } finally {
        if (this.fs.exists(datasetPath)) {
            this.fs.delete(datasetPath, true);
        }
    }
}

Example 9 with ConfigKeyPath

use of org.apache.gobblin.config.store.api.ConfigKeyPath in project incubator-gobblin by apache.

the class SimpleHdfsConfigStoreTest method testGetOwnImportsWithResolution.

@Test
public void testGetOwnImportsWithResolution() throws IOException, URISyntaxException, ConfigStoreCreationException {
    String datasetName = "dataset-test-get-own-imports-resolution";
    Path datasetPath = new Path(CONFIG_DIR_PATH, datasetName);
    try {
        this.fs.mkdirs(datasetPath);
        BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(this.fs.create(new Path(datasetPath, "includes.conf")), Charsets.UTF_8));
        writer.write("/path/to/${?" + TAG_NAME_SYS_PROP_KEY + "}");
        writer.close();
        ConfigKeyPath datasetConfigKey = SingleLinkedListConfigKeyPath.ROOT.createChild(datasetName);
        List<ConfigKeyPath> imports = this._simpleHadoopFilesystemConfigStore.getOwnImports(datasetConfigKey, VERSION);
        Assert.assertEquals(imports.size(), 1);
        Assert.assertEquals(imports.get(0).getAbsolutePathString(), "/path/to/" + TAG_NAME_SYS_PROP_VALUE);
    } finally {
        if (this.fs.exists(datasetPath)) {
            this.fs.delete(datasetPath, true);
        }
    }
}

Example 10 with ConfigKeyPath

use of org.apache.gobblin.config.store.api.ConfigKeyPath in project incubator-gobblin by apache.

the class SimpleHadoopFilesystemConfigStore method getChildren.

/**
 * Retrieves all the children of the given {@link ConfigKeyPath} by doing a {@code ls} on the {@link Path} specified
 * by the {@link ConfigKeyPath}. If the {@link Path} described by the {@link ConfigKeyPath} does not exist, an empty
 * {@link Collection} is returned.
 *
 * @param  configKey      the config key path whose children are necessary.
 * @param  version        specify the configuration version in the configuration store.
 *
 * @return a {@link Collection} of {@link ConfigKeyPath} where each entry is a child of the given configKey.
 *
 * @throws VersionDoesNotExistException if the version specified cannot be found in the {@link ConfigStore}.
 */
@Override
public Collection<ConfigKeyPath> getChildren(ConfigKeyPath configKey, String version) throws VersionDoesNotExistException {
    Preconditions.checkNotNull(configKey, "configKey cannot be null!");
    Preconditions.checkArgument(!Strings.isNullOrEmpty(version), "version cannot be null or empty!");
    List<ConfigKeyPath> children = new ArrayList<>();
    Path datasetDir = getDatasetDirForKey(configKey, version);
    try {
        if (!this.fs.exists(datasetDir)) {
            return children;
        }
        for (FileStatus fileStatus : this.fs.listStatus(datasetDir)) {
            if (fileStatus.isDirectory()) {
                children.add(configKey.createChild(fileStatus.getPath().getName()));
            }
        }
        return children;
    } catch (IOException e) {
        throw new RuntimeException(String.format("Error while getting children for configKey: \"%s\"", configKey), e);
    }
}