Examples with ConfigStore org.apache.gobblin.config.store.api.ConfigStore used on opensource projects

Example 1 with ConfigStore

use of org.apache.gobblin.config.store.api.ConfigStore in project incubator-gobblin by apache.

the class ConfigClient method getConfigStoreAccessor.

private ConfigStoreAccessor getConfigStoreAccessor(URI configKeyURI) throws ConfigStoreFactoryDoesNotExistsException, ConfigStoreCreationException, VersionDoesNotExistException {
    URI matchedFloorKey = getMatchedFloorKeyFromCache(configKeyURI);
    ConfigStoreAccessor result;
    if (matchedFloorKey != null) {
        result = this.configStoreAccessorMap.get(matchedFloorKey);
        return result;
    }
    result = createNewConfigStoreAccessor(configKeyURI);
    ConfigStore cs = result.configStore;
    // put default root URI in cache as well for the URI which missing authority
    if (configKeyURI.getAuthority() == null) {
        // configKeyURI is missing authority/configstore root "etl-hdfs:///datasets/a1/a2"
        try {
            this.configStoreAccessorMap.put(new URI(configKeyURI.getScheme(), null, "/", null, null), result);
        } catch (URISyntaxException e) {
            // should not come here
            throw new RuntimeException("Can not build URI based on " + configKeyURI);
        }
    } else {
        // need to check Config Store's root is the prefix of input configKeyURI
        if (!ConfigClientUtils.isAncestorOrSame(configKeyURI, cs.getStoreURI())) {
            throw new RuntimeException(String.format("Config Store root URI %s is not the prefix of input %s", cs.getStoreURI(), configKeyURI));
        }
    }
    // put to cache
    this.configStoreAccessorMap.put(cs.getStoreURI(), result);
    return result;
}

Example 2 with ConfigStore

use of org.apache.gobblin.config.store.api.ConfigStore in project incubator-gobblin by apache.

the class TestCircularDependency method testLoops.

@Test
public void testLoops() {
    ConfigKeyPath tag = SingleLinkedListConfigKeyPath.ROOT.createChild("tag");
    ConfigKeyPath subTag1 = tag.createChild("subTag1");
    ConfigKeyPath subTag2 = tag.createChild("subTag2");
    ConfigKeyPath subTag3 = tag.createChild("subTag3");
    ConfigStore mockConfigStore = mock(ConfigStore.class, Mockito.RETURNS_SMART_NULLS);
    when(mockConfigStore.getCurrentVersion()).thenReturn(version);
    addConfigStoreChildren(mockConfigStore, version, SingleLinkedListConfigKeyPath.ROOT, tag);
    addConfigStoreChildren(mockConfigStore, version, tag, subTag1, subTag2, subTag3);
    // self import descendant
    // formed loop /tag/subTag1 -> /tag/subTag2 -> /tag/subTag3 -> /tag/subTag1
    addConfigStoreImports(mockConfigStore, version, subTag1, subTag2);
    addConfigStoreImports(mockConfigStore, version, subTag2, subTag3);
    addConfigStoreImports(mockConfigStore, version, subTag3, subTag1);
    ConfigStoreBackedTopology csTopology = new ConfigStoreBackedTopology(mockConfigStore, this.version);
    InMemoryTopology inMemory = new InMemoryTopology(csTopology);
    try {
        inMemory.getImportsRecursively(subTag1);
        Assert.fail("Did not catch expected CircularDependencyException");
    } catch (CircularDependencyException e) {
        Assert.assertTrue(e.getMessage().indexOf("/tag/subTag1") > 0 && e.getMessage().indexOf("/tag/subTag2") > 0 && e.getMessage().indexOf("/tag/subTag3") > 0);
    }
}

Example 3 with ConfigStore

use of org.apache.gobblin.config.store.api.ConfigStore in project incubator-gobblin by apache.

the class TestCircularDependency method testSelfImportDescendant.

@Test
public void testSelfImportDescendant() {
    ConfigKeyPath tag = SingleLinkedListConfigKeyPath.ROOT.createChild("tag");
    ConfigKeyPath highPriorityTag = tag.createChild("highPriorityTag");
    ConfigKeyPath nertzHighPriorityTag = highPriorityTag.createChild("nertzHighPriorityTag");
    ConfigStore mockConfigStore = mock(ConfigStore.class, Mockito.RETURNS_SMART_NULLS);
    when(mockConfigStore.getCurrentVersion()).thenReturn(version);
    addConfigStoreChildren(mockConfigStore, version, SingleLinkedListConfigKeyPath.ROOT, tag);
    addConfigStoreChildren(mockConfigStore, version, tag, highPriorityTag);
    addConfigStoreChildren(mockConfigStore, version, highPriorityTag, nertzHighPriorityTag);
    // self import descendant
    // formed the loop /tag -> /tag/highPriorityTag/nertzHighPriorityTag -> /tag/highPriorityTag -> /tag
    addConfigStoreImports(mockConfigStore, version, tag, nertzHighPriorityTag);
    ConfigStoreBackedTopology csTopology = new ConfigStoreBackedTopology(mockConfigStore, this.version);
    InMemoryTopology inMemory = new InMemoryTopology(csTopology);
    try {
        inMemory.getImportsRecursively(tag);
        Assert.fail("Did not catch expected CircularDependencyException");
    } catch (CircularDependencyException e) {
        Assert.assertTrue(e.getMessage().indexOf("/tag/highPriorityTag/nertzHighPriorityTag") > 0 && e.getMessage().indexOf("/tag/highPriorityTag ") > 0 && e.getMessage().indexOf("/tag ") > 0);
    }
}

Example 4 with ConfigStore

use of org.apache.gobblin.config.store.api.ConfigStore in project incubator-gobblin by apache.

the class TestConfigStoreValueInspector method testResolveConfigOverridingInChild.

@Test
public void testResolveConfigOverridingInChild() {
    ConfigStore mockConfigStore = mock(ConfigStore.class, Mockito.RETURNS_SMART_NULLS);
    when(mockConfigStore.getCurrentVersion()).thenReturn(version);
    ConfigStoreTopologyInspector mockTopology = mock(ConfigStoreTopologyInspector.class, Mockito.RETURNS_SMART_NULLS);
    ConfigStoreBackedValueInspector valueInspector = new ConfigStoreBackedValueInspector(mockConfigStore, version, mockTopology);
    ConfigKeyPath keyPathA = SingleLinkedListConfigKeyPath.ROOT.createChild("a");
    ConfigKeyPath keyPathA_Slash_B = keyPathA.createChild("b");
    when(mockConfigStore.getOwnConfig(keyPathA.getParent(), version)).thenReturn(ConfigFactory.empty());
    when(mockConfigStore.getOwnConfig(keyPathA, version)).thenReturn(ConfigFactory.parseString("key1 = value1InA \n key2 = ${key1}"));
    when(mockConfigStore.getOwnConfig(keyPathA_Slash_B, version)).thenReturn(ConfigFactory.parseString("key1 = value1InB"));
    Assert.assertEquals(valueInspector.getResolvedConfig(keyPathA_Slash_B).getString("key2"), "value1InB");
}

Example 5 with ConfigStore

use of org.apache.gobblin.config.store.api.ConfigStore in project incubator-gobblin by apache.

the class TestConfigStoreValueInspector method testSystemPropertyResolution.

@Test
public void testSystemPropertyResolution() {
    ConfigStore mockConfigStore = mock(ConfigStore.class, Mockito.RETURNS_SMART_NULLS);
    when(mockConfigStore.getCurrentVersion()).thenReturn(version);
    ConfigStoreTopologyInspector mockTopology = mock(ConfigStoreTopologyInspector.class, Mockito.RETURNS_SMART_NULLS);
    ConfigStoreBackedValueInspector valueInspector = new ConfigStoreBackedValueInspector(mockConfigStore, version, mockTopology);
    ConfigKeyPath testConfigKeyPath = SingleLinkedListConfigKeyPath.ROOT.createChild("a");
    when(mockConfigStore.getOwnConfig(testConfigKeyPath.getParent(), version)).thenReturn(ConfigFactory.empty());
    when(mockConfigStore.getOwnConfig(testConfigKeyPath, version)).thenReturn(ConfigFactory.parseString("configProp = ${?" + VALUE_INSPECTOR_SYS_PROP_KEY + "}"));
    Assert.assertEquals(valueInspector.getResolvedConfig(testConfigKeyPath).getString("configProp"), VALUE_INSPECTOR_SYS_PROP_VALUE);
}