Search in sources :

Example 96 with Config

use of com.typesafe.config.Config in project incubator-gobblin by apache.

the class DefaultGobblinBrokerTest method testConfigurationInjection.

@Test
public void testConfigurationInjection() throws Exception {
    String key = "myKey";
    Config config = ConfigFactory.parseMap(ImmutableMap.of(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, "key1"), "value1", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, "key2"), "value2", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.CONTAINER.name(), "key2"), "value2scope", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, key, "key2"), "value2key", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.CONTAINER.name(), key, "key2"), "value2scopekey"));
    SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance());
    SharedResourcesBrokerImpl<GobblinScopeTypes> containerBroker = topBroker.newSubscopedBuilder(GobblinScopeTypes.CONTAINER.defaultScopeInstance()).build();
    // create a shared resource
    TestFactory.SharedResource resource = containerBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.CONTAINER);
    Assert.assertEquals(resource.getConfig().getString("key1"), "value1");
    Assert.assertEquals(resource.getConfig().getString("key2"), "value2scopekey");
}
Also used : GobblinScopeTypes(org.apache.gobblin.broker.gobblin_scopes.GobblinScopeTypes) Config(com.typesafe.config.Config) Test(org.testng.annotations.Test)

Example 97 with Config

use of com.typesafe.config.Config in project incubator-gobblin by apache.

the class DefaultGobblinBrokerTest method testExpiringResource.

@Test
public void testExpiringResource() throws Exception {
    Config config = ConfigFactory.empty();
    SharedResourcesBrokerImpl<SimpleScopeType> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, SimpleScopeType.GLOBAL.defaultScopeInstance());
    InvalidatableResourceFactory factory = new InvalidatableResourceFactory();
    long value = topBroker.getSharedResource(factory, new EmptyKey());
    Assert.assertEquals(topBroker.getSharedResource(factory, new EmptyKey()), new Long(value));
    factory.getLastResourceEntry().setValid(false);
    Assert.assertNotEquals(topBroker.getSharedResource(factory, new EmptyKey()), value);
    value = topBroker.getSharedResource(factory, new EmptyKey());
    Assert.assertEquals(topBroker.getSharedResource(factory, new EmptyKey()), new Long(value));
}
Also used : Config(com.typesafe.config.Config) Test(org.testng.annotations.Test)

Example 98 with Config

use of com.typesafe.config.Config in project incubator-gobblin by apache.

the class GobblinBrokerConfTest method testOverrides.

@Test
public void testOverrides() {
    String key = "myKey";
    // Correct creation behavior
    Config config = ConfigFactory.parseMap(ImmutableMap.<String, String>builder().put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, "key1"), "value1").put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, "key2"), "value2").put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.CONTAINER.name(), "key2"), "value2scope").put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, key, "key2"), "value2key").put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.CONTAINER.name(), key, "key2"), "value2scopekey").put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.JOB.name(), "key2"), "value2scope").put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.JOB.name(), key, "key2"), "value2scopekey").build());
    SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance());
    Config overrides = ConfigFactory.parseMap(ImmutableMap.<String, String>builder().put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, "key1"), "value1_o").put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, "key2"), "value2_o").put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.CONTAINER.name(), "key2"), "value2scope_o").put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, key, "key2"), "value2key_o").put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.CONTAINER.name(), key, "key2"), "value2scopekey_o").put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.JOB.name(), "key2"), "value2scope_o").put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.JOB.name(), key, "key2"), "value2scopekey_o").build());
    SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).withOverridingConfig(overrides).build();
    ScopedConfigView<GobblinScopeTypes, TestResourceKey> configView = jobBroker.getConfigView(GobblinScopeTypes.CONTAINER, new TestResourceKey(key), TestFactory.NAME);
    Assert.assertEquals(configView.getConfig().getString("key1"), "value1");
    Assert.assertEquals(configView.getConfig().getString("key2"), "value2scopekey");
    configView = jobBroker.getConfigView(GobblinScopeTypes.JOB, new TestResourceKey(key), TestFactory.NAME);
    Assert.assertEquals(configView.getConfig().getString("key1"), "value1");
    Assert.assertEquals(configView.getConfig().getString("key2"), "value2scopekey_o");
}
Also used : GobblinScopeTypes(org.apache.gobblin.broker.gobblin_scopes.GobblinScopeTypes) JobScopeInstance(org.apache.gobblin.broker.gobblin_scopes.JobScopeInstance) Config(com.typesafe.config.Config) Test(org.testng.annotations.Test)

Example 99 with Config

use of com.typesafe.config.Config in project incubator-gobblin by apache.

the class SharedResourcesBrokerFactoryTest method testLoadingOfClasspath.

@Test
public void testLoadingOfClasspath() {
    Config config = ConfigFactory.parseMap(ImmutableMap.of(SharedResourcesBrokerFactory.BROKER_CONF_FILE_KEY, "/broker/testBroker.conf"));
    SharedResourcesBrokerImpl<SimpleScopeType> broker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, SimpleScopeType.GLOBAL.defaultScopeInstance());
    ConfigView configView = broker.getConfigView(null, null, "factory");
    Assert.assertTrue(configView.getConfig().hasPath("testKey"));
    Assert.assertEquals(configView.getConfig().getString("testKey"), "testValue");
}
Also used : ConfigView(org.apache.gobblin.broker.iface.ConfigView) Config(com.typesafe.config.Config) Test(org.testng.annotations.Test)

Example 100 with Config

use of com.typesafe.config.Config in project incubator-gobblin by apache.

the class GitConfigMonitor method loadConfigFileWithFlowNameOverrides.

/**
 * Load the config file and override the flow name and flow path properties with the names from the file path
 * @param configFilePath path of the config file relative to the repository root
 * @return the configuration object
 * @throws IOException
 */
private Config loadConfigFileWithFlowNameOverrides(Path configFilePath) throws IOException {
    Config flowConfig = this.pullFileLoader.loadPullFile(configFilePath, emptyConfig, false);
    String flowName = FSSpecStore.getSpecName(configFilePath);
    String flowGroup = FSSpecStore.getSpecGroup(configFilePath);
    return flowConfig.withValue(ConfigurationKeys.FLOW_NAME_KEY, ConfigValueFactory.fromAnyRef(flowName)).withValue(ConfigurationKeys.FLOW_GROUP_KEY, ConfigValueFactory.fromAnyRef(flowGroup));
}
Also used : Config(com.typesafe.config.Config)

Aggregations

Config (com.typesafe.config.Config)489 Test (org.junit.Test)182 FeatureVector (com.airbnb.aerosolve.core.FeatureVector)117 Test (org.testng.annotations.Test)109 Properties (java.util.Properties)59 Map (java.util.Map)42 Path (org.apache.hadoop.fs.Path)41 Set (java.util.Set)37 HashMap (java.util.HashMap)34 URI (java.net.URI)32 HashSet (java.util.HashSet)32 IOException (java.io.IOException)22 Configuration (org.apache.hadoop.conf.Configuration)22 ArrayList (java.util.ArrayList)16 DrillConfig (org.apache.drill.common.config.DrillConfig)15 URL (java.net.URL)14 BaseTest (org.apache.drill.test.BaseTest)13 GobblinScopeTypes (org.apache.gobblin.broker.gobblin_scopes.GobblinScopeTypes)13 ImmutableMap (com.google.common.collect.ImmutableMap)11 File (java.io.File)11