Examples with SimpleScopeType org.apache.gobblin.broker.SimpleScopeType used on opensource projects

Example 1 with SimpleScopeType

use of org.apache.gobblin.broker.SimpleScopeType in project incubator-gobblin by apache.

the class MetricContextFactoryTest method test.

@Test
public void test() throws Exception {
    MetricContextFactory<SimpleScopeType> factory = new MetricContextFactory<>();
    Config config = ConfigFactory.parseMap(ImmutableMap.of(BrokerConfigurationKeyGenerator.generateKey(factory, null, null, MetricContextFactory.TAG_KEY + ".tag1"), "value1", BrokerConfigurationKeyGenerator.generateKey(factory, null, SimpleScopeType.GLOBAL, MetricContextFactory.TAG_KEY + ".tag2"), "value2", BrokerConfigurationKeyGenerator.generateKey(factory, null, SimpleScopeType.LOCAL, MetricContextFactory.TAG_KEY + ".tag3"), "value3"));
    SharedResourcesBroker<SimpleScopeType> rootBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, SimpleScopeType.GLOBAL.defaultScopeInstance());
    SharedResourcesBroker<SimpleScopeType> localBroker = rootBroker.newSubscopedBuilder(SimpleScopeType.LOCAL.defaultScopeInstance()).build();
    MetricContext localContext = localBroker.getSharedResource(factory, new MetricContextKey());
    Map<String, String> tagMap = (Map<String, String>) Tag.toMap(Tag.tagValuesToString(localContext.getTags()));
    Assert.assertEquals(tagMap.get("tag1"), "value1");
    Assert.assertEquals(tagMap.get("tag2"), "value2");
    Assert.assertEquals(tagMap.get("tag3"), "value3");
    MetricContext globalContext = rootBroker.getSharedResource(factory, new MetricContextKey());
    Assert.assertEquals(localContext.getParent().get(), globalContext);
    tagMap = (Map<String, String>) Tag.toMap(Tag.tagValuesToString(globalContext.getTags()));
    Assert.assertEquals(tagMap.get("tag1"), "value1");
    Assert.assertEquals(tagMap.get("tag2"), "value2");
    Assert.assertFalse(tagMap.containsKey("tag3"));
}

Example 2 with SimpleScopeType

use of org.apache.gobblin.broker.SimpleScopeType in project incubator-gobblin by apache.

the class MetricContextFactoryTest method testSubTaggedMetricContext.

@Test
public void testSubTaggedMetricContext() throws Exception {
    MetricContextFactory<SimpleScopeType> factory = new MetricContextFactory<>();
    Config config = ConfigFactory.parseMap(ImmutableMap.of(BrokerConfigurationKeyGenerator.generateKey(factory, null, null, MetricContextFactory.TAG_KEY + ".tag1"), "value1"));
    SharedResourcesBroker<SimpleScopeType> rootBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, SimpleScopeType.GLOBAL.defaultScopeInstance());
    MetricContext metricContext = rootBroker.getSharedResource(factory, new SubTaggedMetricContextKey("myMetricContext", ImmutableMap.of("tag2", "value2")));
    Map<String, String> tagMap = (Map<String, String>) Tag.toMap(Tag.tagValuesToString(metricContext.getTags()));
    Assert.assertEquals(metricContext.getName(), "myMetricContext");
    Assert.assertEquals(tagMap.get("tag1"), "value1");
    Assert.assertEquals(tagMap.get("tag2"), "value2");
    MetricContext metricContext2 = rootBroker.getSharedResource(factory, new SubTaggedMetricContextKey("myMetricContext", ImmutableMap.of("tag2", "value2")));
    Assert.assertEquals(metricContext, metricContext2);
    MetricContext metricContext3 = rootBroker.getSharedResource(factory, new SubTaggedMetricContextKey("myMetricContext", ImmutableMap.of("tag3", "value3")));
    Assert.assertNotEquals(metricContext, metricContext3);
    MetricContext parent = rootBroker.getSharedResource(factory, new MetricContextKey());
    tagMap = (Map<String, String>) Tag.toMap(Tag.tagValuesToString(parent.getTags()));
    Assert.assertEquals(metricContext.getParent().get(), parent);
    Assert.assertEquals(tagMap.get("tag1"), "value1");
    Assert.assertFalse(tagMap.containsKey("tag2"));
}

Example 3 with SimpleScopeType

use of org.apache.gobblin.broker.SimpleScopeType in project incubator-gobblin by apache.

the class MRStressTest method createLimiter.

static Limiter createLimiter(Configuration configuration, SharedResourcesBroker<SimpleScopeType> broker) {
    try {
        Limiter limiter = new NoopLimiter();
        long localQps = configuration.getLong(LOCALLY_ENFORCED_QPS, 0);
        if (localQps > 0) {
            log.info("Setting up local qps " + localQps);
            limiter = new MultiLimiter(limiter, new RateBasedLimiter(localQps));
        }
        if (configuration.getBoolean(USE_THROTTLING_SERVER, false)) {
            log.info("Setting up remote throttling.");
            String resourceId = configuration.get(RESOURCE_ID);
            Limiter globalLimiter = broker.getSharedResource(new RestliLimiterFactory<SimpleScopeType>(), new SharedLimiterKey(resourceId));
            limiter = new MultiLimiter(limiter, globalLimiter);
        }
        return limiter;
    } catch (NotConfiguredException nce) {
        throw new RuntimeException(nce);
    }
}

Example 4 with SimpleScopeType

use of org.apache.gobblin.broker.SimpleScopeType in project incubator-gobblin by apache.

the class RestliLimiterFactoryTest method testRestliLimiterCalledByLimiterFactory.

@Test
public void testRestliLimiterCalledByLimiterFactory() throws Exception {
    SharedResourcesBroker<SimpleScopeType> broker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance());
    MyRequestSender requestSender = new MyRequestSender();
    broker.bindSharedResourceAtScope(new RedirectAwareRestClientRequestSender.Factory<>(), new SharedRestClientKey(RestliLimiterFactory.RESTLI_SERVICE_NAME), SimpleScopeType.GLOBAL, requestSender);
    Limiter limiter = broker.getSharedResource(new SharedLimiterFactory<>(), new SharedLimiterKey("my/resource"));
    Assert.assertNotNull(limiter.acquirePermits(10));
    Assert.assertEquals(requestSender.requestList.size(), 1);
    broker.close();
}

Example 5 with SimpleScopeType

use of org.apache.gobblin.broker.SimpleScopeType in project incubator-gobblin by apache.

the class RestliServiceBasedLimiterTest method testServerFailover.

@Test
public void testServerFailover() throws Exception {
    try (Closer closer = Closer.create()) {
        SharedLimiterKey res1key = new SharedLimiterKey("res1");
        Map<String, String> configMap = Maps.newHashMap();
        TestingServer zkTestingServer = closer.register(new TestingServer(-1));
        configMap.put(ThrottlingGuiceServletConfig.ZK_STRING_KEY, zkTestingServer.getConnectString());
        configMap.put(ThrottlingGuiceServletConfig.HA_CLUSTER_NAME, RestliServiceBasedLimiterTest.class.getSimpleName() + "_cluster");
        Config config = ConfigFactory.parseMap(configMap);
        RestliServer server2500 = createAndStartServer(config, 2500);
        RestliServer server2501 = createAndStartServer(config, 2501);
        SharedResourcesBroker<SimpleScopeType> broker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance());
        RedirectAwareRestClientRequestSender requestSender = new RedirectAwareRestClientRequestSender(broker, Lists.newArrayList(server2500.getServer().getURIPrefix(), server2501.getServer().getURIPrefix()));
        RestliServiceBasedLimiter limiter = RestliServiceBasedLimiter.builder().requestSender(requestSender).resourceLimited(res1key.getResourceLimitedPath()).serviceIdentifier("service").build();
        Assert.assertNotNull(limiter.acquirePermits(20));
        limiter.clearAllStoredPermits();
        server2500.close();
        Assert.assertNotNull(limiter.acquirePermits(20));
        Assert.assertEquals(parsePortOfCurrentServerPrefix(requestSender), 2501);
        limiter.clearAllStoredPermits();
        server2500 = createAndStartServer(config, 2500);
        Assert.assertNotNull(limiter.acquirePermits(20));
        limiter.clearAllStoredPermits();
        // leader is currently 2501
        Assert.assertEquals(parsePortOfCurrentServerPrefix(requestSender), 2501);
        // set request to 2500 (not leader)
        requestSender.updateRestClient(server2500.getServer().getURIPrefix(), "test");
        Assert.assertEquals(parsePortOfCurrentServerPrefix(requestSender), 2500);
        Assert.assertNotNull(limiter.acquirePermits(20));
        // verify request sender switched back to leader
        Assert.assertEquals(parsePortOfCurrentServerPrefix(requestSender), 2501);
        server2501.close();
        Assert.assertNotNull(limiter.acquirePermits(20));
        limiter.clearAllStoredPermits();
        server2500.close();
        Assert.assertNull(limiter.acquirePermits(20));
        limiter.clearAllStoredPermits();
    }
}