Examples with SharedLimiterKey org.apache.gobblin.util.limiter.broker.SharedLimiterKey used on opensource projects

Example 1 with SharedLimiterKey

use of org.apache.gobblin.util.limiter.broker.SharedLimiterKey in project incubator-gobblin by apache.

the class MRStressTest method createLimiter.

static Limiter createLimiter(Configuration configuration, SharedResourcesBroker<SimpleScopeType> broker) {
    try {
        Limiter limiter = new NoopLimiter();
        long localQps = configuration.getLong(LOCALLY_ENFORCED_QPS, 0);
        if (localQps > 0) {
            log.info("Setting up local qps " + localQps);
            limiter = new MultiLimiter(limiter, new RateBasedLimiter(localQps));
        }
        if (configuration.getBoolean(USE_THROTTLING_SERVER, false)) {
            log.info("Setting up remote throttling.");
            String resourceId = configuration.get(RESOURCE_ID);
            Limiter globalLimiter = broker.getSharedResource(new RestliLimiterFactory<SimpleScopeType>(), new SharedLimiterKey(resourceId));
            limiter = new MultiLimiter(limiter, globalLimiter);
        }
        return limiter;
    } catch (NotConfiguredException nce) {
        throw new RuntimeException(nce);
    }
}

Example 2 with SharedLimiterKey

use of org.apache.gobblin.util.limiter.broker.SharedLimiterKey in project incubator-gobblin by apache.

the class LocalStressTest method main.

public static void main(String[] args) throws Exception {
    CommandLine cli = StressTestUtils.parseCommandLine(OPTIONS, args);
    int stressorThreads = Integer.parseInt(cli.getOptionValue(STRESSOR_THREADS.getOpt(), Integer.toString(DEFAULT_STRESSOR_THREADS)));
    int processorThreads = Integer.parseInt(cli.getOptionValue(PROCESSOR_THREADS.getOpt(), Integer.toString(DEFAULT_PROCESSOR_THREADS)));
    int artificialLatency = Integer.parseInt(cli.getOptionValue(ARTIFICIAL_LATENCY.getOpt(), Integer.toString(DEFAULT_ARTIFICIAL_LATENCY)));
    long targetQps = Integer.parseInt(cli.getOptionValue(QPS.getOpt(), Integer.toString(DEFAULT_TARGET_QPS)));
    Configuration configuration = new Configuration();
    StressTestUtils.populateConfigFromCli(configuration, cli);
    String resourceLimited = LocalStressTest.class.getSimpleName();
    Map<String, String> configMap = Maps.newHashMap();
    ThrottlingPolicyFactory factory = new ThrottlingPolicyFactory();
    SharedLimiterKey res1key = new SharedLimiterKey(resourceLimited);
    configMap.put(BrokerConfigurationKeyGenerator.generateKey(factory, res1key, null, ThrottlingPolicyFactory.POLICY_KEY), QPSPolicy.FACTORY_ALIAS);
    configMap.put(BrokerConfigurationKeyGenerator.generateKey(factory, res1key, null, QPSPolicy.QPS), Long.toString(targetQps));
    ThrottlingGuiceServletConfig guiceServletConfig = new ThrottlingGuiceServletConfig();
    guiceServletConfig.initialize(ConfigFactory.parseMap(configMap));
    LimiterServerResource limiterServer = guiceServletConfig.getInjector().getInstance(LimiterServerResource.class);
    RateComputingLimiterContainer limiterContainer = new RateComputingLimiterContainer();
    Class<? extends Stressor> stressorClass = configuration.getClass(StressTestUtils.STRESSOR_CLASS, StressTestUtils.DEFAULT_STRESSOR_CLASS, Stressor.class);
    ExecutorService executorService = Executors.newFixedThreadPool(stressorThreads);
    SharedResourcesBroker broker = guiceServletConfig.getInjector().getInstance(Key.get(SharedResourcesBroker.class, Names.named(LimiterServerResource.BROKER_INJECT_NAME)));
    ThrottlingPolicy policy = (ThrottlingPolicy) broker.getSharedResource(new ThrottlingPolicyFactory(), new SharedLimiterKey(resourceLimited));
    ScheduledExecutorService reportingThread = Executors.newSingleThreadScheduledExecutor();
    reportingThread.scheduleAtFixedRate(new Reporter(limiterContainer, policy), 0, 15, TimeUnit.SECONDS);
    Queue<Future<?>> futures = new LinkedList<>();
    MockRequester requester = new MockRequester(limiterServer, artificialLatency, processorThreads);
    requester.start();
    for (int i = 0; i < stressorThreads; i++) {
        RestliServiceBasedLimiter restliLimiter = RestliServiceBasedLimiter.builder().resourceLimited(resourceLimited).requestSender(requester).serviceIdentifier("stressor" + i).build();
        Stressor stressor = stressorClass.newInstance();
        stressor.configure(configuration);
        futures.add(executorService.submit(new StressorRunner(limiterContainer.decorateLimiter(restliLimiter), stressor)));
    }
    int stressorFailures = 0;
    for (Future<?> future : futures) {
        try {
            future.get();
        } catch (ExecutionException ee) {
            stressorFailures++;
        }
    }
    requester.stop();
    executorService.shutdownNow();
    if (stressorFailures > 0) {
        log.error("There were " + stressorFailures + " failed stressor threads.");
    }
    System.exit(stressorFailures);
}

Example 3 with SharedLimiterKey

use of org.apache.gobblin.util.limiter.broker.SharedLimiterKey in project incubator-gobblin by apache.

the class ThrottlingClientTest method test.

@Test
public void test() throws Exception {
    ThrottlingPolicyFactory factory = new ThrottlingPolicyFactory();
    SharedLimiterKey res1key = new SharedLimiterKey("res1");
    Map<String, String> configMap = ImmutableMap.<String, String>builder().put(BrokerConfigurationKeyGenerator.generateKey(factory, res1key, null, ThrottlingPolicyFactory.POLICY_KEY), CountBasedPolicy.FACTORY_ALIAS).put(BrokerConfigurationKeyGenerator.generateKey(factory, res1key, null, CountBasedPolicy.COUNT_KEY), "50").put(BrokerConfigurationKeyGenerator.generateKey(factory, null, null, ThrottlingPolicyFactory.FAIL_ON_UNKNOWN_RESOURCE_ID), "true").build();
    ThrottlingGuiceServletConfig guiceServletConfig = new ThrottlingGuiceServletConfig();
    guiceServletConfig.initialize(ConfigFactory.parseMap(configMap));
    Injector injector = guiceServletConfig.getInjector();
    EmbeddedRestliServer server = EmbeddedRestliServer.builder().resources(Lists.<Class<? extends BaseResource>>newArrayList(LimiterServerResource.class)).injector(injector).build();
    try {
        server.startAsync();
        server.awaitRunning();
        final HttpClientFactory http = new HttpClientFactory();
        final Client r2Client = new TransportClientAdapter(http.getClient(Collections.<String, String>emptyMap()));
        RestClient restClient = new RestClient(r2Client, server.getURIPrefix());
        PermitsGetRequestBuilder getBuilder = new PermitsRequestBuilders().get();
        PermitRequest res1request = new PermitRequest();
        res1request.setPermits(20);
        res1request.setResource(res1key.getResourceLimitedPath());
        PermitAllocation allocation = getPermitAllocation(res1request, restClient, getBuilder);
        Assert.assertEquals(allocation.getPermits(), new Long(20));
        allocation = getPermitAllocation(res1request, restClient, getBuilder);
        Assert.assertEquals(allocation.getPermits(), new Long(20));
        // out of permits
        try {
            allocation = getPermitAllocation(res1request, restClient, getBuilder);
            Assert.fail();
        } catch (RestLiResponseException exc) {
            Assert.assertEquals(exc.getStatus(), HttpStatus.S_403_FORBIDDEN.getCode());
        }
        PermitRequest invalidRequest = new PermitRequest();
        invalidRequest.setPermits(20);
        invalidRequest.setResource("invalidkey");
        try {
            allocation = getPermitAllocation(invalidRequest, restClient, getBuilder);
            Assert.fail();
        } catch (RestLiResponseException exc) {
            Assert.assertEquals(exc.getStatus(), 422);
        }
    } finally {
        if (server.isRunning()) {
            server.stopAsync();
            server.awaitTerminated();
        }
    }
}

Example 4 with SharedLimiterKey

use of org.apache.gobblin.util.limiter.broker.SharedLimiterKey in project incubator-gobblin by apache.

the class RestliLimiterFactoryTest method testRestliLimiterCalledByLimiterFactory.

@Test
public void testRestliLimiterCalledByLimiterFactory() throws Exception {
    SharedResourcesBroker<SimpleScopeType> broker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance());
    MyRequestSender requestSender = new MyRequestSender();
    broker.bindSharedResourceAtScope(new RedirectAwareRestClientRequestSender.Factory<>(), new SharedRestClientKey(RestliLimiterFactory.RESTLI_SERVICE_NAME), SimpleScopeType.GLOBAL, requestSender);
    Limiter limiter = broker.getSharedResource(new SharedLimiterFactory<>(), new SharedLimiterKey("my/resource"));
    Assert.assertNotNull(limiter.acquirePermits(10));
    Assert.assertEquals(requestSender.requestList.size(), 1);
    broker.close();
}

Example 5 with SharedLimiterKey

use of org.apache.gobblin.util.limiter.broker.SharedLimiterKey in project incubator-gobblin by apache.

the class RestliServiceBasedLimiterTest method testServerFailover.

@Test
public void testServerFailover() throws Exception {
    try (Closer closer = Closer.create()) {
        SharedLimiterKey res1key = new SharedLimiterKey("res1");
        Map<String, String> configMap = Maps.newHashMap();
        TestingServer zkTestingServer = closer.register(new TestingServer(-1));
        configMap.put(ThrottlingGuiceServletConfig.ZK_STRING_KEY, zkTestingServer.getConnectString());
        configMap.put(ThrottlingGuiceServletConfig.HA_CLUSTER_NAME, RestliServiceBasedLimiterTest.class.getSimpleName() + "_cluster");
        Config config = ConfigFactory.parseMap(configMap);
        RestliServer server2500 = createAndStartServer(config, 2500);
        RestliServer server2501 = createAndStartServer(config, 2501);
        SharedResourcesBroker<SimpleScopeType> broker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance());
        RedirectAwareRestClientRequestSender requestSender = new RedirectAwareRestClientRequestSender(broker, Lists.newArrayList(server2500.getServer().getURIPrefix(), server2501.getServer().getURIPrefix()));
        RestliServiceBasedLimiter limiter = RestliServiceBasedLimiter.builder().requestSender(requestSender).resourceLimited(res1key.getResourceLimitedPath()).serviceIdentifier("service").build();
        Assert.assertNotNull(limiter.acquirePermits(20));
        limiter.clearAllStoredPermits();
        server2500.close();
        Assert.assertNotNull(limiter.acquirePermits(20));
        Assert.assertEquals(parsePortOfCurrentServerPrefix(requestSender), 2501);
        limiter.clearAllStoredPermits();
        server2500 = createAndStartServer(config, 2500);
        Assert.assertNotNull(limiter.acquirePermits(20));
        limiter.clearAllStoredPermits();
        // leader is currently 2501
        Assert.assertEquals(parsePortOfCurrentServerPrefix(requestSender), 2501);
        // set request to 2500 (not leader)
        requestSender.updateRestClient(server2500.getServer().getURIPrefix(), "test");
        Assert.assertEquals(parsePortOfCurrentServerPrefix(requestSender), 2500);
        Assert.assertNotNull(limiter.acquirePermits(20));
        // verify request sender switched back to leader
        Assert.assertEquals(parsePortOfCurrentServerPrefix(requestSender), 2501);
        server2501.close();
        Assert.assertNotNull(limiter.acquirePermits(20));
        limiter.clearAllStoredPermits();
        server2500.close();
        Assert.assertNull(limiter.acquirePermits(20));
        limiter.clearAllStoredPermits();
    }
}