Examples with SharedLimiterKey org.apache.gobblin.util.limiter.broker.SharedLimiterKey used on opensource projects

Example 6 with SharedLimiterKey

use of org.apache.gobblin.util.limiter.broker.SharedLimiterKey in project incubator-gobblin by apache.

the class RestliServiceBasedLimiterTest method test.

@Test
public void test() throws Exception {
    ThrottlingPolicyFactory factory = new ThrottlingPolicyFactory();
    SharedLimiterKey res1key = new SharedLimiterKey("res1");
    Map<String, String> configMap = ImmutableMap.<String, String>builder().put(BrokerConfigurationKeyGenerator.generateKey(factory, res1key, null, ThrottlingPolicyFactory.POLICY_KEY), CountBasedPolicy.FACTORY_ALIAS).put(BrokerConfigurationKeyGenerator.generateKey(factory, res1key, null, CountBasedPolicy.COUNT_KEY), "100").build();
    ThrottlingGuiceServletConfig guiceServletConfig = new ThrottlingGuiceServletConfig();
    guiceServletConfig.initialize(ConfigFactory.parseMap(configMap));
    Injector injector = guiceServletConfig.getInjector();
    EmbeddedRestliServer server = EmbeddedRestliServer.builder().resources(Lists.<Class<? extends BaseResource>>newArrayList(LimiterServerResource.class)).injector(injector).build();
    SharedResourcesBroker<SimpleScopeType> broker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance());
    try {
        server.startAsync();
        server.awaitRunning();
        RestliServiceBasedLimiter limiter = RestliServiceBasedLimiter.builder().requestSender(new RedirectAwareRestClientRequestSender(broker, Lists.newArrayList(server.getURIPrefix()))).resourceLimited(res1key.getResourceLimitedPath()).serviceIdentifier("service").build();
        Assert.assertNotNull(limiter.acquirePermits(20));
        Assert.assertNotNull(limiter.acquirePermits(20));
        Assert.assertNull(limiter.acquirePermits(1000));
    } finally {
        if (server.isRunning()) {
            server.stopAsync();
            server.awaitTerminated();
        }
    }
}

Example 7 with SharedLimiterKey

use of org.apache.gobblin.util.limiter.broker.SharedLimiterKey in project incubator-gobblin by apache.

the class LimiterServerResource method get.

/**
 * Request permits from the limiter server. The returned {@link PermitAllocation} specifies the number of permits
 * that the client can use.
 */
@Override
@RestMethod.Get
public void get(ComplexResourceKey<PermitRequest, EmptyRecord> key, @CallbackParam final Callback<PermitAllocation> callback) {
    try (Closeable context = this.requestTimer == null ? NoopCloseable.INSTANCE : this.requestTimer.time()) {
        PermitRequest request = key.getKey();
        String resourceId = request.getResource();
        MetricContext resourceContext = (MetricContext) broker.getSharedResource(new MetricContextFactory(), new SubTaggedMetricContextKey(resourceId, ImmutableMap.of(RESOURCE_ID_TAG, resourceId)));
        Meter permitsRequestedMeter = resourceContext.meter(PERMITS_REQUESTED_METER_NAME);
        Meter permitsGrantedMeter = resourceContext.meter(PERMITS_GRANTED_METER_NAME);
        Timer limiterTimer = resourceContext.timer(LIMITER_TIMER_NAME);
        permitsRequestedMeter.mark(request.getPermits());
        if (this.leaderFinderOpt.isPresent() && !this.leaderFinderOpt.get().isLeader()) {
            URI leaderUri = this.leaderFinderOpt.get().getLeaderMetadata().getUri();
            RestLiServiceException exception = new RestLiServiceException(HttpStatus.S_301_MOVED_PERMANENTLY, String.format("New leader <a href=\"%s\">%s</a>", leaderUri, leaderUri));
            exception.setErrorDetails(new DataMap(ImmutableMap.of(LOCATION_301, leaderUri.toString())));
            throw exception;
        } else {
            ThrottlingPolicy policy = (ThrottlingPolicy) this.broker.getSharedResource(new ThrottlingPolicyFactory(), new SharedLimiterKey(request.getResource()));
            PermitAllocation allocation;
            try (Closeable thisContext = limiterTimer.time()) {
                allocation = policy.computePermitAllocation(request);
            }
            permitsGrantedMeter.mark(allocation.getPermits());
            callback.onSuccess(allocation);
        }
    } catch (NotConfiguredException nce) {
        throw new RestLiServiceException(HttpStatus.S_422_UNPROCESSABLE_ENTITY, "No configuration for the requested resource.");
    } catch (IOException ioe) {
        // Failed to close timer context. This should never happen
        throw new RuntimeException(ioe);
    }
}

Example 8 with SharedLimiterKey

use of org.apache.gobblin.util.limiter.broker.SharedLimiterKey in project incubator-gobblin by apache.

the class PoliciesResource method get.

@Override
public Policy get(String resourceId) {
    try {
        ThrottlingPolicy throttlingPolicy = (ThrottlingPolicy) this.broker.getSharedResource(new ThrottlingPolicyFactory(), new SharedLimiterKey(resourceId));
        Policy restliPolicy = new Policy();
        restliPolicy.setPolicyName(throttlingPolicy.getClass().getSimpleName());
        restliPolicy.setResource(resourceId);
        restliPolicy.setParameters(new StringMap(throttlingPolicy.getParameters()));
        restliPolicy.setPolicyDetails(throttlingPolicy.getDescription());
        MetricContext resourceContext = (MetricContext) broker.getSharedResource(new MetricContextFactory(), new SubTaggedMetricContextKey(resourceId, ImmutableMap.of(RESOURCE_ID_TAG, resourceId)));
        StringMap metrics = new StringMap();
        for (Map.Entry<String, Meter> meter : resourceContext.getMeters().entrySet()) {
            metrics.put(meter.getKey(), Double.toString(meter.getValue().getOneMinuteRate()));
        }
        restliPolicy.setMetrics(metrics);
        return restliPolicy;
    } catch (NotConfiguredException nce) {
        throw new RestLiServiceException(HttpStatus.S_404_NOT_FOUND, "Policy not found for resource " + resourceId);
    }
}

Example 9 with SharedLimiterKey

use of org.apache.gobblin.util.limiter.broker.SharedLimiterKey in project incubator-gobblin by apache.

the class PoliciesResourceTest method test.

@Test
public void test() {
    ThrottlingGuiceServletConfig guiceServletConfig = new ThrottlingGuiceServletConfig();
    ThrottlingPolicyFactory factory = new ThrottlingPolicyFactory();
    SharedLimiterKey res1key = new SharedLimiterKey("res1");
    Map<String, String> configMap = avro.shaded.com.google.common.collect.ImmutableMap.<String, String>builder().put(BrokerConfigurationKeyGenerator.generateKey(factory, res1key, null, ThrottlingPolicyFactory.POLICY_KEY), CountBasedPolicy.FACTORY_ALIAS).put(BrokerConfigurationKeyGenerator.generateKey(factory, res1key, null, CountBasedPolicy.COUNT_KEY), "100").build();
    guiceServletConfig.initialize(ConfigFactory.parseMap(configMap));
    Injector injector = guiceServletConfig.getInjector();
    PoliciesResource policiesResource = injector.getInstance(PoliciesResource.class);
    Policy policy = policiesResource.get("res1");
    Assert.assertEquals(policy.getPolicyName(), CountBasedPolicy.class.getSimpleName());
    Assert.assertEquals(policy.getResource(), "res1");
    Assert.assertEquals(policy.getParameters().get("maxPermits"), "100");
    guiceServletConfig.close();
}

Example 10 with SharedLimiterKey

use of org.apache.gobblin.util.limiter.broker.SharedLimiterKey in project incubator-gobblin by apache.

the class RestliLimiterFactoryTest method testSkipGlobalLimiterOnLimiterFactory.

@Test
public void testSkipGlobalLimiterOnLimiterFactory() throws Exception {
    Map<String, String> configMap = ImmutableMap.of(BrokerConfigurationKeyGenerator.generateKey(new SharedLimiterFactory(), null, null, SharedLimiterFactory.SKIP_GLOBAL_LIMITER_KEY), "true");
    SharedResourcesBroker<SimpleScopeType> broker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(ConfigFactory.parseMap(configMap), SimpleScopeType.GLOBAL.defaultScopeInstance());
    MyRequestSender requestSender = new MyRequestSender();
    broker.bindSharedResourceAtScope(new RedirectAwareRestClientRequestSender.Factory<>(), new SharedRestClientKey(RestliLimiterFactory.RESTLI_SERVICE_NAME), SimpleScopeType.GLOBAL, requestSender);
    Limiter limiter = broker.getSharedResource(new SharedLimiterFactory<>(), new SharedLimiterKey("my/resource"));
    Assert.assertNotNull(limiter.acquirePermits(10));
    Assert.assertEquals(requestSender.requestList.size(), 0);
    broker.close();
}