Examples with SharedRestClientKey org.apache.gobblin.restli.SharedRestClientKey used on opensource projects

Example 1 with SharedRestClientKey

use of org.apache.gobblin.restli.SharedRestClientKey in project incubator-gobblin by apache.

the class MRStressTest method main.

public static void main(String[] args) throws Exception {
    CommandLine cli = StressTestUtils.parseCommandLine(OPTIONS, args);
    Configuration configuration = new Configuration();
    if (cli.hasOption(THROTTLING_SERVER_URI.getOpt())) {
        configuration.setBoolean(USE_THROTTLING_SERVER, true);
        String resourceLimited = cli.getOptionValue(RESOURCE_ID_OPT.getOpt(), "MRStressTest");
        configuration.set(RESOURCE_ID, resourceLimited);
        configuration.set(BrokerConfigurationKeyGenerator.generateKey(new SharedRestClientFactory(), new SharedRestClientKey(RestliLimiterFactory.RESTLI_SERVICE_NAME), null, SharedRestClientFactory.SERVER_URI_KEY), cli.getOptionValue(THROTTLING_SERVER_URI.getOpt()));
    }
    if (cli.hasOption(LOCAL_QPS_OPT.getOpt())) {
        configuration.set(LOCALLY_ENFORCED_QPS, cli.getOptionValue(LOCAL_QPS_OPT.getOpt()));
    }
    Job job = Job.getInstance(configuration, "ThrottlingStressTest");
    job.getConfiguration().setBoolean("mapreduce.job.user.classpath.first", true);
    job.getConfiguration().setBoolean("mapreduce.map.speculative", false);
    job.getConfiguration().set(NUM_MAPPERS, cli.getOptionValue(NUM_MAPPERS_OPT.getOpt(), DEFAULT_MAPPERS));
    StressTestUtils.populateConfigFromCli(job.getConfiguration(), cli);
    job.setJarByClass(MRStressTest.class);
    job.setMapperClass(StresserMapper.class);
    job.setReducerClass(AggregatorReducer.class);
    job.setInputFormatClass(MyInputFormat.class);
    job.setOutputKeyClass(LongWritable.class);
    job.setOutputValueClass(DoubleWritable.class);
    FileOutputFormat.setOutputPath(job, new Path("/tmp/MRStressTest" + System.currentTimeMillis()));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

Example 2 with SharedRestClientKey

use of org.apache.gobblin.restli.SharedRestClientKey in project incubator-gobblin by apache.

the class RestliLimiterFactoryTest method testRestliLimiterCalledByLimiterFactory.

@Test
public void testRestliLimiterCalledByLimiterFactory() throws Exception {
    SharedResourcesBroker<SimpleScopeType> broker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance());
    MyRequestSender requestSender = new MyRequestSender();
    broker.bindSharedResourceAtScope(new RedirectAwareRestClientRequestSender.Factory<>(), new SharedRestClientKey(RestliLimiterFactory.RESTLI_SERVICE_NAME), SimpleScopeType.GLOBAL, requestSender);
    Limiter limiter = broker.getSharedResource(new SharedLimiterFactory<>(), new SharedLimiterKey("my/resource"));
    Assert.assertNotNull(limiter.acquirePermits(10));
    Assert.assertEquals(requestSender.requestList.size(), 1);
    broker.close();
}

Example 3 with SharedRestClientKey

use of org.apache.gobblin.restli.SharedRestClientKey in project incubator-gobblin by apache.

the class RestliLimiterFactory method createResource.

@Override
public SharedResourceFactoryResponse<RestliServiceBasedLimiter> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, SharedLimiterKey> config) throws NotConfiguredException {
    S scope = config.getScope();
    if (scope != scope.rootScope()) {
        return new ResourceCoordinate<>(this, config.getKey(), scope.rootScope());
    }
    String serviceIdentifier = config.getConfig().hasPath(SERVICE_IDENTIFIER_KEY) ? config.getConfig().getString(SERVICE_IDENTIFIER_KEY) : "UNKNOWN";
    String resourceLimited = config.getKey().getResourceLimitedPath();
    MetricContextKey metricContextKey = new SubTaggedMetricContextKey(RestliServiceBasedLimiter.class.getSimpleName() + "_" + resourceLimited, ImmutableMap.of("resourceLimited", resourceLimited));
    return new ResourceInstance<>(RestliServiceBasedLimiter.builder().resourceLimited(resourceLimited).serviceIdentifier(serviceIdentifier).metricContext(broker.getSharedResource(new MetricContextFactory<S>(), metricContextKey)).requestSender(broker.getSharedResource(new RedirectAwareRestClientRequestSender.Factory<S>(), new SharedRestClientKey(RESTLI_SERVICE_NAME))).build());
}

Example 4 with SharedRestClientKey

use of org.apache.gobblin.restli.SharedRestClientKey in project incubator-gobblin by apache.

the class RestliLimiterFactoryTest method testSkipGlobalLimiterOnLimiterFactory.

@Test
public void testSkipGlobalLimiterOnLimiterFactory() throws Exception {
    Map<String, String> configMap = ImmutableMap.of(BrokerConfigurationKeyGenerator.generateKey(new SharedLimiterFactory(), null, null, SharedLimiterFactory.SKIP_GLOBAL_LIMITER_KEY), "true");
    SharedResourcesBroker<SimpleScopeType> broker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(ConfigFactory.parseMap(configMap), SimpleScopeType.GLOBAL.defaultScopeInstance());
    MyRequestSender requestSender = new MyRequestSender();
    broker.bindSharedResourceAtScope(new RedirectAwareRestClientRequestSender.Factory<>(), new SharedRestClientKey(RestliLimiterFactory.RESTLI_SERVICE_NAME), SimpleScopeType.GLOBAL, requestSender);
    Limiter limiter = broker.getSharedResource(new SharedLimiterFactory<>(), new SharedLimiterKey("my/resource"));
    Assert.assertNotNull(limiter.acquirePermits(10));
    Assert.assertEquals(requestSender.requestList.size(), 0);
    broker.close();
}

Example 5 with SharedRestClientKey

use of org.apache.gobblin.restli.SharedRestClientKey in project incubator-gobblin by apache.

the class RestliLimiterFactoryTest method testFactory.

@Test
public void testFactory() throws Exception {
    SharedResourcesBroker<SimpleScopeType> broker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance());
    MyRequestSender requestSender = new MyRequestSender();
    broker.bindSharedResourceAtScope(new RedirectAwareRestClientRequestSender.Factory<>(), new SharedRestClientKey(RestliLimiterFactory.RESTLI_SERVICE_NAME), SimpleScopeType.GLOBAL, requestSender);
    RestliServiceBasedLimiter limiter = broker.getSharedResource(new RestliLimiterFactory<>(), new SharedLimiterKey("my/resource"));
    Assert.assertNotNull(limiter.acquirePermits(10));
    Assert.assertEquals(requestSender.requestList.size(), 1);
    broker.close();
}