Examples with SharedResourcesBroker org.apache.gobblin.broker.iface.SharedResourcesBroker used on opensource projects

Example 1 with SharedResourcesBroker

use of org.apache.gobblin.broker.iface.SharedResourcesBroker in project incubator-gobblin by apache.

the class LocalStressTest method main.

public static void main(String[] args) throws Exception {
    CommandLine cli = StressTestUtils.parseCommandLine(OPTIONS, args);
    int stressorThreads = Integer.parseInt(cli.getOptionValue(STRESSOR_THREADS.getOpt(), Integer.toString(DEFAULT_STRESSOR_THREADS)));
    int processorThreads = Integer.parseInt(cli.getOptionValue(PROCESSOR_THREADS.getOpt(), Integer.toString(DEFAULT_PROCESSOR_THREADS)));
    int artificialLatency = Integer.parseInt(cli.getOptionValue(ARTIFICIAL_LATENCY.getOpt(), Integer.toString(DEFAULT_ARTIFICIAL_LATENCY)));
    long targetQps = Integer.parseInt(cli.getOptionValue(QPS.getOpt(), Integer.toString(DEFAULT_TARGET_QPS)));
    Configuration configuration = new Configuration();
    StressTestUtils.populateConfigFromCli(configuration, cli);
    String resourceLimited = LocalStressTest.class.getSimpleName();
    Map<String, String> configMap = Maps.newHashMap();
    ThrottlingPolicyFactory factory = new ThrottlingPolicyFactory();
    SharedLimiterKey res1key = new SharedLimiterKey(resourceLimited);
    configMap.put(BrokerConfigurationKeyGenerator.generateKey(factory, res1key, null, ThrottlingPolicyFactory.POLICY_KEY), QPSPolicy.FACTORY_ALIAS);
    configMap.put(BrokerConfigurationKeyGenerator.generateKey(factory, res1key, null, QPSPolicy.QPS), Long.toString(targetQps));
    ThrottlingGuiceServletConfig guiceServletConfig = new ThrottlingGuiceServletConfig();
    guiceServletConfig.initialize(ConfigFactory.parseMap(configMap));
    LimiterServerResource limiterServer = guiceServletConfig.getInjector().getInstance(LimiterServerResource.class);
    RateComputingLimiterContainer limiterContainer = new RateComputingLimiterContainer();
    Class<? extends Stressor> stressorClass = configuration.getClass(StressTestUtils.STRESSOR_CLASS, StressTestUtils.DEFAULT_STRESSOR_CLASS, Stressor.class);
    ExecutorService executorService = Executors.newFixedThreadPool(stressorThreads);
    SharedResourcesBroker broker = guiceServletConfig.getInjector().getInstance(Key.get(SharedResourcesBroker.class, Names.named(LimiterServerResource.BROKER_INJECT_NAME)));
    ThrottlingPolicy policy = (ThrottlingPolicy) broker.getSharedResource(new ThrottlingPolicyFactory(), new SharedLimiterKey(resourceLimited));
    ScheduledExecutorService reportingThread = Executors.newSingleThreadScheduledExecutor();
    reportingThread.scheduleAtFixedRate(new Reporter(limiterContainer, policy), 0, 15, TimeUnit.SECONDS);
    Queue<Future<?>> futures = new LinkedList<>();
    MockRequester requester = new MockRequester(limiterServer, artificialLatency, processorThreads);
    requester.start();
    for (int i = 0; i < stressorThreads; i++) {
        RestliServiceBasedLimiter restliLimiter = RestliServiceBasedLimiter.builder().resourceLimited(resourceLimited).requestSender(requester).serviceIdentifier("stressor" + i).build();
        Stressor stressor = stressorClass.newInstance();
        stressor.configure(configuration);
        futures.add(executorService.submit(new StressorRunner(limiterContainer.decorateLimiter(restliLimiter), stressor)));
    }
    int stressorFailures = 0;
    for (Future<?> future : futures) {
        try {
            future.get();
        } catch (ExecutionException ee) {
            stressorFailures++;
        }
    }
    requester.stop();
    executorService.shutdownNow();
    if (stressorFailures > 0) {
        log.error("There were " + stressorFailures + " failed stressor threads.");
    }
    System.exit(stressorFailures);
}

Example 2 with SharedResourcesBroker

use of org.apache.gobblin.broker.iface.SharedResourcesBroker in project incubator-gobblin by apache.

the class DataPublisherFactoryTest method testGetNonThreadSafePublisher.

@Test
public void testGetNonThreadSafePublisher() throws IOException {
    SharedResourcesBroker broker = SharedResourcesBrokerFactory.<SimpleScopeType>createDefaultTopLevelBroker(ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance());
    DataPublisher publisher1 = DataPublisherFactory.get(TestNonThreadsafeDataPublisher.class.getName(), null, broker);
    DataPublisher publisher2 = DataPublisherFactory.get(TestNonThreadsafeDataPublisher.class.getName(), null, broker);
    // should get different publishers
    Assert.assertNotEquals(publisher1, publisher2);
    // Check capabilities
    Assert.assertTrue(publisher1.supportsCapability(DataPublisher.REUSABLE, Collections.EMPTY_MAP));
    Assert.assertFalse(publisher1.supportsCapability(Capability.THREADSAFE, Collections.EMPTY_MAP));
}

Example 3 with SharedResourcesBroker

use of org.apache.gobblin.broker.iface.SharedResourcesBroker in project incubator-gobblin by apache.

the class DataPublisherFactoryTest method testMultiThreadedGetNonThreadSafePublisher.

@Test()
public void testMultiThreadedGetNonThreadSafePublisher() throws InterruptedException, ExecutionException, IOException {
    SharedResourcesBroker broker = SharedResourcesBrokerFactory.<SimpleScopeType>createDefaultTopLevelBroker(ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance());
    ExecutorService service = Executors.newFixedThreadPool(40);
    List<Future<?>> futures = new ArrayList<>();
    for (int i = 0; i < 100000; i++) {
        futures.add(service.submit(new GetNonThreadSafePublisher(broker)));
    }
    for (Future f : futures) {
        f.get();
    }
    service.shutdown();
    service.awaitTermination(100, TimeUnit.SECONDS);
}

Example 4 with SharedResourcesBroker

use of org.apache.gobblin.broker.iface.SharedResourcesBroker in project incubator-gobblin by apache.

the class DefaultGobblinBrokerTest method testExplicitBinding.

@Test
public void testExplicitBinding() throws Exception {
    Config config = ConfigFactory.empty();
    SharedResourcesBrokerImpl<SimpleScopeType> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, SimpleScopeType.GLOBAL.defaultScopeInstance());
    SharedResourceFactory<Long, EmptyKey, SimpleScopeType> factory = new SharedResourceFactory<Long, EmptyKey, SimpleScopeType>() {

        @Override
        public String getName() {
            return "myTestFactory";
        }

        @Override
        public SharedResourceFactoryResponse<Long> createResource(SharedResourcesBroker<SimpleScopeType> broker, ScopedConfigView<SimpleScopeType, EmptyKey> config) throws NotConfiguredException {
            throw new UnsupportedOperationException();
        }

        @Override
        public SimpleScopeType getAutoScope(SharedResourcesBroker<SimpleScopeType> broker, ConfigView<SimpleScopeType, EmptyKey> config) {
            return broker.selfScope().getType();
        }
    };
    topBroker.bindSharedResourceAtScope(factory, new EmptyKey(), SimpleScopeType.GLOBAL, 10l);
    Assert.assertEquals(topBroker.getSharedResource(factory, new EmptyKey()), new Long(10));
}