Examples with SimpleScopeType org.apache.gobblin.broker.SimpleScopeType used on opensource projects

Example 6 with SimpleScopeType

use of org.apache.gobblin.broker.SimpleScopeType in project incubator-gobblin by apache.

the class RestliServiceBasedLimiterTest method test.

@Test
public void test() throws Exception {
    ThrottlingPolicyFactory factory = new ThrottlingPolicyFactory();
    SharedLimiterKey res1key = new SharedLimiterKey("res1");
    Map<String, String> configMap = ImmutableMap.<String, String>builder().put(BrokerConfigurationKeyGenerator.generateKey(factory, res1key, null, ThrottlingPolicyFactory.POLICY_KEY), CountBasedPolicy.FACTORY_ALIAS).put(BrokerConfigurationKeyGenerator.generateKey(factory, res1key, null, CountBasedPolicy.COUNT_KEY), "100").build();
    ThrottlingGuiceServletConfig guiceServletConfig = new ThrottlingGuiceServletConfig();
    guiceServletConfig.initialize(ConfigFactory.parseMap(configMap));
    Injector injector = guiceServletConfig.getInjector();
    EmbeddedRestliServer server = EmbeddedRestliServer.builder().resources(Lists.<Class<? extends BaseResource>>newArrayList(LimiterServerResource.class)).injector(injector).build();
    SharedResourcesBroker<SimpleScopeType> broker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance());
    try {
        server.startAsync();
        server.awaitRunning();
        RestliServiceBasedLimiter limiter = RestliServiceBasedLimiter.builder().requestSender(new RedirectAwareRestClientRequestSender(broker, Lists.newArrayList(server.getURIPrefix()))).resourceLimited(res1key.getResourceLimitedPath()).serviceIdentifier("service").build();
        Assert.assertNotNull(limiter.acquirePermits(20));
        Assert.assertNotNull(limiter.acquirePermits(20));
        Assert.assertNull(limiter.acquirePermits(1000));
    } finally {
        if (server.isRunning()) {
            server.stopAsync();
            server.awaitTerminated();
        }
    }
}

Example 7 with SimpleScopeType

use of org.apache.gobblin.broker.SimpleScopeType in project incubator-gobblin by apache.

the class FileSystemFactoryTest method test.

@Test
public void test() throws Exception {
    SharedResourcesBrokerImpl<SimpleScopeType> broker = SharedResourcesBrokerFactory.<SimpleScopeType>createDefaultTopLevelBroker(ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance());
    FileSystemKey key = new FileSystemKey(new URI("file:///"), new Configuration());
    FileSystemFactory<SimpleScopeType> factory = new FileSystemFactory<>();
    FileSystem fs = broker.getSharedResource(factory, key);
    verifyInstrumentedOnce(fs);
    SharedResourcesBroker<SimpleScopeType> subBroker = broker.newSubscopedBuilder(SimpleScopeType.LOCAL.defaultScopeInstance()).build();
    FileSystem subBrokerFs = FileSystemFactory.get(new URI("file:///"), new Configuration(), subBroker);
    Assert.assertEquals(fs, subBrokerFs);
}

Example 8 with SimpleScopeType

use of org.apache.gobblin.broker.SimpleScopeType in project incubator-gobblin by apache.

the class SharedLimiterFactoryTest method testMultiLevelLimiter.

@Test
public void testMultiLevelLimiter() throws Exception {
    SharedResourcesBrokerImpl<SimpleScopeType> broker = getBrokerForConfigMap(ImmutableMap.of(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, SimpleScopeType.GLOBAL, SharedLimiterFactory.LIMITER_CLASS_KEY), "CountBasedLimiter", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, SimpleScopeType.GLOBAL, CountBasedLimiter.Factory.COUNT_KEY), "10", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, SimpleScopeType.LOCAL, SharedLimiterFactory.LIMITER_CLASS_KEY), "CountBasedLimiter", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, SimpleScopeType.LOCAL, CountBasedLimiter.Factory.COUNT_KEY), "5"));
    SharedResourcesBroker<SimpleScopeType> localBroker1 = broker.newSubscopedBuilder(new SimpleScope<>(SimpleScopeType.LOCAL, "local1")).build();
    SharedResourcesBroker<SimpleScopeType> localBroker2 = broker.newSubscopedBuilder(new SimpleScope<>(SimpleScopeType.LOCAL, "local2")).build();
    SharedLimiterFactory<SimpleScopeType> factory = new SharedLimiterFactory<>();
    Limiter limiter1 = ((ResourceInstance<Limiter>) factory.createResource(localBroker1, broker.getConfigView(SimpleScopeType.LOCAL, new SharedLimiterKey("resource"), factory.getName()))).getResource();
    Limiter limiter2 = ((ResourceInstance<Limiter>) factory.createResource(localBroker2, broker.getConfigView(SimpleScopeType.LOCAL, new SharedLimiterKey("resource"), factory.getName()))).getResource();
    Assert.assertTrue(limiter1 instanceof MultiLimiter);
    Assert.assertTrue(limiter2 instanceof MultiLimiter);
    Assert.assertEquals(((CountBasedLimiter) ((MultiLimiter) limiter1).getUnderlyingLimiters().get(0)).getCountLimit(), 5);
    Assert.assertEquals(((CountBasedLimiter) ((MultiLimiter) limiter1).getUnderlyingLimiters().get(1)).getCountLimit(), 10);
    Assert.assertEquals(((CountBasedLimiter) ((MultiLimiter) limiter2).getUnderlyingLimiters().get(0)).getCountLimit(), 5);
    Assert.assertEquals(((CountBasedLimiter) ((MultiLimiter) limiter2).getUnderlyingLimiters().get(1)).getCountLimit(), 10);
    Assert.assertNotEquals(((MultiLimiter) limiter1).getUnderlyingLimiters().get(0), ((MultiLimiter) limiter2).getUnderlyingLimiters().get(0));
    Assert.assertEquals(((MultiLimiter) limiter1).getUnderlyingLimiters().get(1), ((MultiLimiter) limiter2).getUnderlyingLimiters().get(1));
}

Example 9 with SimpleScopeType

use of org.apache.gobblin.broker.SimpleScopeType in project incubator-gobblin by apache.

the class DataPublisherFactoryTest method testGetNonThreadSafePublisher.

@Test
public void testGetNonThreadSafePublisher() throws IOException {
    SharedResourcesBroker broker = SharedResourcesBrokerFactory.<SimpleScopeType>createDefaultTopLevelBroker(ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance());
    DataPublisher publisher1 = DataPublisherFactory.get(TestNonThreadsafeDataPublisher.class.getName(), null, broker);
    DataPublisher publisher2 = DataPublisherFactory.get(TestNonThreadsafeDataPublisher.class.getName(), null, broker);
    // should get different publishers
    Assert.assertNotEquals(publisher1, publisher2);
    // Check capabilities
    Assert.assertTrue(publisher1.supportsCapability(DataPublisher.REUSABLE, Collections.EMPTY_MAP));
    Assert.assertFalse(publisher1.supportsCapability(Capability.THREADSAFE, Collections.EMPTY_MAP));
}

Example 10 with SimpleScopeType

use of org.apache.gobblin.broker.SimpleScopeType in project incubator-gobblin by apache.

the class DataPublisherFactoryTest method testMultiThreadedGetNonThreadSafePublisher.

@Test()
public void testMultiThreadedGetNonThreadSafePublisher() throws InterruptedException, ExecutionException, IOException {
    SharedResourcesBroker broker = SharedResourcesBrokerFactory.<SimpleScopeType>createDefaultTopLevelBroker(ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance());
    ExecutorService service = Executors.newFixedThreadPool(40);
    List<Future<?>> futures = new ArrayList<>();
    for (int i = 0; i < 100000; i++) {
        futures.add(service.submit(new GetNonThreadSafePublisher(broker)));
    }
    for (Future f : futures) {
        f.get();
    }
    service.shutdown();
    service.awaitTermination(100, TimeUnit.SECONDS);
}