Examples with CountBasedLimiter org.apache.gobblin.util.limiter.CountBasedLimiter used on opensource projects

Example 1 with CountBasedLimiter

use of org.apache.gobblin.util.limiter.CountBasedLimiter in project incubator-gobblin by apache.

the class ThrottledFileSystemTest method testListing.

@Test
public void testListing() throws Exception {
    FileSystem fs = Mockito.mock(FileSystem.class);
    Mockito.when(fs.listStatus(Mockito.any(Path.class))).thenAnswer(new Answer<FileStatus[]>() {

        @Override
        public FileStatus[] answer(InvocationOnMock invocation) throws Throwable {
            Path path = (Path) invocation.getArguments()[0];
            int files = Integer.parseInt(path.getName());
            FileStatus status = new FileStatus(0, false, 0, 0, 0, new Path("/"));
            FileStatus[] out = new FileStatus[files];
            for (int i = 0; i < files; i++) {
                out[i] = status;
            }
            return out;
        }
    });
    Limiter limiter = new CountBasedLimiter(5);
    ThrottledFileSystem throttledFileSystem = new ThrottledFileSystem(fs, limiter, "testService");
    Assert.assertEquals(throttledFileSystem.getServiceName(), "testService");
    // use 1 permit
    Assert.assertNotNull(throttledFileSystem.listStatus(new Path("/files/99")));
    // use 3 permits
    Assert.assertNotNull(throttledFileSystem.listStatus(new Path("/files/250")));
    try {
        // requires 2 permits
        throttledFileSystem.listStatus(new Path("/files/150"));
        Assert.fail();
    } catch (NotEnoughPermitsException expected) {
    // Expected
    }
    // requires 1 permit
    Assert.assertNotNull(throttledFileSystem.listStatus(new Path("/files/99")));
}

Example 2 with CountBasedLimiter

use of org.apache.gobblin.util.limiter.CountBasedLimiter in project incubator-gobblin by apache.

the class CountBasedLimiterTest method testThrottling.

@Test
public void testThrottling() throws InterruptedException {
    Limiter limiter = new CountBasedLimiter(10);
    limiter.start();
    for (int i = 0; i < 10; i++) {
        Assert.assertTrue(limiter.acquirePermits(1) != null);
    }
    Assert.assertTrue(limiter.acquirePermits(1) == null);
    limiter.stop();
}

Example 3 with CountBasedLimiter

use of org.apache.gobblin.util.limiter.CountBasedLimiter in project incubator-gobblin by apache.

the class SharedLimiterFactoryTest method testMultiLevelLimiter.

@Test
public void testMultiLevelLimiter() throws Exception {
    SharedResourcesBrokerImpl<SimpleScopeType> broker = getBrokerForConfigMap(ImmutableMap.of(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, SimpleScopeType.GLOBAL, SharedLimiterFactory.LIMITER_CLASS_KEY), "CountBasedLimiter", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, SimpleScopeType.GLOBAL, CountBasedLimiter.Factory.COUNT_KEY), "10", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, SimpleScopeType.LOCAL, SharedLimiterFactory.LIMITER_CLASS_KEY), "CountBasedLimiter", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, SimpleScopeType.LOCAL, CountBasedLimiter.Factory.COUNT_KEY), "5"));
    SharedResourcesBroker<SimpleScopeType> localBroker1 = broker.newSubscopedBuilder(new SimpleScope<>(SimpleScopeType.LOCAL, "local1")).build();
    SharedResourcesBroker<SimpleScopeType> localBroker2 = broker.newSubscopedBuilder(new SimpleScope<>(SimpleScopeType.LOCAL, "local2")).build();
    SharedLimiterFactory<SimpleScopeType> factory = new SharedLimiterFactory<>();
    Limiter limiter1 = ((ResourceInstance<Limiter>) factory.createResource(localBroker1, broker.getConfigView(SimpleScopeType.LOCAL, new SharedLimiterKey("resource"), factory.getName()))).getResource();
    Limiter limiter2 = ((ResourceInstance<Limiter>) factory.createResource(localBroker2, broker.getConfigView(SimpleScopeType.LOCAL, new SharedLimiterKey("resource"), factory.getName()))).getResource();
    Assert.assertTrue(limiter1 instanceof MultiLimiter);
    Assert.assertTrue(limiter2 instanceof MultiLimiter);
    Assert.assertEquals(((CountBasedLimiter) ((MultiLimiter) limiter1).getUnderlyingLimiters().get(0)).getCountLimit(), 5);
    Assert.assertEquals(((CountBasedLimiter) ((MultiLimiter) limiter1).getUnderlyingLimiters().get(1)).getCountLimit(), 10);
    Assert.assertEquals(((CountBasedLimiter) ((MultiLimiter) limiter2).getUnderlyingLimiters().get(0)).getCountLimit(), 5);
    Assert.assertEquals(((CountBasedLimiter) ((MultiLimiter) limiter2).getUnderlyingLimiters().get(1)).getCountLimit(), 10);
    Assert.assertNotEquals(((MultiLimiter) limiter1).getUnderlyingLimiters().get(0), ((MultiLimiter) limiter2).getUnderlyingLimiters().get(0));
    Assert.assertEquals(((MultiLimiter) limiter1).getUnderlyingLimiters().get(1), ((MultiLimiter) limiter2).getUnderlyingLimiters().get(1));
}

Example 4 with CountBasedLimiter

use of org.apache.gobblin.util.limiter.CountBasedLimiter in project incubator-gobblin by apache.

the class ThrottledFileSystemTest method testSimpleCalls.

@Test
public void testSimpleCalls() throws Exception {
    FileSystem fs = Mockito.mock(FileSystem.class);
    Mockito.when(fs.getFileStatus(Mockito.any(Path.class))).thenReturn(new FileStatus(0, false, 0, 0, 0, new Path("/")));
    Limiter limiter = new CountBasedLimiter(2);
    ThrottledFileSystem throttledFileSystem = new ThrottledFileSystem(fs, limiter, "testService");
    Assert.assertNotNull(throttledFileSystem.getFileStatus(new Path("/myFile")));
    Assert.assertNotNull(throttledFileSystem.getFileStatus(new Path("/myFile")));
    try {
        throttledFileSystem.getFileStatus(new Path("/myFile"));
        Assert.fail();
    } catch (NotEnoughPermitsException expected) {
    // Expected
    }
}

Example 5 with CountBasedLimiter

use of org.apache.gobblin.util.limiter.CountBasedLimiter in project incubator-gobblin by apache.

the class ThrottledInputStreamTest method test.

@Test
public void test() throws Exception {
    ByteArrayInputStream inputStream = new ByteArrayInputStream("abcde".getBytes(Charsets.UTF_8));
    MeteredInputStream meteredInputStream = MeteredInputStream.builder().in(inputStream).updateFrequency(1).build();
    Limiter limiter = new CountBasedLimiter(4);
    InputStream throttled = new ThrottledInputStream(meteredInputStream, limiter, meteredInputStream);
    try {
        String output = IOUtils.toString(throttled, Charsets.UTF_8);
        Assert.fail();
    } catch (RuntimeException re) {
    // Expected
    }
    meteredInputStream.reset();
    limiter = new CountBasedLimiter(5);
    throttled = new ThrottledInputStream(meteredInputStream, limiter, meteredInputStream);
    Assert.assertEquals(IOUtils.toString(throttled, Charsets.UTF_8), "abcde");
}