Examples with Limiter org.apache.gobblin.util.limiter.Limiter used on opensource projects

Example 11 with Limiter

use of org.apache.gobblin.util.limiter.Limiter in project incubator-gobblin by apache.

the class RatedControlledFileSystemTest method setUp.

@BeforeClass
public void setUp() throws IOException, ExecutionException {
    Limiter limiter = new RateBasedLimiter(20);
    this.rateControlledFs = new TestRateControlledFileSystem(FileSystem.getLocal(new Configuration()), 20, limiter);
    this.rateControlledFs.startRateControl();
}

Example 12 with Limiter

use of org.apache.gobblin.util.limiter.Limiter in project incubator-gobblin by apache.

the class ThrottledFileSystemTest method testSimpleCalls.

@Test
public void testSimpleCalls() throws Exception {
    FileSystem fs = Mockito.mock(FileSystem.class);
    Mockito.when(fs.getFileStatus(Mockito.any(Path.class))).thenReturn(new FileStatus(0, false, 0, 0, 0, new Path("/")));
    Limiter limiter = new CountBasedLimiter(2);
    ThrottledFileSystem throttledFileSystem = new ThrottledFileSystem(fs, limiter, "testService");
    Assert.assertNotNull(throttledFileSystem.getFileStatus(new Path("/myFile")));
    Assert.assertNotNull(throttledFileSystem.getFileStatus(new Path("/myFile")));
    try {
        throttledFileSystem.getFileStatus(new Path("/myFile"));
        Assert.fail();
    } catch (NotEnoughPermitsException expected) {
    // Expected
    }
}

Example 13 with Limiter

use of org.apache.gobblin.util.limiter.Limiter in project incubator-gobblin by apache.

the class ThrottledInputStreamTest method test.

@Test
public void test() throws Exception {
    ByteArrayInputStream inputStream = new ByteArrayInputStream("abcde".getBytes(Charsets.UTF_8));
    MeteredInputStream meteredInputStream = MeteredInputStream.builder().in(inputStream).updateFrequency(1).build();
    Limiter limiter = new CountBasedLimiter(4);
    InputStream throttled = new ThrottledInputStream(meteredInputStream, limiter, meteredInputStream);
    try {
        String output = IOUtils.toString(throttled, Charsets.UTF_8);
        Assert.fail();
    } catch (RuntimeException re) {
    // Expected
    }
    meteredInputStream.reset();
    limiter = new CountBasedLimiter(5);
    throttled = new ThrottledInputStream(meteredInputStream, limiter, meteredInputStream);
    Assert.assertEquals(IOUtils.toString(throttled, Charsets.UTF_8), "abcde");
}

Example 14 with Limiter

use of org.apache.gobblin.util.limiter.Limiter in project incubator-gobblin by apache.

the class SharedLimiterFactoryTest method testCountLimiter.

@Test
public void testCountLimiter() throws Exception {
    SharedResourcesBrokerImpl<SimpleScopeType> broker = getBrokerForConfigMap(ImmutableMap.of(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, SharedLimiterFactory.LIMITER_CLASS_KEY), "CountBasedLimiter", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, CountBasedLimiter.Factory.COUNT_KEY), "10"));
    SharedLimiterFactory<SimpleScopeType> factory = new SharedLimiterFactory<>();
    Assert.assertEquals(factory.getAutoScope(broker, broker.getConfigView(SimpleScopeType.LOCAL, new SharedLimiterKey("resource"), factory.getName())), SimpleScopeType.GLOBAL);
    Limiter limiter = ((ResourceInstance<Limiter>) factory.createResource(broker, broker.getConfigView(SimpleScopeType.GLOBAL, new SharedLimiterKey("resource"), factory.getName()))).getResource();
    Assert.assertTrue(limiter instanceof CountBasedLimiter);
    Assert.assertEquals(((CountBasedLimiter) limiter).getCountLimit(), 10);
}

Example 15 with Limiter

use of org.apache.gobblin.util.limiter.Limiter in project incubator-gobblin by apache.

the class SharedLimiterFactoryTest method testEmptyConfig.

@Test
public void testEmptyConfig() throws Exception {
    SharedResourcesBrokerImpl<SimpleScopeType> broker = getBrokerForConfigMap(ImmutableMap.<String, String>of());
    SharedLimiterFactory<SimpleScopeType> factory = new SharedLimiterFactory<>();
    Assert.assertEquals(factory.getAutoScope(broker, broker.getConfigView(SimpleScopeType.LOCAL, new SharedLimiterKey("resource"), factory.getName())), SimpleScopeType.GLOBAL);
    Limiter limiter = ((ResourceInstance<Limiter>) factory.createResource(broker, broker.getConfigView(SimpleScopeType.GLOBAL, new SharedLimiterKey("resource"), factory.getName()))).getResource();
    Assert.assertTrue(limiter instanceof NoopLimiter);
}