use of com.twitter.distributedlog.exceptions.OverCapacityException in project distributedlog by twitter.
the class TestAsyncReaderWriter method writeRecordsWithOutstandingWriteLimit.
public void writeRecordsWithOutstandingWriteLimit(int stream, int global, boolean shouldFail) throws Exception {
DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
confLocal.addConfiguration(testConf);
confLocal.setOutputBufferSize(0);
confLocal.setImmediateFlushEnabled(true);
confLocal.setPerWriterOutstandingWriteLimit(stream);
confLocal.setOutstandingWriteLimitDarkmode(false);
DistributedLogManager dlm;
if (global > -1) {
dlm = createNewDLM(confLocal, runtime.getMethodName(), new SimplePermitLimiter(false, global, new NullStatsLogger(), true, new FixedValueFeature("", 0)));
} else {
dlm = createNewDLM(confLocal, runtime.getMethodName());
}
BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());
ArrayList<Future<DLSN>> results = new ArrayList<Future<DLSN>>(1000);
for (int i = 0; i < 1000; i++) {
results.add(writer.write(DLMTestUtil.getLogRecordInstance(1L)));
}
for (Future<DLSN> result : results) {
try {
Await.result(result);
if (shouldFail) {
fail("should fail due to no outstanding writes permitted");
}
} catch (OverCapacityException ex) {
assertTrue(shouldFail);
}
}
writer.closeAndComplete();
dlm.close();
}
use of com.twitter.distributedlog.exceptions.OverCapacityException in project distributedlog by twitter.
the class ServiceRequestLimiter method build.
@Override
public RequestLimiter<StreamOp> build() {
int rpsStreamAcquireLimit = dynConf.getRpsStreamAcquireServiceLimit();
int rpsSoftServiceLimit = dynConf.getRpsSoftServiceLimit();
int rpsHardServiceLimit = dynConf.getRpsHardServiceLimit();
int bpsStreamAcquireLimit = dynConf.getBpsStreamAcquireServiceLimit();
int bpsSoftServiceLimit = dynConf.getBpsSoftServiceLimit();
int bpsHardServiceLimit = dynConf.getBpsHardServiceLimit();
RequestLimiterBuilder rpsHardLimiterBuilder = RequestLimiterBuilder.newRpsLimiterBuilder().statsLogger(limiterStatLogger.scope("rps_hard_limit")).limit(rpsHardServiceLimit).overlimit(new OverlimitFunction<StreamOp>() {
@Override
public void apply(StreamOp request) throws OverCapacityException {
throw new OverCapacityException("RPS limit exceeded for the service instance");
}
});
RequestLimiterBuilder rpsSoftLimiterBuilder = RequestLimiterBuilder.newRpsLimiterBuilder().statsLogger(limiterStatLogger.scope("rps_soft_limit")).limit(rpsSoftServiceLimit);
RequestLimiterBuilder bpsHardLimiterBuilder = RequestLimiterBuilder.newBpsLimiterBuilder().statsLogger(limiterStatLogger.scope("bps_hard_limit")).limit(bpsHardServiceLimit).overlimit(new OverlimitFunction<StreamOp>() {
@Override
public void apply(StreamOp request) throws OverCapacityException {
throw new OverCapacityException("BPS limit exceeded for the service instance");
}
});
RequestLimiterBuilder bpsSoftLimiterBuilder = RequestLimiterBuilder.newBpsLimiterBuilder().statsLogger(limiterStatLogger.scope("bps_soft_limit")).limit(bpsSoftServiceLimit);
ChainedRequestLimiter.Builder<StreamOp> builder = new ChainedRequestLimiter.Builder<StreamOp>();
builder.addLimiter(new StreamAcquireLimiter(streamManager, serviceRps, rpsStreamAcquireLimit, limiterStatLogger.scope("rps_acquire")));
builder.addLimiter(new StreamAcquireLimiter(streamManager, serviceBps, bpsStreamAcquireLimit, limiterStatLogger.scope("bps_acquire")));
builder.addLimiter(bpsHardLimiterBuilder.build());
builder.addLimiter(bpsSoftLimiterBuilder.build());
builder.addLimiter(rpsHardLimiterBuilder.build());
builder.addLimiter(rpsSoftLimiterBuilder.build());
builder.statsLogger(limiterStatLogger);
return builder.build();
}
use of com.twitter.distributedlog.exceptions.OverCapacityException in project distributedlog by twitter.
the class StreamImpl method submit.
/**
* Execute the StreamOp. If reacquire is needed, this may initiate reacquire and queue the op for
* execution once complete.
*
* @param op
* stream operation to execute.
*/
@Override
public void submit(StreamOp op) {
// Let stream acquire thread know a write has been attempted.
writeSinceLastAcquire = true;
try {
limiter.apply(op);
} catch (OverCapacityException ex) {
op.fail(ex);
return;
}
// Timeout stream op if requested.
if (serviceTimeoutMs > 0) {
scheduleTimeout(op);
}
boolean notifyAcquireThread = false;
boolean completeOpNow = false;
boolean success = true;
if (StreamStatus.isUnavailable(status)) {
// Stream is closed, fail the op immediately
op.fail(new StreamUnavailableException("Stream " + name + " is closed."));
return;
}
if (StreamStatus.INITIALIZED == status && writer != null) {
completeOpNow = true;
success = true;
} else {
synchronized (this) {
if (StreamStatus.isUnavailable(status)) {
// complete the write op as {@link #executeOp(op, success)} will handle closed case.
completeOpNow = true;
success = true;
}
if (StreamStatus.INITIALIZED == status) {
completeOpNow = true;
success = true;
} else if (StreamStatus.BACKOFF == status && lastAcquireFailureWatch.elapsed(TimeUnit.MILLISECONDS) < nextAcquireWaitTimeMs) {
completeOpNow = true;
success = false;
} else if (failFastOnStreamNotReady) {
notifyAcquireThread = true;
completeOpNow = false;
success = false;
op.fail(new StreamNotReadyException("Stream " + name + " is not ready; status = " + status));
} else {
// closing & initializing
notifyAcquireThread = true;
pendingOps.add(op);
pendingOpsCounter.inc();
if (1 == pendingOps.size()) {
if (op instanceof HeartbeatOp) {
((HeartbeatOp) op).setWriteControlRecord(true);
}
}
}
}
}
if (notifyAcquireThread && !suspended) {
scheduleTryAcquireOnce(0L);
}
if (completeOpNow) {
executeOp(op, success);
}
}
use of com.twitter.distributedlog.exceptions.OverCapacityException in project distributedlog by twitter.
the class TestWriteLimiter method testStreamOnly.
@Test
public void testStreamOnly() throws Exception {
SimplePermitLimiter streamLimiter = createPermitLimiter(false, 1);
SimplePermitLimiter globalLimiter = createPermitLimiter(false, Integer.MAX_VALUE);
WriteLimiter limiter = new WriteLimiter("test", streamLimiter, globalLimiter);
limiter.acquire();
try {
limiter.acquire();
fail("should have thrown stream limit exception");
} catch (OverCapacityException ex) {
}
assertPermits(streamLimiter, 1, globalLimiter, 1);
}
use of com.twitter.distributedlog.exceptions.OverCapacityException in project distributedlog by twitter.
the class TestWriteLimiter method testGlobalOnly.
@Test
public void testGlobalOnly() throws Exception {
SimplePermitLimiter streamLimiter = createPermitLimiter(false, Integer.MAX_VALUE);
SimplePermitLimiter globalLimiter = createPermitLimiter(false, 1);
WriteLimiter limiter = new WriteLimiter("test", streamLimiter, globalLimiter);
limiter.acquire();
try {
limiter.acquire();
fail("should have thrown global limit exception");
} catch (OverCapacityException ex) {
}
assertPermits(streamLimiter, 1, globalLimiter, 1);
limiter.release();
assertPermits(streamLimiter, 0, globalLimiter, 0);
}
Aggregations