use of org.apache.bookkeeper.client.LedgerHandle in project distributedlog by twitter.
the class TestLedgerAllocatorPool method testRescueAllocators.
@Test(timeout = 60000)
public void testRescueAllocators() throws Exception {
String allocationPath = "/rescueAllocators";
int numAllocators = 3;
LedgerAllocatorPool pool = new LedgerAllocatorPool(allocationPath, numAllocators, dlConf, zkc, bkc, allocationExecutor);
List<ZKTransaction> pendingTxns = Lists.newArrayListWithExpectedSize(numAllocators);
List<String> allocatePaths = Lists.newArrayListWithExpectedSize(numAllocators);
for (int i = 0; i < numAllocators; i++) {
ZKTransaction txn = newTxn();
pool.allocate();
LedgerHandle lh = FutureUtils.result(pool.tryObtain(txn, NULL_LISTENER));
// get the corresponding ledger allocator
SimpleLedgerAllocator sla = pool.getLedgerAllocator(lh);
String slaPath = sla.allocatePath;
logger.info("Allocated ledger {} from path {}", lh.getId(), slaPath);
pendingTxns.add(txn);
allocatePaths.add(slaPath);
}
for (int i = 0; i < numAllocators; i++) {
ZKTransaction txn = pendingTxns.get(i);
String slaPath = allocatePaths.get(i);
// execute the transaction to confirm/abort obtain
FutureUtils.result(txn.execute());
// introduce error to individual ledger allocator
byte[] data = zkc.get().getData(slaPath, false, new Stat());
zkc.get().setData(slaPath, data, -1);
}
int numSuccess = 0;
Set<String> allocatedPathSet = new HashSet<String>();
while (numSuccess < 2 * numAllocators) {
try {
pool.allocate();
ZKTransaction txn = newTxn();
LedgerHandle lh = FutureUtils.result(pool.tryObtain(txn, NULL_LISTENER));
// get the corresponding ledger allocator
SimpleLedgerAllocator sla = pool.getLedgerAllocator(lh);
String slaPath = sla.allocatePath;
logger.info("Allocated ledger {} from path {}", lh.getId(), slaPath);
allocatedPathSet.add(slaPath);
FutureUtils.result(txn.execute());
++numSuccess;
} catch (IOException ioe) {
// continue
}
}
assertEquals(2 * numAllocators, numSuccess);
assertEquals(numAllocators, allocatedPathSet.size());
Utils.close(pool);
}
use of org.apache.bookkeeper.client.LedgerHandle in project distributedlog by twitter.
the class DLAuditor method calculateStreamSpaceUsage.
private long calculateStreamSpaceUsage(final com.twitter.distributedlog.DistributedLogManagerFactory factory, final String stream) throws IOException {
DistributedLogManager dlm = factory.createDistributedLogManager(stream, com.twitter.distributedlog.DistributedLogManagerFactory.ClientSharingOption.SharedClients);
long totalBytes = 0;
try {
List<LogSegmentMetadata> segments = dlm.getLogSegments();
for (LogSegmentMetadata segment : segments) {
try {
LedgerHandle lh = getBookKeeperClient(factory).get().openLedgerNoRecovery(segment.getLedgerId(), BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8));
totalBytes += lh.getLength();
lh.close();
} catch (BKException e) {
logger.error("Failed to open ledger {} : ", segment.getLedgerId(), e);
throw new IOException("Failed to open ledger " + segment.getLedgerId(), e);
} catch (InterruptedException e) {
logger.warn("Interrupted on opening ledger {} : ", segment.getLedgerId(), e);
Thread.currentThread().interrupt();
throw new DLInterruptedException("Interrupted on opening ledger " + segment.getLedgerId(), e);
}
}
} finally {
dlm.close();
}
return totalBytes;
}
use of org.apache.bookkeeper.client.LedgerHandle in project distributedlog by twitter.
the class LedgerAllocatorPool method tryObtain.
@Override
public Future<LedgerHandle> tryObtain(final Transaction<Object> txn, final Transaction.OpListener<LedgerHandle> listener) {
final SimpleLedgerAllocator allocator;
synchronized (this) {
if (allocatingList.isEmpty()) {
return Future.exception(new IOException("No ledger allocator available under " + poolPath + "."));
} else {
allocator = allocatingList.removeFirst();
}
}
final Promise<LedgerHandle> tryObtainPromise = new Promise<LedgerHandle>();
final FutureEventListener<LedgerHandle> tryObtainListener = new FutureEventListener<LedgerHandle>() {
@Override
public void onSuccess(LedgerHandle lh) {
synchronized (LedgerAllocatorPool.this) {
obtainMap.put(lh, allocator);
reverseObtainMap.put(allocator, lh);
tryObtainPromise.setValue(lh);
}
}
@Override
public void onFailure(Throwable cause) {
try {
rescueAllocator(allocator);
} catch (IOException ioe) {
logger.info("Failed to rescue allocator {}", allocator.allocatePath, ioe);
}
tryObtainPromise.setException(cause);
}
};
allocator.tryObtain(txn, new Transaction.OpListener<LedgerHandle>() {
@Override
public void onCommit(LedgerHandle lh) {
confirmObtain(allocator);
listener.onCommit(lh);
}
@Override
public void onAbort(Throwable t) {
abortObtain(allocator);
listener.onAbort(t);
}
}).addEventListener(tryObtainListener);
return tryObtainPromise;
}
use of org.apache.bookkeeper.client.LedgerHandle in project distributedlog by twitter.
the class TestRollLogSegments method testCaughtUpReaderOnLogSegmentRolling.
@FlakyTest
@Test(timeout = 60000)
public void testCaughtUpReaderOnLogSegmentRolling() throws Exception {
String name = "distrlog-caughtup-reader-on-logsegment-rolling";
DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
confLocal.loadConf(conf);
confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
confLocal.setImmediateFlushEnabled(false);
confLocal.setOutputBufferSize(4 * 1024 * 1024);
confLocal.setTraceReadAheadMetadataChanges(true);
confLocal.setEnsembleSize(1);
confLocal.setWriteQuorumSize(1);
confLocal.setAckQuorumSize(1);
confLocal.setReadLACLongPollTimeout(99999999);
DistributedLogManager dlm = createNewDLM(confLocal, name);
BKSyncLogWriter writer = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
// 1) writer added 5 entries.
final int numEntries = 5;
for (int i = 1; i <= numEntries; i++) {
writer.write(DLMTestUtil.getLogRecordInstance(i));
writer.setReadyToFlush();
writer.flushAndSync();
}
BKDistributedLogManager readDLM = (BKDistributedLogManager) createNewDLM(confLocal, name);
final BKAsyncLogReaderDLSN reader = (BKAsyncLogReaderDLSN) readDLM.getAsyncLogReader(DLSN.InitialDLSN);
// 2) reader should be able to read 5 entries.
for (long i = 1; i <= numEntries; i++) {
LogRecordWithDLSN record = Await.result(reader.readNext());
DLMTestUtil.verifyLogRecord(record);
assertEquals(i, record.getTransactionId());
assertEquals(record.getTransactionId() - 1, record.getSequenceId());
}
BKLogSegmentWriter perStreamWriter = writer.segmentWriter;
BookKeeperClient bkc = readDLM.getReaderBKC();
LedgerHandle readLh = bkc.get().openLedgerNoRecovery(getLedgerHandle(perStreamWriter).getId(), BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8));
// Writer moved to lac = 9, while reader knows lac = 8 and moving to wait on 9
checkAndWaitWriterReaderPosition(perStreamWriter, 9, reader, 9, readLh, 8);
// write 6th record
writer.write(DLMTestUtil.getLogRecordInstance(numEntries + 1));
writer.setReadyToFlush();
// Writer moved to lac = 10, while reader knows lac = 9 and moving to wait on 10
checkAndWaitWriterReaderPosition(perStreamWriter, 10, reader, 10, readLh, 9);
// write records without commit to simulate similar failure cases
writer.write(DLMTestUtil.getLogRecordInstance(numEntries + 2));
writer.setReadyToFlush();
// Writer moved to lac = 11, while reader knows lac = 10 and moving to wait on 11
checkAndWaitWriterReaderPosition(perStreamWriter, 11, reader, 11, readLh, 10);
while (null == reader.bkLedgerManager.readAheadWorker.getMetadataNotification()) {
Thread.sleep(1000);
}
logger.info("Waiting for long poll getting interrupted with metadata changed");
// simulate a recovery without closing ledger causing recording wrong last dlsn
BKLogWriteHandler writeHandler = writer.getCachedWriteHandler();
writeHandler.completeAndCloseLogSegment(writeHandler.inprogressZNodeName(perStreamWriter.getLogSegmentId(), perStreamWriter.getStartTxId(), perStreamWriter.getLogSegmentSequenceNumber()), perStreamWriter.getLogSegmentSequenceNumber(), perStreamWriter.getLogSegmentId(), perStreamWriter.getStartTxId(), perStreamWriter.getLastTxId(), perStreamWriter.getPositionWithinLogSegment() - 1, 9, 0);
BKSyncLogWriter anotherWriter = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
anotherWriter.write(DLMTestUtil.getLogRecordInstance(numEntries + 3));
anotherWriter.setReadyToFlush();
anotherWriter.flushAndSync();
anotherWriter.closeAndComplete();
for (long i = numEntries + 1; i <= numEntries + 3; i++) {
LogRecordWithDLSN record = Await.result(reader.readNext());
DLMTestUtil.verifyLogRecord(record);
assertEquals(i, record.getTransactionId());
}
Utils.close(reader);
readDLM.close();
}
use of org.apache.bookkeeper.client.LedgerHandle in project distributedlog by twitter.
the class TestLedgerAllocator method testCloseAllocatorAfterConfirm.
/**
* {@link https://issues.apache.org/jira/browse/DL-26}
*/
@DistributedLogAnnotations.FlakyTest
@Ignore
@Test(timeout = 60000)
public void testCloseAllocatorAfterConfirm() throws Exception {
String allocationPath = "/allocation2";
SimpleLedgerAllocator allocator = createAllocator(allocationPath);
allocator.allocate();
ZKTransaction txn = newTxn();
// close during obtaining ledger.
LedgerHandle lh = FutureUtils.result(allocator.tryObtain(txn, NULL_LISTENER));
FutureUtils.result(txn.execute());
Utils.close(allocator);
byte[] data = zkc.get().getData(allocationPath, false, null);
assertEquals(0, data.length);
// the ledger is not deleted.
bkc.get().openLedger(lh.getId(), BookKeeper.DigestType.CRC32, dlConf.getBKDigestPW().getBytes(UTF_8));
}
Aggregations