use of org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo in project pulsar by yahoo.
the class ManagedLedgerImpl method initialize.
synchronized void initialize(final ManagedLedgerInitializeLedgerCallback callback, final Object ctx) {
log.info("Opening managed ledger {}", name);
// Fetch the list of existing ledgers in the managed ledger
store.getManagedLedgerInfo(name, new MetaStoreCallback<ManagedLedgerInfo>() {
@Override
public void operationComplete(ManagedLedgerInfo mlInfo, Stat stat) {
ledgersStat = stat;
for (LedgerInfo ls : mlInfo.getLedgerInfoList()) {
ledgers.put(ls.getLedgerId(), ls);
}
// Last ledger stat may be zeroed, we must update it
if (ledgers.size() > 0) {
final long id = ledgers.lastKey();
OpenCallback opencb = (rc, lh, ctx1) -> {
executor.submitOrdered(name, safeRun(() -> {
mbean.endDataLedgerOpenOp();
if (log.isDebugEnabled()) {
log.debug("[{}] Opened ledger {}: ", name, id, BKException.getMessage(rc));
}
if (rc == BKException.Code.OK) {
LedgerInfo info = LedgerInfo.newBuilder().setLedgerId(id).setEntries(lh.getLastAddConfirmed() + 1).setSize(lh.getLength()).setTimestamp(System.currentTimeMillis()).build();
ledgers.put(id, info);
initializeBookKeeper(callback);
} else if (rc == BKException.Code.NoSuchLedgerExistsException) {
log.warn("[{}] Ledger not found: {}", name, ledgers.lastKey());
ledgers.remove(ledgers.lastKey());
initializeBookKeeper(callback);
} else {
log.error("[{}] Failed to open ledger {}: {}", name, id, BKException.getMessage(rc));
callback.initializeFailed(new ManagedLedgerException(BKException.getMessage(rc)));
return;
}
}));
};
if (log.isDebugEnabled()) {
log.debug("[{}] Opening legder {}", name, id);
}
mbean.startDataLedgerOpenOp();
bookKeeper.asyncOpenLedger(id, config.getDigestType(), config.getPassword(), opencb, null);
} else {
initializeBookKeeper(callback);
}
}
@Override
public void operationFailed(MetaStoreException e) {
callback.initializeFailed(new ManagedLedgerException(e));
}
});
}
use of org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo in project pulsar by yahoo.
the class ManagedLedgerImpl method internalTrimConsumedLedgers.
/**
* Checks whether there are ledger that have been fully consumed and deletes them
*
* @throws Exception
*/
void internalTrimConsumedLedgers() {
// Ensure only one trimming operation is active
if (!trimmerMutex.tryLock()) {
scheduleDeferredTrimming();
return;
}
List<LedgerInfo> ledgersToDelete = Lists.newArrayList();
synchronized (this) {
if (log.isDebugEnabled()) {
log.debug("[{}] Start TrimConsumedLedgers. ledgers={} totalSize={}", name, ledgers.keySet(), TOTAL_SIZE_UPDATER.get(this));
}
if (STATE_UPDATER.get(this) == State.Closed) {
log.debug("[{}] Ignoring trimming request since the managed ledger was already closed", name);
trimmerMutex.unlock();
return;
}
long slowestReaderLedgerId = -1;
if (cursors.isEmpty()) {
// At this point the lastLedger will be pointing to the
// ledger that has just been closed, therefore the +1 to
// include lastLedger in the trimming.
slowestReaderLedgerId = currentLedger.getId() + 1;
} else {
PositionImpl slowestReaderPosition = cursors.getSlowestReaderPosition();
if (slowestReaderPosition != null) {
slowestReaderLedgerId = slowestReaderPosition.getLedgerId();
} else {
trimmerMutex.unlock();
return;
}
}
if (log.isDebugEnabled()) {
log.debug("[{}] Slowest consumer ledger id: {}", name, slowestReaderLedgerId);
}
// skip ledger if retention constraint met
for (LedgerInfo ls : ledgers.headMap(slowestReaderLedgerId, false).values()) {
boolean expired = hasLedgerRetentionExpired(ls.getTimestamp());
boolean overRetentionQuota = TOTAL_SIZE_UPDATER.get(this) > ((long) config.getRetentionSizeInMB()) * 1024 * 1024;
if (ls.getLedgerId() == currentLedger.getId() || (!expired && !overRetentionQuota)) {
if (log.isDebugEnabled()) {
if (!expired) {
log.debug("[{}] ledger id skipped for deletion as unexpired: {}", name, ls.getLedgerId());
}
if (!overRetentionQuota) {
log.debug("[{}] ledger id: {} skipped for deletion as size: {} under quota: {} MB", name, ls.getLedgerId(), TOTAL_SIZE_UPDATER.get(this), config.getRetentionSizeInMB());
}
}
break;
}
ledgersToDelete.add(ls);
ledgerCache.remove(ls.getLedgerId());
}
if (ledgersToDelete.isEmpty()) {
trimmerMutex.unlock();
return;
}
if (// Give up now and schedule a new trimming
STATE_UPDATER.get(this) == State.CreatingLedger || !ledgersListMutex.tryLock()) {
// Avoid deadlocks with other operations updating the ledgers list
scheduleDeferredTrimming();
trimmerMutex.unlock();
return;
}
// Update metadata
for (LedgerInfo ls : ledgersToDelete) {
ledgers.remove(ls.getLedgerId());
NUMBER_OF_ENTRIES_UPDATER.addAndGet(this, -ls.getEntries());
TOTAL_SIZE_UPDATER.addAndGet(this, -ls.getSize());
entryCache.invalidateAllEntries(ls.getLedgerId());
}
if (log.isDebugEnabled()) {
log.debug("[{}] Updating of ledgers list after trimming", name);
}
ManagedLedgerInfo mlInfo = ManagedLedgerInfo.newBuilder().addAllLedgerInfo(ledgers.values()).build();
store.asyncUpdateLedgerIds(name, mlInfo, ledgersStat, new MetaStoreCallback<Void>() {
@Override
public void operationComplete(Void result, Stat stat) {
log.info("[{}] End TrimConsumedLedgers. ledgers={} totalSize={}", name, ledgers.size(), TOTAL_SIZE_UPDATER.get(ManagedLedgerImpl.this));
ledgersStat = stat;
ledgersListMutex.unlock();
trimmerMutex.unlock();
for (LedgerInfo ls : ledgersToDelete) {
log.info("[{}] Removing ledger {} - size: {}", name, ls.getLedgerId(), ls.getSize());
bookKeeper.asyncDeleteLedger(ls.getLedgerId(), (rc, ctx) -> {
if (rc == BKException.Code.NoSuchLedgerExistsException) {
log.warn("[{}] Ledger was already deleted {}", name, ls.getLedgerId());
} else if (rc != BKException.Code.OK) {
log.error("[{}] Error deleting ledger {}", name, ls.getLedgerId(), BKException.getMessage(rc));
} else {
if (log.isDebugEnabled()) {
log.debug("[{}] Deleted ledger {}", name, ls.getLedgerId());
}
}
}, null);
}
}
@Override
public void operationFailed(MetaStoreException e) {
log.warn("[{}] Failed to update the list of ledgers after trimming", name, e);
ledgersListMutex.unlock();
trimmerMutex.unlock();
}
});
}
}
use of org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo in project pulsar by yahoo.
the class ManagedLedgerImpl method deleteAllLedgers.
private void deleteAllLedgers(DeleteLedgerCallback callback, Object ctx) {
List<LedgerInfo> ledgers = Lists.newArrayList(ManagedLedgerImpl.this.ledgers.values());
AtomicInteger ledgersToDelete = new AtomicInteger(ledgers.size());
if (ledgers.isEmpty()) {
// No ledgers to delete, proceed with deleting metadata
deleteMetadata(callback, ctx);
return;
}
for (LedgerInfo ls : ledgers) {
if (log.isDebugEnabled()) {
log.debug("[{}] Deleting ledger {}", name, ls);
}
bookKeeper.asyncDeleteLedger(ls.getLedgerId(), (rc, ctx1) -> {
switch(rc) {
case BKException.Code.NoSuchLedgerExistsException:
log.warn("[{}] Ledger {} not found when deleting it", name, ls.getLedgerId());
case BKException.Code.OK:
if (ledgersToDelete.decrementAndGet() == 0) {
// All ledgers deleted, now remove ML metadata
deleteMetadata(callback, ctx);
}
break;
default:
// Handle error
log.warn("[{}] Failed to delete ledger {} -- {}", name, ls.getLedgerId(), BKException.getMessage(rc));
int toDelete = ledgersToDelete.get();
if (toDelete != -1 && ledgersToDelete.compareAndSet(toDelete, -1)) {
// Trigger callback only once
callback.deleteLedgerFailed(new ManagedLedgerException(BKException.getMessage(rc)), ctx);
}
}
}, null);
}
}
use of org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo in project pulsar by yahoo.
the class ManagedLedgerImpl method initializeBookKeeper.
private synchronized void initializeBookKeeper(final ManagedLedgerInitializeLedgerCallback callback) {
if (log.isDebugEnabled()) {
log.debug("[{}] initializing bookkeeper; ledgers {}", name, ledgers);
}
// Calculate total entries and size
Iterator<LedgerInfo> iterator = ledgers.values().iterator();
while (iterator.hasNext()) {
LedgerInfo li = iterator.next();
if (li.getEntries() > 0) {
NUMBER_OF_ENTRIES_UPDATER.addAndGet(this, li.getEntries());
TOTAL_SIZE_UPDATER.addAndGet(this, li.getSize());
} else {
iterator.remove();
bookKeeper.asyncDeleteLedger(li.getLedgerId(), (rc, ctx) -> {
if (log.isDebugEnabled()) {
log.debug("[{}] Deleted empty ledger ledgerId={} rc={}", name, li.getLedgerId(), rc);
}
}, null);
}
}
final MetaStoreCallback<Void> storeLedgersCb = new MetaStoreCallback<Void>() {
@Override
public void operationComplete(Void v, Stat stat) {
ledgersStat = stat;
initializeCursors(callback);
}
@Override
public void operationFailed(MetaStoreException e) {
callback.initializeFailed(new ManagedLedgerException(e));
}
};
// Create a new ledger to start writing
this.lastLedgerCreationInitiationTimestamp = System.nanoTime();
mbean.startDataLedgerCreateOp();
bookKeeper.asyncCreateLedger(config.getEnsembleSize(), config.getWriteQuorumSize(), config.getAckQuorumSize(), config.getDigestType(), config.getPassword(), (rc, lh, ctx) -> {
executor.submitOrdered(name, safeRun(() -> {
mbean.endDataLedgerCreateOp();
if (rc != BKException.Code.OK) {
callback.initializeFailed(new ManagedLedgerException(BKException.getMessage(rc)));
return;
}
log.info("[{}] Created ledger {}", name, lh.getId());
STATE_UPDATER.set(this, State.LedgerOpened);
lastLedgerCreatedTimestamp = System.currentTimeMillis();
currentLedger = lh;
lastConfirmedEntry = new PositionImpl(lh.getId(), -1);
LedgerInfo info = LedgerInfo.newBuilder().setLedgerId(lh.getId()).setTimestamp(0).build();
ledgers.put(lh.getId(), info);
// Save it back to ensure all nodes exist
ManagedLedgerInfo mlInfo = ManagedLedgerInfo.newBuilder().addAllLedgerInfo(ledgers.values()).build();
store.asyncUpdateLedgerIds(name, mlInfo, ledgersStat, storeLedgersCb);
}));
}, null);
}
use of org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo in project pulsar by yahoo.
the class ManagedLedgerTest method testTimestampOnWorkingLedger.
@Test
public void testTimestampOnWorkingLedger() throws Exception {
ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
ManagedLedgerConfig conf = new ManagedLedgerConfig();
conf.setMaxEntriesPerLedger(1);
conf.setRetentionSizeInMB(10);
conf.setRetentionTime(1, TimeUnit.HOURS);
ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("my_test_ledger", conf);
ml.openCursor("c1");
ml.addEntry("msg1".getBytes());
Iterator<LedgerInfo> iter = ml.getLedgersInfoAsList().iterator();
long ts = -1;
while (iter.hasNext()) {
LedgerInfo i = iter.next();
if (iter.hasNext()) {
assertTrue(ts <= i.getTimestamp(), i.toString());
ts = i.getTimestamp();
} else {
// the last timestamp can be
// 0 if it is still opened
// >0 if it is closed after the addEntry see OpAddEntry#addComplete()
assertTrue(i.getTimestamp() == 0 || ts <= i.getTimestamp(), i.toString());
}
}
ml.addEntry("msg02".getBytes());
ml.close();
// Thread.sleep(1000);
iter = ml.getLedgersInfoAsList().iterator();
ts = -1;
while (iter.hasNext()) {
LedgerInfo i = iter.next();
if (iter.hasNext()) {
assertTrue(ts <= i.getTimestamp(), i.toString());
ts = i.getTimestamp();
} else {
assertTrue(i.getTimestamp() > 0, "well closed LedgerInfo should set a timestamp > 0");
}
}
}
Aggregations