Examples with GenericCallback org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.GenericCallback used on opensource projects

Example 6 with GenericCallback

use of org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.GenericCallback in project bookkeeper by apache.

the class GcLedgersTest method testGcLedgersIfReadLedgerMetadataFailsForDeletedLedgers.

/*
     * In this test scenario all the created ledgers are deleted, but LedgerManager.readLedgerMetadata fails with
     * ZKException. So even in this case, GarbageCollector.gc shouldn't delete ledgers data.
     *
     * ScanAndCompareGarbageCollector/GC should clean data of ledger only if both the LedgerManager.getLedgerRanges says
     * that ledger is not existing and also ledgerManager.readLedgerMetadata fails with error
     * NoSuchLedgerExistsException, but is shouldn't delete if the readLedgerMetadata fails with any other error.
     */
@Test
public void testGcLedgersIfReadLedgerMetadataFailsForDeletedLedgers() throws Exception {
    baseConf.setVerifyMetadataOnGc(true);
    final SortedSet<Long> createdLedgers = Collections.synchronizedSortedSet(new TreeSet<Long>());
    final SortedSet<Long> cleaned = Collections.synchronizedSortedSet(new TreeSet<Long>());
    // Create few ledgers
    final int numLedgers = 5;
    createLedgers(numLedgers, createdLedgers);
    LedgerManager mockLedgerManager = new CleanupLedgerManager(getLedgerManager()) {

        @Override
        public void readLedgerMetadata(long ledgerId, GenericCallback<LedgerMetadata> readCb) {
            readCb.operationComplete(BKException.Code.ZKException, null);
        }
    };
    final GarbageCollector garbageCollector = new ScanAndCompareGarbageCollector(mockLedgerManager, new MockLedgerStorage(), baseConf, NullStatsLogger.INSTANCE);
    GarbageCollector.GarbageCleaner cleaner = new GarbageCollector.GarbageCleaner() {

        @Override
        public void clean(long ledgerId) {
            LOG.info("Cleaned {}", ledgerId);
            cleaned.add(ledgerId);
        }
    };
    validateLedgerRangeIterator(createdLedgers);
    for (long ledgerId : createdLedgers) {
        removeLedger(ledgerId);
    }
    garbageCollector.gc(cleaner);
    assertTrue("Should have cleaned nothing", cleaned.isEmpty());
}

Example 7 with GenericCallback

use of org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.GenericCallback in project bookkeeper by apache.

the class TestWatchEnsembleChange method testWatchMetadataRemoval.

private void testWatchMetadataRemoval(LedgerManagerFactory factory) throws Exception {
    @Cleanup final LedgerManager manager = factory.newLedgerManager();
    @Cleanup LedgerIdGenerator idGenerator = factory.newLedgerIdGenerator();
    final ByteBuffer bbLedgerId = ByteBuffer.allocate(8);
    final CountDownLatch createLatch = new CountDownLatch(1);
    final CountDownLatch removeLatch = new CountDownLatch(1);
    idGenerator.generateLedgerId(new GenericCallback<Long>() {

        @Override
        public void operationComplete(int rc, final Long lid) {
            manager.createLedgerMetadata(lid, new LedgerMetadata(4, 2, 2, digestType, "fpj was here".getBytes()), new BookkeeperInternalCallbacks.GenericCallback<Void>() {

                @Override
                public void operationComplete(int rc, Void result) {
                    bbLedgerId.putLong(lid);
                    bbLedgerId.flip();
                    createLatch.countDown();
                }
            });
        }
    });
    assertTrue(createLatch.await(2000, TimeUnit.MILLISECONDS));
    final long createdLid = bbLedgerId.getLong();
    manager.registerLedgerMetadataListener(createdLid, new LedgerMetadataListener() {

        @Override
        public void onChanged(long ledgerId, LedgerMetadata metadata) {
            assertEquals(ledgerId, createdLid);
            assertEquals(metadata, null);
            removeLatch.countDown();
        }
    });
    manager.removeLedgerMetadata(createdLid, Version.ANY, new BookkeeperInternalCallbacks.GenericCallback<Void>() {

        @Override
        public void operationComplete(int rc, Void result) {
            assertEquals(rc, BKException.Code.OK);
        }
    });
    assertTrue(removeLatch.await(2000, TimeUnit.MILLISECONDS));
}

Example 8 with GenericCallback

use of org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.GenericCallback in project bookkeeper by apache.

the class CleanupLedgerManager method close.

@Override
@SuppressWarnings("unchecked")
public void close() throws IOException {
    Set<GenericCallback> keys;
    closeLock.writeLock().lock();
    try {
        if (closed) {
            return;
        }
        closed = true;
        keys = new HashSet<GenericCallback>(callbacks.keySet());
    } finally {
        closeLock.writeLock().unlock();
    }
    for (GenericCallback key : keys) {
        GenericCallback callback = callbacks.remove(key);
        if (null != callback) {
            callback.operationComplete(BKException.Code.ClientClosedException, null);
        }
    }
    underlying.close();
}

Example 9 with GenericCallback

use of org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.GenericCallback in project distributedlog by twitter.

the class BKLogHandler method asyncGetLedgerListInternal.

private void asyncGetLedgerListInternal(final Comparator<LogSegmentMetadata> comparator, final LogSegmentFilter segmentFilter, final Watcher watcher, final GenericCallback<List<LogSegmentMetadata>> finalCallback, final AtomicInteger numAttemptsLeft, final AtomicLong backoffMillis) {
    final Stopwatch stopwatch = Stopwatch.createStarted();
    try {
        if (LOG.isTraceEnabled()) {
            LOG.trace("Async getting ledger list for {}.", getFullyQualifiedName());
        }
        final GenericCallback<List<LogSegmentMetadata>> callback = new GenericCallback<List<LogSegmentMetadata>>() {

            @Override
            public void operationComplete(int rc, List<LogSegmentMetadata> result) {
                long elapsedMicros = stopwatch.stop().elapsed(TimeUnit.MICROSECONDS);
                if (KeeperException.Code.OK.intValue() != rc) {
                    getListStat.registerFailedEvent(elapsedMicros);
                } else {
                    if (LogSegmentFilter.DEFAULT_FILTER == segmentFilter) {
                        isFullListFetched.set(true);
                    }
                    getListStat.registerSuccessfulEvent(elapsedMicros);
                }
                finalCallback.operationComplete(rc, result);
            }
        };
        zooKeeperClient.get().getChildren(logMetadata.getLogSegmentsPath(), watcher, new AsyncCallback.Children2Callback() {

            @Override
            public void processResult(final int rc, final String path, final Object ctx, final List<String> children, final Stat stat) {
                if (KeeperException.Code.OK.intValue() != rc) {
                    if ((KeeperException.Code.CONNECTIONLOSS.intValue() == rc || KeeperException.Code.SESSIONEXPIRED.intValue() == rc || KeeperException.Code.SESSIONMOVED.intValue() == rc) && numAttemptsLeft.decrementAndGet() > 0) {
                        long backoffMs = backoffMillis.get();
                        backoffMillis.set(Math.min(conf.getZKRetryBackoffMaxMillis(), 2 * backoffMs));
                        scheduler.schedule(new Runnable() {

                            @Override
                            public void run() {
                                asyncGetLedgerListInternal(comparator, segmentFilter, watcher, finalCallback, numAttemptsLeft, backoffMillis);
                            }
                        }, backoffMs, TimeUnit.MILLISECONDS);
                        return;
                    }
                    callback.operationComplete(rc, null);
                    return;
                }
                if (LOG.isTraceEnabled()) {
                    LOG.trace("Got ledger list from {} : {}", logMetadata.getLogSegmentsPath(), children);
                }
                ledgerListWatchSet.set(true);
                Set<String> segmentsReceived = new HashSet<String>();
                segmentsReceived.addAll(segmentFilter.filter(children));
                Set<String> segmentsAdded;
                final Set<String> removedSegments = Collections.synchronizedSet(new HashSet<String>());
                final Map<String, LogSegmentMetadata> addedSegments = Collections.synchronizedMap(new HashMap<String, LogSegmentMetadata>());
                Pair<Set<String>, Set<String>> segmentChanges = logSegmentCache.diff(segmentsReceived);
                segmentsAdded = segmentChanges.getLeft();
                removedSegments.addAll(segmentChanges.getRight());
                if (segmentsAdded.isEmpty()) {
                    if (LOG.isTraceEnabled()) {
                        LOG.trace("No segments added for {}.", getFullyQualifiedName());
                    }
                    // update the cache before fetch
                    logSegmentCache.update(removedSegments, addedSegments);
                    List<LogSegmentMetadata> segmentList;
                    try {
                        segmentList = getCachedLogSegments(comparator);
                    } catch (UnexpectedException e) {
                        callback.operationComplete(KeeperException.Code.DATAINCONSISTENCY.intValue(), null);
                        return;
                    }
                    callback.operationComplete(KeeperException.Code.OK.intValue(), segmentList);
                    notifyUpdatedLogSegments(segmentList);
                    if (!removedSegments.isEmpty()) {
                        notifyOnOperationComplete();
                    }
                    return;
                }
                final AtomicInteger numChildren = new AtomicInteger(segmentsAdded.size());
                final AtomicInteger numFailures = new AtomicInteger(0);
                for (final String segment : segmentsAdded) {
                    metadataStore.getLogSegment(logMetadata.getLogSegmentPath(segment)).addEventListener(new FutureEventListener<LogSegmentMetadata>() {

                        @Override
                        public void onSuccess(LogSegmentMetadata result) {
                            addedSegments.put(segment, result);
                            complete();
                        }

                        @Override
                        public void onFailure(Throwable cause) {
                            // 2. In progress segment has been completed => inprogress ZNode does not exist
                            if (cause instanceof KeeperException && KeeperException.Code.NONODE == ((KeeperException) cause).code()) {
                                removedSegments.add(segment);
                                complete();
                            } else {
                                // fail fast
                                if (1 == numFailures.incrementAndGet()) {
                                    int rcToReturn = KeeperException.Code.SYSTEMERROR.intValue();
                                    if (cause instanceof KeeperException) {
                                        rcToReturn = ((KeeperException) cause).code().intValue();
                                    } else if (cause instanceof ZKException) {
                                        rcToReturn = ((ZKException) cause).getKeeperExceptionCode().intValue();
                                    }
                                    // :( properly we need dlog related response code.
                                    callback.operationComplete(rcToReturn, null);
                                    return;
                                }
                            }
                        }

                        private void complete() {
                            if (0 == numChildren.decrementAndGet() && numFailures.get() == 0) {
                                // update the cache only when fetch completed
                                logSegmentCache.update(removedSegments, addedSegments);
                                List<LogSegmentMetadata> segmentList;
                                try {
                                    segmentList = getCachedLogSegments(comparator);
                                } catch (UnexpectedException e) {
                                    callback.operationComplete(KeeperException.Code.DATAINCONSISTENCY.intValue(), null);
                                    return;
                                }
                                callback.operationComplete(KeeperException.Code.OK.intValue(), segmentList);
                                notifyUpdatedLogSegments(segmentList);
                                notifyOnOperationComplete();
                            }
                        }
                    });
                }
            }
        }, null);
    } catch (ZooKeeperClient.ZooKeeperConnectionException e) {
        getListStat.registerFailedEvent(stopwatch.stop().elapsed(TimeUnit.MICROSECONDS));
        finalCallback.operationComplete(KeeperException.Code.CONNECTIONLOSS.intValue(), null);
    } catch (InterruptedException e) {
        getListStat.registerFailedEvent(stopwatch.stop().elapsed(TimeUnit.MICROSECONDS));
        finalCallback.operationComplete(KeeperException.Code.CONNECTIONLOSS.intValue(), null);
    }
}

Example 10 with GenericCallback

use of org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.GenericCallback in project bookkeeper by apache.

the class PerChannelBookieClient method initTLSHandshake.

void initTLSHandshake() {
    // create TLS handler
    PerChannelBookieClient parentObj = PerChannelBookieClient.this;
    SslHandler handler = parentObj.shFactory.newTLSHandler();
    channel.pipeline().addFirst(parentObj.shFactory.getHandlerName(), handler);
    handler.handshakeFuture().addListener(new GenericFutureListener<Future<Channel>>() {

        @Override
        public void operationComplete(Future<Channel> future) throws Exception {
            int rc;
            Queue<GenericCallback<PerChannelBookieClient>> oldPendingOps;
            synchronized (PerChannelBookieClient.this) {
                if (future.isSuccess() && state == ConnectionState.CONNECTING) {
                    LOG.error("Connection state changed before TLS handshake completed {}/{}", addr, state);
                    rc = BKException.Code.BookieHandleNotAvailableException;
                    closeChannel(channel);
                    channel = null;
                    if (state != ConnectionState.CLOSED) {
                        state = ConnectionState.DISCONNECTED;
                    }
                } else if (future.isSuccess() && state == ConnectionState.START_TLS) {
                    rc = BKException.Code.OK;
                    LOG.info("Successfully connected to bookie using TLS: " + addr);
                    state = ConnectionState.CONNECTED;
                    AuthHandler.ClientSideHandler authHandler = future.get().pipeline().get(AuthHandler.ClientSideHandler.class);
                    authHandler.authProvider.onProtocolUpgrade();
                    activeTlsChannelCounter.inc();
                } else if (future.isSuccess() && (state == ConnectionState.CLOSED || state == ConnectionState.DISCONNECTED)) {
                    LOG.warn("Closed before TLS handshake completed, clean up: {}, current state {}", channel, state);
                    closeChannel(channel);
                    rc = BKException.Code.BookieHandleNotAvailableException;
                    channel = null;
                } else if (future.isSuccess() && state == ConnectionState.CONNECTED) {
                    LOG.debug("Already connected with another channel({}), so close the new channel({})", channel, channel);
                    closeChannel(channel);
                    // pendingOps should have been completed when other channel connected
                    return;
                } else {
                    LOG.error("TLS handshake failed with bookie: {}/{}, current state {} : ", channel, addr, state, future.cause());
                    rc = BKException.Code.SecurityException;
                    closeChannel(channel);
                    channel = null;
                    if (state != ConnectionState.CLOSED) {
                        state = ConnectionState.DISCONNECTED;
                    }
                    failedTlsHandshakeCounter.inc();
                }
                // trick to not do operations under the lock, take the list
                // of pending ops and assign it to a new variable, while
                // emptying the pending ops by just assigning it to a new
                // list
                oldPendingOps = pendingOps;
                pendingOps = new ArrayDeque<>();
            }
            for (GenericCallback<PerChannelBookieClient> pendingOp : oldPendingOps) {
                pendingOp.operationComplete(rc, PerChannelBookieClient.this);
            }
        }
    });
}