Examples with OwnershipAcquireFailedException com.twitter.distributedlog.exceptions.OwnershipAcquireFailedException used on opensource projects

Example 1 with OwnershipAcquireFailedException

use of com.twitter.distributedlog.exceptions.OwnershipAcquireFailedException in project distributedlog by twitter.

the class TestZKSessionLock method testLockWhenSiblingUseDifferentLockId.

private void testLockWhenSiblingUseDifferentLockId(long timeout, final boolean isUnlock) throws Exception {
    String lockPath = "/test-lock-when-sibling-use-different-lock-id-" + timeout + "-" + isUnlock + "-" + System.currentTimeMillis();
    String clientId0 = "client-id-0";
    String clientId1 = "client-id-1";
    createLockPath(zkc.get(), lockPath);
    final ZKSessionLock lock0_0 = new ZKSessionLock(zkc0, lockPath, clientId0, lockStateExecutor);
    final ZKSessionLock lock0_1 = new ZKSessionLock(zkc0, lockPath, clientId0, lockStateExecutor);
    final ZKSessionLock lock1 = new ZKSessionLock(zkc, lockPath, clientId1, lockStateExecutor);
    lock0_0.tryLock(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
    // lock1 wait for the lock ownership.
    final CountDownLatch lock1DoneLatch = new CountDownLatch(1);
    Thread lock1Thread = new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                lock1.tryLock(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
                lock1DoneLatch.countDown();
            } catch (LockingException e) {
                logger.error("Failed on locking lock1 : ", e);
            }
        }
    }, "lock1-thread");
    lock1Thread.start();
    // check lock1 is waiting for lock0_0
    List<String> children = awaitWaiters(2, zkc, lockPath);
    assertEquals(2, children.size());
    assertEquals(State.CLAIMED, lock0_0.getLockState());
    assertEquals(lock0_0.getLockId(), Await.result(asyncParseClientID(zkc0.get(), lockPath, children.get(0))));
    awaitState(State.WAITING, lock1);
    assertEquals(lock1.getLockId(), Await.result(asyncParseClientID(zkc.get(), lockPath, children.get(1))));
    final CountDownLatch lock0DoneLatch = new CountDownLatch(1);
    final AtomicReference<String> ownerFromLock0 = new AtomicReference<String>(null);
    Thread lock0Thread = null;
    if (timeout == 0) {
        try {
            lock0_1.tryLock(0, TimeUnit.MILLISECONDS);
            fail("Should fail on locking if sibling is using differnt lock id.");
        } catch (OwnershipAcquireFailedException oafe) {
            assertEquals(clientId0, oafe.getCurrentOwner());
        }
        assertEquals(State.CLOSED, lock0_1.getLockState());
        children = getLockWaiters(zkc, lockPath);
        assertEquals(2, children.size());
        assertEquals(State.CLAIMED, lock0_0.getLockState());
        assertEquals(lock0_0.getLockId(), Await.result(asyncParseClientID(zkc0.get(), lockPath, children.get(0))));
        assertEquals(State.WAITING, lock1.getLockState());
        assertEquals(lock1.getLockId(), Await.result(asyncParseClientID(zkc.get(), lockPath, children.get(1))));
    } else {
        lock0Thread = new Thread(new Runnable() {

            @Override
            public void run() {
                try {
                    lock0_1.tryLock(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
                    if (isUnlock) {
                        lock0DoneLatch.countDown();
                    }
                } catch (OwnershipAcquireFailedException oafe) {
                    if (!isUnlock) {
                        ownerFromLock0.set(oafe.getCurrentOwner());
                        lock0DoneLatch.countDown();
                    }
                } catch (LockingException le) {
                    logger.error("Failed on locking lock0_1 : ", le);
                }
            }
        }, "lock0-thread");
        lock0Thread.start();
        // check lock1 is waiting for lock0_0
        children = awaitWaiters(3, zkc, lockPath);
        assertEquals(3, children.size());
        assertEquals(State.CLAIMED, lock0_0.getLockState());
        assertEquals(lock0_0.getLockId(), Await.result(asyncParseClientID(zkc0.get(), lockPath, children.get(0))));
        awaitState(State.WAITING, lock1);
        assertEquals(lock1.getLockId(), Await.result(asyncParseClientID(zkc.get(), lockPath, children.get(1))));
        awaitState(State.WAITING, lock0_1);
        assertEquals(lock0_1.getLockId(), Await.result(asyncParseClientID(zkc0.get(), lockPath, children.get(2))));
    }
    if (isUnlock) {
        lock0_0.unlock();
    } else {
        ZooKeeperClientUtils.expireSession(zkc0, zkServers, sessionTimeoutMs);
    }
    lock1DoneLatch.await();
    lock1Thread.join();
    // check the state of lock0_0
    if (isUnlock) {
        assertEquals(State.CLOSED, lock0_0.getLockState());
    } else {
        assertEquals(State.EXPIRED, lock0_0.getLockState());
    }
    if (timeout == 0) {
        children = getLockWaiters(zkc, lockPath);
        assertEquals(1, children.size());
        assertEquals(State.CLAIMED, lock1.getLockState());
        assertEquals(lock1.getLockId(), Await.result(asyncParseClientID(zkc.get(), lockPath, children.get(0))));
    } else {
        assertNotNull(lock0Thread);
        if (!isUnlock) {
            // both lock0_0 and lock0_1 would be expired
            lock0DoneLatch.await();
            lock0Thread.join();
            assertEquals(clientId0, ownerFromLock0.get());
            assertEquals(State.CLOSED, lock0_1.getLockState());
            children = getLockWaiters(zkc, lockPath);
            assertEquals(1, children.size());
            assertEquals(State.CLAIMED, lock1.getLockState());
            assertEquals(lock1.getLockId(), Await.result(asyncParseClientID(zkc.get(), lockPath, children.get(0))));
        } else {
            children = getLockWaiters(zkc, lockPath);
            assertEquals(2, children.size());
            assertEquals(State.CLAIMED, lock1.getLockState());
            assertEquals(lock1.getLockId(), Await.result(asyncParseClientID(zkc.get(), lockPath, children.get(0))));
            assertEquals(State.WAITING, lock0_1.getLockState());
            assertEquals(lock0_1.getLockId(), Await.result(asyncParseClientID(zkc0.get(), lockPath, children.get(1))));
        }
    }
    lock1.unlock();
    if (timeout != 0 && isUnlock) {
        lock0DoneLatch.await();
        lock0Thread.join();
        children = getLockWaiters(zkc, lockPath);
        assertEquals(1, children.size());
        assertEquals(State.CLAIMED, lock0_1.getLockState());
        assertEquals(lock0_1.getLockId(), Await.result(asyncParseClientID(zkc0.get(), lockPath, children.get(0))));
    }
}

Example 2 with OwnershipAcquireFailedException

use of com.twitter.distributedlog.exceptions.OwnershipAcquireFailedException in project distributedlog by twitter.

the class StreamImpl method doExecuteOp.

private void doExecuteOp(final StreamOp op, boolean success) {
    final AsyncLogWriter writer;
    final Throwable lastException;
    synchronized (this) {
        writer = this.writer;
        lastException = this.lastException;
    }
    if (null != writer && success) {
        op.execute(writer, sequencer, txnLock).addEventListener(new FutureEventListener<Void>() {

            @Override
            public void onSuccess(Void value) {
            // nop
            }

            @Override
            public void onFailure(Throwable cause) {
                boolean countAsException = true;
                if (cause instanceof DLException) {
                    final DLException dle = (DLException) cause;
                    switch(dle.getCode()) {
                        case FOUND:
                            assert (cause instanceof OwnershipAcquireFailedException);
                            countAsException = false;
                            handleOwnershipAcquireFailedException(op, (OwnershipAcquireFailedException) cause);
                            break;
                        case ALREADY_CLOSED:
                            assert (cause instanceof AlreadyClosedException);
                            op.fail(cause);
                            handleAlreadyClosedException((AlreadyClosedException) cause);
                            break;
                        // exceptions that mostly from client (e.g. too large record)
                        case NOT_IMPLEMENTED:
                        case METADATA_EXCEPTION:
                        case LOG_EMPTY:
                        case LOG_NOT_FOUND:
                        case TRUNCATED_TRANSACTION:
                        case END_OF_STREAM:
                        case TRANSACTION_OUT_OF_ORDER:
                        case INVALID_STREAM_NAME:
                        case TOO_LARGE_RECORD:
                        case STREAM_NOT_READY:
                        case OVER_CAPACITY:
                            op.fail(cause);
                            break;
                        // exceptions that *could* / *might* be recovered by creating a new writer
                        default:
                            handleRecoverableDLException(op, cause);
                            break;
                    }
                } else {
                    handleUnknownException(op, cause);
                }
                if (countAsException) {
                    countException(cause, streamExceptionStatLogger);
                }
            }
        });
    } else {
        op.fail(lastException);
    }
}

Example 3 with OwnershipAcquireFailedException

use of com.twitter.distributedlog.exceptions.OwnershipAcquireFailedException in project distributedlog by twitter.

the class ResponseUtils method exceptionToHeader.

public static ResponseHeader exceptionToHeader(Throwable t) {
    ResponseHeader response = new ResponseHeader();
    if (t instanceof DLException) {
        DLException dle = (DLException) t;
        if (dle instanceof OwnershipAcquireFailedException) {
            response.setLocation(((OwnershipAcquireFailedException) dle).getCurrentOwner());
        }
        response.setCode(dle.getCode());
        response.setErrMsg(dle.getMessage());
    } else {
        response.setCode(StatusCode.INTERNAL_SERVER_ERROR);
        response.setErrMsg("Internal server error : " + t.getMessage());
    }
    return response;
}

Example 4 with OwnershipAcquireFailedException

use of com.twitter.distributedlog.exceptions.OwnershipAcquireFailedException in project distributedlog by twitter.

the class ZKSessionLock method asyncTryLock.

@Override
public Future<LockWaiter> asyncTryLock(final long timeout, final TimeUnit unit) {
    final Promise<String> result = new Promise<String>();
    final boolean wait = DistributedLogConstants.LOCK_IMMEDIATE != timeout;
    if (wait) {
        asyncTryLock(wait, result);
    } else {
        // try to check locks first
        zk.getChildren(lockPath, null, new AsyncCallback.Children2Callback() {

            @Override
            public void processResult(final int rc, String path, Object ctx, final List<String> children, Stat stat) {
                lockStateExecutor.submit(lockPath, new SafeRunnable() {

                    @Override
                    public void safeRun() {
                        if (!lockState.inState(State.INIT)) {
                            result.setException(new LockStateChangedException(lockPath, lockId, State.INIT, lockState.getState()));
                            return;
                        }
                        if (KeeperException.Code.OK.intValue() != rc) {
                            result.setException(KeeperException.create(KeeperException.Code.get(rc)));
                            return;
                        }
                        FailpointUtils.checkFailPointNoThrow(FailpointUtils.FailPointName.FP_LockTryAcquire);
                        Collections.sort(children, MEMBER_COMPARATOR);
                        if (children.size() > 0) {
                            asyncParseClientID(zk, lockPath, children.get(0)).addEventListener(new FutureEventListener<Pair<String, Long>>() {

                                @Override
                                public void onSuccess(Pair<String, Long> owner) {
                                    if (!checkOrClaimLockOwner(owner, result)) {
                                        acquireFuture.updateIfEmpty(new Return<Boolean>(false));
                                    }
                                }

                                @Override
                                public void onFailure(final Throwable cause) {
                                    lockStateExecutor.submit(lockPath, new SafeRunnable() {

                                        @Override
                                        public void safeRun() {
                                            result.setException(cause);
                                        }
                                    });
                                }
                            });
                        } else {
                            asyncTryLock(wait, result);
                        }
                    }
                });
            }
        }, null);
    }
    final Promise<Boolean> waiterAcquireFuture = new Promise<Boolean>(new com.twitter.util.Function<Throwable, BoxedUnit>() {

        @Override
        public BoxedUnit apply(Throwable t) {
            acquireFuture.raise(t);
            return BoxedUnit.UNIT;
        }
    });
    return result.map(new AbstractFunction1<String, LockWaiter>() {

        @Override
        public LockWaiter apply(final String currentOwner) {
            final Exception acquireException = new OwnershipAcquireFailedException(lockPath, currentOwner);
            FutureUtils.within(acquireFuture, timeout, unit, acquireException, lockStateExecutor, lockPath).addEventListener(new FutureEventListener<Boolean>() {

                @Override
                public void onSuccess(Boolean acquired) {
                    completeOrFail(acquireException);
                }

                @Override
                public void onFailure(final Throwable acquireCause) {
                    completeOrFail(acquireException);
                }

                private void completeOrFail(final Throwable acquireCause) {
                    if (isLockHeld()) {
                        waiterAcquireFuture.setValue(true);
                    } else {
                        asyncUnlock().addEventListener(new FutureEventListener<BoxedUnit>() {

                            @Override
                            public void onSuccess(BoxedUnit value) {
                                waiterAcquireFuture.setException(acquireCause);
                            }

                            @Override
                            public void onFailure(Throwable cause) {
                                waiterAcquireFuture.setException(acquireCause);
                            }
                        });
                    }
                }
            });
            ;
            return new LockWaiter(lockId.getLeft(), currentOwner, waiterAcquireFuture);
        }
    });
}

Example 5 with OwnershipAcquireFailedException

use of com.twitter.distributedlog.exceptions.OwnershipAcquireFailedException in project distributedlog by twitter.

the class TestDistributedLock method testLockReacquireFailure.

private void testLockReacquireFailure(boolean checkOwnershipAndReacquire) throws Exception {
    String lockPath = "/test-lock-re-acquire-failure-" + checkOwnershipAndReacquire + "-" + System.currentTimeMillis();
    String clientId = "test-lock-re-acquire";
    createLockPath(zkc.get(), lockPath);
    SessionLockFactory lockFactory0 = createLockFactory(clientId, zkc0);
    ZKDistributedLock lock0 = new ZKDistributedLock(lockStateExecutor, lockFactory0, lockPath, Long.MAX_VALUE, NullStatsLogger.INSTANCE);
    FutureUtils.result(lock0.asyncAcquire());
    final CountDownLatch lock1DoneLatch = new CountDownLatch(1);
    SessionLockFactory lockFactory1 = createLockFactory(clientId, zkc);
    final ZKDistributedLock lock1 = new ZKDistributedLock(lockStateExecutor, lockFactory1, lockPath, Long.MAX_VALUE, NullStatsLogger.INSTANCE);
    Thread lock1Thread = new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                FutureUtils.result(lock1.asyncAcquire());
                lock1DoneLatch.countDown();
            } catch (IOException e) {
                logger.error("Error on acquiring lock1 : ", e);
            }
        }
    }, "lock1-thread");
    lock1Thread.start();
    List<String> children;
    do {
        Thread.sleep(1);
        children = getLockWaiters(zkc, lockPath);
    } while (children.size() < 2);
    assertEquals(2, children.size());
    assertTrue(lock0.haveLock());
    assertFalse(lock1.haveLock());
    assertEquals(((ZKSessionLock) lock0.getInternalLock()).getLockId(), Await.result(asyncParseClientID(zkc0.get(), lockPath, children.get(0))));
    assertEquals(((ZKSessionLock) lock1.getInternalLock()).getLockId(), Await.result(asyncParseClientID(zkc.get(), lockPath, children.get(1))));
    logger.info("Expiring session on lock0");
    ZooKeeperClientUtils.expireSession(zkc0, zkServers, sessionTimeoutMs);
    logger.info("Session on lock0 is expired");
    lock1DoneLatch.await();
    assertFalse(lock0.haveLock());
    assertTrue(lock1.haveLock());
    if (checkOwnershipAndReacquire) {
        try {
            checkLockAndReacquire(lock0, true);
            fail("Should fail check write lock since lock is already held by other people");
        } catch (OwnershipAcquireFailedException oafe) {
            assertEquals(((ZKSessionLock) lock1.getInternalLock()).getLockId().getLeft(), oafe.getCurrentOwner());
        }
        try {
            checkLockAndReacquire(lock0, false);
            fail("Should fail check write lock since lock is already held by other people");
        } catch (OwnershipAcquireFailedException oafe) {
            assertEquals(((ZKSessionLock) lock1.getInternalLock()).getLockId().getLeft(), oafe.getCurrentOwner());
        }
    } else {
        logger.info("Waiting lock0 to attempt acquisition after session expired");
        // session expire will trigger lock re-acquisition
        Future<ZKDistributedLock> asyncLockAcquireFuture;
        do {
            Thread.sleep(1);
            asyncLockAcquireFuture = lock0.getLockReacquireFuture();
        } while (null == asyncLockAcquireFuture);
        try {
            Await.result(asyncLockAcquireFuture);
            fail("Should fail check write lock since lock is already held by other people");
        } catch (OwnershipAcquireFailedException oafe) {
            assertEquals(((ZKSessionLock) lock1.getInternalLock()).getLockId().getLeft(), oafe.getCurrentOwner());
        }
        try {
            checkLockAndReacquire(lock0, false);
            fail("Should fail check write lock since lock is already held by other people");
        } catch (OwnershipAcquireFailedException oafe) {
            assertEquals(((ZKSessionLock) lock1.getInternalLock()).getLockId().getLeft(), oafe.getCurrentOwner());
        }
    }
    children = getLockWaiters(zkc, lockPath);
    assertEquals(1, children.size());
    assertFalse(lock0.haveLock());
    assertTrue(lock1.haveLock());
    assertEquals(((ZKSessionLock) lock1.getInternalLock()).getLockId(), Await.result(asyncParseClientID(zkc.get(), lockPath, children.get(0))));
    FutureUtils.result(lock0.asyncClose());
    FutureUtils.result(lock1.asyncClose());
    children = getLockWaiters(zkc, lockPath);
    assertEquals(0, children.size());
}