Examples with ZKTransaction com.twitter.distributedlog.zk.ZKTransaction used on opensource projects

Example 6 with ZKTransaction

use of com.twitter.distributedlog.zk.ZKTransaction in project distributedlog by twitter.

the class TestLedgerAllocator method testObtainMultipleLedgers.

@Test(timeout = 60000)
public void testObtainMultipleLedgers() throws Exception {
    String allocationPath = "/" + runtime.getMethodName();
    SimpleLedgerAllocator allocator = createAllocator(allocationPath);
    int numLedgers = 10;
    Set<LedgerHandle> allocatedLedgers = new HashSet<LedgerHandle>();
    for (int i = 0; i < numLedgers; i++) {
        allocator.allocate();
        ZKTransaction txn = newTxn();
        LedgerHandle lh = FutureUtils.result(allocator.tryObtain(txn, NULL_LISTENER));
        FutureUtils.result(txn.execute());
        allocatedLedgers.add(lh);
    }
    assertEquals(numLedgers, allocatedLedgers.size());
}

Example 7 with ZKTransaction

use of com.twitter.distributedlog.zk.ZKTransaction in project distributedlog by twitter.

the class TestLedgerAllocator method testSuccessAllocatorShouldDeleteUnusedledger.

@Test(timeout = 60000)
public void testSuccessAllocatorShouldDeleteUnusedledger() throws Exception {
    String allocationPath = "/allocation-delete-unused-ledger";
    zkc.get().create(allocationPath, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    Stat stat = new Stat();
    byte[] data = zkc.get().getData(allocationPath, false, stat);
    Versioned<byte[]> allocationData = new Versioned<byte[]>(data, new ZkVersion(stat.getVersion()));
    SimpleLedgerAllocator allocator1 = new SimpleLedgerAllocator(allocationPath, allocationData, newQuorumConfigProvider(dlConf), zkc, bkc);
    allocator1.allocate();
    // wait until allocated
    ZKTransaction txn1 = newTxn();
    LedgerHandle lh1 = FutureUtils.result(allocator1.tryObtain(txn1, NULL_LISTENER));
    // Second allocator kicks in
    stat = new Stat();
    data = zkc.get().getData(allocationPath, false, stat);
    allocationData = new Versioned<byte[]>(data, new ZkVersion(stat.getVersion()));
    SimpleLedgerAllocator allocator2 = new SimpleLedgerAllocator(allocationPath, allocationData, newQuorumConfigProvider(dlConf), zkc, bkc);
    allocator2.allocate();
    // wait until allocated
    ZKTransaction txn2 = newTxn();
    LedgerHandle lh2 = FutureUtils.result(allocator2.tryObtain(txn2, NULL_LISTENER));
    // should fail to commit txn1 as version is changed by second allocator
    try {
        FutureUtils.result(txn1.execute());
        fail("Should fail commit obtaining ledger handle from first allocator as allocator is modified by second allocator.");
    } catch (ZKException ke) {
    // as expected
    }
    FutureUtils.result(txn2.execute());
    Utils.close(allocator1);
    Utils.close(allocator2);
    // ledger handle should be deleted
    try {
        lh1.close();
        fail("LedgerHandle allocated by allocator1 should be deleted.");
    } catch (BKException bke) {
    // as expected
    }
    try {
        bkc.get().openLedger(lh1.getId(), BookKeeper.DigestType.CRC32, dlConf.getBKDigestPW().getBytes());
        fail("LedgerHandle allocated by allocator1 should be deleted.");
    } catch (BKException.BKNoSuchLedgerExistsException nslee) {
    // as expected
    }
    long eid = lh2.addEntry("hello world".getBytes());
    lh2.close();
    LedgerHandle readLh = bkc.get().openLedger(lh2.getId(), BookKeeper.DigestType.CRC32, dlConf.getBKDigestPW().getBytes());
    Enumeration<LedgerEntry> entries = readLh.readEntries(eid, eid);
    int i = 0;
    while (entries.hasMoreElements()) {
        LedgerEntry entry = entries.nextElement();
        assertEquals("hello world", new String(entry.getEntry(), UTF_8));
        ++i;
    }
    assertEquals(1, i);
}

Example 8 with ZKTransaction

use of com.twitter.distributedlog.zk.ZKTransaction in project distributedlog by twitter.

the class BKLogWriteHandler method doCompleteAndCloseLogSegment.

protected void doCompleteAndCloseLogSegment(final String inprogressZnodeName, long logSegmentSeqNo, long ledgerId, long firstTxId, long lastTxId, int recordCount, long lastEntryId, long lastSlotId, final Promise<LogSegmentMetadata> promise) {
    try {
        lock.checkOwnershipAndReacquire();
    } catch (IOException ioe) {
        FutureUtils.setException(promise, ioe);
        return;
    }
    LOG.debug("Completing and Closing Log Segment {} {}", firstTxId, lastTxId);
    final String inprogressZnodePath = inprogressZNode(inprogressZnodeName);
    LogSegmentMetadata inprogressLogSegment = readLogSegmentFromCache(inprogressZnodeName);
    // validate log segment
    if (inprogressLogSegment.getLedgerId() != ledgerId) {
        FutureUtils.setException(promise, new IOException("Active ledger has different ID to inprogress. " + inprogressLogSegment.getLedgerId() + " found, " + ledgerId + " expected"));
        return;
    }
    // validate the transaction id
    if (inprogressLogSegment.getFirstTxId() != firstTxId) {
        FutureUtils.setException(promise, new IOException("Transaction id not as expected, " + inprogressLogSegment.getFirstTxId() + " found, " + firstTxId + " expected"));
        return;
    }
    // validate the log sequence number
    if (validateLogSegmentSequenceNumber) {
        synchronized (inprogressLSSNs) {
            if (inprogressLSSNs.isEmpty()) {
                FutureUtils.setException(promise, new UnexpectedException("Didn't find matched inprogress log segments when completing inprogress " + inprogressLogSegment));
                return;
            }
            long leastInprogressLSSN = inprogressLSSNs.getFirst();
            // it should also be same as the least inprogress log segment sequence number tracked in {@link inprogressLSSNs}
            if ((inprogressLogSegment.getLogSegmentSequenceNumber() != logSegmentSeqNo) || (leastInprogressLSSN != logSegmentSeqNo)) {
                FutureUtils.setException(promise, new UnexpectedException("Didn't find matched inprogress log segments when completing inprogress " + inprogressLogSegment));
                return;
            }
        }
    }
    // store max sequence number.
    long maxSeqNo = Math.max(logSegmentSeqNo, maxLogSegmentSequenceNo.getSequenceNumber());
    if (maxLogSegmentSequenceNo.getSequenceNumber() == logSegmentSeqNo || (maxLogSegmentSequenceNo.getSequenceNumber() == logSegmentSeqNo + 1)) {
        // ignore the case that a new inprogress log segment is pre-allocated
        // before completing current inprogress one
        LOG.info("Try storing max sequence number {} in completing {}.", new Object[] { logSegmentSeqNo, inprogressZnodePath });
    } else {
        LOG.warn("Unexpected max ledger sequence number {} found while completing log segment {} for {}", new Object[] { maxLogSegmentSequenceNo.getSequenceNumber(), logSegmentSeqNo, getFullyQualifiedName() });
        if (validateLogSegmentSequenceNumber) {
            FutureUtils.setException(promise, new DLIllegalStateException("Unexpected max log segment sequence number " + maxLogSegmentSequenceNo.getSequenceNumber() + " for " + getFullyQualifiedName() + ", expected " + (logSegmentSeqNo - 1)));
            return;
        }
    }
    // Prepare the completion
    final String nameForCompletedLedger = completedLedgerZNodeName(firstTxId, lastTxId, logSegmentSeqNo);
    final String pathForCompletedLedger = completedLedgerZNode(firstTxId, lastTxId, logSegmentSeqNo);
    long startSequenceId;
    try {
        startSequenceId = computeStartSequenceId(inprogressLogSegment);
    } catch (IOException ioe) {
        FutureUtils.setException(promise, ioe);
        return;
    }
    // write completed ledger znode
    final LogSegmentMetadata completedLogSegment = inprogressLogSegment.completeLogSegment(pathForCompletedLedger, lastTxId, recordCount, lastEntryId, lastSlotId, startSequenceId);
    setLastLedgerRollingTimeMillis(completedLogSegment.getCompletionTime());
    // prepare the transaction
    ZKTransaction txn = new ZKTransaction(zooKeeperClient);
    // create completed log segment
    writeLogSegment(txn, zooKeeperClient.getDefaultACL(), nameForCompletedLedger, completedLogSegment, pathForCompletedLedger);
    // delete inprogress log segment
    deleteLogSegment(txn, inprogressZnodeName, inprogressZnodePath);
    // store max sequence number
    storeMaxSequenceNumber(txn, maxLogSegmentSequenceNo, maxSeqNo, false);
    // update max txn id.
    LOG.debug("Trying storing LastTxId in Finalize Path {} LastTxId {}", pathForCompletedLedger, lastTxId);
    storeMaxTxId(txn, maxTxId, lastTxId);
    txn.execute().addEventListener(FutureEventListenerRunnable.of(new FutureEventListener<Void>() {

        @Override
        public void onSuccess(Void value) {
            LOG.info("Completed {} to {} for {} : {}", new Object[] { inprogressZnodeName, nameForCompletedLedger, getFullyQualifiedName(), completedLogSegment });
            FutureUtils.setValue(promise, completedLogSegment);
        }

        @Override
        public void onFailure(Throwable cause) {
            FutureUtils.setException(promise, cause);
        }
    }, scheduler));
}

Example 9 with ZKTransaction

use of com.twitter.distributedlog.zk.ZKTransaction in project distributedlog by twitter.

the class BKLogWriteHandler method doStartLogSegment.

protected void doStartLogSegment(final long txId, final boolean bestEffort, final boolean allowMaxTxID, final Promise<BKLogSegmentWriter> promise) {
    // validate the tx id
    if ((txId < 0) || (!allowMaxTxID && (txId == DistributedLogConstants.MAX_TXID))) {
        FutureUtils.setException(promise, new IOException("Invalid Transaction Id " + txId));
        return;
    }
    if (this.sanityCheckTxnId) {
        long highestTxIdWritten = maxTxId.get();
        if (txId < highestTxIdWritten) {
            if (highestTxIdWritten == DistributedLogConstants.MAX_TXID) {
                LOG.error("We've already marked the stream as ended and attempting to start a new log segment");
                FutureUtils.setException(promise, new EndOfStreamException("Writing to a stream after it has been marked as completed"));
                return;
            } else {
                LOG.error("We've already seen TxId {} the max TXId is {}", txId, highestTxIdWritten);
                FutureUtils.setException(promise, new TransactionIdOutOfOrderException(txId, highestTxIdWritten));
                return;
            }
        }
    }
    try {
        ledgerAllocator.allocate();
    } catch (IOException e) {
        // failed to issue an allocation request
        failStartLogSegment(promise, bestEffort, e);
        return;
    }
    // start the transaction from zookeeper
    final ZKTransaction txn = new ZKTransaction(zooKeeperClient);
    // failpoint injected before creating ledger
    try {
        FailpointUtils.checkFailPoint(FailpointUtils.FailPointName.FP_StartLogSegmentBeforeLedgerCreate);
    } catch (IOException ioe) {
        failStartLogSegment(promise, bestEffort, ioe);
        return;
    }
    ledgerAllocator.tryObtain(txn, new Transaction.OpListener<LedgerHandle>() {

        @Override
        public void onCommit(LedgerHandle lh) {
        // no-op
        }

        @Override
        public void onAbort(Throwable t) {
        // no-op
        }
    }).addEventListener(new FutureEventListener<LedgerHandle>() {

        @Override
        public void onSuccess(LedgerHandle lh) {
            // try-obtain succeed
            createInprogressLogSegment(txn, txId, lh, bestEffort, promise);
        }

        @Override
        public void onFailure(Throwable cause) {
            failStartLogSegment(promise, bestEffort, cause);
        }
    });
}

Example 10 with ZKTransaction

use of com.twitter.distributedlog.zk.ZKTransaction in project distributedlog by twitter.

the class TestLedgerAllocator method testCloseAllocatorAfterConfirm.

/**
 * {@link https://issues.apache.org/jira/browse/DL-26}
 */
@DistributedLogAnnotations.FlakyTest
@Ignore
@Test(timeout = 60000)
public void testCloseAllocatorAfterConfirm() throws Exception {
    String allocationPath = "/allocation2";
    SimpleLedgerAllocator allocator = createAllocator(allocationPath);
    allocator.allocate();
    ZKTransaction txn = newTxn();
    // close during obtaining ledger.
    LedgerHandle lh = FutureUtils.result(allocator.tryObtain(txn, NULL_LISTENER));
    FutureUtils.result(txn.execute());
    Utils.close(allocator);
    byte[] data = zkc.get().getData(allocationPath, false, null);
    assertEquals(0, data.length);
    // the ledger is not deleted.
    bkc.get().openLedger(lh.getId(), BookKeeper.DigestType.CRC32, dlConf.getBKDigestPW().getBytes(UTF_8));
}