Examples with TxnIterator org.apache.zookeeper.server.persistence.TxnLog.TxnIterator used on opensource projects

Example 1 with TxnIterator

use of org.apache.zookeeper.server.persistence.TxnLog.TxnIterator in project zookeeper by apache.

the class LoadFromLogTest method testLoad.

/**
     * test that all transactions from the Log are loaded, and only once
     * @throws Exception an exception might be thrown here
     */
@Test
public void testLoad() throws Exception {
    final String hostPort = HOST + PortAssignment.unique();
    // setup a single server cluster
    File tmpDir = ClientBase.createTmpDir();
    ClientBase.setupTestEnv();
    ZooKeeperServer zks = new ZooKeeperServer(tmpDir, tmpDir, 3000);
    SyncRequestProcessor.setSnapCount(100);
    final int PORT = Integer.parseInt(hostPort.split(":")[1]);
    ServerCnxnFactory f = ServerCnxnFactory.createFactory(PORT, -1);
    f.startup(zks);
    Assert.assertTrue("waiting for server being up ", ClientBase.waitForServerUp(hostPort, CONNECTION_TIMEOUT));
    ZooKeeper zk = ClientBase.createZKClient(hostPort);
    // generate some transactions that will get logged
    try {
        for (int i = 0; i < NUM_MESSAGES; i++) {
            zk.create("/invalidsnap-" + i, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
        }
    } finally {
        zk.close();
    }
    f.shutdown();
    Assert.assertTrue("waiting for server to shutdown", ClientBase.waitForServerDown(hostPort, CONNECTION_TIMEOUT));
    // now verify that the FileTxnLog reads every transaction only once
    File logDir = new File(tmpDir, FileTxnSnapLog.version + FileTxnSnapLog.VERSION);
    FileTxnLog txnLog = new FileTxnLog(logDir);
    TxnIterator itr = txnLog.read(0);
    // Check that storage space return some value
    FileTxnIterator fileItr = (FileTxnIterator) itr;
    long storageSize = fileItr.getStorageSize();
    LOG.info("Txnlog size: " + storageSize + " bytes");
    Assert.assertTrue("Storage size is greater than zero ", (storageSize > 0));
    long expectedZxid = 0;
    long lastZxid = 0;
    TxnHeader hdr;
    do {
        hdr = itr.getHeader();
        expectedZxid++;
        Assert.assertTrue("not the same transaction. lastZxid=" + lastZxid + ", zxid=" + hdr.getZxid(), lastZxid != hdr.getZxid());
        Assert.assertTrue("excepting next transaction. expected=" + expectedZxid + ", retreived=" + hdr.getZxid(), (hdr.getZxid() == expectedZxid));
        lastZxid = hdr.getZxid();
    } while (itr.next());
    Assert.assertTrue("processed all transactions. " + expectedZxid + " == " + TOTAL_TRANSACTIONS, (expectedZxid == TOTAL_TRANSACTIONS));
    zks.shutdown();
}

Example 2 with TxnIterator

use of org.apache.zookeeper.server.persistence.TxnLog.TxnIterator in project zookeeper by apache.

the class LoadFromLogTest method testLoadFailure.

/**
     * test that we fail to load txnlog of a request zxid that is older
     * than what exist on disk
     * @throws Exception an exception might be thrown here
     */
@Test
public void testLoadFailure() throws Exception {
    final String hostPort = HOST + PortAssignment.unique();
    // setup a single server cluster
    File tmpDir = ClientBase.createTmpDir();
    ClientBase.setupTestEnv();
    ZooKeeperServer zks = new ZooKeeperServer(tmpDir, tmpDir, 3000);
    // So we have at least 4 logs
    SyncRequestProcessor.setSnapCount(50);
    final int PORT = Integer.parseInt(hostPort.split(":")[1]);
    ServerCnxnFactory f = ServerCnxnFactory.createFactory(PORT, -1);
    f.startup(zks);
    Assert.assertTrue("waiting for server being up ", ClientBase.waitForServerUp(hostPort, CONNECTION_TIMEOUT));
    ZooKeeper zk = ClientBase.createZKClient(hostPort);
    // generate some transactions that will get logged
    try {
        for (int i = 0; i < NUM_MESSAGES; i++) {
            zk.create("/data-", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT_SEQUENTIAL);
        }
    } finally {
        zk.close();
    }
    f.shutdown();
    Assert.assertTrue("waiting for server to shutdown", ClientBase.waitForServerDown(hostPort, CONNECTION_TIMEOUT));
    File logDir = new File(tmpDir, FileTxnSnapLog.version + FileTxnSnapLog.VERSION);
    File[] logFiles = FileTxnLog.getLogFiles(logDir.listFiles(), 0);
    // Verify that we have at least 4 txnlog
    Assert.assertTrue(logFiles.length > 4);
    // Delete the first log file, so we will fail to read it back from disk
    Assert.assertTrue("delete the first log file", logFiles[0].delete());
    // Find zxid for the second log
    long secondStartZxid = Util.getZxidFromName(logFiles[1].getName(), "log");
    FileTxnLog txnLog = new FileTxnLog(logDir);
    TxnIterator itr = txnLog.read(1, false);
    // Oldest log is already remove, so this should point to the start of
    // of zxid on the second log
    Assert.assertEquals(secondStartZxid, itr.getHeader().getZxid());
    itr = txnLog.read(secondStartZxid, false);
    Assert.assertEquals(secondStartZxid, itr.getHeader().getZxid());
    Assert.assertTrue(itr.next());
    // Trying to get a second txn on second txnlog give us the
    // the start of second log, since the first one is removed
    long nextZxid = itr.getHeader().getZxid();
    itr = txnLog.read(nextZxid, false);
    Assert.assertEquals(secondStartZxid, itr.getHeader().getZxid());
    // Trying to get a first txn on the third give us the
    // the start of second log, since the first one is removed
    long thirdStartZxid = Util.getZxidFromName(logFiles[2].getName(), "log");
    itr = txnLog.read(thirdStartZxid, false);
    Assert.assertEquals(secondStartZxid, itr.getHeader().getZxid());
    Assert.assertTrue(itr.next());
    nextZxid = itr.getHeader().getZxid();
    itr = txnLog.read(nextZxid, false);
    Assert.assertEquals(secondStartZxid, itr.getHeader().getZxid());
}

Example 3 with TxnIterator

use of org.apache.zookeeper.server.persistence.TxnLog.TxnIterator in project zookeeper by apache.

the class CRCTest method testChecksums.

/** test checksums for the logs and snapshots.
     * the reader should fail on reading
     * a corrupt snapshot and a corrupt log
     * file
     * @throws Exception
     */
@Test
public void testChecksums() throws Exception {
    File tmpDir = ClientBase.createTmpDir();
    ClientBase.setupTestEnv();
    ZooKeeperServer zks = new ZooKeeperServer(tmpDir, tmpDir, 3000);
    SyncRequestProcessor.setSnapCount(150);
    final int PORT = Integer.parseInt(HOSTPORT.split(":")[1]);
    ServerCnxnFactory f = ServerCnxnFactory.createFactory(PORT, -1);
    f.startup(zks);
    LOG.info("starting up the zookeeper server .. waiting");
    Assert.assertTrue("waiting for server being up", ClientBase.waitForServerUp(HOSTPORT, CONNECTION_TIMEOUT));
    ZooKeeper zk = ClientBase.createZKClient(HOSTPORT);
    try {
        for (int i = 0; i < 2000; i++) {
            zk.create("/crctest- " + i, ("/crctest- " + i).getBytes(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
        }
    } finally {
        zk.close();
    }
    f.shutdown();
    zks.shutdown();
    Assert.assertTrue("waiting for server down", ClientBase.waitForServerDown(HOSTPORT, ClientBase.CONNECTION_TIMEOUT));
    File versionDir = new File(tmpDir, "version-2");
    File[] list = versionDir.listFiles();
    //there should be only two files
    // one the snapshot and the other logFile
    File snapFile = null;
    File logFile = null;
    for (File file : list) {
        LOG.info("file is " + file);
        if (file.getName().startsWith("log")) {
            logFile = file;
            corruptFile(logFile);
        }
    }
    FileTxnLog flog = new FileTxnLog(versionDir);
    TxnIterator itr = flog.read(1);
    //we will get a checksum failure
    try {
        while (itr.next()) {
        }
        Assert.assertTrue(false);
    } catch (IOException ie) {
        LOG.info("crc corruption", ie);
    }
    itr.close();
    // find the last snapshot
    FileSnap snap = new FileSnap(versionDir);
    List<File> snapFiles = snap.findNRecentSnapshots(2);
    snapFile = snapFiles.get(0);
    corruptFile(snapFile);
    boolean cfile = false;
    try {
        cfile = getCheckSum(snap, snapFile);
    } catch (IOException ie) {
        //the last snapshot seems incompelte
        // corrupt the last but one
        // and use that
        snapFile = snapFiles.get(1);
        corruptFile(snapFile);
        cfile = getCheckSum(snap, snapFile);
    }
    Assert.assertTrue(cfile);
}

Example 4 with TxnIterator

use of org.apache.zookeeper.server.persistence.TxnLog.TxnIterator in project zookeeper by apache.

the class TruncateTest method testTruncationStreamReset.

@Test
public void testTruncationStreamReset() throws Exception {
    File tmpdir = ClientBase.createTmpDir();
    FileTxnSnapLog snaplog = new FileTxnSnapLog(tmpdir, tmpdir);
    ZKDatabase zkdb = new ZKDatabase(snaplog);
    // make sure to snapshot, so that we have something there when
    // truncateLog reloads the db
    snaplog.save(zkdb.getDataTree(), zkdb.getSessionWithTimeOuts());
    for (int i = 1; i <= 100; i++) {
        append(zkdb, i);
    }
    zkdb.truncateLog(1);
    append(zkdb, 200);
    zkdb.close();
    // verify that the truncation and subsequent append were processed
    // correctly
    FileTxnLog txnlog = new FileTxnLog(new File(tmpdir, "version-2"));
    TxnIterator iter = txnlog.read(1);
    TxnHeader hdr = iter.getHeader();
    Record txn = iter.getTxn();
    Assert.assertEquals(1, hdr.getZxid());
    Assert.assertTrue(txn instanceof SetDataTxn);
    iter.next();
    hdr = iter.getHeader();
    txn = iter.getTxn();
    Assert.assertEquals(200, hdr.getZxid());
    Assert.assertTrue(txn instanceof SetDataTxn);
    iter.close();
    ClientBase.recursiveDelete(tmpdir);
}

Example 5 with TxnIterator

use of org.apache.zookeeper.server.persistence.TxnLog.TxnIterator in project zookeeper by apache.

the class FileTxnLog method getLastLoggedZxid.

/**
     * get the last zxid that was logged in the transaction logs
     * @return the last zxid logged in the transaction logs
     */
public long getLastLoggedZxid() {
    File[] files = getLogFiles(logDir.listFiles(), 0);
    long maxLog = files.length > 0 ? Util.getZxidFromName(files[files.length - 1].getName(), "log") : -1;
    // if a log file is more recent we must scan it to find
    // the highest zxid
    long zxid = maxLog;
    TxnIterator itr = null;
    try {
        FileTxnLog txn = new FileTxnLog(logDir);
        itr = txn.read(maxLog);
        while (true) {
            if (!itr.next())
                break;
            TxnHeader hdr = itr.getHeader();
            zxid = hdr.getZxid();
        }
    } catch (IOException e) {
        LOG.warn("Unexpected exception", e);
    } finally {
        close(itr);
    }
    return zxid;
}