Examples with RedoableOp com.zimbra.cs.redolog.op.RedoableOp used on opensource projects

Example 1 with RedoableOp

use of com.zimbra.cs.redolog.op.RedoableOp in project zm-mailbox by Zimbra.

the class DesktopMailboxTest method init.

@BeforeClass
public static void init() throws Exception {
    //TODO: allow paths to be specified so we can run tests outside of ZimbraServer
    MailboxTestUtil.initServer();
    MailboxManager.setInstance(new MailboxManager() {

        @Override
        protected Mailbox instantiateMailbox(MailboxData data) {
            //mock the behaviors we need to test in DesktopMailbox
            return new Mailbox(data) {

                @Override
                protected boolean needRedo(OperationContext octxt, RedoableOp recorder) {
                    return false;
                }
            };
        }
    });
    Provisioning prov = Provisioning.getInstance();
    prov.createAccount("test@zimbra.com", "secret", new HashMap<String, Object>());
}

Example 2 with RedoableOp

use of com.zimbra.cs.redolog.op.RedoableOp in project zm-mailbox by Zimbra.

the class RedoLogVerify method scanLog.

public boolean scanLog(File logfile) throws IOException {
    boolean good = false;
    FileLogReader logReader = new FileLogReader(logfile, false);
    logReader.open();
    if (!mParams.quiet) {
        FileHeader header = logReader.getHeader();
        mOut.println("HEADER");
        mOut.println("------");
        mOut.print(header);
        mOut.println("------");
    }
    boolean hasMailboxIdsFilter = !mParams.mboxIds.isEmpty();
    RedoableOp op = null;
    long lastPosition = 0;
    long lastOpStartOffset = 0;
    try {
        while ((op = logReader.getNextOp()) != null) {
            lastOpStartOffset = logReader.getLastOpStartOffset();
            lastPosition = logReader.position();
            if (hasMailboxIdsFilter) {
                int mboxId = op.getMailboxId();
                if (op instanceof StoreIncomingBlob) {
                    List<Integer> list = ((StoreIncomingBlob) op).getMailboxIdList();
                    if (list != null) {
                        boolean match = false;
                        for (Integer mid : list) {
                            if (mParams.mboxIds.contains(mid)) {
                                match = true;
                                break;
                            }
                        }
                        if (!match)
                            continue;
                    }
                // If list==null, it's a store incoming blob op targeted at unknown set of mailboxes.
                // It applies to our filtered mailboxes.
                } else if (!mParams.mboxIds.contains(mboxId)) {
                    continue;
                }
            }
            if (!mParams.quiet) {
                printOp(mOut, op, mParams.hideOffset, lastOpStartOffset, lastPosition - lastOpStartOffset);
                if (mParams.showBlob) {
                    InputStream dataStream = op.getAdditionalDataStream();
                    if (dataStream != null) {
                        mOut.println("<START OF BLOB>");
                        ByteUtil.copy(dataStream, true, mOut, false);
                        mOut.println();
                        mOut.println("<END OF BLOB>");
                    }
                }
            }
        }
        good = true;
    } catch (IOException e) {
        // The IOException could be a real I/O problem or it could mean
        // there was a server crash previously and there were half-written
        // log entries.
        mOut.println();
        mOut.printf("Error while parsing data starting at offset 0x%08x", lastPosition);
        mOut.println();
        long size = logReader.getSize();
        long diff = size - lastPosition;
        mOut.printf("%d bytes remaining in the file", diff);
        mOut.println();
        mOut.println();
        if (op != null) {
            mOut.println("Last suceessfully parsed redo op:");
            printOp(mOut, op, false, lastOpStartOffset, lastPosition - lastOpStartOffset);
            mOut.println();
        }
        // hexdump data around the bad bytes
        int bytesPerLine = 16;
        int linesBefore = 10;
        int linesAfter = 10;
        long startPos = Math.max(lastPosition - (lastPosition % bytesPerLine) - linesBefore * bytesPerLine, 0);
        int count = (int) Math.min((linesBefore + linesAfter + 1) * bytesPerLine, lastPosition - startPos + diff);
        RandomAccessFile raf = null;
        try {
            raf = new RandomAccessFile(logfile, "r");
            raf.seek(startPos);
            byte[] buf = new byte[count];
            raf.read(buf, 0, count);
            mOut.printf("Data near error offset %08x:", lastPosition);
            mOut.println();
            hexdump(mOut, buf, 0, count, startPos, lastPosition);
            mOut.println();
        } catch (IOException eh) {
            mOut.println("Error opening log file " + logfile.getAbsolutePath() + " for hexdump");
            eh.printStackTrace(mOut);
        } finally {
            if (raf != null)
                raf.close();
        }
        throw e;
    } finally {
        logReader.close();
    }
    return good;
}

Example 3 with RedoableOp

use of com.zimbra.cs.redolog.op.RedoableOp in project zm-mailbox by Zimbra.

the class RedoLogManager method start.

public synchronized void start() {
    mEnabled = true;
    try {
        File logdir = mLogFile.getParentFile();
        if (!logdir.exists()) {
            if (!logdir.mkdirs())
                throw new IOException("Unable to create directory " + logdir.getAbsolutePath());
        }
        if (!mArchiveDir.exists()) {
            if (!mArchiveDir.mkdirs())
                throw new IOException("Unable to create directory " + mArchiveDir.getAbsolutePath());
        }
    } catch (IOException e) {
        signalFatalError(e);
    }
    setInCrashRecovery(true);
    // mSupportsCrashRecovery is false.
    try {
        mRolloverMgr.crashRecovery();
    } catch (IOException e) {
        ZimbraLog.redolog.fatal("Exception during crash recovery");
        signalFatalError(e);
    }
    long fsyncInterval = RedoConfig.redoLogFsyncIntervalMS();
    mLogWriter = createLogWriter(this, mLogFile, fsyncInterval);
    ArrayList<RedoableOp> postStartupRecoveryOps = new ArrayList<RedoableOp>(100);
    int numRecoveredOps = 0;
    if (mSupportsCrashRecovery) {
        mRecoveryMode = true;
        ZimbraLog.redolog.info("Starting pre-startup crash recovery");
        // Run crash recovery.
        try {
            mLogWriter.open();
            mRolloverMgr.initSequence(mLogWriter.getSequence());
            RedoPlayer redoPlayer = new RedoPlayer(true);
            try {
                numRecoveredOps = redoPlayer.runCrashRecovery(this, postStartupRecoveryOps);
            } finally {
                redoPlayer.shutdown();
            }
            mLogWriter.close();
        } catch (Exception e) {
            ZimbraLog.redolog.fatal("Exception during crash recovery");
            signalFatalError(e);
        }
        ZimbraLog.redolog.info("Finished pre-startup crash recovery");
        mRecoveryMode = false;
    }
    setInCrashRecovery(false);
    // Reopen log after crash recovery.
    try {
        mLogWriter.open();
        mRolloverMgr.initSequence(mLogWriter.getSequence());
        mInitialLogSize = mLogWriter.getSize();
    } catch (IOException e) {
        ZimbraLog.redolog.fatal("Unable to open redo log");
        signalFatalError(e);
    }
    if (numRecoveredOps > 0) {
        // file after rollover will still list these uncommitted ops.
        if (postStartupRecoveryOps.size() > 0) {
            synchronized (mActiveOps) {
                for (Iterator iter = postStartupRecoveryOps.iterator(); iter.hasNext(); ) {
                    RedoableOp op = (RedoableOp) iter.next();
                    assert (op.isStartMarker());
                    mActiveOps.put(op.getTransactionId(), op);
                }
            }
        }
        // Force rollover to clear the current log file.
        forceRollover();
        // requests.
        if (postStartupRecoveryOps.size() > 0) {
            synchronized (mShuttingDownGuard) {
                mInPostStartupCrashRecovery = true;
            }
            Thread psrThread = new PostStartupCrashRecoveryThread(postStartupRecoveryOps);
            psrThread.start();
        }
    }
}

Example 4 with RedoableOp

use of com.zimbra.cs.redolog.op.RedoableOp in project zm-mailbox by Zimbra.

the class RedoLogManager method getChangedMailboxesSince.

/**
     * Returns the set of mailboxes that had any committed changes since a
     * particular CommitId in the past, by scanning redologs.  Also returns
     * the last CommitId seen during the scanning process.
     * @param cid
     * @return can be null if server is shutting down
     * @throws IOException
     * @throws MailServiceException
     */
public Pair<Set<Integer>, CommitId> getChangedMailboxesSince(CommitId cid) throws IOException, MailServiceException {
    Set<Integer> mailboxes = new HashSet<Integer>();
    // Grab a read lock to prevent rollover.
    ReadLock readLock = mRWLock.readLock();
    try {
        readLock.lockInterruptibly();
    } catch (InterruptedException e) {
        synchronized (mShuttingDownGuard) {
            if (!mShuttingDown)
                ZimbraLog.redolog.error("InterruptedException during redo log scan for CommitId", e);
            else
                ZimbraLog.redolog.debug("Redo log scan for CommitId interrupted for shutdown");
        }
        return null;
    }
    File linkDir = null;
    File[] logs;
    try {
        try {
            long seq = cid.getRedoSeq();
            File[] archived = getArchivedLogsFromSequence(seq);
            if (archived != null) {
                logs = new File[archived.length + 1];
                System.arraycopy(archived, 0, logs, 0, archived.length);
                logs[archived.length] = mLogFile;
            } else {
                logs = new File[] { mLogFile };
            }
            // Make sure the first log has the sequence in cid.
            FileLogReader firstLog = new FileLogReader(logs[0]);
            if (firstLog.getHeader().getSequence() != seq) {
                // Most likely, the CommitId is too old.
                throw MailServiceException.INVALID_COMMIT_ID(cid.toString());
            }
            // Create a temp directory and make hard links to all redologs.
            // This prevents the logs from disappearing while being scanned.
            String dirName = "tmp-scan-" + System.currentTimeMillis();
            linkDir = new File(mLogFile.getParentFile(), dirName);
            if (linkDir.exists()) {
                int suffix = 1;
                while (linkDir.exists()) {
                    linkDir = new File(mLogFile.getParentFile(), dirName + "-" + suffix);
                }
            }
            if (!linkDir.mkdir())
                throw new IOException("Unable to create temp dir " + linkDir.getAbsolutePath());
            for (int i = 0; i < logs.length; i++) {
                File src = logs[i];
                File dest = new File(linkDir, logs[i].getName());
                IO.link(src.getAbsolutePath(), dest.getAbsolutePath());
                logs[i] = dest;
            }
        } finally {
            // We can let rollover happen now.
            readLock.unlock();
        }
        // Scan redologs to get list with IDs of mailboxes that have
        // committed changes since the given commit id.
        long lastSeq = -1;
        CommitTxn lastCommitTxn = null;
        boolean foundMarker = false;
        for (File logfile : logs) {
            FileLogReader logReader = new FileLogReader(logfile);
            logReader.open();
            lastSeq = logReader.getHeader().getSequence();
            try {
                RedoableOp op = null;
                while ((op = logReader.getNextOp()) != null) {
                    if (ZimbraLog.redolog.isDebugEnabled())
                        ZimbraLog.redolog.debug("Read: " + op);
                    if (!(op instanceof CommitTxn))
                        continue;
                    lastCommitTxn = (CommitTxn) op;
                    if (foundMarker) {
                        int mboxId = op.getMailboxId();
                        if (mboxId > 0)
                            mailboxes.add(mboxId);
                    } else {
                        if (cid.matches(lastCommitTxn))
                            foundMarker = true;
                    }
                }
            } catch (IOException e) {
                ZimbraLog.redolog.warn("IOException while reading redolog file", e);
            } finally {
                logReader.close();
            }
        }
        if (!foundMarker) {
            // Most likely, the CommitId is too old.
            throw MailServiceException.INVALID_COMMIT_ID(cid.toString());
        }
        CommitId lastCommitId = new CommitId(lastSeq, lastCommitTxn);
        return new Pair<Set<Integer>, CommitId>(mailboxes, lastCommitId);
    } finally {
        if (linkDir != null) {
            // Clean up the temp dir with links.
            try {
                if (linkDir.exists())
                    FileUtil.deleteDir(linkDir);
            } catch (IOException e) {
                ZimbraLog.redolog.warn("Unable to delete temporary directory " + linkDir.getAbsolutePath(), e);
            }
        }
    }
}

Example 5 with RedoableOp

use of com.zimbra.cs.redolog.op.RedoableOp in project zm-mailbox by Zimbra.

the class RedoLogManager method checkpoint.

/**
     * Should be called with write lock on mRWLock held.
     */
private void checkpoint() {
    LinkedHashSet<TransactionId> txns = null;
    synchronized (mActiveOps) {
        if (mActiveOps.size() == 0)
            return;
        // Create an empty LinkedHashSet and insert keys from mActiveOps
        // by iterating the keyset.
        txns = new LinkedHashSet<TransactionId>();
        for (Iterator<Map.Entry<TransactionId, RedoableOp>> it = mActiveOps.entrySet().iterator(); it.hasNext(); ) {
            Map.Entry<TransactionId, RedoableOp> entry = it.next();
            txns.add(entry.getKey());
        }
    }
    Checkpoint ckpt = new Checkpoint(txns);
    logOnly(ckpt, true);
}