Examples with UnderlyingData com.zimbra.cs.mailbox.MailItem.UnderlyingData used on opensource projects

Example 1 with UnderlyingData

use of com.zimbra.cs.mailbox.MailItem.UnderlyingData in project zm-mailbox by Zimbra.

the class MailboxErrorUtil method handleCascadeFailure.

/**
 * Attempt to delete a list of ids whih previously failed due to foreign key constraint violation. Intended to find which items had the FK issue, and provide a bit more detail about them.
 * Throws the original exception argument, or wraps it with additional details if individual deletes fail
 */
static void handleCascadeFailure(Mailbox mbox, List<Integer> cascadeIds, ServiceException e) throws ServiceException {
    if (causeMatchesFKFailure(e)) {
        ZimbraLog.mailbox.error("deleting cascadeIds failed due to foreign key constraint failed; attempting to delete individually and find failure");
        LinkedList<Integer> failures = new LinkedList<Integer>();
        for (Integer id : cascadeIds) {
            try {
                List<Integer> singleItemList = Collections.singletonList(id);
                ZimbraLog.mailbox.debug("attempting to delete id [" + id + "]");
                DbMailItem.delete(mbox, singleItemList, false);
                ZimbraLog.mailbox.debug("deleted [" + id + "] OK");
            } catch (ServiceException se) {
                ZimbraLog.mailbox.error("deleted FAILED for [" + id + "] due to exception", se);
                failures.add(id);
            }
        }
        if (!failures.isEmpty()) {
            StringBuilder sb = new StringBuilder();
            // find the id,type,subject for each entry. this should help us figure out what's not being removed correctly...
            for (Integer id : failures) {
                MailItem item = mbox.getItemById(id, MailItem.Type.UNKNOWN);
                String logMsg = "failure item id[" + id + "] type[" + item.getType() + ":" + item.getClass().getSimpleName() + "] subject[" + item.getSubject() + "] size[" + item.getSize() + "] folder[" + item.getFolderId() + "] parent[" + item.getParentId() + "]";
                sb.append(logMsg).append("\r\n");
                ZimbraLog.mailbox.error(logMsg);
                if (item instanceof Conversation) {
                    Conversation conv = (Conversation) item;
                    List<Message> children = conv.getMessages();
                    if (children != null && children.size() > 0) {
                        ZimbraLog.mailbox.error("converstaion[" + conv.getId() + "] still has " + children.size() + " children.");
                        for (Message msg : children) {
                            logMsg = "child[" + msg + "] type[" + msg.getType() + "] subject[" + msg.getSubject() + "] in folder[" + msg.getFolderId() + ":" + msg.getFolder().getName() + "] still associated with conv [" + conv.getId() + "]";
                            sb.append(logMsg);
                            ZimbraLog.mailbox.error(logMsg);
                        }
                    }
                } else if (item instanceof Folder) {
                    Folder folder = (Folder) item;
                    List<UnderlyingData> children = DbMailItem.getByFolder(folder, MailItem.Type.UNKNOWN, SortBy.NONE);
                    if (children != null && children.size() > 0) {
                        ZimbraLog.mailbox.error("folder[" + folder.getId() + "] still has " + children.size() + " children.");
                        for (UnderlyingData data : children) {
                            logMsg = "child[" + item + "] type[" + data.type + "] subject[" + data.getSubject() + "] still present in folder [" + folder.getId() + "]";
                            sb.append(logMsg);
                            ZimbraLog.mailbox.error(logMsg);
                        }
                    }
                } else {
                    ZimbraLog.mailbox.warn("cascade failure in unexpected type [" + item.getType() + ":" + item.getClass().getSimpleName() + "] other than folder or conversation");
                }
            }
            throw ServiceException.FAILURE(e.getMessage() + "---" + sb.toString(), e);
        } else {
            throw ServiceException.FAILURE(e.getMessage() + "--- no additional data available from attempting individual deletes.", e);
        }
    } else {
        throw e;
    }
}

Example 2 with UnderlyingData

use of com.zimbra.cs.mailbox.MailItem.UnderlyingData in project zm-mailbox by Zimbra.

the class MailboxIndex method indexItemList.

/**
 * Index a potentially very large list of {@link MailItem}s. Iterate through the list of items, fetch each one and
 * call generateIndexData(). Buffer the items, IndexData into a chunk and when the chunk gets sufficiently large,
 * run a Mailbox transaction to actually do the indexing
 *
 * @param ids item IDs to index
 * @param status progress will be written to the status
 * @throws ServiceException {@link ServiceException#INTERRUPTED} if {@link #cancelReIndex()} is called
 */
private void indexItemList(Collection<Integer> ids, BatchStatus status) throws ServiceException {
    assert (mailbox.lock.isUnlocked());
    status.setTotal(ids.size());
    if (ids.isEmpty()) {
        return;
    }
    // we re-index 'chunks' of items -- up to a certain size or count
    List<Mailbox.IndexItemEntry> chunk = new ArrayList<Mailbox.IndexItemEntry>();
    long chunkByteSize = 0;
    int i = 0;
    for (int id : ids) {
        i++;
        status.addProcessed(1);
        // Fetch the item and generate the list of Lucene documents to index. Do this without holding the Mailbox
        // lock. Once we've accumulated a "chunk" of items, do a mailbox transaction to actually add them to the
        // index.
        ZimbraLog.index.debug("Tokenizing id=%d", id);
        MailItem item = null;
        try {
            mailbox.beginReadTransaction("IndexItemList-Fetch", null);
            item = mailbox.getItemById(id, MailItem.Type.UNKNOWN, false);
        } catch (MailServiceException.NoSuchItemException e) {
            // fallback to dumpster
            try {
                item = mailbox.getItemById(id, MailItem.Type.UNKNOWN, true);
            } catch (MailServiceException.NoSuchItemException again) {
                // The item has just been deleted.
                ZimbraLog.index.debug("deferred item no longer exist id=%d", id);
                removeDeferredId(id);
                continue;
            }
        } catch (MailServiceException e) {
            // fetch without metadata because reindex will regenerate metadata
            if (MailServiceException.INVALID_METADATA.equals(e.getCode()) && isReIndexInProgress()) {
                UnderlyingData ud = DbMailItem.getById(mailbox, id, MailItem.Type.UNKNOWN, false);
                // ignore corrupted metadata
                ud.metadata = null;
                item = mailbox.getItem(ud);
            } else {
                throw e;
            }
        } catch (Exception e) {
            ZimbraLog.index.warn("Failed to fetch deferred item id=%d", id, e);
            status.addFailed(1);
            continue;
        } finally {
            mailbox.endTransaction(item != null);
        }
        try {
            chunk.add(new Mailbox.IndexItemEntry(item, item.generateIndexData()));
        } catch (MailItem.TemporaryIndexingException e) {
            ZimbraLog.index.warn("Temporary index failure id=%d", id, e);
            lastFailedTime = System.currentTimeMillis();
            status.addFailed(1);
            continue;
        }
        chunkByteSize += item.getSize();
        if (i == ids.size() || chunkByteSize > MAX_TX_BYTES || chunk.size() >= MAX_TX_ITEMS) {
            // we have a chunk of items and their corresponding index data -- add them to the index
            try {
                ZimbraLog.index.debug("Batch progress %d/%d", i, ids.size());
                if (status.isCancelled()) {
                    throw ServiceException.INTERRUPTED("cancelled");
                }
                try {
                    boolean success = false;
                    try {
                        mailbox.beginTransaction("IndexItemList-Commit", null);
                        for (Mailbox.IndexItemEntry entry : chunk) {
                            mailbox.addIndexItemToCurrentChange(entry);
                        }
                        success = true;
                    } finally {
                        mailbox.endTransaction(success);
                    }
                } catch (ServiceException e) {
                    ZimbraLog.index.warn("Failed to index chunk=%s", chunk, e);
                    status.addFailed(chunk.size());
                }
            } finally {
                chunk.clear();
                chunkByteSize = 0;
            }
        }
    }
}

Example 3 with UnderlyingData

use of com.zimbra.cs.mailbox.MailItem.UnderlyingData in project zm-mailbox by Zimbra.

the class MailboxUpgrade method createTagRows.

private static Map<Integer, String> createTagRows(DbConnection conn, Mailbox mbox, boolean checkDuplicates) throws ServiceException {
    PreparedStatement stmt = null;
    ResultSet rs = null;
    try {
        stmt = conn.prepareStatement("SELECT id, name, mod_metadata, metadata FROM " + DbMailItem.getMailItemTableName(mbox) + " WHERE " + DbMailItem.IN_THIS_MAILBOX_AND + "type = " + MailItem.Type.TAG.toByte());
        DbMailItem.setMailboxId(stmt, mbox, 1);
        rs = stmt.executeQuery();
        Map<Integer, String> tagNames = Maps.newHashMap();
        while (rs.next()) {
            UnderlyingData data = new UnderlyingData();
            data.id = rs.getInt(1);
            data.name = rs.getString(2);
            data.modMetadata = rs.getInt(3);
            Color color = Color.fromMetadata(new Metadata(rs.getString(4)).getLong(Metadata.FN_COLOR, MailItem.DEFAULT_COLOR));
            try {
                DbTag.createTag(conn, mbox, data, color.getMappedColor() == MailItem.DEFAULT_COLOR ? null : color, true);
            } catch (ServiceException se) {
                if (checkDuplicates && se.getCode().equals(MailServiceException.ALREADY_EXISTS)) {
                    // tag name already exist. append a random number to the tag name.
                    SecureRandom sr = new SecureRandom();
                    data.name = data.name + sr.nextInt();
                    DbTag.createTag(conn, mbox, data, color.getMappedColor() == MailItem.DEFAULT_COLOR ? null : color, true);
                } else {
                    throw se;
                }
            }
            tagNames.put(data.id, data.name);
        }
        return tagNames;
    } catch (SQLException e) {
        throw ServiceException.FAILURE("creating TAG rows in mbox " + mbox.getId(), e);
    } finally {
        DbPool.closeResults(rs);
        DbPool.closeStatement(stmt);
    }
}

Example 4 with UnderlyingData

use of com.zimbra.cs.mailbox.MailItem.UnderlyingData in project zm-mailbox by Zimbra.

the class DbTag method verifyTagCounts.

// TODO remove JUnit dependency from non test class
private static void verifyTagCounts(DbConnection conn, Mailbox mbox, Map<Integer, UnderlyingData> tdata) throws ServiceException {
    PreparedStatement stmt = null;
    ResultSet rs = null;
    try {
        String mailboxesMatchAnd = DebugConfig.disableMailboxGroups ? "" : "ti.mailbox_id = mi.mailbox_id AND ";
        stmt = conn.prepareStatement("SELECT ti.tag_id, COUNT(ti.item_id), " + Db.clauseIFNULL("SUM(mi.unread)", "0") + " FROM " + getTaggedItemTableName(mbox, "ti") + " INNER JOIN " + DbMailItem.getMailItemTableName(mbox, "mi") + " ON " + mailboxesMatchAnd + "mi.id = ti.item_id" + " WHERE " + inThisMailboxAnd("ti") + "ti.tag_id > 0 AND " + Db.getInstance().bitAND("mi.flags", String.valueOf(Flag.BITMASK_DELETED)) + " = 0" + " GROUP BY ti.tag_id");
        DbMailItem.setMailboxId(stmt, mbox, 1);
        rs = stmt.executeQuery();
        Map<Integer, UnderlyingData> tcheck = new HashMap<Integer, UnderlyingData>(tdata);
        while (rs.next()) {
            int id = rs.getInt(1), size = rs.getInt(2), unread = rs.getInt(3);
            UnderlyingData data = tcheck.remove(id);
            Assert.assertNotNull("no TAG row for id " + id, data);
            Assert.assertEquals("size for tag " + data.name, size, data.size);
            Assert.assertEquals("unread for tag " + data.name, unread, data.unreadCount);
        }
        for (UnderlyingData data : tcheck.values()) {
            Assert.assertEquals("size for tag " + data.name, 0, data.size);
            Assert.assertEquals("unread for tag " + data.name, 0, data.unreadCount);
        }
    } catch (SQLException e) {
        throw ServiceException.FAILURE("consistency checking TAGGED_ITEM vs. TAG", e);
    } finally {
        DbPool.closeResults(rs);
        DbPool.closeStatement(stmt);
    }
}

Example 5 with UnderlyingData

use of com.zimbra.cs.mailbox.MailItem.UnderlyingData in project zm-mailbox by Zimbra.

the class DbTag method debugConsistencyCheck.

@VisibleForTesting
public static void debugConsistencyCheck(Mailbox mbox) throws ServiceException {
    DbConnection conn = mbox.lock.isWriteLockedByCurrentThread() ? mbox.getOperationConnection() : DbPool.getConnection(mbox);
    PreparedStatement stmt = null;
    ResultSet rs = null;
    try {
        Map<Integer, UnderlyingData> tdata = Maps.newHashMap();
        // make sure the item counts match the tag totals
        stmt = conn.prepareStatement("SELECT " + TAG_FIELDS + " FROM " + getTagTableName(mbox) + " WHERE " + DbMailItem.IN_THIS_MAILBOX_AND + "id > 0");
        DbMailItem.setMailboxId(stmt, mbox, 1);
        rs = stmt.executeQuery();
        while (rs.next()) {
            UnderlyingData data = asUnderlyingData(rs);
            tdata.put(data.id, data);
        }
        DbPool.closeResults(rs);
        rs = null;
        DbPool.closeStatement(stmt);
        stmt = null;
        // catch spurious TAGGED_ITEM entries
        validateTaggedItem(conn, mbox, tdata);
        // make sure the TAGGED_ITEM table is accurate
        verifyTaggedItem(conn, mbox, tdata);
        // make sure the item counts match the tag totals
        verifyTagCounts(conn, mbox, tdata);
    } catch (SQLException e) {
        throw ServiceException.FAILURE("consistency checking tags/flags for mailbox " + mbox.getId(), e);
    } finally {
        DbPool.closeResults(rs);
        DbPool.closeStatement(stmt);
        if (!mbox.lock.isWriteLockedByCurrentThread()) {
            conn.close();
        }
    }
}