Examples with StagedBlob com.zimbra.cs.store.StagedBlob used on opensource projects

Example 16 with StagedBlob

use of com.zimbra.cs.store.StagedBlob in project zm-mailbox by Zimbra.

the class Mailbox method addMessageInternal.

private Message addMessageInternal(OperationContext octxt, ParsedMessage pm, int folderId, boolean noICal, int flags, String[] tags, int conversationId, String rcptEmail, Message.DraftInfo dinfo, CustomMetadata customData, DeliveryContext dctxt, StagedBlob staged) throws IOException, ServiceException {
    assert lock.isWriteLockedByCurrentThread();
    if (pm == null) {
        throw ServiceException.INVALID_REQUEST("null ParsedMessage when adding message to mailbox " + mId, null);
    }
    if (Math.abs(conversationId) <= HIGHEST_SYSTEM_ID) {
        conversationId = ID_AUTO_INCREMENT;
    }
    CreateMessage redoPlayer = (octxt == null ? null : (CreateMessage) octxt.getPlayer());
    boolean needRedo = needRedo(octxt, redoPlayer);
    boolean isRedo = redoPlayer != null;
    Blob blob = dctxt.getIncomingBlob();
    if (blob == null) {
        throw ServiceException.FAILURE("Incoming blob not found.", null);
    }
    // make sure we're parsing headers using the target account's charset
    pm.setDefaultCharset(getAccount().getPrefMailDefaultCharset());
    // quick check to make sure we don't deliver 5 copies of the same message
    String msgidHeader = pm.getMessageID();
    boolean isSent = ((flags & Flag.BITMASK_FROM_ME) != 0);
    if (!isRedo && msgidHeader != null && !isSent && mSentMessageIDs.containsKey(msgidHeader)) {
        Integer sentMsgID = mSentMessageIDs.get(msgidHeader);
        if (conversationId == ID_AUTO_INCREMENT) {
            conversationId = getConversationIdFromReferent(pm.getMimeMessage(), sentMsgID.intValue());
            ZimbraLog.mailbox.debug("duplicate detected but not deduped (%s); will try to slot into conversation %d", msgidHeader, conversationId);
        }
    }
    // caller can't set system flags other than \Draft, \Sent and \Post
    flags &= ~Flag.FLAGS_SYSTEM | Flag.BITMASK_DRAFT | Flag.BITMASK_FROM_ME | Flag.BITMASK_POST;
    // caller can't specify non-message flags
    flags &= Flag.FLAGS_GENERIC | Flag.FLAGS_MESSAGE;
    String digest;
    int msgSize;
    try {
        digest = blob.getDigest();
        msgSize = (int) blob.getRawSize();
    } catch (IOException e) {
        throw ServiceException.FAILURE("Unable to get message properties.", e);
    }
    CreateMessage redoRecorder = new CreateMessage(mId, rcptEmail, pm.getReceivedDate(), dctxt.getShared(), digest, msgSize, folderId, noICal, flags, tags, customData);
    StoreIncomingBlob storeRedoRecorder = null;
    // strip out unread flag for internal storage (don't do this before redoRecorder initialization)
    boolean unread = (flags & Flag.BITMASK_UNREAD) > 0;
    flags &= ~Flag.BITMASK_UNREAD;
    // "having attachments" is currently tracked via flags
    if (pm.hasAttachments()) {
        flags |= Flag.BITMASK_ATTACHED;
    } else {
        flags &= ~Flag.BITMASK_ATTACHED;
    }
    // priority is calculated from headers
    flags &= ~(Flag.BITMASK_HIGH_PRIORITY | Flag.BITMASK_LOW_PRIORITY);
    flags |= pm.getPriorityBitmask();
    boolean isSpam = folderId == ID_FOLDER_SPAM;
    boolean isDraft = (flags & Flag.BITMASK_DRAFT) != 0;
    // draft replies get slotted in the same conversation as their parent, if possible
    if (isDraft && !isRedo && conversationId == ID_AUTO_INCREMENT && dinfo != null && !Strings.isNullOrEmpty(dinfo.origId)) {
        try {
            ItemId iid = new ItemId(dinfo.origId, getAccountId());
            if (iid.getId() > 0 && iid.belongsTo(this)) {
                conversationId = getMessageById(octxt, iid.getId()).getConversationId();
            }
        } catch (ServiceException e) {
        }
    }
    Message msg = null;
    boolean success = false;
    CustomMetadata.CustomMetadataList extended = MetadataCallback.preDelivery(pm);
    if (customData != null) {
        if (extended == null) {
            extended = customData.asList();
        } else {
            extended.addSection(customData);
        }
    }
    Threader threader = pm.getThreader(this);
    String subject = pm.getNormalizedSubject();
    try {
        beginTransaction("addMessage", octxt, redoRecorder);
        if (isRedo) {
            rcptEmail = redoPlayer.getRcptEmail();
        }
        Tag.NormalizedTags ntags = new Tag.NormalizedTags(this, tags);
        Folder folder = getFolderById(folderId);
        // step 0: preemptively check for quota issues (actual update is done in Message.create)
        if (!getAccount().isMailAllowReceiveButNotSendWhenOverQuota()) {
            checkSizeChange(getSize() + staged.getSize());
        }
        // step 1: get an ID assigned for the new message
        int messageId = getNextItemId(!isRedo ? ID_AUTO_INCREMENT : redoPlayer.getMessageId());
        List<Conversation> mergeConvs = null;
        if (isRedo) {
            conversationId = redoPlayer.getConvId();
            // fetch the conversations that were merged in as a result of the original delivery...
            List<Integer> mergeConvIds = redoPlayer.getMergedConvIds();
            mergeConvs = new ArrayList<Conversation>(mergeConvIds.size());
            for (int mergeId : mergeConvIds) {
                try {
                    mergeConvs.add(getConversationById(mergeId));
                } catch (NoSuchItemException nsie) {
                    ZimbraLog.mailbox.debug("could not find merge conversation %d", mergeId);
                }
            }
        }
        // step 2: figure out where the message belongs
        Conversation conv = null;
        if (threader.isEnabled()) {
            boolean isReply = pm.isReply();
            if (conversationId != ID_AUTO_INCREMENT) {
                try {
                    // fetch the requested conversation
                    //   (we'll ensure that it's receiving new mail after the new message is added to it)
                    conv = getConversationById(conversationId);
                    ZimbraLog.mailbox.debug("fetched explicitly-specified conversation %d", conv.getId());
                } catch (NoSuchItemException nsie) {
                    if (!isRedo) {
                        ZimbraLog.mailbox.debug("could not find explicitly-specified conversation %d", conversationId);
                        conversationId = ID_AUTO_INCREMENT;
                    }
                }
            } else if (!isRedo && !isSpam && (isReply || (!isSent && !subject.isEmpty()))) {
                List<Conversation> matches = threader.lookupConversation();
                if (matches != null && !matches.isEmpty()) {
                    // file the message into the largest conversation, then later merge any other matching convs
                    Collections.sort(matches, new MailItem.SortSizeDescending());
                    conv = matches.remove(0);
                    mergeConvs = matches;
                }
            }
        }
        if (conv != null && conv.isTagged(Flag.FlagInfo.MUTED)) {
            // adding a message to a muted conversation marks it muted and read
            unread = false;
            flags |= Flag.BITMASK_MUTED;
        }
        // step 3: create the message and update the cache
        //         and if the message is also an invite, deal with the calendar item
        Conversation convTarget = conv instanceof VirtualConversation ? null : conv;
        if (convTarget != null) {
            ZimbraLog.mailbox.debug("  placing message in existing conversation %d", convTarget.getId());
        }
        CalendarPartInfo cpi = pm.getCalendarPartInfo();
        ZVCalendar iCal = null;
        if (cpi != null && CalendarItem.isAcceptableInvite(getAccount(), cpi)) {
            iCal = cpi.cal;
        }
        msg = Message.create(messageId, folder, convTarget, pm, staged, unread, flags, ntags, dinfo, noICal, iCal, extended);
        redoRecorder.setMessageId(msg.getId());
        // step 4: create a conversation for the message, if necessary
        if (threader.isEnabled() && convTarget == null) {
            if (conv == null && conversationId == ID_AUTO_INCREMENT) {
                conv = VirtualConversation.create(this, msg);
                ZimbraLog.mailbox.debug("placed message %d in vconv %d", msg.getId(), conv.getId());
                redoRecorder.setConvFirstMsgId(-1);
            } else {
                Message[] contents = null;
                VirtualConversation vconv = null;
                if (!isRedo) {
                    vconv = (VirtualConversation) conv;
                    contents = (vconv == null ? new Message[] { msg } : new Message[] { vconv.getMessage(), msg });
                } else {
                    // Executing redo.
                    int convFirstMsgId = redoPlayer.getConvFirstMsgId();
                    Message convFirstMsg = null;
                    // If there was a virtual conversation, then...
                    if (convFirstMsgId > 0) {
                        try {
                            convFirstMsg = getMessageById(octxt, redoPlayer.getConvFirstMsgId());
                        } catch (MailServiceException e) {
                            if (!MailServiceException.NO_SUCH_MSG.equals(e.getCode())) {
                                throw e;
                            }
                        // The first message of conversation may have been deleted
                        // by user between the time of original operation and redo.
                        // Handle the case by skipping the updating of its
                        // conversation ID.
                        }
                        // if it is still a standalone message.
                        if (convFirstMsg != null && convFirstMsg.getConversationId() < 0) {
                            contents = new Message[] { convFirstMsg, msg };
                            vconv = new VirtualConversation(this, convFirstMsg);
                        }
                    }
                    if (contents == null) {
                        contents = new Message[] { msg };
                    }
                }
                redoRecorder.setConvFirstMsgId(vconv != null ? vconv.getMessageId() : -1);
                conv = createConversation(conversationId, contents);
                if (vconv != null) {
                    ZimbraLog.mailbox.debug("removed vconv %d", vconv.getId());
                    vconv.removeChild(vconv.getMessage());
                }
                //   associate the first message's reference hashes with the new conversation
                if (contents.length == 2) {
                    threader.changeThreadingTargets(contents[0], conv);
                }
            }
        } else {
            // conversation feature turned off
            redoRecorder.setConvFirstMsgId(-1);
        }
        redoRecorder.setConvId(conv != null && !(conv instanceof VirtualConversation) ? conv.getId() : -1);
        // if we're threading by references, associate the new message's reference hashes with its conversation
        if (!isSpam && !isDraft) {
            threader.recordAddedMessage(conv);
        }
        if (conv != null && mergeConvs != null) {
            redoRecorder.setMergedConversations(mergeConvs);
            for (Conversation smaller : mergeConvs) {
                ZimbraLog.mailbox.info("merging conversation %d for references threading", smaller.getId());
                //                    try {
                conv.merge(smaller);
            //                    } catch (ServiceException e) {
            //                        if (!e.getCode().equals(MailServiceException.NO_SUCH_MSG)) {
            //                            throw e;
            //                        }
            //                    }
            }
        }
        // conversations may have shifted, so the threader's cached state is now questionable
        threader.reset();
        // step 5: write the redolog entries
        if (dctxt.getShared()) {
            if (dctxt.isFirst() && needRedo) {
                // Log entry in redolog for blob save.  Blob bytes are logged in the StoreIncoming entry.
                // Subsequent CreateMessage ops will reference this blob.
                storeRedoRecorder = new StoreIncomingBlob(digest, msgSize, dctxt.getMailboxIdList());
                storeRedoRecorder.start(getOperationTimestampMillis());
                storeRedoRecorder.setBlobBodyInfo(blob.getFile());
                storeRedoRecorder.log();
            }
            // Link to the file created by StoreIncomingBlob.
            redoRecorder.setMessageLinkInfo(blob.getPath());
        } else {
            // Store the blob data inside the CreateMessage op.
            redoRecorder.setMessageBodyInfo(blob.getFile());
        }
        // step 6: link to existing blob
        MailboxBlob mblob = StoreManager.getInstance().link(staged, this, messageId, getOperationChangeID());
        markOtherItemDirty(mblob);
        // when we created the Message, we used the staged locator/size/digest;
        //   make sure that data actually matches the final blob in the store
        msg.updateBlobData(mblob);
        if (dctxt.getMailboxBlob() == null) {
            // Set mailbox blob for in case we want to add the message to the
            // message cache after delivery.
            dctxt.setMailboxBlob(mblob);
        }
        // step 7: queue new message for indexing
        index.add(msg);
        success = true;
        // step 8: send lawful intercept message
        try {
            Notification.getInstance().interceptIfNecessary(this, pm.getMimeMessage(), "add message", folder);
        } catch (ServiceException e) {
            ZimbraLog.mailbox.error("unable to send legal intercept message", e);
        }
    } finally {
        if (storeRedoRecorder != null) {
            if (success) {
                storeRedoRecorder.commit();
            } else {
                storeRedoRecorder.abort();
            }
        }
        endTransaction(success);
        if (success) {
            // Everything worked.  Update the blob field in ParsedMessage
            // so the next recipient in the multi-recipient case will link
            // to this blob as opposed to saving its own copy.
            dctxt.setFirst(false);
        }
    }
    // step 8: remember the Message-ID header so that we can avoid receiving duplicates
    if (isSent && !isRedo && msgidHeader != null) {
        mSentMessageIDs.put(msgidHeader, msg.getId());
    }
    return msg;
}

Example 17 with StagedBlob

use of com.zimbra.cs.store.StagedBlob in project zm-mailbox by Zimbra.

the class Mailbox method saveDraft.

/**
     * Saves draft.
     *
     * @param autoSendTime time at which the draft needs to be auto-sent. Note that this method does not schedule
     *                     the task for auto-sending the draft. It just persists this time for tracking purposes.
     * @see com.zimbra.cs.service.mail.SaveDraft#handle(com.zimbra.common.soap.Element, java.util.Map)
     */
public Message saveDraft(OperationContext octxt, ParsedMessage pm, int id, String origId, String replyType, String identityId, String accountId, long autoSendTime) throws IOException, ServiceException {
    Message.DraftInfo dinfo = null;
    if ((replyType != null && origId != null) || identityId != null || accountId != null || autoSendTime != 0) {
        dinfo = new Message.DraftInfo(replyType, origId, identityId, accountId, autoSendTime);
    }
    if (id == ID_AUTO_INCREMENT) {
        // special-case saving a new draft
        return addMessage(octxt, pm, ID_FOLDER_DRAFTS, true, Flag.BITMASK_DRAFT | Flag.BITMASK_FROM_ME, null, ID_AUTO_INCREMENT, ":API:", dinfo, null, null);
    }
    // write the draft content directly to the mailbox's blob staging area
    StoreManager sm = StoreManager.getInstance();
    StagedBlob staged;
    InputStream is = pm.getRawInputStream();
    try {
        staged = sm.stage(is, this);
    } finally {
        ByteUtil.closeStream(is);
    }
    String digest = staged.getDigest();
    int size = (int) staged.getSize();
    SaveDraft redoRecorder = new SaveDraft(mId, id, digest, size);
    InputStream redoStream = null;
    boolean success = false;
    try {
        beginTransaction("saveDraft", octxt, redoRecorder);
        SaveDraft redoPlayer = (SaveDraft) currentChange().getRedoPlayer();
        Message msg = getMessageById(id);
        if (!msg.isTagged(Flag.FlagInfo.DRAFT)) {
            throw MailServiceException.IMMUTABLE_OBJECT(id);
        }
        if (!checkItemChangeID(msg)) {
            throw MailServiceException.MODIFY_CONFLICT();
        }
        // content changed, so we're obliged to change the IMAP uid
        int imapID = getNextItemId(redoPlayer == null ? ID_AUTO_INCREMENT : redoPlayer.getImapId());
        redoRecorder.setImapId(imapID);
        redoRecorder.setMessageBodyInfo(new ParsedMessageDataSource(pm), size);
        msg.setDraftAutoSendTime(autoSendTime);
        if (dinfo != null) {
            if (replyType != null) {
                msg.setDraftReplyType(replyType);
            }
            if (origId != null) {
                msg.setDraftOrigId(origId);
            }
            if (identityId != null) {
                msg.setDraftIdentityId(identityId);
            }
            if (accountId != null) {
                msg.setDraftAccountId(accountId);
            }
            if (autoSendTime != 0) {
                msg.setDraftAutoSendTime(autoSendTime);
            }
        }
        // update the content and increment the revision number
        msg.setContent(staged, pm);
        index.add(msg);
        success = true;
        try {
            Notification.getInstance().interceptIfNecessary(this, pm.getMimeMessage(), "save draft", msg.getFolder());
        } catch (ServiceException e) {
            ZimbraLog.mailbox.error("Unable to send lawful intercept message.", e);
        }
        return msg;
    } finally {
        endTransaction(success);
        ByteUtil.closeStream(redoStream);
        sm.quietDelete(staged);
    }
}

Example 18 with StagedBlob

use of com.zimbra.cs.store.StagedBlob in project zm-mailbox by Zimbra.

the class Mailbox method addDocumentRevision.

public Document addDocumentRevision(OperationContext octxt, int docId, ParsedDocument pd) throws IOException, ServiceException {
    StoreManager sm = StoreManager.getInstance();
    StagedBlob staged = sm.stage(pd.getBlob(), this);
    AddDocumentRevision redoRecorder = new AddDocumentRevision(mId, pd.getDigest(), pd.getSize(), 0);
    boolean success = false;
    try {
        beginTransaction("addDocumentRevision", octxt, redoRecorder);
        Document doc = getDocumentById(docId);
        redoRecorder.setDocument(pd);
        redoRecorder.setDocId(docId);
        redoRecorder.setItemType(doc.getType());
        // TODO: simplify the redoRecorder by not subclassing from CreateMessage
        // Get the redolog data from the mailbox blob.  This is less than ideal in the
        // HTTP store case because it will result in network access, and possibly an
        // extra write to local disk.  If this becomes a problem, we should update the
        // ParsedDocument constructor to take a DataSource instead of an InputStream.
        MailboxBlob mailboxBlob = doc.setContent(staged, pd);
        redoRecorder.setMessageBodyInfo(new MailboxBlobDataSource(mailboxBlob), mailboxBlob.getSize());
        index.add(doc);
        success = true;
        return doc;
    } catch (IOException ioe) {
        throw ServiceException.FAILURE("error writing document blob", ioe);
    } finally {
        endTransaction(success);
        sm.quietDelete(staged);
    }
}

Example 19 with StagedBlob

use of com.zimbra.cs.store.StagedBlob in project zm-mailbox by Zimbra.

the class AbstractExternalStoreManagerTest method testUncachedFile.

@Test
public void testUncachedFile() throws Exception {
    ParsedMessage pm = ThreaderTest.getRootMessage();
    byte[] mimeBytes = TestUtil.readInputStream(pm.getRawInputStream());
    Mailbox mbox = MailboxManager.getInstance().getMailboxByAccountId(MockProvisioning.DEFAULT_ACCOUNT_ID);
    StoreManager sm = StoreManager.getInstance();
    Blob blob = sm.storeIncoming(pm.getRawInputStream());
    StagedBlob staged = sm.stage(blob, mbox);
    MailboxBlob mblob = sm.link(staged, mbox, 0, 0);
    mblob = sm.getMailboxBlob(mbox, 0, 0, staged.getLocator());
    Blob localBlob = mblob.getLocalBlob();
    InputStream stream = sm.getContent(localBlob);
    Assert.assertTrue("input stream external", stream instanceof BlobInputStream);
    if (sm instanceof ExternalStoreManager) {
        ((ExternalStoreManager) sm).clearCache();
    }
    blob.getFile().delete();
    Assert.assertFalse(blob.getFile().exists());
    //now get it again. this would bomb if it only looked in cache
    stream = sm.getContent(mblob.getLocalBlob());
    Assert.assertTrue("input stream external", stream instanceof ExternalBlobInputStream);
    ExternalBlobInputStream extStream = (ExternalBlobInputStream) stream;
    File file = extStream.getRootFile();
    Assert.assertTrue(file.exists());
    Assert.assertTrue("stream content = mime content", TestUtil.bytesEqual(mimeBytes, stream));
}

Example 20 with StagedBlob

use of com.zimbra.cs.store.StagedBlob in project zm-mailbox by Zimbra.

the class CalendarItem method storeUpdatedBlob.

private MailboxBlob storeUpdatedBlob(MimeMessage mm) throws ServiceException, IOException {
    ParsedMessage pm = new ParsedMessage(mm, mMailbox.attachmentsIndexingEnabled());
    StoreManager sm = StoreManager.getInstance();
    InputStream is = null;
    try {
        is = pm.getRawInputStream();
        if (is != null) {
            StagedBlob sblob = sm.stage(is, mMailbox);
            return setContent(sblob, pm);
        } else {
            ZimbraLog.calendar.warn("Invalid state: updating blob with null data for calendar item " + getId() + " in mailbox " + getMailboxId());
            return setContent(null, pm);
        }
    } finally {
        ByteUtil.closeStream(is);
    }
}