Examples with Blob com.zimbra.cs.store.Blob used on opensource projects

Example 1 with Blob

use of com.zimbra.cs.store.Blob in project zm-mailbox by Zimbra.

the class Mailbox method addMessage.

public Message addMessage(OperationContext octxt, InputStream in, long sizeHint, Long receivedDate, DeliveryOptions dopt, DeliveryContext dctxt, ItemData id) throws IOException, ServiceException {
    int bufLen = Provisioning.getInstance().getLocalServer().getMailDiskStreamingThreshold();
    CopyInputStream cs = new CopyInputStream(in, sizeHint, bufLen, bufLen);
    in = cs;
    Blob blob = null;
    try {
        BufferStream bs = cs.getBufferStream();
        ParsedMessage pm = null;
        Rfc822ValidationInputStream validator = null;
        if (LC.zimbra_lmtp_validate_messages.booleanValue()) {
            validator = new Rfc822ValidationInputStream(cs, LC.zimbra_lmtp_max_line_length.longValue());
            in = validator;
        }
        blob = StoreManager.getInstance().storeIncoming(in);
        if (id != null && id.ud != null && id.ud.getBlobDigest() != null && !id.ud.getBlobDigest().isEmpty()) {
            blob.setDigest(id.ud.getBlobDigest());
        }
        if (validator != null && !validator.isValid()) {
            StoreManager.getInstance().delete(blob);
            throw ServiceException.INVALID_REQUEST("Message content is invalid.", null);
        }
        pm = new ParsedMessage(new ParsedMessageOptions(blob, bs.isPartial() ? null : bs.getBuffer(), receivedDate, attachmentsIndexingEnabled()));
        cs.release();
        if (dctxt == null) {
            dctxt = new DeliveryContext();
        }
        dctxt.setIncomingBlob(blob);
        return addMessage(octxt, pm, dopt, dctxt);
    } finally {
        cs.release();
        StoreManager.getInstance().quietDelete(blob);
    }
}

Example 2 with Blob

use of com.zimbra.cs.store.Blob in project zm-mailbox by Zimbra.

the class CreateMessage method redo.

@Override
public void redo() throws Exception {
    int mboxId = getMailboxId();
    Mailbox mbox = MailboxManager.getInstance().getMailboxById(mboxId);
    OperationContext octxt = getOperationContext();
    if (mTags == null && mTagIds != null) {
        mTags = TagUtil.tagIdStringToNames(mbox, octxt, mTagIds);
    }
    DeliveryContext dctxt = new DeliveryContext(mShared, Arrays.asList(mboxId));
    if (mMsgBodyType == MSGBODY_LINK) {
        Blob blob = StoreIncomingBlob.fetchBlob(mPath);
        if (blob == null)
            throw new RedoException("Missing link source blob " + mPath + " (digest=" + mDigest + ")", this);
        dctxt.setIncomingBlob(blob);
        ParsedMessage pm = null;
        try {
            ParsedMessageOptions opt = new ParsedMessageOptions().setContent(blob.getFile()).setReceivedDate(mReceivedDate).setAttachmentIndexing(mbox.attachmentsIndexingEnabled()).setSize(mMsgSize).setDigest(mDigest);
            pm = new ParsedMessage(opt);
            mbox.addMessage(octxt, pm, getDeliveryOptions(), dctxt);
        } catch (MailServiceException e) {
            if (e.getCode() == MailServiceException.ALREADY_EXISTS) {
                mLog.info("Message " + mMsgId + " is already in mailbox " + mboxId);
                return;
            } else {
                throw e;
            }
        } finally {
            if (pm != null) {
                ByteUtil.closeStream(pm.getBlobInputStream());
            }
        }
    } else {
        // mMsgBodyType == MSGBODY_INLINE
        // Just one recipient.  Blob data is stored inline.
        InputStream in = null;
        try {
            in = mData.getInputStream();
            if (mData.getLength() != mMsgSize) {
                in = new GZIPInputStream(in);
            }
            mbox.addMessage(octxt, in, mMsgSize, mReceivedDate, getDeliveryOptions(), dctxt);
        } catch (MailServiceException e) {
            if (e.getCode() == MailServiceException.ALREADY_EXISTS) {
                mLog.info("Message " + mMsgId + " is already in mailbox " + mboxId);
                return;
            } else {
                throw e;
            }
        } finally {
            ByteUtil.closeStream(in);
        }
    }
}

Example 3 with Blob

use of com.zimbra.cs.store.Blob in project zm-mailbox by Zimbra.

the class ZimbraLmtpBackend method deliverMessageToLocalMailboxes.

private void deliverMessageToLocalMailboxes(Blob blob, BlobInputStream bis, byte[] data, MimeMessage mm, LmtpEnvelope env) throws ServiceException, IOException {
    List<LmtpAddress> recipients = env.getLocalRecipients();
    String envSender = env.getSender().getEmailAddress();
    boolean shared = recipients.size() > 1;
    List<Integer> targetMailboxIds = new ArrayList<Integer>(recipients.size());
    Map<LmtpAddress, RecipientDetail> rcptMap = new HashMap<LmtpAddress, RecipientDetail>(recipients.size());
    try {
        // Examine attachments indexing option for all recipients and
        // prepare ParsedMessage versions needed.  Parsing is done before
        // attempting delivery to any recipient.  Therefore, parse error
        // will result in non-delivery to all recipients.
        // ParsedMessage for users with attachments indexing
        ParsedMessage pmAttachIndex = null;
        // ParsedMessage for users without attachments indexing
        ParsedMessage pmNoAttachIndex = null;
        // message id for logging
        String msgId = null;
        for (LmtpAddress recipient : recipients) {
            String rcptEmail = recipient.getEmailAddress();
            Account account;
            Mailbox mbox;
            boolean attachmentsIndexingEnabled;
            try {
                account = Provisioning.getInstance().get(AccountBy.name, rcptEmail);
                if (account == null) {
                    ZimbraLog.mailbox.warn("No account found delivering mail to " + rcptEmail);
                    continue;
                }
                mbox = MailboxManager.getInstance().getMailboxByAccount(account);
                if (mbox == null) {
                    ZimbraLog.mailbox.warn("No mailbox found delivering mail to " + rcptEmail);
                    continue;
                }
                attachmentsIndexingEnabled = mbox.attachmentsIndexingEnabled();
            } catch (ServiceException se) {
                if (se.isReceiversFault()) {
                    ZimbraLog.mailbox.info("Recoverable exception getting mailbox for " + rcptEmail, se);
                    rcptMap.put(recipient, new RecipientDetail(null, null, null, false, DeliveryAction.defer));
                } else {
                    ZimbraLog.mailbox.warn("Unrecoverable exception getting mailbox for " + rcptEmail, se);
                }
                continue;
            }
            if (account != null && mbox != null) {
                ParsedMessageOptions pmo;
                if (mm != null) {
                    pmo = new ParsedMessageOptions().setContent(mm).setDigest(blob.getDigest()).setSize(blob.getRawSize());
                } else {
                    pmo = new ParsedMessageOptions(blob, data);
                }
                ParsedMessage pm;
                if (attachmentsIndexingEnabled) {
                    if (pmAttachIndex == null) {
                        pmo.setAttachmentIndexing(true);
                        ZimbraLog.lmtp.debug("Creating ParsedMessage from %s with attachment indexing enabled", data == null ? "file" : "memory");
                        pmAttachIndex = new ParsedMessage(pmo);
                    }
                    pm = pmAttachIndex;
                } else {
                    if (pmNoAttachIndex == null) {
                        pmo.setAttachmentIndexing(false);
                        ZimbraLog.lmtp.debug("Creating ParsedMessage from %s with attachment indexing disabled", data == null ? "file" : "memory");
                        pmNoAttachIndex = new ParsedMessage(pmo);
                    }
                    pm = pmNoAttachIndex;
                }
                msgId = pm.getMessageID();
                if (account.isPrefMailLocalDeliveryDisabled()) {
                    ZimbraLog.lmtp.debug("Local delivery disabled for account %s", rcptEmail);
                    rcptMap.put(recipient, new RecipientDetail(account, mbox, pm, false, DeliveryAction.discard));
                    continue;
                }
                // For non-shared delivery (i.e. only one recipient),
                // always deliver regardless of backup mode.
                DeliveryAction da = DeliveryAction.deliver;
                boolean endSharedDelivery = false;
                if (shared) {
                    if (mbox.beginSharedDelivery()) {
                        endSharedDelivery = true;
                    } else {
                        // Skip delivery to mailboxes in backup mode.
                        da = DeliveryAction.defer;
                    }
                }
                rcptMap.put(recipient, new RecipientDetail(account, mbox, pm, endSharedDelivery, da));
                if (da == DeliveryAction.deliver) {
                    targetMailboxIds.add(mbox.getId());
                }
            }
        }
        ZimbraLog.removeAccountFromContext();
        if (ZimbraLog.lmtp.isInfoEnabled()) {
            ZimbraLog.lmtp.info("Delivering message: size=%s, nrcpts=%d, sender=%s, msgid=%s", env.getSize() == 0 ? "unspecified" : Integer.toString(env.getSize()) + " bytes", recipients.size(), env.getSender(), msgId == null ? "" : msgId);
        }
        DeliveryContext sharedDeliveryCtxt = new DeliveryContext(shared, targetMailboxIds);
        sharedDeliveryCtxt.setIncomingBlob(blob);
        // version each recipient needs.  Deliver!
        for (LmtpAddress recipient : recipients) {
            String rcptEmail = recipient.getEmailAddress();
            LmtpReply reply = LmtpReply.TEMPORARY_FAILURE;
            RecipientDetail rd = rcptMap.get(recipient);
            if (rd == null) {
                // Account or mailbox not found.
                ZimbraLog.lmtp.info("rejecting message from=%s,to=%s: account or mailbox not found", envSender, rcptEmail);
                recipient.setDeliveryStatus(LmtpReply.PERMANENT_FAILURE);
                continue;
            }
            if (rd.account != null) {
                ZimbraLog.addAccountNameToContext(rd.account.getName());
            }
            if (rd.mbox != null) {
                ZimbraLog.addMboxToContext(rd.mbox.getId());
            }
            boolean success = false;
            try {
                switch(rd.action) {
                    case discard:
                        ZimbraLog.lmtp.info("accepted and discarded message from=%s,to=%s: local delivery is disabled", envSender, rcptEmail);
                        if (rd.account.getPrefMailForwardingAddress() != null) {
                            // mail forwarding is set up
                            for (LmtpCallback callback : callbacks) {
                                ZimbraLog.lmtp.debug("Executing callback %s", callback.getClass().getName());
                                callback.forwardWithoutDelivery(rd.account, rd.mbox, envSender, rcptEmail, rd.pm);
                            }
                        }
                        reply = LmtpReply.DELIVERY_OK;
                        break;
                    case deliver:
                        Account account = rd.account;
                        Mailbox mbox = rd.mbox;
                        ParsedMessage pm = rd.pm;
                        List<ItemId> addedMessageIds = null;
                        ReentrantLock lock = mailboxDeliveryLocks.get(mbox.getId());
                        boolean acquiredLock;
                        try {
                            // Wait for the lock, up to the timeout
                            acquiredLock = lock.tryLock(LC.zimbra_mailbox_lock_timeout.intValue(), TimeUnit.SECONDS);
                        } catch (InterruptedException e) {
                            acquiredLock = false;
                        }
                        if (!acquiredLock) {
                            ZimbraLog.lmtp.info("try again for message from=%s,to=%s: another mail delivery in progress.", envSender, rcptEmail);
                            reply = LmtpReply.TEMPORARY_FAILURE;
                            break;
                        }
                        try {
                            if (dedupe(pm, mbox)) {
                                // message was already delivered to this mailbox
                                ZimbraLog.lmtp.info("Not delivering message with duplicate Message-ID %s", pm.getMessageID());
                            } else if (mbox.dedupeForSelfMsg(pm)) {
                                ZimbraLog.mailbox.info("not delivering message, because it is a duplicate of sent message %s", pm.getMessageID());
                            } else if (recipient.getSkipFilters()) {
                                msgId = pm.getMessageID();
                                int folderId = Mailbox.ID_FOLDER_INBOX;
                                if (recipient.getFolder() != null) {
                                    try {
                                        Folder folder = mbox.getFolderByPath(null, recipient.getFolder());
                                        folderId = folder.getId();
                                    } catch (ServiceException se) {
                                        if (se.getCode().equals(MailServiceException.NO_SUCH_FOLDER)) {
                                            Folder folder = mbox.createFolder(null, recipient.getFolder(), new Folder.FolderOptions().setDefaultView(MailItem.Type.MESSAGE));
                                            folderId = folder.getId();
                                        } else {
                                            throw se;
                                        }
                                    }
                                }
                                int flags = Flag.BITMASK_UNREAD;
                                if (recipient.getFlags() != null) {
                                    flags = Flag.toBitmask(recipient.getFlags());
                                }
                                DeliveryOptions dopt = new DeliveryOptions().setFolderId(folderId);
                                dopt.setFlags(flags).setTags(recipient.getTags()).setRecipientEmail(rcptEmail);
                                Message msg = mbox.addMessage(null, pm, dopt, sharedDeliveryCtxt);
                                addedMessageIds = Lists.newArrayList(new ItemId(msg));
                            } else if (!DebugConfig.disableIncomingFilter) {
                                // Get msgid first, to avoid having to reopen and reparse the blob
                                // file if Mailbox.addMessageInternal() closes it.
                                pm.getMessageID();
                                addedMessageIds = RuleManager.applyRulesToIncomingMessage(null, mbox, pm, (int) blob.getRawSize(), rcptEmail, env, sharedDeliveryCtxt, Mailbox.ID_FOLDER_INBOX, false, true);
                            } else {
                                pm.getMessageID();
                                DeliveryOptions dopt = new DeliveryOptions().setFolderId(Mailbox.ID_FOLDER_INBOX);
                                dopt.setFlags(Flag.BITMASK_UNREAD).setRecipientEmail(rcptEmail);
                                Message msg = mbox.addMessage(null, pm, dopt, sharedDeliveryCtxt);
                                addedMessageIds = Lists.newArrayList(new ItemId(msg));
                            }
                            success = true;
                            if (addedMessageIds != null && addedMessageIds.size() > 0) {
                                addToDedupeCache(pm, mbox);
                            }
                        } finally {
                            lock.unlock();
                        }
                        if (addedMessageIds != null && addedMessageIds.size() > 0) {
                            // Execute callbacks
                            for (LmtpCallback callback : callbacks) {
                                for (ItemId id : addedMessageIds) {
                                    if (id.belongsTo(mbox)) {
                                        // Message was added to the local mailbox, as opposed to a mountpoint.
                                        ZimbraLog.lmtp.debug("Executing callback %s", callback.getClass().getName());
                                        try {
                                            Message msg = mbox.getMessageById(null, id.getId());
                                            callback.afterDelivery(account, mbox, envSender, rcptEmail, msg);
                                        } catch (OutOfMemoryError oome) {
                                            Zimbra.halt("LMTP callback failed", oome);
                                        } catch (Throwable t) {
                                            ZimbraLog.lmtp.warn("LMTP callback threw an exception", t);
                                        }
                                    }
                                }
                            }
                        }
                        reply = LmtpReply.DELIVERY_OK;
                        break;
                    case defer:
                        // Delivery to mailbox skipped.  Let MTA retry again later.
                        // This case happens for shared delivery to a mailbox in
                        // backup mode.
                        ZimbraLog.lmtp.info("try again for message from=%s,to=%s: mailbox skipped", envSender, rcptEmail);
                        reply = LmtpReply.TEMPORARY_FAILURE;
                        break;
                }
            } catch (DeliveryServiceException e) {
                ZimbraLog.lmtp.info("rejecting message from=%s,to=%s: sieve filter rule", envSender, rcptEmail);
                reply = LmtpReply.PERMANENT_MESSAGE_REFUSED;
            } catch (ServiceException e) {
                if (e.getCode().equals(MailServiceException.QUOTA_EXCEEDED)) {
                    ZimbraLog.lmtp.info("rejecting message from=%s,to=%s: overquota", envSender, rcptEmail);
                    if (config.isPermanentFailureWhenOverQuota()) {
                        reply = LmtpReply.PERMANENT_FAILURE_OVER_QUOTA;
                    } else {
                        reply = LmtpReply.TEMPORARY_FAILURE_OVER_QUOTA;
                    }
                } else if (e.isReceiversFault()) {
                    ZimbraLog.lmtp.info("try again for message from=%s,to=%s", envSender, rcptEmail, e);
                    reply = LmtpReply.TEMPORARY_FAILURE;
                } else {
                    ZimbraLog.lmtp.info("rejecting message from=%s,to=%s", envSender, rcptEmail, e);
                    reply = LmtpReply.PERMANENT_FAILURE;
                }
            } catch (Exception e) {
                reply = LmtpReply.TEMPORARY_FAILURE;
                ZimbraLog.lmtp.warn("try again for message from=%s,to=%s", envSender, rcptEmail, e);
            } finally {
                if (rd.action == DeliveryAction.deliver && !success) {
                    // Message was not delivered.  Remove it from the dedupe
                    // cache so we don't dedupe it on LMTP retry.
                    removeFromDedupeCache(msgId, rd.mbox);
                }
                recipient.setDeliveryStatus(reply);
                if (shared && rd != null && rd.esd) {
                    rd.mbox.endSharedDelivery();
                    rd.esd = false;
                }
            }
        }
        // If this message is being streamed from disk, cache it
        ParsedMessage mimeSource = pmAttachIndex != null ? pmAttachIndex : pmNoAttachIndex;
        MailboxBlob mblob = sharedDeliveryCtxt.getMailboxBlob();
        if (mblob != null && mimeSource != null) {
            if (bis == null) {
                bis = mimeSource.getBlobInputStream();
            }
            if (bis != null) {
                try {
                    // Update the MimeMessage with the blob that's stored inside the mailbox,
                    // since the incoming blob will be deleted.
                    Blob storedBlob = mblob.getLocalBlob();
                    bis.fileMoved(storedBlob.getFile());
                    MessageCache.cacheMessage(mblob.getDigest(), mimeSource.getOriginalMessage(), mimeSource.getMimeMessage());
                } catch (IOException e) {
                    ZimbraLog.lmtp.warn("Unable to cache message for " + mblob, e);
                }
            }
        }
    } finally {
        // called, we check and fix those here.
        if (shared) {
            for (RecipientDetail rd : rcptMap.values()) {
                if (rd.esd && rd.mbox != null)
                    rd.mbox.endSharedDelivery();
            }
        }
    }
}

Example 4 with Blob

use of com.zimbra.cs.store.Blob in project zm-mailbox by Zimbra.

the class StoreManagerBasedTempBlobStore method store.

// BlobStore API
@Override
public StoredBlob store(IncomingBlob ib, String id, int version) throws IOException, ServiceException {
    IncomingBlob myIb = ib;
    synchronized (incomingBlobs) {
        IncomingBlob existingBlob = incomingBlobs.remove(myIb.getId());
        if (existingBlob == null) {
            // because the underlying file is gone
            throw ServiceException.INVALID_REQUEST("Cannot accept incoming blob " + myIb.getId(), null);
        }
        assert existingBlob == myIb : "Wrong blob removed: " + myIb.getId();
    }
    Blob blob = myIb.getBlob();
    StoredBlob sb = new StoredBlob(id, blob, myIb.getContext(), blob.getRawSize());
    if (storedCached) {
        synchronized (storedBlobs) {
            TreeMap<Integer, StoredBlob> versionMap = storedBlobs.get(id);
            if (versionMap == null) {
                versionMap = new TreeMap<Integer, StoredBlob>();
                storedBlobs.put(id, versionMap);
            }
            Integer versionInt = Integer.valueOf(version);
            assert !versionMap.containsKey(versionInt) : "Version " + version + " already exists";
            versionMap.put(versionInt, sb);
        }
    }
    return sb;
}

Example 5 with Blob

use of com.zimbra.cs.store.Blob in project zm-mailbox by Zimbra.

the class ZimbraMailAdapterTest method testUpdateIncomingBlob.

@Test
public void testUpdateIncomingBlob() throws Exception {
    int mboxId = 10;
    Mailbox mbox = PowerMockito.mock(Mailbox.class);
    Mockito.when(mbox.getId()).thenReturn(mboxId);
    List<Integer> targetMailboxIds = new ArrayList<Integer>(1);
    targetMailboxIds.add(mboxId);
    DeliveryContext sharedDeliveryCtxt = new DeliveryContext(true, targetMailboxIds);
    String testStr = "test";
    ParsedMessage pm = new ParsedMessage(testStr.getBytes(), false);
    IncomingMessageHandler handler = PowerMockito.mock(IncomingMessageHandler.class);
    Mockito.when(handler.getDeliveryContext()).thenReturn(sharedDeliveryCtxt);
    Mockito.when(handler.getParsedMessage()).thenReturn(pm);
    ZimbraMailAdapter mailAdapter = new ZimbraMailAdapter(mbox, handler);
    mailAdapter.updateIncomingBlob();
    Assert.assertNotNull(sharedDeliveryCtxt.getMailBoxSpecificBlob(mboxId));
    Assert.assertNull(sharedDeliveryCtxt.getIncomingBlob());
    DeliveryContext nonSharedDeliveryCtxt = new DeliveryContext(false, targetMailboxIds);
    Mockito.when(handler.getDeliveryContext()).thenReturn(nonSharedDeliveryCtxt);
    mailAdapter.updateIncomingBlob();
    Assert.assertNull(nonSharedDeliveryCtxt.getMailBoxSpecificBlob(mboxId));
    Assert.assertNotNull(nonSharedDeliveryCtxt.getIncomingBlob());
    Mockito.when(handler.getDeliveryContext()).thenReturn(sharedDeliveryCtxt);
    Blob blobFile = sharedDeliveryCtxt.getMailBoxSpecificBlob(mboxId);
    mailAdapter.cloneParsedMessage();
    mailAdapter.updateIncomingBlob();
    Assert.assertNotNull(sharedDeliveryCtxt.getMailBoxSpecificBlob(mboxId));
    Assert.assertNull(sharedDeliveryCtxt.getIncomingBlob());
    Assert.assertNotSame(blobFile, sharedDeliveryCtxt.getMailBoxSpecificBlob(mboxId));
    Mockito.when(handler.getDeliveryContext()).thenReturn(nonSharedDeliveryCtxt);
    blobFile = nonSharedDeliveryCtxt.getMailBoxSpecificBlob(mboxId);
    mailAdapter.cloneParsedMessage();
    mailAdapter.updateIncomingBlob();
    Assert.assertNull(nonSharedDeliveryCtxt.getMailBoxSpecificBlob(mboxId));
    Assert.assertNotNull(nonSharedDeliveryCtxt.getIncomingBlob());
    Assert.assertEquals(blobFile, nonSharedDeliveryCtxt.getMailBoxSpecificBlob(mboxId));
}