Examples with BufferStream com.zimbra.common.util.BufferStream used on opensource projects

Example 1 with BufferStream

use of com.zimbra.common.util.BufferStream in project zm-mailbox by Zimbra.

the class MessageContent method readContent.

private void readContent(InputStream is, int sizeHint) throws IOException, ServiceException {
    if (sizeHint < StoreManager.getDiskStreamingThreshold()) {
        BufferStream bs = new BufferStream(sizeHint);
        long realSize = bs.readFrom(is);
        if (realSize != sizeHint) {
        // ZimbraLog.datasource.debug("Content size mismatch: expected %d but got %d bytes", sizeHint, realSize);
        }
        data = bs.toByteArray();
        bs.close();
    } else {
        blob = StoreManager.getInstance().storeIncoming(is);
    }
}

Example 2 with BufferStream

use of com.zimbra.common.util.BufferStream in project zm-mailbox by Zimbra.

the class Mailbox method addMessage.

public Message addMessage(OperationContext octxt, InputStream in, long sizeHint, Long receivedDate, DeliveryOptions dopt, DeliveryContext dctxt, ItemData id) throws IOException, ServiceException {
    int bufLen = Provisioning.getInstance().getLocalServer().getMailDiskStreamingThreshold();
    CopyInputStream cs = new CopyInputStream(in, sizeHint, bufLen, bufLen);
    in = cs;
    Blob blob = null;
    try {
        BufferStream bs = cs.getBufferStream();
        ParsedMessage pm = null;
        Rfc822ValidationInputStream validator = null;
        if (LC.zimbra_lmtp_validate_messages.booleanValue()) {
            validator = new Rfc822ValidationInputStream(cs, LC.zimbra_lmtp_max_line_length.longValue());
            in = validator;
        }
        blob = StoreManager.getInstance().storeIncoming(in);
        if (id != null && id.ud != null && id.ud.getBlobDigest() != null && !id.ud.getBlobDigest().isEmpty()) {
            blob.setDigest(id.ud.getBlobDigest());
        }
        if (validator != null && !validator.isValid()) {
            StoreManager.getInstance().delete(blob);
            throw ServiceException.INVALID_REQUEST("Message content is invalid.", null);
        }
        pm = new ParsedMessage(new ParsedMessageOptions(blob, bs.isPartial() ? null : bs.getBuffer(), receivedDate, attachmentsIndexingEnabled()));
        cs.release();
        if (dctxt == null) {
            dctxt = new DeliveryContext();
        }
        dctxt.setIncomingBlob(blob);
        return addMessage(octxt, pm, dopt, dctxt);
    } finally {
        cs.release();
        StoreManager.getInstance().quietDelete(blob);
    }
}

Example 3 with BufferStream

use of com.zimbra.common.util.BufferStream in project zm-mailbox by Zimbra.

the class ArchiveFormatter method saveItem.

private ArchiveOutputStream saveItem(UserServletContext context, MailItem mi, Map<Integer, String> fldrs, Map<Integer, Integer> cnts, boolean version, ArchiveOutputStream aos, CharsetEncoder charsetEncoder, Set<String> names) throws ServiceException {
    String ext = null, name = null;
    String extra = null;
    Integer fid = mi.getFolderId();
    String fldr;
    InputStream is = null;
    String metaParam = context.params.get(UserServlet.QP_META);
    boolean meta = metaParam == null ? getDefaultMeta() : !metaParam.equals("0");
    if (!version && mi.isTagged(Flag.FlagInfo.VERSIONED)) {
        for (MailItem rev : context.targetMailbox.getAllRevisions(context.opContext, mi.getId(), mi.getType())) {
            if (mi.getVersion() != rev.getVersion())
                aos = saveItem(context, rev, fldrs, cnts, true, aos, charsetEncoder, names);
        }
    }
    switch(mi.getType()) {
        case APPOINTMENT:
            Appointment appt = (Appointment) mi;
            if (!appt.isPublic() && !appt.allowPrivateAccess(context.getAuthAccount(), context.isUsingAdminPrivileges())) {
                return aos;
            }
            if (meta) {
                name = appt.getSubject();
                ext = "appt";
            } else {
                ext = "ics";
            }
            break;
        case CHAT:
            ext = "chat";
            break;
        case CONTACT:
            Contact ct = (Contact) mi;
            name = ct.getFileAsString();
            if (!meta) {
                ext = "vcf";
            }
            break;
        case FLAG:
            return aos;
        case FOLDER:
        case MOUNTPOINT:
        case SEARCHFOLDER:
            if (mi.getId() == Mailbox.ID_FOLDER_ROOT) {
                name = "ROOT";
            } else if (mi.getId() == Mailbox.ID_FOLDER_USER_ROOT) {
                name = "USER_ROOT";
            } else {
                name = mi.getName();
            }
            break;
        case MESSAGE:
            Message msg = (Message) mi;
            if (msg.hasCalendarItemInfos()) {
                Set<ItemId> calItems = Sets.newHashSet();
                for (Iterator<CalendarItemInfo> it = msg.getCalendarItemInfoIterator(); it.hasNext(); ) {
                    ItemId iid = it.next().getCalendarItemId();
                    if (iid != null) {
                        calItems.add(iid);
                    }
                }
                for (ItemId i : calItems) {
                    if (extra == null) {
                        extra = "calendar=" + i.toString();
                    } else {
                        extra += ',' + i.toString();
                    }
                }
            }
            ext = "eml";
            break;
        case NOTE:
            ext = "note";
            break;
        case TASK:
            Task task = (Task) mi;
            if (!task.isPublic() && !task.allowPrivateAccess(context.getAuthAccount(), context.isUsingAdminPrivileges())) {
                return aos;
            }
            ext = "task";
            break;
        case VIRTUAL_CONVERSATION:
            return aos;
        case WIKI:
            ext = "wiki";
            break;
    }
    fldr = fldrs.get(fid);
    if (fldr == null) {
        Folder f = mi.getMailbox().getFolderById(context.opContext, fid);
        cnts.put(fid, 1);
        fldr = f.getPath();
        if (fldr.startsWith("/")) {
            fldr = fldr.substring(1);
        }
        fldr = sanitize(fldr, charsetEncoder);
        fldr = ILLEGAL_FOLDER_CHARS.matcher(fldr).replaceAll("_");
        fldrs.put(fid, fldr);
    } else if (!(mi instanceof Folder)) {
        final int BATCH = 500;
        int cnt = cnts.get(fid) + 1;
        cnts.put(fid, cnt);
        cnt /= BATCH;
        if (cnt > 0) {
            fldr = fldr + '!' + cnt;
        }
    }
    int targetBaseLength = 0;
    if (context.noHierarchy()) {
        // Parent hierarchy is not needed, so construct the folder names without parent hierarchy.
        // e.g> represent "inbox/subfolder/target" as "target".
        String targetPath = null;
        if (context.itemPath.endsWith("/")) {
            // inbox/subfolder/target/
            targetPath = context.itemPath.substring(0, context.itemPath.lastIndexOf("/"));
        } else {
            // inbox/subfolder/target
            targetPath = context.itemPath;
        }
        // "inbox/subfolder".length()
        targetBaseLength = targetPath.lastIndexOf('/');
        if (targetBaseLength >= fldr.length()) {
            // fldr is "inbox/subfolder"
            fldr = "";
        } else if (targetBaseLength > 0) {
            // fldr is "inbox/subfolder/target"
            fldr = fldr.substring(targetBaseLength + 1);
        }
    }
    try {
        ArchiveOutputEntry aoe;
        byte[] data = null;
        String path = mi instanceof Contact ? getEntryName(mi, fldr, name, ext, charsetEncoder, names) : getEntryName(mi, fldr, name, ext, charsetEncoder, !(mi instanceof Document));
        long miSize = mi.getSize();
        if (miSize == 0 && mi.getDigest() != null) {
            ZimbraLog.misc.debug("blob db size 0 for item %d", mi.getId());
            return aos;
        }
        try {
            is = mi.getContentStream();
        } catch (Exception e) {
            ZimbraLog.misc.error("missing blob for item %d: expected %d", mi.getId(), miSize);
            return aos;
        }
        if (aos == null) {
            aos = getOutputStream(context, charsetEncoder.charset().name());
        }
        if ((mi instanceof CalendarItem) && (context.getStartTime() != TIME_UNSPECIFIED || context.getEndTime() != TIME_UNSPECIFIED)) {
            Collection<Instance> instances = ((CalendarItem) mi).expandInstances(context.getStartTime(), context.getEndTime(), false);
            if (instances.isEmpty()) {
                return aos;
            }
        }
        aoe = aos.newOutputEntry(path + ".meta", mi.getType().toString(), mi.getType().toByte(), mi.getDate());
        if (mi instanceof Message && (mi.getFlagBitmask() & Flag.ID_UNREAD) != 0) {
            aoe.setUnread();
        }
        if (meta) {
            ItemData itemData = new ItemData(mi, extra);
            if (context.noHierarchy()) {
                // itemData.path is of the form /Inbox/subfolder/target and after this step it becomes /target.
                if (targetBaseLength > 0 && ((targetBaseLength + 1) < itemData.path.length())) {
                    itemData.path = itemData.path.substring(targetBaseLength + 1);
                }
            }
            byte[] metaData = itemData.encode();
            aoe.setSize(metaData.length);
            aos.putNextEntry(aoe);
            aos.write(metaData);
            aos.closeEntry();
        } else if (mi instanceof CalendarItem) {
            Browser browser = HttpUtil.guessBrowser(context.req);
            List<CalendarItem> calItems = new ArrayList<CalendarItem>();
            boolean needAppleICalHacks = Browser.APPLE_ICAL.equals(browser);
            boolean useOutlookCompatMode = Browser.IE.equals(browser);
            OperationContext octxt = new OperationContext(context.getAuthAccount(), context.isUsingAdminPrivileges());
            StringWriter writer = new StringWriter();
            calItems.add((CalendarItem) mi);
            context.targetMailbox.writeICalendarForCalendarItems(writer, octxt, calItems, useOutlookCompatMode, true, needAppleICalHacks, true);
            data = writer.toString().getBytes(charsetEncoder.charset());
        } else if (mi instanceof Contact) {
            VCard vcf = VCard.formatContact((Contact) mi);
            data = vcf.getFormatted().getBytes(charsetEncoder.charset());
        } else if (mi instanceof Message) {
            if (context.hasPart()) {
                MimeMessage mm = ((Message) mi).getMimeMessage();
                Set<String> attachmentNames = new HashSet<String>();
                for (String part : context.getPart().split(",")) {
                    BufferStream bs;
                    MimePart mp = Mime.getMimePart(mm, part);
                    long sz;
                    if (mp == null) {
                        throw MailServiceException.NO_SUCH_PART(part);
                    }
                    name = Mime.getFilename(mp);
                    ext = null;
                    sz = mp.getSize();
                    if (sz == -1) {
                        sz = miSize;
                    }
                    if (name == null) {
                        name = "attachment";
                    } else {
                        int dot = name.lastIndexOf('.');
                        if (dot != -1 && dot < name.length() - 1) {
                            ext = name.substring(dot + 1);
                            name = name.substring(0, dot);
                        }
                    }
                    bs = new BufferStream(sz, 1024 * 1024);
                    InputStream stream = mp.getInputStream();
                    try {
                        bs.readFrom(stream);
                    } finally {
                        // close the stream, it could be an instance of PipedInputStream.
                        ByteUtil.closeStream(stream);
                    }
                    aoe = aos.newOutputEntry(getEntryName(mi, "", name, ext, charsetEncoder, attachmentNames), mi.getType().toString(), mi.getType().toByte(), mi.getDate());
                    sz = bs.getSize();
                    aoe.setSize(sz);
                    aos.putNextEntry(aoe);
                    bs.copyTo(aos.getOutputStream());
                    bs.close();
                    aos.closeEntry();
                }
                return aos;
            }
        }
        aoe = aos.newOutputEntry(path, mi.getType().toString(), mi.getType().toByte(), mi.getDate());
        if (data != null) {
            aoe.setSize(data.length);
            aos.putNextEntry(aoe);
            aos.write(data);
            aos.closeEntry();
        } else if (is != null) {
            if (context.shouldReturnBody()) {
                byte[] buf = new byte[aos.getRecordSize() * 20];
                int in;
                long remain = miSize;
                aoe.setSize(miSize);
                aos.putNextEntry(aoe);
                while (remain > 0 && (in = is.read(buf)) >= 0) {
                    aos.write(buf, 0, remain < in ? (int) remain : in);
                    remain -= in;
                }
                if (remain != 0) {
                    ZimbraLog.misc.error("mismatched blob size for item %d: expected %d", mi.getId(), miSize);
                    if (remain > 0) {
                        Arrays.fill(buf, (byte) ' ');
                        while (remain > 0) {
                            aos.write(buf, 0, remain < buf.length ? (int) remain : buf.length);
                            remain -= buf.length;
                        }
                    }
                    aos.closeEntry();
                    aoe = aos.newOutputEntry(path + ".err", mi.getType().toString(), mi.getType().toByte(), mi.getDate());
                    aoe.setSize(0);
                    aos.putNextEntry(aoe);
                }
            } else {
                // Read headers into memory to compute size
                byte[] headerData = HeadersOnlyInputStream.getHeaders(is);
                aoe.setSize(headerData.length);
                aos.putNextEntry(aoe);
                aos.write(headerData);
            }
            aos.closeEntry();
        }
    } catch (Exception e) {
        throw ServiceException.FAILURE("archive error", e);
    } finally {
        ByteUtil.closeStream(is);
    }
    return aos;
}

Example 4 with BufferStream

use of com.zimbra.common.util.BufferStream in project zm-mailbox by Zimbra.

the class ZimletProps2JsServlet method getBuffer.

protected byte[] getBuffer(HttpServletRequest req, Locale locale, String uri) throws IOException {
    BufferStream bos = new BufferStream(24 * 1024);
    DataOutputStream out = new DataOutputStream(bos);
    out.writeBytes("// Locale: " + Props2Js.getCommentSafeString(locale.toString()) + '\n');
    // tokenize the list of patterns
    List<String> patternsList = this.getBasenamePatternsList(req);
    List<List<String>> basenamePatterns = new LinkedList<List<String>>();
    for (String patterns : patternsList) {
        StringTokenizer tokenizer = new StringTokenizer(patterns, ",");
        List<String> basenamesList = new LinkedList<String>();
        basenamePatterns.add(basenamesList);
        while (tokenizer.hasMoreTokens()) {
            String pattern = tokenizer.nextToken().trim();
            basenamesList.add(pattern);
        }
    }
    // This gets the base directory for the resource bundle
    // basename. For example, if the URI is:
    //
    //   .../messages/I18nMsg.js
    //
    // then the basedir is "/messages/" and if the URI is:
    //
    //   .../keys/ZmKeys.js
    //
    // then the basedir is "/keys/".
    //
    // NOTE: The <url-pattern>s in the web.xml file restricts
    //       which URLs map to this servlet so there's no risk
    //       that the basedir will be other than what we expect.
    int lastSlash = uri.lastIndexOf('/');
    int prevSlash = uri.substring(0, lastSlash).lastIndexOf('/');
    String basedir = uri.substring(prevSlash, lastSlash + 1);
    String dirname = this.getDirPath("");
    String filenames = uri.substring(uri.lastIndexOf('/') + 1);
    String classnames = filenames.substring(0, filenames.indexOf('.'));
    StringTokenizer tokenizer = new StringTokenizer(classnames, ",");
    if (ZimbraLog.zimlet.isDebugEnabled()) {
        for (List<String> basenames : basenamePatterns) {
            ZimbraLog.zimlet.debug("!!! basenames: %s", basenames);
        }
        ZimbraLog.zimlet.debug("!!! basedir: %s", basedir);
    }
    while (tokenizer.hasMoreTokens()) {
        String classname = tokenizer.nextToken();
        if (ZimbraLog.zimlet.isDebugEnabled()) {
            ZimbraLog.zimlet.debug("!!! classname: %s", classname);
        }
        load(req, out, locale, basenamePatterns, basedir, dirname, classname);
    }
    return bos.toByteArray();
}

Example 5 with BufferStream

use of com.zimbra.common.util.BufferStream in project zm-mailbox by Zimbra.

the class SoapServlet method doWork.

private void doWork(HttpServletRequest req, HttpServletResponse resp) throws IOException {
    int len = req.getContentLength();
    byte[] buffer;
    boolean isResumed = true;
    // resuming from a Jetty Continuation does *not* reset the HttpRequest's input stream -
    // therefore we store the read buffer in the Continuation, and use the stored buffer
    // if we're resuming
    buffer = (byte[]) req.getAttribute("com.zimbra.request.buffer");
    if (buffer == null) {
        isResumed = false;
        // Look up max request size
        int maxSize = 0;
        try {
            maxSize = Provisioning.getInstance().getLocalServer().getIntAttr(Provisioning.A_zimbraSoapRequestMaxSize, 0);
        } catch (ServiceException e) {
            ZimbraLog.soap.warn("Unable to look up %s.  Not limiting the request size.", Provisioning.A_zimbraSoapRequestMaxSize, e);
        }
        if (maxSize <= 0) {
            maxSize = Integer.MAX_VALUE;
        }
        // Read the request
        boolean success;
        if (len > maxSize) {
            success = false;
        } else {
            BufferStream bs = null;
            try {
                bs = new BufferStream(len, maxSize, maxSize);
                int in = (int) bs.readFrom(req.getInputStream(), len >= 0 ? len : Integer.MAX_VALUE);
                if (len > 0 && in < len)
                    throw new EOFException("SOAP content truncated " + in + "!=" + len);
                success = in <= maxSize;
                buffer = bs.toByteArray();
            } finally {
                ByteUtil.closeStream(bs);
            }
        }
        // Handle requests that exceed the size limit
        if (!success) {
            String sizeString = (len < 0 ? "" : " size " + len);
            String msg = String.format("Request%s exceeded limit of %d bytes set for %s.", sizeString, maxSize, Provisioning.A_zimbraSoapRequestMaxSize);
            ServiceException e = ServiceException.INVALID_REQUEST(msg, null);
            ZimbraLog.soap.warn(null, e);
            Element fault = SoapProtocol.Soap12.soapFault(e);
            Element envelope = SoapProtocol.Soap12.soapEnvelope(fault);
            sendResponse(req, resp, envelope);
            return;
        }
        req.setAttribute("com.zimbra.request.buffer", buffer);
    }
    HashMap<String, Object> context = new HashMap<String, Object>();
    context.put(SERVLET_CONTEXT, getServletContext());
    context.put(SERVLET_REQUEST, req);
    context.put(SERVLET_RESPONSE, resp);
    try {
        Boolean isAdminReq = isAdminRequest(req);
        context.put(IS_ADMIN_REQUEST, isAdminReq);
    } catch (ServiceException se) {
        ZimbraLog.soap.warn("unable to determine isAdminReq", se);
    }
    // setup IPs in the context and add to logging context
    RemoteIP remoteIp = new RemoteIP(req, ZimbraServlet.getTrustedIPs());
    context.put(SoapEngine.SOAP_REQUEST_IP, remoteIp.getClientIP());
    context.put(SoapEngine.ORIG_REQUEST_IP, remoteIp.getOrigIP());
    context.put(SoapEngine.REQUEST_IP, remoteIp.getRequestIP());
    remoteIp.addToLoggingContext();
    //checkAuthToken(req.getCookies(), context);
    context.put(SoapEngine.REQUEST_PORT, req.getServerPort());
    Element envelope = null;
    try {
        envelope = mEngine.dispatch(req.getRequestURI(), buffer, context);
        if (context.containsKey(INVALIDATE_COOKIES)) {
            ZAuthToken.clearCookies(resp);
        }
    } catch (Throwable e) {
        if (e instanceof OutOfMemoryError) {
            Zimbra.halt("handler exception", e);
        }
        if (ZimbraLog.soap.isTraceEnabled() && !context.containsKey(SoapEngine.SOAP_REQUEST_LOGGED)) {
            ZimbraLog.soap.trace(!isResumed ? "C:\n%s" : "C: (resumed)\n%s", new String(buffer, Charsets.UTF_8));
        }
        // don't interfere with Jetty Continuations -- pass the exception right up
        if (e.getClass().getName().equals("org.eclipse.jetty.continuation.ContinuationThrowable"))
            throw (Error) e;
        ZimbraLog.soap.warn("handler exception", e);
        Element fault = SoapProtocol.Soap12.soapFault(ServiceException.FAILURE(e.toString(), e));
        envelope = SoapProtocol.Soap12.soapEnvelope(fault);
    }
    if (ZimbraLog.soap.isTraceEnabled()) {
        ZimbraLog.soap.trace("S:\n%s", envelope.prettyPrint());
    }
    sendResponse(req, resp, envelope);
}