Examples with DeferredSerialization org.voltcore.utils.DeferredSerialization used on opensource projects

Example 1 with DeferredSerialization

use of org.voltcore.utils.DeferredSerialization in project voltdb by VoltDB.

the class TLSNIOWriteStream method serializeQueuedWrites.

@Override
int serializeQueuedWrites(NetworkDBBPool pool) throws IOException {
    checkForGatewayExceptions();
    final int frameMax = Math.min(CipherExecutor.FRAME_SIZE, applicationBufferSize());
    int processedWrites = 0;
    final Deque<DeferredSerialization> oldlist = getQueuedWrites();
    if (oldlist.isEmpty())
        return 0;
    ByteBuf accum = m_ce.allocator().buffer(frameMax).clear();
    DeferredSerialization ds = null;
    int bytesQueued = 0;
    int frameMsgs = 0;
    while ((ds = oldlist.poll()) != null) {
        ++processedWrites;
        final int serializedSize = ds.getSerializedSize();
        if (serializedSize == DeferredSerialization.EMPTY_MESSAGE_LENGTH)
            continue;
        // the encryption gateway
        if (serializedSize > frameMax) {
            // messages and an incomplete partial fragment of one
            if (accum.writerIndex() > 0) {
                m_ecryptgw.offer(new EncryptFrame(accum, frameMsgs));
                frameMsgs = 0;
                bytesQueued += accum.writerIndex();
                accum = m_ce.allocator().buffer(frameMax).clear();
            }
            ByteBuf big = m_ce.allocator().buffer(serializedSize).writerIndex(serializedSize);
            ByteBuffer jbb = big.nioBuffer();
            ds.serialize(jbb);
            checkSloppySerialization(jbb, ds);
            bytesQueued += big.writerIndex();
            m_ecryptgw.offer(new EncryptFrame(big, 1));
            frameMsgs = 0;
            continue;
        } else if (accum.writerIndex() + serializedSize > frameMax) {
            m_ecryptgw.offer(new EncryptFrame(accum, frameMsgs));
            frameMsgs = 0;
            bytesQueued += accum.writerIndex();
            accum = m_ce.allocator().buffer(frameMax).clear();
        }
        ByteBuf packet = accum.slice(accum.writerIndex(), serializedSize);
        ByteBuffer jbb = packet.nioBuffer();
        ds.serialize(jbb);
        checkSloppySerialization(jbb, ds);
        accum.writerIndex(accum.writerIndex() + serializedSize);
        ++frameMsgs;
    }
    if (accum.writerIndex() > 0) {
        m_ecryptgw.offer(new EncryptFrame(accum, frameMsgs));
        bytesQueued += accum.writerIndex();
    } else {
        accum.release();
    }
    updateQueued(bytesQueued, true);
    return processedWrites;
}

Example 2 with DeferredSerialization

use of org.voltcore.utils.DeferredSerialization in project voltdb by VoltDB.

the class TLSNIOWriteStream method shutdown.

@Override
synchronized void shutdown() {
    m_isShutdown = true;
    try {
        DeferredSerialization ds = null;
        while ((ds = m_queuedWrites.poll()) != null) {
            ds.cancel();
        }
        int waitFor = 1 - Math.min(m_inFlight.availablePermits(), -4);
        for (int i = 0; i < waitFor; ++i) {
            try {
                if (m_inFlight.tryAcquire(1, TimeUnit.SECONDS)) {
                    m_inFlight.release();
                    break;
                }
            } catch (InterruptedException e) {
                break;
            }
        }
        m_ecryptgw.die();
        EncryptFrame frame = null;
        while ((frame = m_encrypted.poll()) != null) {
            frame.frame.release();
        }
        for (EncryptFrame ef : m_partial) {
            ef.frame.release();
        }
        m_partial.clear();
        m_outbuf.release();
        // we have to use ledger because we have no idea how much encrypt delta
        // corresponds to what is left in the output buffer
        final int unqueue = -m_queuedBytes;
        updateQueued(unqueue, false);
    } finally {
        m_inFlight.drainPermits();
        m_inFlight.release();
    }
}

Example 3 with DeferredSerialization

use of org.voltcore.utils.DeferredSerialization in project voltdb by VoltDB.

the class NIOWriteStream method enqueue.

/**
     * Queue a ByteBuffer for writing to the network. If the ByteBuffer is not direct then it will
     * be copied to a DirectByteBuffer if it is less then DBBPool.MAX_ALLOCATION_SIZE. This method
     * is a backup for code that isn't able to defer its serialization to a network thread
     * for whatever reason. It is reasonably efficient if a DirectByteBuffer is passed in,
     * but it would be better to keep allocations of DirectByteBuffers inside the network pools.
     * @param b
     */
@Override
public void enqueue(final ByteBuffer[] b) {
    assert (b != null);
    for (ByteBuffer buf : b) {
        //Don't queue direct buffers, they leak memory without a container
        assert (!buf.isDirect());
        if (buf.remaining() == 0) {
            throw new IllegalArgumentException("Attempted to queue a zero length buffer");
        }
    }
    synchronized (this) {
        if (m_isShutdown) {
            return;
        }
        updateLastPendingWriteTimeAndQueueBackpressure();
        m_queuedWrites.offer(new DeferredSerialization() {

            @Override
            public void serialize(ByteBuffer outbuf) {
                for (ByteBuffer buf : b) {
                    outbuf.put(buf);
                }
            }

            @Override
            public void cancel() {
            }

            @Override
            public int getSerializedSize() {
                int sum = 0;
                for (ByteBuffer buf : b) {
                    buf.position(0);
                    sum += buf.remaining();
                }
                return sum;
            }
        });
        m_port.setInterests(SelectionKey.OP_WRITE, 0);
    }
}

Example 4 with DeferredSerialization

use of org.voltcore.utils.DeferredSerialization in project voltdb by VoltDB.

the class NIOWriteStreamBase method serializeQueuedWrites.

/**
     * Serialize all queued writes into the queue of pending buffers, which are allocated from
     * thread local memory pool.
     * @return number of queued writes processed
     * @throws IOException
     */
int serializeQueuedWrites(final NetworkDBBPool pool) throws IOException {
    int processedWrites = 0;
    final Deque<DeferredSerialization> oldlist = getQueuedWrites();
    if (oldlist.isEmpty())
        return 0;
    DeferredSerialization ds = null;
    int bytesQueued = 0;
    while ((ds = oldlist.poll()) != null) {
        processedWrites++;
        final int serializedSize = ds.getSerializedSize();
        if (serializedSize == DeferredSerialization.EMPTY_MESSAGE_LENGTH)
            continue;
        BBContainer outCont = m_queuedBuffers.peekLast();
        ByteBuffer outbuf = null;
        if (outCont == null || !outCont.b().hasRemaining()) {
            outCont = pool.acquire();
            outCont.b().clear();
            m_queuedBuffers.offer(outCont);
        }
        outbuf = outCont.b();
        if (outbuf.remaining() >= serializedSize) {
            // Fast path, serialize to direct buffer creating no garbage
            final int oldLimit = outbuf.limit();
            outbuf.limit(outbuf.position() + serializedSize);
            final ByteBuffer slice = outbuf.slice();
            ds.serialize(slice);
            checkSloppySerialization(slice, ds);
            slice.position(0);
            bytesQueued += slice.remaining();
            outbuf.position(outbuf.limit());
            outbuf.limit(oldLimit);
        } else {
            // Slow path serialize to heap, and then put in buffers
            ByteBuffer buf = ByteBuffer.allocate(serializedSize);
            ds.serialize(buf);
            checkSloppySerialization(buf, ds);
            buf.position(0);
            bytesQueued += buf.remaining();
            // Copy data allocated in heap buffer to direct buffer
            while (buf.hasRemaining()) {
                if (!outbuf.hasRemaining()) {
                    outCont = pool.acquire();
                    outbuf = outCont.b();
                    outbuf.clear();
                    m_queuedBuffers.offer(outCont);
                }
                if (outbuf.remaining() >= buf.remaining()) {
                    outbuf.put(buf);
                } else {
                    final int oldLimit = buf.limit();
                    buf.limit(buf.position() + outbuf.remaining());
                    outbuf.put(buf);
                    buf.limit(oldLimit);
                }
            }
        }
    }
    updateQueued(bytesQueued, true);
    return processedWrites;
}

Example 5 with DeferredSerialization

use of org.voltcore.utils.DeferredSerialization in project voltdb by VoltDB.

the class PicoNIOWriteStream method shutdown.

/**
     * Free the pool resources that are held by this WriteStream. The pool itself is thread local
     * and will be freed when the thread terminates.
     */
@Override
synchronized void shutdown() {
    super.shutdown();
    DeferredSerialization ds = null;
    while ((ds = m_queuedWrites.poll()) != null) {
        ds.cancel();
    }
}