Examples with BBContainer org.voltcore.utils.DBBPool.BBContainer used on opensource projects

Example 46 with BBContainer

use of org.voltcore.utils.DBBPool.BBContainer in project voltdb by VoltDB.

the class TableSaveFile method close.

public void close() throws IOException {
    Thread chunkReader;
    synchronized (this) {
        m_hasMoreChunks = false;
        chunkReader = m_chunkReaderThread;
    }
    if (chunkReader != null) {
        chunkReader.interrupt();
        try {
            chunkReader.join();
        } catch (InterruptedException e) {
            throw new IOException(e);
        }
    }
    synchronized (this) {
        while (!m_availableChunks.isEmpty()) {
            m_availableChunks.poll().discard();
        }
        notifyAll();
    }
    /*
         * Free buffers used to pull snapshot data in process
         */
    BBContainer cont;
    while ((cont = m_buffers.poll()) != null) {
        cont.discard();
    }
}

Example 47 with BBContainer

use of org.voltcore.utils.DBBPool.BBContainer in project voltdb by VoltDB.

the class TableSaveFile method getNextChunk.

// Will get the next chunk of the table that is just over the chunk size
public synchronized BBContainer getNextChunk() throws IOException {
    if (m_chunkReaderException != null) {
        throw m_chunkReaderException;
    }
    if (!m_hasMoreChunks) {
        final Container c = m_availableChunks.poll();
        return c;
    }
    if (m_chunkReader == null) {
        m_chunkReader = new ChunkReader();
        m_chunkReaderThread = new Thread(m_chunkReader, "ChunkReader");
        m_chunkReaderThread.start();
    }
    Container c = null;
    while (c == null && (m_hasMoreChunks || !m_availableChunks.isEmpty())) {
        c = m_availableChunks.poll();
        if (c == null) {
            try {
                wait();
            } catch (InterruptedException e) {
                throw new IOException(e);
            }
        }
    }
    if (c != null) {
        m_chunkReads.release();
    } else {
        if (m_chunkReaderException != null) {
            throw m_chunkReaderException;
        }
    }
    return c;
}

Example 48 with BBContainer

use of org.voltcore.utils.DBBPool.BBContainer in project voltdb by VoltDB.

the class CSVSnapshotFilter method filter.

@Override
public Callable<BBContainer> filter(final Callable<BBContainer> input) {
    return new Callable<BBContainer>() {

        @Override
        public BBContainer call() throws Exception {
            BBContainer cont = input.call();
            if (cont == null) {
                return null;
            }
            try {
                ByteBuffer buf = ByteBuffer.allocate(m_schemaBytes.length + cont.b().remaining() - 4);
                buf.put(m_schemaBytes);
                cont.b().position(4);
                buf.put(cont.b());
                VoltTable vt = PrivateVoltTableFactory.createVoltTableFromBuffer(buf, true);
                Pair<Integer, byte[]> p = VoltTableUtil.toCSV(vt, m_columnTypes, m_delimiter, m_fullDelimiters, m_lastNumCharacters);
                m_lastNumCharacters = p.getFirst();
                final BBContainer origin = cont;
                cont = null;
                return new BBContainer(ByteBuffer.wrap(p.getSecond())) {

                    @Override
                    public void discard() {
                        checkDoubleFree();
                        origin.discard();
                    }
                };
            } finally {
                if (cont != null) {
                    cont.discard();
                }
            }
        }
    };
}

Example 49 with BBContainer

use of org.voltcore.utils.DBBPool.BBContainer in project voltdb by VoltDB.

the class ExportDataSource method pushExportBufferImpl.

private void pushExportBufferImpl(long uso, ByteBuffer buffer, boolean sync, boolean endOfStream, boolean poll) throws Exception {
    final java.util.concurrent.atomic.AtomicBoolean deleted = new java.util.concurrent.atomic.AtomicBoolean(false);
    if (endOfStream) {
        assert (!m_endOfStream);
        assert (buffer == null);
        assert (!sync);
        m_endOfStream = endOfStream;
        if (m_committedBuffers.isEmpty()) {
            exportLog.info("Pushed EOS buffer with 0 bytes remaining");
            if (m_pollFuture != null) {
                m_pollFuture.set(null);
                m_pollFuture = null;
            }
            if (m_onDrain != null) {
                m_drainTraceForDebug = new Exception("Push USO " + uso + " endOfStream " + endOfStream + " poll " + poll);
                m_onDrain.run();
            }
        } else {
            exportLog.info("EOS for " + m_tableName + " partition " + m_partitionId + " with first unpolled uso " + m_firstUnpolledUso + " and remaining bytes " + m_committedBuffers.sizeInBytes());
        }
        return;
    }
    assert (!m_endOfStream);
    if (buffer != null) {
        //the USO in stream block
        if (buffer.capacity() > 8) {
            final BBContainer cont = DBBPool.wrapBB(buffer);
            if (m_lastReleaseOffset > 0 && m_lastReleaseOffset >= (uso + (buffer.capacity() - 8))) {
                //What ack from future is known?
                if (exportLog.isDebugEnabled()) {
                    exportLog.debug("Dropping already acked USO: " + m_lastReleaseOffset + " Buffer info: " + uso + " Size: " + buffer.capacity());
                }
                cont.discard();
                return;
            }
            try {
                m_committedBuffers.offer(new StreamBlock(new BBContainer(buffer) {

                    @Override
                    public void discard() {
                        checkDoubleFree();
                        cont.discard();
                        deleted.set(true);
                    }
                }, uso, false));
            } catch (IOException e) {
                VoltDB.crashLocalVoltDB("Unable to write to export overflow.", true, e);
            }
        } else {
            /*
                 * TupleStreamWrapper::setBytesUsed propagates the USO by sending
                 * over an empty stream block. The block will be deleted
                 * on the native side when this method returns
                 */
            exportLog.info("Syncing first unpolled USO to " + uso + " for table " + m_tableName + " partition " + m_partitionId);
            m_firstUnpolledUso = uso;
        }
    }
    if (sync) {
        try {
            //Don't do a real sync, just write the in memory buffers
            //to a file. @Quiesce or blocking snapshot will do the sync
            m_committedBuffers.sync(true);
        } catch (IOException e) {
            VoltDB.crashLocalVoltDB("Unable to write to export overflow.", true, e);
        }
    }
    if (poll) {
        pollImpl(m_pollFuture);
    }
}

Example 50 with BBContainer

use of org.voltcore.utils.DBBPool.BBContainer in project voltdb by VoltDB.

the class DefaultSnapshotDataTarget method write.

/*
     * Prepend length is basically synonymous with writing actual tuple data and not
     * the header.
     */
private ListenableFuture<?> write(final Callable<BBContainer> tupleDataC, final boolean prependLength) {
    /*
         * Unwrap the data to be written. For the traditional
         * snapshot data target this should be a noop.
         */
    BBContainer tupleDataTemp;
    try {
        tupleDataTemp = tupleDataC.call();
        /*
             * Can be null if the dedupe filter nulled out the buffer
             */
        if (tupleDataTemp == null) {
            return Futures.immediateFuture(null);
        }
    } catch (Throwable t) {
        return Futures.immediateFailedFuture(t);
    }
    final BBContainer tupleDataCont = tupleDataTemp;
    if (m_writeFailed) {
        tupleDataCont.discard();
        return null;
    }
    ByteBuffer tupleData = tupleDataCont.b();
    m_outstandingWriteTasks.incrementAndGet();
    Future<BBContainer> compressionTask = null;
    if (prependLength) {
        BBContainer cont = DBBPool.allocateDirectAndPool(SnapshotSiteProcessor.m_snapshotBufferCompressedLen);
        //Skip 4-bytes so the partition ID is not compressed
        //That way if we detect a corruption we know what partition is bad
        tupleData.position(tupleData.position() + 4);
        /*
             * Leave 12 bytes, it's going to be a 4-byte length prefix, a 4-byte partition id,
             * and a 4-byte CRC32C of just the header bytes, in addition to the compressed payload CRC
             * that is 16 bytes, but 4 of those are done by CompressionService
             */
        cont.b().position(12);
        compressionTask = CompressionService.compressAndCRC32cBufferAsync(tupleData, cont);
    }
    final Future<BBContainer> compressionTaskFinal = compressionTask;
    ListenableFuture<?> writeTask = m_es.submit(new Callable<Object>() {

        @Override
        public Object call() throws Exception {
            try {
                if (m_acceptOneWrite) {
                    m_acceptOneWrite = false;
                } else {
                    if (m_simulateBlockedWrite != null) {
                        m_simulateBlockedWrite.await();
                    }
                    if (m_simulateFullDiskWritingChunk) {
                        //Make sure to consume the result of the compression
                        compressionTaskFinal.get().discard();
                        throw new IOException("Disk full");
                    }
                }
                final ByteBuffer tupleData = tupleDataCont.b();
                int totalWritten = 0;
                if (prependLength) {
                    BBContainer payloadContainer = compressionTaskFinal.get();
                    try {
                        final ByteBuffer payloadBuffer = payloadContainer.b();
                        payloadBuffer.position(0);
                        ByteBuffer lengthPrefix = ByteBuffer.allocate(12);
                        m_bytesAllowedBeforeSync.acquire(payloadBuffer.remaining());
                        //Length prefix does not include 4 header items, just compressd payload
                        //that follows
                        //length prefix
                        lengthPrefix.putInt(payloadBuffer.remaining() - 16);
                        // partitionId
                        lengthPrefix.putInt(tupleData.getInt(0));
                        /*
                             * Checksum the header and put it in the payload buffer
                             */
                        PureJavaCrc32C crc = new PureJavaCrc32C();
                        crc.update(lengthPrefix.array(), 0, 8);
                        lengthPrefix.putInt((int) crc.getValue());
                        lengthPrefix.flip();
                        payloadBuffer.put(lengthPrefix);
                        payloadBuffer.position(0);
                        enforceSnapshotRateLimit(payloadBuffer.remaining());
                        /*
                             * Write payload to file
                             */
                        while (payloadBuffer.hasRemaining()) {
                            totalWritten += m_channel.write(payloadBuffer);
                        }
                    } finally {
                        payloadContainer.discard();
                    }
                } else {
                    while (tupleData.hasRemaining()) {
                        totalWritten += m_channel.write(tupleData);
                    }
                }
                m_bytesWritten += totalWritten;
                m_bytesWrittenSinceLastSync.addAndGet(totalWritten);
            } catch (IOException e) {
                m_writeException = e;
                SNAP_LOG.error("Error while attempting to write snapshot data to file " + m_file, e);
                m_writeFailed = true;
                throw e;
            } finally {
                try {
                    tupleDataCont.discard();
                } finally {
                    m_outstandingWriteTasksLock.lock();
                    try {
                        if (m_outstandingWriteTasks.decrementAndGet() == 0) {
                            m_noMoreOutstandingWriteTasksCondition.signalAll();
                        }
                    } finally {
                        m_outstandingWriteTasksLock.unlock();
                    }
                }
            }
            return null;
        }
    });
    return writeTask;
}