Examples with ByteArrayOutputStream org.apache.hadoop.hbase.io.ByteArrayOutputStream used on opensource projects

Example 6 with ByteArrayOutputStream

use of org.apache.hadoop.hbase.io.ByteArrayOutputStream in project hbase by apache.

the class RowIndexCodecV1 method decodeKeyValues.

@Override
public ByteBuffer decodeKeyValues(DataInputStream source, HFileBlockDecodingContext decodingCtx) throws IOException {
    ByteBuffer sourceAsBuffer = ByteBufferUtils.drainInputStreamToBuffer(// waste
    source);
    sourceAsBuffer.mark();
    if (!decodingCtx.getHFileContext().isIncludesTags()) {
        sourceAsBuffer.position(sourceAsBuffer.limit() - Bytes.SIZEOF_INT);
        int onDiskSize = sourceAsBuffer.getInt();
        sourceAsBuffer.reset();
        ByteBuffer dup = sourceAsBuffer.duplicate();
        dup.position(sourceAsBuffer.position());
        dup.limit(sourceAsBuffer.position() + onDiskSize);
        return dup.slice();
    } else {
        RowIndexSeekerV1 seeker = new RowIndexSeekerV1(CellComparator.COMPARATOR, decodingCtx);
        seeker.setCurrentBuffer(new SingleByteBuff(sourceAsBuffer));
        List<Cell> kvs = new ArrayList<>();
        kvs.add(seeker.getCell());
        while (seeker.next()) {
            kvs.add(seeker.getCell());
        }
        boolean includesMvcc = decodingCtx.getHFileContext().isIncludesMvcc();
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        DataOutputStream out = new DataOutputStream(baos);
        for (Cell cell : kvs) {
            KeyValue currentCell = KeyValueUtil.copyToNewKeyValue(cell);
            out.write(currentCell.getBuffer(), currentCell.getOffset(), currentCell.getLength());
            if (includesMvcc) {
                WritableUtils.writeVLong(out, cell.getSequenceId());
            }
        }
        out.flush();
        return ByteBuffer.wrap(baos.getBuffer(), 0, baos.size());
    }
}

Example 7 with ByteArrayOutputStream

use of org.apache.hadoop.hbase.io.ByteArrayOutputStream in project hbase by apache.

the class AsyncFSOutputHelper method createOutput.

/**
   * Create {@link FanOutOneBlockAsyncDFSOutput} for {@link DistributedFileSystem}, and a simple
   * implementation for other {@link FileSystem} which wraps around a {@link FSDataOutputStream}.
   */
public static AsyncFSOutput createOutput(FileSystem fs, Path f, boolean overwrite, boolean createParent, short replication, long blockSize, final EventLoop eventLoop) throws IOException {
    if (fs instanceof DistributedFileSystem) {
        return FanOutOneBlockAsyncDFSOutputHelper.createOutput((DistributedFileSystem) fs, f, overwrite, createParent, replication, blockSize, eventLoop);
    }
    final FSDataOutputStream fsOut;
    int bufferSize = fs.getConf().getInt(CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY, CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT);
    if (createParent) {
        fsOut = fs.create(f, overwrite, bufferSize, replication, blockSize, null);
    } else {
        fsOut = fs.createNonRecursive(f, overwrite, bufferSize, replication, blockSize, null);
    }
    final ExecutorService flushExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setDaemon(true).setNameFormat("AsyncFSOutputFlusher-" + f.toString().replace("%", "%%")).build());
    return new AsyncFSOutput() {

        private final ByteArrayOutputStream out = new ByteArrayOutputStream();

        @Override
        public void write(final byte[] b, final int off, final int len) {
            if (eventLoop.inEventLoop()) {
                out.write(b, off, len);
            } else {
                eventLoop.submit(() -> out.write(b, off, len)).syncUninterruptibly();
            }
        }

        @Override
        public void write(byte[] b) {
            write(b, 0, b.length);
        }

        @Override
        public void recoverAndClose(CancelableProgressable reporter) throws IOException {
            fsOut.close();
        }

        @Override
        public DatanodeInfo[] getPipeline() {
            return new DatanodeInfo[0];
        }

        private void flush0(CompletableFuture<Long> future, boolean sync) {
            try {
                synchronized (out) {
                    fsOut.write(out.getBuffer(), 0, out.size());
                    out.reset();
                }
            } catch (IOException e) {
                eventLoop.execute(() -> future.completeExceptionally(e));
                return;
            }
            try {
                if (sync) {
                    fsOut.hsync();
                } else {
                    fsOut.hflush();
                }
                long pos = fsOut.getPos();
                eventLoop.execute(() -> future.complete(pos));
            } catch (IOException e) {
                eventLoop.execute(() -> future.completeExceptionally(e));
            }
        }

        @Override
        public CompletableFuture<Long> flush(boolean sync) {
            CompletableFuture<Long> future = new CompletableFuture<>();
            flushExecutor.execute(() -> flush0(future, sync));
            return future;
        }

        @Override
        public void close() throws IOException {
            try {
                flushExecutor.submit(() -> {
                    synchronized (out) {
                        fsOut.write(out.getBuffer(), 0, out.size());
                        out.reset();
                    }
                    return null;
                }).get();
            } catch (InterruptedException e) {
                throw new InterruptedIOException();
            } catch (ExecutionException e) {
                Throwables.propagateIfPossible(e.getCause(), IOException.class);
                throw new IOException(e.getCause());
            } finally {
                flushExecutor.shutdown();
            }
            fsOut.close();
        }

        @Override
        public int buffered() {
            return out.size();
        }

        @Override
        public void writeInt(int i) {
            out.writeInt(i);
        }

        @Override
        public void write(ByteBuffer bb) {
            out.write(bb, bb.position(), bb.remaining());
        }
    };
}