Examples with CodedInputStream org.apache.hbase.thirdparty.com.google.protobuf.CodedInputStream used on opensource projects

Example 6 with CodedInputStream

use of org.apache.hbase.thirdparty.com.google.protobuf.CodedInputStream in project hbase by apache.

the class ProtobufUtil method mergeFrom.

/**
 * This version of protobuf's mergeFrom avoids the hard-coded 64MB limit for decoding
 * buffers when working with ByteStrings
 * @param builder current message builder
 * @param bs ByteString containing the
 * @throws IOException
 */
public static void mergeFrom(Message.Builder builder, ByteString bs) throws IOException {
    final CodedInputStream codedInput = bs.newCodedInput();
    codedInput.setSizeLimit(bs.size());
    builder.mergeFrom(codedInput);
    codedInput.checkLastTagWas(0);
}

Example 7 with CodedInputStream

use of org.apache.hbase.thirdparty.com.google.protobuf.CodedInputStream in project hbase by apache.

the class ProtobufUtil method mergeFrom.

/**
 * This version of protobuf's mergeFrom avoids the hard-coded 64MB limit for decoding
 * buffers when working with byte arrays
 * @param builder current message builder
 * @param b byte array
 * @throws IOException
 */
public static void mergeFrom(Message.Builder builder, byte[] b) throws IOException {
    final CodedInputStream codedInput = CodedInputStream.newInstance(b);
    codedInput.setSizeLimit(b.length);
    builder.mergeFrom(codedInput);
    codedInput.checkLastTagWas(0);
}

Example 8 with CodedInputStream

use of org.apache.hbase.thirdparty.com.google.protobuf.CodedInputStream in project hbase by apache.

the class ProtobufUtil method mergeFrom.

/**
 * This version of protobuf's mergeFrom avoids the hard-coded 64MB limit for decoding
 * buffers where the message size is known
 * @param builder current message builder
 * @param in InputStream containing protobuf data
 * @param size known size of protobuf data
 * @throws IOException
 */
public static void mergeFrom(Message.Builder builder, InputStream in, int size) throws IOException {
    final CodedInputStream codedInput = CodedInputStream.newInstance(in);
    codedInput.setSizeLimit(size);
    builder.mergeFrom(codedInput);
    codedInput.checkLastTagWas(0);
}

Example 9 with CodedInputStream

use of org.apache.hbase.thirdparty.com.google.protobuf.CodedInputStream in project hbase by apache.

the class ServerRpcConnection method processRequest.

/**
 * @param buf
 *          Has the request header and the request param and optionally
 *          encoded data buffer all in this one array.
 * @throws IOException
 * @throws InterruptedException
 */
protected void processRequest(ByteBuff buf) throws IOException, InterruptedException {
    long totalRequestSize = buf.limit();
    int offset = 0;
    // Here we read in the header. We avoid having pb
    // do its default 4k allocation for CodedInputStream. We force it to use
    // backing array.
    CodedInputStream cis;
    if (buf.hasArray()) {
        cis = UnsafeByteOperations.unsafeWrap(buf.array(), 0, buf.limit()).newCodedInput();
    } else {
        cis = UnsafeByteOperations.unsafeWrap(new ByteBuffByteInput(buf, 0, buf.limit()), 0, buf.limit()).newCodedInput();
    }
    cis.enableAliasing(true);
    int headerSize = cis.readRawVarint32();
    offset = cis.getTotalBytesRead();
    Message.Builder builder = RequestHeader.newBuilder();
    ProtobufUtil.mergeFrom(builder, cis, headerSize);
    RequestHeader header = (RequestHeader) builder.build();
    offset += headerSize;
    TextMapGetter<RPCTInfo> getter = new TextMapGetter<RPCTInfo>() {

        @Override
        public Iterable<String> keys(RPCTInfo carrier) {
            return carrier.getHeadersMap().keySet();
        }

        @Override
        public String get(RPCTInfo carrier, String key) {
            return carrier.getHeadersMap().get(key);
        }
    };
    Context traceCtx = GlobalOpenTelemetry.getPropagators().getTextMapPropagator().extract(Context.current(), header.getTraceInfo(), getter);
    Span span = TraceUtil.createRemoteSpan("RpcServer.process", traceCtx);
    try (Scope scope = span.makeCurrent()) {
        int id = header.getCallId();
        if (RpcServer.LOG.isTraceEnabled()) {
            RpcServer.LOG.trace("RequestHeader " + TextFormat.shortDebugString(header) + " totalRequestSize: " + totalRequestSize + " bytes");
        }
        // total request.
        if ((totalRequestSize + this.rpcServer.callQueueSizeInBytes.sum()) > this.rpcServer.maxQueueSizeInBytes) {
            final ServerCall<?> callTooBig = createCall(id, this.service, null, null, null, null, totalRequestSize, null, 0, this.callCleanup);
            this.rpcServer.metrics.exception(RpcServer.CALL_QUEUE_TOO_BIG_EXCEPTION);
            callTooBig.setResponse(null, null, RpcServer.CALL_QUEUE_TOO_BIG_EXCEPTION, "Call queue is full on " + this.rpcServer.server.getServerName() + ", is hbase.ipc.server.max.callqueue.size too small?");
            callTooBig.sendResponseIfReady();
            return;
        }
        MethodDescriptor md = null;
        Message param = null;
        CellScanner cellScanner = null;
        try {
            if (header.hasRequestParam() && header.getRequestParam()) {
                md = this.service.getDescriptorForType().findMethodByName(header.getMethodName());
                if (md == null) {
                    throw new UnsupportedOperationException(header.getMethodName());
                }
                builder = this.service.getRequestPrototype(md).newBuilderForType();
                cis.resetSizeCounter();
                int paramSize = cis.readRawVarint32();
                offset += cis.getTotalBytesRead();
                if (builder != null) {
                    ProtobufUtil.mergeFrom(builder, cis, paramSize);
                    param = builder.build();
                }
                offset += paramSize;
            } else {
                // currently header must have request param, so we directly throw
                // exception here
                String msg = "Invalid request header: " + TextFormat.shortDebugString(header) + ", should have param set in it";
                RpcServer.LOG.warn(msg);
                throw new DoNotRetryIOException(msg);
            }
            if (header.hasCellBlockMeta()) {
                buf.position(offset);
                ByteBuff dup = buf.duplicate();
                dup.limit(offset + header.getCellBlockMeta().getLength());
                cellScanner = this.rpcServer.cellBlockBuilder.createCellScannerReusingBuffers(this.codec, this.compressionCodec, dup);
            }
        } catch (Throwable t) {
            InetSocketAddress address = this.rpcServer.getListenerAddress();
            String msg = (address != null ? address : "(channel closed)") + " is unable to read call parameter from client " + getHostAddress();
            RpcServer.LOG.warn(msg, t);
            this.rpcServer.metrics.exception(t);
            // version
            if (t instanceof LinkageError) {
                t = new DoNotRetryIOException(t);
            }
            // If the method is not present on the server, do not retry.
            if (t instanceof UnsupportedOperationException) {
                t = new DoNotRetryIOException(t);
            }
            ServerCall<?> readParamsFailedCall = createCall(id, this.service, null, null, null, null, totalRequestSize, null, 0, this.callCleanup);
            readParamsFailedCall.setResponse(null, null, t, msg + "; " + t.getMessage());
            readParamsFailedCall.sendResponseIfReady();
            return;
        }
        int timeout = 0;
        if (header.hasTimeout() && header.getTimeout() > 0) {
            timeout = Math.max(this.rpcServer.minClientRequestTimeout, header.getTimeout());
        }
        ServerCall<?> call = createCall(id, this.service, md, header, param, cellScanner, totalRequestSize, this.addr, timeout, this.callCleanup);
        if (!this.rpcServer.scheduler.dispatch(new CallRunner(this.rpcServer, call))) {
            this.rpcServer.callQueueSizeInBytes.add(-1 * call.getSize());
            this.rpcServer.metrics.exception(RpcServer.CALL_QUEUE_TOO_BIG_EXCEPTION);
            call.setResponse(null, null, RpcServer.CALL_QUEUE_TOO_BIG_EXCEPTION, "Call queue is full on " + this.rpcServer.server.getServerName() + ", too many items queued ?");
            call.sendResponseIfReady();
        }
    }
}

Example 10 with CodedInputStream

use of org.apache.hbase.thirdparty.com.google.protobuf.CodedInputStream in project hbase by apache.

the class SnapshotManifest method readDataManifest.

/*
   * Read the SnapshotDataManifest file
   */
private SnapshotDataManifest readDataManifest() throws IOException {
    try (FSDataInputStream in = workingDirFs.open(new Path(workingDir, DATA_MANIFEST_NAME))) {
        CodedInputStream cin = CodedInputStream.newInstance(in);
        cin.setSizeLimit(manifestSizeLimit);
        return SnapshotDataManifest.parseFrom(cin);
    } catch (FileNotFoundException e) {
        return null;
    } catch (InvalidProtocolBufferException e) {
        throw new CorruptedSnapshotException("unable to parse data manifest " + e.getMessage(), e);
    }
}