Examples with CodedInputStream com.google.protobuf.CodedInputStream used on opensource projects

Example 16 with CodedInputStream

use of com.google.protobuf.CodedInputStream in project compiler by boalang.

the class BoaAstIntrinsics method getcomments.

/**
	 * Given a ChangedFile, return the comments for that file at that revision.
	 *
	 * @param f the ChangedFile to get a snapshot of the comments for
	 * @return the comments list, or an empty list on any sort of error
	 */
@FunctionSpec(name = "getcomments", returnType = "CommentsRoot", formalParameters = { "ChangedFile" })
public static CommentsRoot getcomments(final ChangedFile f) {
    // since we know only certain kinds have comments, filter before looking up
    final ChangedFile.FileKind kind = f.getKind();
    if (kind != ChangedFile.FileKind.SOURCE_JAVA_ERROR && kind != ChangedFile.FileKind.SOURCE_JAVA_JLS2 && kind != ChangedFile.FileKind.SOURCE_JAVA_JLS3 && kind != ChangedFile.FileKind.SOURCE_JAVA_JLS4 && kind != ChangedFile.FileKind.SOURCE_JAVA_JLS8)
        return emptyComments;
    final String rowName = f.getKey() + "!!" + f.getName();
    if (commentsMap == null)
        openCommentMap();
    try {
        final BytesWritable value = new BytesWritable();
        if (commentsMap.get(new Text(rowName), value) != null) {
            final CodedInputStream _stream = CodedInputStream.newInstance(value.getBytes(), 0, value.getLength());
            final CommentsRoot root = CommentsRoot.parseFrom(_stream);
            return root;
        }
    } catch (final InvalidProtocolBufferException e) {
        e.printStackTrace();
    } catch (final IOException e) {
        e.printStackTrace();
    } catch (final RuntimeException e) {
        e.printStackTrace();
    } catch (final Error e) {
        e.printStackTrace();
    }
    System.err.println("error with comments: " + rowName);
    return emptyComments;
}

Example 17 with CodedInputStream

use of com.google.protobuf.CodedInputStream in project hive by apache.

the class EncodedReaderImpl method readIndexStreams.

@Override
public void readIndexStreams(OrcIndex index, StripeInformation stripe, List<OrcProto.Stream> streams, boolean[] physicalFileIncludes, boolean[] sargColumns) throws IOException {
    long stripeOffset = stripe.getOffset();
    DiskRangeList indexRanges = planIndexReading(fileSchema, streams, true, physicalFileIncludes, sargColumns, version, index.getBloomFilterKinds());
    if (indexRanges == null) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Nothing to read for stripe [" + stripe + "]");
        }
        return;
    }
    ReadContext[] colCtxs = new ReadContext[physicalFileIncludes.length];
    int colRgIx = -1;
    for (int i = 0; i < physicalFileIncludes.length; ++i) {
        if (!physicalFileIncludes[i] && (sargColumns == null || !sargColumns[i]))
            continue;
        colCtxs[i] = new ReadContext(i, ++colRgIx);
        if (isTracingEnabled) {
            LOG.trace("Creating context: " + colCtxs[i].toString());
        }
        // Bogus encoding.
        trace.logColumnRead(i, colRgIx, ColumnEncoding.Kind.DIRECT);
    }
    long offset = 0;
    for (OrcProto.Stream stream : streams) {
        long length = stream.getLength();
        int colIx = stream.getColumn();
        OrcProto.Stream.Kind streamKind = stream.getKind();
        // See planIndexReading - only read non-row-index streams if involved in SARGs.
        if ((StreamName.getArea(streamKind) == StreamName.Area.INDEX) && ((sargColumns != null && sargColumns[colIx]) || (physicalFileIncludes[colIx] && streamKind == Kind.ROW_INDEX))) {
            trace.logAddStream(colIx, streamKind, offset, length, -1, true);
            colCtxs[colIx].addStream(offset, stream, -1);
            if (isTracingEnabled) {
                LOG.trace("Adding stream for column " + colIx + ": " + streamKind + " at " + offset + ", " + length);
            }
        }
        offset += length;
    }
    boolean hasFileId = this.fileKey != null;
    // 2. Now, read all of the ranges from cache or disk.
    IdentityHashMap<ByteBuffer, Boolean> toRelease = new IdentityHashMap<>();
    MutateHelper toRead = getDataFromCacheAndDisk(indexRanges, stripeOffset, hasFileId, toRelease);
    // 3. For uncompressed case, we need some special processing before read.
    DiskRangeList iter = preReadUncompressedStreams(stripeOffset, colCtxs, toRead, toRelease);
    // 4. Decompress the data.
    boolean hasError = true;
    try {
        for (int colIx = 0; colIx < colCtxs.length; ++colIx) {
            ReadContext ctx = colCtxs[colIx];
            // This column is not included.
            if (ctx == null)
                continue;
            for (int streamIx = 0; streamIx < ctx.streamCount; ++streamIx) {
                StreamContext sctx = ctx.streams[streamIx];
                try {
                    if (isTracingEnabled) {
                        LOG.trace("Getting index stream " + sctx.kind + " for column " + ctx.colIx + " at " + sctx.offset + ", " + sctx.length);
                    }
                    ColumnStreamData csd = POOLS.csdPool.take();
                    long endCOffset = sctx.offset + sctx.length;
                    DiskRangeList lastCached = readEncodedStream(stripeOffset, iter, sctx.offset, endCOffset, csd, endCOffset, sctx.offset, toRelease);
                    if (lastCached != null) {
                        iter = lastCached;
                    }
                    if (isTracingEnabled) {
                        traceLogBuffersUsedToParse(csd);
                    }
                    CodedInputStream cis = CodedInputStream.newInstance(new IndexStream(csd.getCacheBuffers(), sctx.length));
                    cis.setSizeLimit(InStream.PROTOBUF_MESSAGE_MAX_LIMIT);
                    switch(sctx.kind) {
                        case ROW_INDEX:
                            OrcProto.RowIndex tmp = index.getRowGroupIndex()[colIx] = OrcProto.RowIndex.parseFrom(cis);
                            if (isTracingEnabled) {
                                LOG.trace("Index is " + tmp.toString().replace('\n', ' '));
                            }
                            break;
                        case BLOOM_FILTER:
                        case BLOOM_FILTER_UTF8:
                            index.getBloomFilterIndex()[colIx] = OrcProto.BloomFilterIndex.parseFrom(cis);
                            break;
                        default:
                            throw new AssertionError("Unexpected index stream type " + sctx.kind);
                    }
                    // We are done with the buffers; unlike data blocks, we are also the consumer. Release.
                    for (MemoryBuffer buf : csd.getCacheBuffers()) {
                        if (buf == null)
                            continue;
                        cacheWrapper.releaseBuffer(buf);
                    }
                } catch (Exception ex) {
                    DiskRangeList drl = toRead == null ? null : toRead.next;
                    LOG.error("Error getting stream " + sctx.kind + " for column " + ctx.colIx + " at " + sctx.offset + ", " + sctx.length + "; toRead " + RecordReaderUtils.stringifyDiskRanges(drl), ex);
                    throw (ex instanceof IOException) ? (IOException) ex : new IOException(ex);
                }
            }
        }
        if (isTracingEnabled) {
            LOG.trace("Disk ranges after preparing all the data " + RecordReaderUtils.stringifyDiskRanges(toRead.next));
        }
        hasError = false;
    } finally {
        // Release the unreleased buffers. See class comment about refcounts.
        try {
            releaseInitialRefcounts(toRead.next);
            releaseBuffers(toRelease.keySet(), true);
        } catch (Throwable t) {
            if (!hasError)
                throw new IOException(t);
            LOG.error("Error during the cleanup after another error; ignoring", t);
        }
    }
}

Example 18 with CodedInputStream

use of com.google.protobuf.CodedInputStream in project weicoder by wdcode.

the class ProtobufEngine method toBean.

/**
 * 反序列化获得对象
 * @param b 字节数组
 * @param c 反序列化的对象
 * @param <E> 范型
 * @return 获得的对象
 */
public static <E> E toBean(byte[] b, Class<E> c) {
    // 实例化对象
    E bean = BeanUtil.newInstance(c);
    try {
        // 声明编码输入流 用于读取字段数据
        CodedInputStream input = CodedInputStream.newInstance(b, 0, b.length);
        // 获得所有字段
        List<Field> fields = BeanUtil.getFields(c);
        // 读取标签
        int tag = input.readTag();
        // 根据标签获得第几个属性
        int num = WireFormat.getTagFieldNumber(tag);
        // 循环所有属性 字段标示从1开始 所以i从1 开始
        for (int i = 1; i < fields.size(); i++) {
            // 读取标签为0时跳出循环
            if (tag == 0) {
                break;
            }
            // 如果字段位置大于当前，跳过本次循环
            if (num > i) {
                continue;
            }
            // 获得本字段
            Field field = fields.get(i - 1);
            // 获得字段类型
            Class<?> type = field.getType();
            // 判断字段类型 根据字段类型赋值
            if (type.equals(String.class)) {
                // 字符串
                BeanUtil.setFieldValue(bean, field, input.readStringRequireUtf8());
            } else if (type.equals(int.class) || type.equals(Integer.class)) {
                // 整型
                BeanUtil.setFieldValue(bean, field, input.readInt32());
            } else if (type.equals(long.class) || type.equals(Long.class)) {
                // 长整型
                BeanUtil.setFieldValue(bean, field, input.readInt64());
            } else if (type.equals(boolean.class) || type.equals(Boolean.class)) {
                // 布尔
                BeanUtil.setFieldValue(bean, field, input.readBool());
            } else if (type.equals(float.class) || type.equals(Float.class)) {
                // float型
                BeanUtil.setFieldValue(bean, field, input.readFloat());
            } else if (type.equals(double.class) || type.equals(Double.class)) {
                // Double型
                BeanUtil.setFieldValue(bean, field, input.readDouble());
            } else if (type.equals(ByteString.class)) {
                // 字节字符串
                BeanUtil.setFieldValue(bean, field, input.readBytes());
            } else if (type.equals(byte[].class)) {
                // 字节流
                BeanUtil.setFieldValue(bean, field, input.readByteArray());
            }
            // 重新读取标签和字段位置
            tag = input.readTag();
            num = WireFormat.getTagFieldNumber(tag);
        }
        // 校验标签是否为0
        input.checkLastTagWas(0);
    } catch (Exception e) {
        Logs.error(e);
    }
    // 返回结果
    return bean;
}

Example 19 with CodedInputStream

use of com.google.protobuf.CodedInputStream in project tez by apache.

the class DAGClientAMProtocolBlockingPBServerImpl method submitDAG.

@Override
public SubmitDAGResponseProto submitDAG(RpcController controller, SubmitDAGRequestProto request) throws ServiceException {
    UserGroupInformation user = getRPCUser();
    if (!real.getACLManager().checkAMModifyAccess(user)) {
        throw new AccessControlException("User " + user + " cannot perform AM modify operation");
    }
    real.updateLastHeartbeatTime();
    try {
        if (request.hasSerializedRequestPath()) {
            // need to deserialize large request from hdfs
            Path requestPath = new Path(request.getSerializedRequestPath());
            try (FSDataInputStream fsDataInputStream = stagingFs.open(requestPath)) {
                CodedInputStream in = CodedInputStream.newInstance(fsDataInputStream);
                in.setSizeLimit(Integer.MAX_VALUE);
                request = SubmitDAGRequestProto.parseFrom(in);
            } catch (IOException e) {
                throw wrapException(e);
            }
        }
        DAGPlan dagPlan = request.getDAGPlan();
        Map<String, LocalResource> additionalResources = null;
        if (request.hasAdditionalAmResources()) {
            additionalResources = DagTypeConverters.convertFromPlanLocalResources(request.getAdditionalAmResources());
        }
        String dagId = real.submitDAG(dagPlan, additionalResources);
        return SubmitDAGResponseProto.newBuilder().setDagId(dagId).build();
    } catch (TezException e) {
        throw wrapException(e);
    }
}

Example 20 with CodedInputStream

use of com.google.protobuf.CodedInputStream in project hpcourse by cscenter.

the class RandomClient method main.

public static void main(String[] args) throws IOException {
    String hostName = args.length == 2 ? args[0] : "localhost";
    int portNumber = args.length == 2 ? Integer.parseInt(args[1]) : Server.DEFAULT_PORT;
    try (Socket socket = new Socket(hostName, portNumber)) {
        Protocol.ServerRequest.Builder serverRequestBuilder = Protocol.ServerRequest.newBuilder();
        Protocol.ServerRequest.Builder requestBuilder = serverRequestBuilder.setClientId(UUID.randomUUID().toString()).setRequestId(randomGenerator.nextInt(1000000));
        int randomNumber = randomGenerator.nextInt();
        CodedInputStream inputStream = CodedInputStream.newInstance(socket.getInputStream());
        StringBuilder sb = new StringBuilder().append("=====||");
        if (randomNumber % 3 == 0) {
            sb.append("Submit new task||");
            Protocol.SubmitTask.Builder submitTaskBuilder = Protocol.SubmitTask.newBuilder();
            Protocol.Task.Builder taskBuilder = Protocol.Task.newBuilder();
            Protocol.ServerRequest request = requestBuilder.setSubmit(submitTaskBuilder.setTask(taskBuilder.setA(getRandomParam()).setB(getRandomParam()).setP(getRandomParam()).setM(getRandomParam()).setN(randomGenerator.nextInt(1_000_000_000)).build()).build()).build();
            sendRequest(request, socket);
            int messageLength = inputStream.readRawVarint32();
            Protocol.SubmitTaskResponse response = Protocol.SubmitTaskResponse.parseFrom(inputStream.readRawBytes(messageLength));
            sb.append(response.getStatus().getValueDescriptor().toString()).append("||").append(response.getSubmittedTaskId()).append("||");
        } else if (randomNumber % 3 == 1) {
            int taskId = randomGenerator.nextInt(10);
            sb.append("Subscribe to task: ").append(taskId).append("||");
            Protocol.ServerRequest request = requestBuilder.setSubscribe(Protocol.Subscribe.newBuilder().setTaskId(taskId).build()).build();
            sendRequest(request, socket);
            int messageLength = inputStream.readRawVarint32();
            Protocol.SubscribeResponse response = Protocol.SubscribeResponse.parseFrom(inputStream.readRawBytes(messageLength));
            sb.append(response.getStatus().getValueDescriptor().toString()).append("\n").append(response.getValue()).append("\n");
        } else {
            sb.append("List tasks||");
            Protocol.ServerRequest request = requestBuilder.setList(Protocol.ListTasks.newBuilder()).build();
            sendRequest(request, socket);
            int messageLength = inputStream.readRawVarint32();
            Protocol.ListTasksResponse response = Protocol.ListTasksResponse.parseFrom(inputStream.readRawBytes(messageLength));
            List<Protocol.ListTasksResponse.TaskDescription> list = response.getTasksList();
            for (Protocol.ListTasksResponse.TaskDescription taskDescription : list) {
                sb.append(taskDescription.getClientId()).append(" @ ").append(String.valueOf(taskDescription.getTaskId())).append(" @ ").append(taskDescription.hasResult() ? taskDescription.getResult() : "Not ready").append("||");
            }
        }
        sb.append("-----------");
        System.out.println(sb);
    } catch (UnknownHostException e) {
        System.err.println("Don't know about host " + hostName);
        System.exit(1);
    } catch (IOException e) {
        System.err.println("Couldn't get I/O for the connection to " + hostName);
        System.exit(1);
    }
}