Examples with CodedInputStream org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedInputStream used on opensource projects

Example 6 with CodedInputStream

use of org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedInputStream in project hbase by apache.

the class ProtobufUtil method mergeDelimitedFrom.

/**
   * This version of protobuf's mergeDelimitedFrom avoid the hard-coded 64MB limit for decoding
   * buffers
   * @param builder current message builder
   * @param in Inputsream with delimited protobuf data
   * @throws IOException
   */
public static void mergeDelimitedFrom(Message.Builder builder, InputStream in) throws IOException {
    // This used to be builder.mergeDelimitedFrom(in);
    // but is replaced to allow us to bump the protobuf size limit.
    final int firstByte = in.read();
    if (firstByte != -1) {
        final int size = CodedInputStream.readRawVarint32(firstByte, in);
        final InputStream limitedInput = new LimitInputStream(in, size);
        final CodedInputStream codedInput = CodedInputStream.newInstance(limitedInput);
        codedInput.setSizeLimit(size);
        builder.mergeFrom(codedInput);
        codedInput.checkLastTagWas(0);
    }
}

Example 7 with CodedInputStream

use of org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedInputStream in project hbase by apache.

the class SnapshotManifest method readDataManifest.

/*
   * Read the SnapshotDataManifest file
   */
private SnapshotDataManifest readDataManifest() throws IOException {
    FSDataInputStream in = null;
    try {
        in = fs.open(new Path(workingDir, DATA_MANIFEST_NAME));
        CodedInputStream cin = CodedInputStream.newInstance(in);
        cin.setSizeLimit(manifestSizeLimit);
        return SnapshotDataManifest.parseFrom(cin);
    } catch (FileNotFoundException e) {
        return null;
    } catch (InvalidProtocolBufferException e) {
        throw new CorruptedSnapshotException("unable to parse data manifest " + e.getMessage(), e);
    } finally {
        if (in != null)
            in.close();
    }
}

Example 8 with CodedInputStream

use of org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedInputStream in project hbase by apache.

the class SnapshotManifestV2 method loadRegionManifests.

static List<SnapshotRegionManifest> loadRegionManifests(final Configuration conf, final Executor executor, final FileSystem fs, final Path snapshotDir, final SnapshotDescription desc, final int manifestSizeLimit) throws IOException {
    FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir, new PathFilter() {

        @Override
        public boolean accept(Path path) {
            return path.getName().startsWith(SNAPSHOT_MANIFEST_PREFIX);
        }
    });
    if (manifestFiles == null || manifestFiles.length == 0)
        return null;
    final ExecutorCompletionService<SnapshotRegionManifest> completionService = new ExecutorCompletionService<>(executor);
    for (final FileStatus st : manifestFiles) {
        completionService.submit(new Callable<SnapshotRegionManifest>() {

            @Override
            public SnapshotRegionManifest call() throws IOException {
                FSDataInputStream stream = fs.open(st.getPath());
                CodedInputStream cin = CodedInputStream.newInstance(stream);
                cin.setSizeLimit(manifestSizeLimit);
                try {
                    return SnapshotRegionManifest.parseFrom(cin);
                } finally {
                    stream.close();
                }
            }
        });
    }
    ArrayList<SnapshotRegionManifest> regionsManifest = new ArrayList<>(manifestFiles.length);
    try {
        for (int i = 0; i < manifestFiles.length; ++i) {
            regionsManifest.add(completionService.take().get());
        }
    } catch (InterruptedException e) {
        throw new InterruptedIOException(e.getMessage());
    } catch (ExecutionException e) {
        Throwable t = e.getCause();
        if (t instanceof InvalidProtocolBufferException) {
            throw (InvalidProtocolBufferException) t;
        } else {
            IOException ex = new IOException("ExecutionException");
            ex.initCause(e.getCause());
            throw ex;
        }
    }
    return regionsManifest;
}