Examples with BinaryDecoder co.cask.cdap.common.io.BinaryDecoder used on opensource projects

Example 16 with BinaryDecoder

use of co.cask.cdap.common.io.BinaryDecoder in project cdap by caskdata.

the class MetricsMessageCallback method onReceived.

@Override
public long onReceived(Iterator<FetchedMessage> messages) {
    // Decode the metrics records.
    ByteBufferInputStream is = new ByteBufferInputStream(null);
    List<MetricValues> records = Lists.newArrayList();
    long nextOffset = 0L;
    while (messages.hasNext()) {
        FetchedMessage input = messages.next();
        nextOffset = input.getNextOffset();
        try {
            MetricValues metricValues = recordReader.read(new BinaryDecoder(is.reset(input.getPayload())), recordSchema);
            records.add(metricValues);
        } catch (IOException e) {
            LOG.warn("Failed to decode message to MetricValue. Skipped. {}", e.getMessage());
        }
    }
    if (records.isEmpty()) {
        LOG.info("No records to process.");
        return nextOffset;
    }
    long now = System.currentTimeMillis();
    try {
        addProcessingStats(records, now);
        metricStore.add(records);
    } catch (Exception e) {
        // SimpleKafkaConsumer will log the error, and continue on past these messages
        throw new RuntimeException("Failed to add metrics data to a store", e);
    }
    recordsProcessed += records.size();
    // avoid logging more than once a minute
    if (now > lastLoggedMillis + TimeUnit.MINUTES.toMillis(1)) {
        lastLoggedMillis = now;
        LOG.debug("{} metrics records processed. Last record time: {}.", recordsProcessed, records.get(records.size() - 1).getTimestamp());
    }
    return nextOffset;
}

Example 17 with BinaryDecoder

use of co.cask.cdap.common.io.BinaryDecoder in project cdap by caskdata.

the class FlowletProgramRunner method createInputDatumDecoder.

private <T> Function<ByteBuffer, T> createInputDatumDecoder(final TypeToken<T> dataType, final Schema schema, final SchemaCache schemaCache) {
    final ReflectionDatumReader<T> datumReader = new ReflectionDatumReader<>(schema, dataType);
    final ByteBufferInputStream byteBufferInput = new ByteBufferInputStream(null);
    final BinaryDecoder decoder = new BinaryDecoder(byteBufferInput);
    return new Function<ByteBuffer, T>() {

        @Nullable
        @Override
        public T apply(ByteBuffer input) {
            byteBufferInput.reset(input);
            try {
                final Schema sourceSchema = schemaCache.get(input);
                Preconditions.checkNotNull(sourceSchema, "Fail to find source schema.");
                return datumReader.read(decoder, sourceSchema);
            } catch (IOException e) {
                throw Throwables.propagate(e);
            }
        }

        @Override
        public String toString() {
            return Objects.toStringHelper(this).add("dataType", dataType).add("schema", schema).toString();
        }
    };
}

Example 18 with BinaryDecoder

use of co.cask.cdap.common.io.BinaryDecoder in project cdap by caskdata.

the class StreamEventCodec method decodePayload.

/**
   * Reverse of {@link #encodePayload(StreamEvent)}.
   *
   * @param payload The byte array containing the queue payload.
   * @return A {@link StreamEvent} reconstructed from payload.
   */
public StreamEvent decodePayload(byte[] payload) {
    ByteBuffer buffer = ByteBuffer.wrap(payload);
    SchemaHash schemaHash = new SchemaHash(buffer);
    Preconditions.checkArgument(schemaHash.equals(STREAM_EVENT_SCHEMA.getSchemaHash()), "Schema from payload not matching StreamEvent schema.");
    Decoder decoder = new BinaryDecoder(new ByteBufferInputStream(buffer));
    try {
        StreamEventData data = StreamEventDataCodec.decode(decoder);
        // Read the timestamp
        long timestamp = decoder.readLong();
        return new StreamEvent(data, timestamp);
    } catch (IOException e) {
        // It should never happens, otherwise something very wrong.
        throw Throwables.propagate(e);
    }
}

Example 19 with BinaryDecoder

use of co.cask.cdap.common.io.BinaryDecoder in project cdap by caskdata.

the class StreamDataFileIndex method loadIndex.

private Map.Entry<LongList, LongList> loadIndex(InputStream input) throws IOException {
    byte[] magic = new byte[INDEX_MAGIC_HEADER.length];
    ByteStreams.readFully(input, magic);
    if (!Arrays.equals(magic, INDEX_MAGIC_HEADER)) {
        throw new IOException("Unsupported index file format. Expected magic bytes as 'I' '1'");
    }
    // Decode the properties map. In current version, it is not used.
    StreamUtils.decodeMap(new BinaryDecoder(input));
    // Read in all index (timestamp, position pairs).
    LongList timestamps = new LongArrayList(1000);
    LongList positions = new LongArrayList(1000);
    byte[] buf = new byte[Longs.BYTES * 2];
    while (ByteStreams.read(input, buf, 0, buf.length) == buf.length) {
        timestamps.add(Bytes.toLong(buf, 0));
        positions.add(Bytes.toLong(buf, Longs.BYTES));
    }
    return Maps.immutableEntry(timestamps, positions);
}

Example 20 with BinaryDecoder

use of co.cask.cdap.common.io.BinaryDecoder in project cdap by caskdata.

the class StreamDataFileReader method readHeader.

private void readHeader() throws IOException {
    // Read the header of the event file
    // First 2 bytes should be 'E' '1'
    byte[] magic = new byte[StreamDataFileConstants.MAGIC_HEADER_SIZE];
    ByteStreams.readFully(eventInput, magic);
    int fileVersion = decodeFileVersion(magic);
    // Read the properties map.
    Map<String, String> properties = StreamUtils.decodeMap(new BinaryDecoder(eventInput));
    verifySchema(properties);
    // Create event template
    if (fileVersion >= 2) {
        eventTemplate = createEventTemplate(properties);
    } else {
        eventTemplate = new StreamEvent(ImmutableMap.<String, String>of(), ByteBuffers.EMPTY_BUFFER, -1L);
    }
    position = eventInput.getPos();
}