Examples with SchemaHash co.cask.cdap.api.data.schema.SchemaHash used on opensource projects

Example 1 with SchemaHash

use of co.cask.cdap.api.data.schema.SchemaHash in project cdap by caskdata.

the class QueueToStreamConsumer method poll.

@Override
public DequeueResult<StreamEvent> poll(int maxEvents, long timeout, TimeUnit timeoutUnit) throws IOException, InterruptedException {
    final DequeueResult<byte[]> result = consumer.dequeue(maxEvents);
    // Decode byte array into stream event
    ImmutableList.Builder<StreamEvent> builder = ImmutableList.builder();
    for (byte[] content : result) {
        try {
            builder.add(STREAM_EVENT_CODEC.decodePayload(content));
        } catch (Throwable t) {
            // If failed to decode, it maybe using old (pre 2.1) stream codec. Try to decode with old one.
            ByteBuffer buffer = ByteBuffer.wrap(content);
            SchemaHash schemaHash = new SchemaHash(buffer);
            Preconditions.checkArgument(schemaHash.equals(StreamEventDataCodec.STREAM_DATA_SCHEMA.getSchemaHash()), "Schema from payload not matching with StreamEventData schema.");
            Decoder decoder = new BinaryDecoder(new ByteBufferInputStream(buffer));
            // In old schema, timestamp is not recorded.
            builder.add(new StreamEvent(StreamEventDataCodec.decode(decoder), 0));
        }
    }
    final List<StreamEvent> events = builder.build();
    return new DequeueResult<StreamEvent>() {

        @Override
        public boolean isEmpty() {
            return events.isEmpty();
        }

        @Override
        public void reclaim() {
            result.reclaim();
        }

        @Override
        public int size() {
            return events.size();
        }

        @Override
        public Iterator<StreamEvent> iterator() {
            return events.iterator();
        }
    };
}

Example 2 with SchemaHash

use of co.cask.cdap.api.data.schema.SchemaHash in project cdap by caskdata.

the class StreamEventCodecTest method testEncodeDecodeWithDatumDecoder.

@Test
public void testEncodeDecodeWithDatumDecoder() throws UnsupportedTypeException, IOException {
    StreamEvent event = new StreamEvent(Maps.<String, String>newHashMap(), ByteBuffer.wrap("Event string".getBytes(Charsets.UTF_8)));
    StreamEventCodec codec = new StreamEventCodec();
    ByteBuffer payload = ByteBuffer.wrap(codec.encodePayload(event));
    SchemaHash schemaHash = new SchemaHash(payload);
    Schema schema = new ReflectionSchemaGenerator().generate(StreamEvent.class);
    Assert.assertEquals(schema.getSchemaHash(), schemaHash);
    StreamEvent decoded = new ReflectionDatumReader<>(schema, TypeToken.of(StreamEvent.class)).read(new BinaryDecoder(new ByteBufferInputStream(payload)), schema);
    Assert.assertEquals(event.getHeaders(), decoded.getHeaders());
    Assert.assertEquals(event.getBody(), decoded.getBody());
}

Example 3 with SchemaHash

use of co.cask.cdap.api.data.schema.SchemaHash in project cdap by caskdata.

the class StreamEventCodec method decodePayload.

/**
   * Reverse of {@link #encodePayload(StreamEvent)}.
   *
   * @param payload The byte array containing the queue payload.
   * @return A {@link StreamEvent} reconstructed from payload.
   */
public StreamEvent decodePayload(byte[] payload) {
    ByteBuffer buffer = ByteBuffer.wrap(payload);
    SchemaHash schemaHash = new SchemaHash(buffer);
    Preconditions.checkArgument(schemaHash.equals(STREAM_EVENT_SCHEMA.getSchemaHash()), "Schema from payload not matching StreamEvent schema.");
    Decoder decoder = new BinaryDecoder(new ByteBufferInputStream(buffer));
    try {
        StreamEventData data = StreamEventDataCodec.decode(decoder);
        // Read the timestamp
        long timestamp = decoder.readLong();
        return new StreamEvent(data, timestamp);
    } catch (IOException e) {
        // It should never happens, otherwise something very wrong.
        throw Throwables.propagate(e);
    }
}