Examples with StreamEvent co.cask.cdap.api.flow.flowlet.StreamEvent used on opensource projects

Example 1 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class QueueToStreamConsumer method poll.

@Override
public DequeueResult<StreamEvent> poll(int maxEvents, long timeout, TimeUnit timeoutUnit) throws IOException, InterruptedException {
    final DequeueResult<byte[]> result = consumer.dequeue(maxEvents);
    // Decode byte array into stream event
    ImmutableList.Builder<StreamEvent> builder = ImmutableList.builder();
    for (byte[] content : result) {
        try {
            builder.add(STREAM_EVENT_CODEC.decodePayload(content));
        } catch (Throwable t) {
            // If failed to decode, it maybe using old (pre 2.1) stream codec. Try to decode with old one.
            ByteBuffer buffer = ByteBuffer.wrap(content);
            SchemaHash schemaHash = new SchemaHash(buffer);
            Preconditions.checkArgument(schemaHash.equals(StreamEventDataCodec.STREAM_DATA_SCHEMA.getSchemaHash()), "Schema from payload not matching with StreamEventData schema.");
            Decoder decoder = new BinaryDecoder(new ByteBufferInputStream(buffer));
            // In old schema, timestamp is not recorded.
            builder.add(new StreamEvent(StreamEventDataCodec.decode(decoder), 0));
        }
    }
    final List<StreamEvent> events = builder.build();
    return new DequeueResult<StreamEvent>() {

        @Override
        public boolean isEmpty() {
            return events.isEmpty();
        }

        @Override
        public void reclaim() {
            result.reclaim();
        }

        @Override
        public int size() {
            return events.size();
        }

        @Override
        public Iterator<StreamEvent> iterator() {
            return events.iterator();
        }
    };
}

Example 2 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class StreamEventTypeAdapter method read.

@Override
public StreamEvent read(JsonReader in) throws IOException {
    long timestamp = -1;
    Map<String, String> headers = null;
    ByteBuffer body = null;
    in.beginObject();
    while (in.peek() == JsonToken.NAME) {
        String key = in.nextName();
        if ("timestamp".equals(key)) {
            timestamp = in.nextLong();
        } else if ("headers".equals(key)) {
            headers = mapTypeAdapter.read(in);
        } else if ("body".equals(key)) {
            body = ByteBuffer.wrap(Bytes.toBytesBinary(in.nextString()));
        } else {
            in.skipValue();
        }
    }
    if (timestamp >= 0 && headers != null && body != null) {
        in.endObject();
        return new StreamEvent(headers, body, timestamp);
    }
    throw new IOException(String.format("Failed to read StreamEvent. Timestamp: %d, headers: %s, body: %s", timestamp, headers, body));
}

Example 3 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class StreamHandlerTest method testSimpleStreamEnqueue.

@Test
public void testSimpleStreamEnqueue() throws Exception {
    // Create new stream.
    HttpURLConnection urlConn = openURL(createURL("streams/test_stream_enqueue"), HttpMethod.PUT);
    Assert.assertEquals(HttpResponseStatus.OK.code(), urlConn.getResponseCode());
    urlConn.disconnect();
    // Enqueue 10 entries
    for (int i = 0; i < 10; ++i) {
        urlConn = openURL(createURL("streams/test_stream_enqueue"), HttpMethod.POST);
        urlConn.setDoOutput(true);
        urlConn.addRequestProperty("test_stream_enqueue.header1", Integer.toString(i));
        urlConn.getOutputStream().write(Integer.toString(i).getBytes(Charsets.UTF_8));
        Assert.assertEquals(HttpResponseStatus.OK.code(), urlConn.getResponseCode());
        urlConn.disconnect();
    }
    // Fetch 10 entries
    urlConn = openURL(createURL("streams/test_stream_enqueue/events?limit=10"), HttpMethod.GET);
    List<StreamEvent> events = GSON.fromJson(new String(ByteStreams.toByteArray(urlConn.getInputStream()), Charsets.UTF_8), new TypeToken<List<StreamEvent>>() {
    }.getType());
    for (int i = 0; i < 10; i++) {
        StreamEvent event = events.get(i);
        int actual = Integer.parseInt(Charsets.UTF_8.decode(event.getBody()).toString());
        Assert.assertEquals(i, actual);
        Assert.assertEquals(Integer.toString(i), event.getHeaders().get("header1"));
    }
    urlConn.disconnect();
}

Example 4 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class DelimitedStringsRecordFormatTest method testFormatRecordWithMapping.

@Test
public void testFormatRecordWithMapping() throws UnsupportedTypeException {
    Schema schema = Schema.recordOf("event", Schema.Field.of("f3", Schema.unionOf(Schema.of(Schema.Type.FLOAT), Schema.of(Schema.Type.NULL))), Schema.Field.of("f4", Schema.unionOf(Schema.of(Schema.Type.DOUBLE), Schema.of(Schema.Type.NULL))));
    DelimitedStringsRecordFormat format = new DelimitedStringsRecordFormat();
    FormatSpecification spec = new FormatSpecification(DelimitedStringsRecordFormat.class.getCanonicalName(), schema, ImmutableMap.of(DelimitedStringsRecordFormat.DELIMITER, ",", DelimitedStringsRecordFormat.MAPPING, "2:f3,3:f4"));
    format.initialize(spec);
    boolean booleanVal = false;
    int intVal = Integer.MAX_VALUE;
    float floatVal = Float.MAX_VALUE;
    double doubleVal = Double.MAX_VALUE;
    byte[] bytesVal = new byte[] { 0, 1, 2 };
    String stringVal = "foo bar";
    String[] arrayVal = new String[] { "extra1", "extra2", "extra3" };
    String body = new StringBuilder().append(booleanVal).append(",").append(intVal).append(",").append(floatVal).append(",").append(doubleVal).append(",").append(Bytes.toStringBinary(bytesVal)).append(",").append(stringVal).append(",").append(arrayVal[0]).append(",").append(arrayVal[1]).append(",").append(arrayVal[2]).toString();
    StructuredRecord output = format.read(new StreamEvent(ByteBuffer.wrap(Bytes.toBytes(body))));
    Assert.assertEquals(2, output.getSchema().getFields().size());
    Assert.assertNull(output.get("f1"));
    Assert.assertNull(output.get("f2"));
    Assert.assertEquals(floatVal, output.get("f3"), 0.0001f);
    Assert.assertEquals(doubleVal, output.get("f4"), 0.0001d);
    Assert.assertNull(output.get("f5"));
    Assert.assertNull(output.get("f6"));
    Assert.assertNull(output.get("f7"));
    // now try with null fields.
    output = format.read(new StreamEvent(ByteBuffer.wrap(Bytes.toBytes("true,,3.14159,,,hello world,extra1"))));
    Assert.assertEquals(2, output.getSchema().getFields().size());
    Assert.assertNull(output.get("f1"));
    Assert.assertNull(output.get("f2"));
    Assert.assertEquals(3.14159f, output.get("f3"), 0.0001f);
    Assert.assertNull(output.get("f4"));
    Assert.assertNull(output.get("f5"));
    Assert.assertNull(output.get("f6"));
    Assert.assertNull(output.get("f7"));
}

Example 5 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class DelimitedStringsRecordFormatTest method testDelimiter.

@Test
public void testDelimiter() throws UnsupportedTypeException, UnexpectedFormatException {
    DelimitedStringsRecordFormat format = new DelimitedStringsRecordFormat();
    FormatSpecification spec = new FormatSpecification(DelimitedStringsRecordFormat.class.getCanonicalName(), null, ImmutableMap.of(DelimitedStringsRecordFormat.DELIMITER, " "));
    format.initialize(spec);
    String body = "userX actionY itemZ";
    StructuredRecord output = format.read(new StreamEvent(ByteBuffer.wrap(Bytes.toBytes(body))));
    String[] actual = output.get("body");
    String[] expected = body.split(" ");
    Assert.assertArrayEquals(expected, actual);
}