Examples with StreamEvent co.cask.cdap.api.flow.flowlet.StreamEvent used on opensource projects

Example 61 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class StreamDataFileReader method readHeader.

private void readHeader() throws IOException {
    // Read the header of the event file
    // First 2 bytes should be 'E' '1'
    byte[] magic = new byte[StreamDataFileConstants.MAGIC_HEADER_SIZE];
    ByteStreams.readFully(eventInput, magic);
    int fileVersion = decodeFileVersion(magic);
    // Read the properties map.
    Map<String, String> properties = StreamUtils.decodeMap(new BinaryDecoder(eventInput));
    verifySchema(properties);
    // Create event template
    if (fileVersion >= 2) {
        eventTemplate = createEventTemplate(properties);
    } else {
        eventTemplate = new StreamEvent(ImmutableMap.<String, String>of(), ByteBuffers.EMPTY_BUFFER, -1L);
    }
    position = eventInput.getPos();
}

Example 62 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class StreamSerDe method deserialize.

@Override
public Object deserialize(Writable writable) throws SerDeException {
    // The writable should always contains a StreamEvent object provided by the StreamRecordReader
    ObjectWritable objectWritable = (ObjectWritable) writable;
    StreamEvent streamEvent = (StreamEvent) objectWritable.get();
    // timestamp and headers are always guaranteed to be first.
    List<Object> event = Lists.newArrayList();
    event.add(streamEvent.getTimestamp());
    event.add(streamEvent.getHeaders());
    try {
        // The format should always format the stream event into a record.
        event.addAll(deserializer.translateRecord(streamFormat.read(streamEvent)));
        return event;
    } catch (Throwable t) {
        LOG.info("Unable to format the stream body.", t);
        throw new SerDeException("Unable to format the stream body.", t);
    }
}

Example 63 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class GenerateClientUsageExample method streamClient.

public void streamClient() throws Exception {
    // Construct the client used to interact with CDAP
    StreamClient streamClient = new StreamClient(clientConfig);
    // Fetch the stream list
    List streams = streamClient.list(NamespaceId.DEFAULT);
    // Create a stream, using the Purchase example
    StreamId streamId = NamespaceId.DEFAULT.stream("purchases");
    streamClient.create(streamId);
    // Fetch a stream's properties
    StreamProperties config = streamClient.getConfig(streamId);
    // Send events to a stream
    streamClient.sendEvent(streamId, "Tom bought 5 apples for $10");
    // Read all events from a stream (results in events)
    List<StreamEvent> events = Lists.newArrayList();
    streamClient.getEvents(streamId, 0, Long.MAX_VALUE, Integer.MAX_VALUE, events);
    // Read first 5 events from a stream (results in events)
    events = Lists.newArrayList();
    streamClient.getEvents(streamId, 0, Long.MAX_VALUE, 5, events);
    // Read 2nd and 3rd events from a stream, after first calling getEvents
    long startTime = events.get(1).getTimestamp();
    long endTime = events.get(2).getTimestamp() + 1;
    events.clear();
    streamClient.getEvents(streamId, startTime, endTime, Integer.MAX_VALUE, events);
    // Write asynchronously to a stream
    streamId = NamespaceId.DEFAULT.stream("testAsync");
    events = Lists.newArrayList();
    streamClient.create(streamId);
    // Send 10 async writes
    int msgCount = 10;
    for (int i = 0; i < msgCount; i++) {
        streamClient.asyncSendEvent(streamId, "Testing " + i);
    }
    // Read them back; need to read it multiple times as the writes happen asynchronously
    while (events.size() != msgCount) {
        events.clear();
        streamClient.getEvents(streamId, 0, Long.MAX_VALUE, msgCount, events);
    }
    // Check that there are no more events
    events.clear();
    while (events.isEmpty()) {
        events.clear();
        streamClient.getEvents(streamId, 0, Long.MAX_VALUE, msgCount, events);
    }
// End write asynchronously
}

Example 64 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class GetStreamEventsCommand method perform.

@Override
public void perform(Arguments arguments, PrintStream output) throws Exception {
    long currentTime = System.currentTimeMillis();
    StreamId streamId = cliConfig.getCurrentNamespace().stream(arguments.get(ArgumentName.STREAM.toString()));
    long startTime = getTimestamp(arguments.getOptional(ArgumentName.START_TIME.toString(), "min"), currentTime);
    long endTime = getTimestamp(arguments.getOptional(ArgumentName.END_TIME.toString(), "max"), currentTime);
    int limit = arguments.getIntOptional(ArgumentName.LIMIT.toString(), Integer.MAX_VALUE);
    // Get a list of stream events and prints it.
    List<StreamEvent> events = streamClient.getEvents(streamId, startTime, endTime, limit, Lists.<StreamEvent>newArrayList());
    Table table = Table.builder().setHeader("timestamp", "headers", "body size", "body").setRows(events, new RowMaker<StreamEvent>() {

        @Override
        public List<?> makeRow(StreamEvent event) {
            long bodySize = event.getBody().remaining();
            return Lists.newArrayList(event.getTimestamp(), event.getHeaders().isEmpty() ? "" : formatHeader(event.getHeaders()), bodySize, getBody(event.getBody()));
        }
    }).build();
    cliConfig.getTableRenderer().render(cliConfig, output, table);
    output.printf("Fetched %d events from stream '%s'", events.size(), streamId.getEntityName());
    output.println();
}

Example 65 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class StreamClientTestRun method testSendFile.

private void testSendFile(String msgPrefix, int msgCount) throws Exception {
    StreamId streamId = namespaceId.stream("testSendFile");
    streamClient.create(streamId);
    // Generate msgCount lines of events
    File file = TMP_FOLDER.newFile();
    try (BufferedWriter writer = Files.newWriter(file, Charsets.UTF_8)) {
        for (int i = 0; i < msgCount; i++) {
            writer.write(msgPrefix + i);
            writer.newLine();
        }
    }
    streamClient.sendFile(streamId, "text/plain", file);
    // Reads the msgCount events back
    List<StreamEvent> events = Lists.newArrayList();
    streamClient.getEvents(streamId, 0, Long.MAX_VALUE, Integer.MAX_VALUE, events);
    Assert.assertEquals(msgCount, events.size());
    for (int i = 0; i < msgCount; i++) {
        StreamEvent event = events.get(i);
        Assert.assertEquals(msgPrefix + i, Bytes.toString(event.getBody()));
        Assert.assertEquals("text/plain", event.getHeaders().get("content.type"));
    }
}