Examples with StreamEvent co.cask.cdap.api.flow.flowlet.StreamEvent used on opensource projects

Example 66 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class StreamEventCodec method decodePayload.

/**
 * Reverse of {@link #encodePayload(StreamEvent)}.
 *
 * @param payload The byte array containing the queue payload.
 * @return A {@link StreamEvent} reconstructed from payload.
 */
public StreamEvent decodePayload(byte[] payload) {
    ByteBuffer buffer = ByteBuffer.wrap(payload);
    SchemaHash schemaHash = new SchemaHash(buffer);
    Preconditions.checkArgument(schemaHash.equals(STREAM_EVENT_SCHEMA.getSchemaHash()), "Schema from payload not matching StreamEvent schema.");
    Decoder decoder = new BinaryDecoder(new ByteBufferInputStream(buffer));
    try {
        StreamEventData data = StreamEventDataCodec.decode(decoder);
        // Read the timestamp
        long timestamp = decoder.readLong();
        return new StreamEvent(data, timestamp);
    } catch (IOException e) {
        // It should never happens, otherwise something very wrong.
        throw Throwables.propagate(e);
    }
}

Example 67 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class MultiLiveStreamFileReaderTestBase method testOffsets.

@Test
public void testOffsets() throws Exception {
    String streamName = "offsets";
    StreamId streamId = NamespaceId.DEFAULT.stream(streamName);
    Location location = getLocationFactory().create(streamName);
    location.mkdirs();
    // Create a stream with 1 partition.
    StreamConfig config = new StreamConfig(streamId, Long.MAX_VALUE, 10000, Long.MAX_VALUE, location, null, 1000);
    // Write out 200 events in 5 files, with interleaving timestamps
    for (int i = 0; i < 5; i++) {
        FileWriter<StreamEvent> writer = createWriter(config, "bucket" + i);
        for (int j = 0; j < 200; j++) {
            long timestamp = j * 5 + i;
            writer.append(StreamFileTestUtils.createEvent(timestamp, "Testing " + timestamp));
        }
        writer.close();
    }
    // Create a multi reader
    List<StreamFileOffset> sources = Lists.newArrayList();
    Location partitionLocation = StreamUtils.createPartitionLocation(config.getLocation(), 0, Long.MAX_VALUE);
    for (int i = 0; i < 5; i++) {
        Location eventFile = StreamUtils.createStreamLocation(partitionLocation, "bucket" + i, 0, StreamFileType.EVENT);
        sources.add(new StreamFileOffset(eventFile, 0L, 0));
    }
    MultiLiveStreamFileReader reader = new MultiLiveStreamFileReader(config, sources);
    // Reads some events
    List<StreamEvent> events = Lists.newArrayList();
    long expectedTimestamp = 0L;
    // Read 250 events, in batch size of 10.
    for (int i = 0; i < 25; i++) {
        Assert.assertEquals(10, reader.read(events, 10, 0, TimeUnit.SECONDS));
        Assert.assertEquals(10, events.size());
        for (StreamEvent event : events) {
            Assert.assertEquals(expectedTimestamp, event.getTimestamp());
            Assert.assertEquals("Testing " + expectedTimestamp, Charsets.UTF_8.decode(event.getBody()).toString());
            expectedTimestamp++;
        }
        events.clear();
    }
    // Capture the offsets
    Iterable<StreamFileOffset> offsets = ImmutableList.copyOf(Iterables.transform(reader.getPosition(), new Function<StreamFileOffset, StreamFileOffset>() {

        @Override
        public StreamFileOffset apply(StreamFileOffset input) {
            return new StreamFileOffset(input);
        }
    }));
    reader.close();
    // Create another multi reader with the offsets
    sources.clear();
    for (StreamFileOffset offset : offsets) {
        sources.add(offset);
    }
    // Read 750 events, in batch size of 10.
    reader = new MultiLiveStreamFileReader(config, sources);
    for (int i = 0; i < 75; i++) {
        Assert.assertEquals(10, reader.read(events, 10, 0, TimeUnit.SECONDS));
        Assert.assertEquals(10, events.size());
        for (StreamEvent event : events) {
            Assert.assertEquals(expectedTimestamp, event.getTimestamp());
            Assert.assertEquals("Testing " + expectedTimestamp, Charsets.UTF_8.decode(event.getBody()).toString());
            expectedTimestamp++;
        }
        events.clear();
    }
    Assert.assertEquals(0, reader.read(events, 10, 2, TimeUnit.SECONDS));
    reader.close();
}

Example 68 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class MultiLiveStreamFileReaderTestBase method testLiveFileReader.

@Test
public void testLiveFileReader() throws Exception {
    String streamName = "liveReader";
    StreamId streamId = NamespaceId.DEFAULT.stream(streamName);
    Location location = getLocationFactory().create(streamName);
    location.mkdirs();
    // Create a stream with 5 seconds partition.
    StreamConfig config = new StreamConfig(streamId, 5000, 1000, Long.MAX_VALUE, location, null, 1000);
    // Write 5 events in the first partition
    try (FileWriter<StreamEvent> writer = createWriter(config, "live.0")) {
        for (int i = 0; i < 5; i++) {
            writer.append(StreamFileTestUtils.createEvent(i, "Testing " + i));
        }
    }
    // Writer 5 events in the forth partition (ts = 15 to 19)
    try (FileWriter<StreamEvent> writer = createWriter(config, "live.0")) {
        for (int i = 0; i < 5; i++) {
            writer.append(StreamFileTestUtils.createEvent(i + 15, "Testing " + (i + 15)));
        }
    }
    // Create a LiveStreamFileReader to read 10 events. It should be able to read them all.
    Location partitionLocation = StreamUtils.createPartitionLocation(config.getLocation(), 0, config.getPartitionDuration());
    Location eventLocation = StreamUtils.createStreamLocation(partitionLocation, "live.0", 0, StreamFileType.EVENT);
    List<StreamEvent> events = new ArrayList<>();
    try (LiveStreamFileReader reader = new LiveStreamFileReader(config, new StreamFileOffset(eventLocation, 0, 0))) {
        while (events.size() < 10) {
            // It shouldn't have empty read.
            Assert.assertTrue(reader.read(events, Integer.MAX_VALUE, 0, TimeUnit.SECONDS) > 0);
        }
    }
    Assert.assertEquals(10, events.size());
    // First 5 events must have timestamps 0-4
    Iterator<StreamEvent> itor = events.iterator();
    for (int i = 0; i < 5; i++) {
        Assert.assertEquals(i, itor.next().getTimestamp());
    }
    // Next 5 events must have timestamps 15-19
    for (int i = 15; i < 20; i++) {
        Assert.assertEquals(i, itor.next().getTimestamp());
    }
}

Example 69 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class StreamFileJanitorTestBase method testCleanupTTL.

@Test
public void testCleanupTTL() throws Exception {
    // Create a stream with 5 seconds TTL, partition duration of 2 seconds
    String streamName = "testCleanupTTL";
    StreamId streamId = NamespaceId.DEFAULT.stream(streamName);
    authorizer.grant(Authorizable.fromEntityId(streamId), ALICE, EnumSet.of(Action.ADMIN));
    StreamAdmin streamAdmin = getStreamAdmin();
    Properties properties = new Properties();
    properties.setProperty(Constants.Stream.PARTITION_DURATION, "2000");
    properties.setProperty(Constants.Stream.TTL, "5000");
    streamAdmin.create(streamId, properties);
    // Truncate to increment generation to 1. This make verification condition easier (won't affect correctness).
    streamAdmin.truncate(streamId);
    StreamConfig config = streamAdmin.getConfig(streamId);
    SecurityRequestContext.setUserId(BOB.getName());
    // Write data with different timestamps that spans across 5 partitions
    FileWriter<StreamEvent> writer = createWriter(streamId);
    for (int i = 0; i < 10; i++) {
        writer.append(StreamFileTestUtils.createEvent(i * 1000, "Testing " + i));
    }
    writer.close();
    // Should see 5 partitions
    Location generationLocation = StreamUtils.createGenerationLocation(config.getLocation(), 1);
    Assert.assertEquals(5, generationLocation.list().size());
    // Perform clean with current time = 10000 (10 seconds since epoch).
    // Since TTL = 5 seconds, 2 partitions will be remove (Ends at 2000 and ends at 4000).
    janitor.clean(config.getLocation(), config.getTTL(), 10000);
    Assert.assertEquals(3, generationLocation.list().size());
    // Cleanup again with current time = 16000, all partitions should be deleted.
    janitor.clean(config.getLocation(), config.getTTL(), 16000);
    Assert.assertTrue(generationLocation.list().isEmpty());
}

Example 70 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class StreamFileJanitorTestBase method testCleanupDeletedStream.

/**
 * Test clean up for all streams and verify user who does not have privileges on the streams is able to clean up.
 */
@Test
public void testCleanupDeletedStream() throws Exception {
    StreamId streamId = NamespaceId.DEFAULT.stream("cleanupDelete");
    StreamAdmin streamAdmin = getStreamAdmin();
    authorizer.grant(Authorizable.fromEntityId(streamId), ALICE, EnumSet.of(Action.ADMIN));
    streamAdmin.create(streamId);
    // Write some data
    try (FileWriter<StreamEvent> writer = createWriter(streamId)) {
        for (int i = 0; i < 10; i++) {
            writer.append(StreamFileTestUtils.createEvent(i * 1000, "Testing " + i));
        }
    }
    // Delete the stream
    streamAdmin.drop(streamId);
    SecurityRequestContext.setUserId(BOB.getName());
    // Run janitor. Should be running fine without exception.
    // Even Bob does not have privilege on the stream, he should be able to clean up the streams
    janitor.cleanAll();
}