Examples with StreamEvent co.cask.cdap.api.flow.flowlet.StreamEvent used on opensource projects

Example 76 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class StreamDataFileTestBase method testIndexIterator.

@Test
public void testIndexIterator() throws Exception {
    Location dir = StreamFileTestUtils.createTempDir(getLocationFactory());
    Location eventFile = dir.getTempFile(".dat");
    Location indexFile = dir.getTempFile(".idx");
    // Write 1000 events with different timestamps, and create index for every 100 timestamps.
    StreamDataFileWriter writer = new StreamDataFileWriter(Locations.newOutputSupplier(eventFile), Locations.newOutputSupplier(indexFile), 100L);
    for (int i = 0; i < 1000; i++) {
        writer.append(StreamFileTestUtils.createEvent(1000 + i, "Testing " + i));
    }
    writer.close();
    // Iterate the index
    StreamDataFileIndex index = new StreamDataFileIndex(Locations.newInputSupplier(indexFile));
    StreamDataFileIndexIterator iterator = index.indexIterator();
    long ts = 1000;
    while (iterator.nextIndexEntry()) {
        Assert.assertEquals(ts, iterator.currentTimestamp());
        StreamDataFileReader reader = StreamDataFileReader.createWithOffset(Locations.newInputSupplier(eventFile), Locations.newInputSupplier(indexFile), iterator.currentPosition());
        List<StreamEvent> events = Lists.newArrayList();
        Assert.assertEquals(1, reader.read(events, 1, 0, TimeUnit.SECONDS));
        Assert.assertEquals("Testing " + (ts - 1000), Charsets.UTF_8.decode(events.get(0).getBody()).toString());
        ts += 100;
    }
    Assert.assertEquals(2000, ts);
}

Example 77 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class StreamDataFileTestBase method testOffsetAtEnd.

@Test
public void testOffsetAtEnd() throws IOException, InterruptedException {
    // Test for offset at the end of file
    Location dir = StreamFileTestUtils.createTempDir(getLocationFactory());
    Location eventFile = dir.getTempFile(".dat");
    Location indexFile = dir.getTempFile(".idx");
    // Write 1 event.
    StreamDataFileWriter writer = new StreamDataFileWriter(Locations.newOutputSupplier(eventFile), Locations.newOutputSupplier(indexFile), 100L);
    writer.append(StreamFileTestUtils.createEvent(1, "Testing"));
    writer.close();
    // Read 1 event.
    List<StreamEvent> events = Lists.newArrayList();
    StreamDataFileReader reader = StreamDataFileReader.create(Locations.newInputSupplier(eventFile));
    Assert.assertEquals(1, reader.read(events, 10, 0, TimeUnit.SECONDS));
    // Create a reader with the offset pointing to EOF timestamp.
    long offset = reader.getPosition();
    reader = StreamDataFileReader.createWithOffset(Locations.newInputSupplier(eventFile), Locations.newInputSupplier(indexFile), offset);
    Assert.assertEquals(-1, reader.read(events, 10, 0, TimeUnit.SECONDS));
    // Create a read with offset way pass EOF
    reader = StreamDataFileReader.createWithOffset(Locations.newInputSupplier(eventFile), Locations.newInputSupplier(indexFile), eventFile.length() + 100);
    Assert.assertEquals(-1, reader.read(events, 10, 0, TimeUnit.SECONDS));
}

Example 78 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class StreamDataFileTestBase method testMaxEvents.

@Test
public void testMaxEvents() throws Exception {
    Location dir = StreamFileTestUtils.createTempDir(getLocationFactory());
    Location eventFile = dir.getTempFile(".dat");
    Location indexFile = dir.getTempFile(".idx");
    // Write 1000 events with 100 different timestamps, and create index for every 100ms timestamps.
    StreamDataFileWriter writer = new StreamDataFileWriter(Locations.newOutputSupplier(eventFile), Locations.newOutputSupplier(indexFile), 100L);
    for (int i = 0; i < 100; i++) {
        for (int j = 0; j < 10; j++) {
            writer.append(StreamFileTestUtils.createEvent(i, "Testing " + (i * 10 + j)));
        }
    }
    writer.close();
    // Reads events one by one
    List<StreamEvent> events = Lists.newArrayList();
    StreamDataFileReader reader = StreamDataFileReader.create(Locations.newInputSupplier(eventFile));
    int expectedId = 0;
    while (reader.read(events, 1, 1, TimeUnit.SECONDS) >= 0) {
        Assert.assertEquals(1, events.size());
        StreamEvent event = events.get(0);
        long expectedTimestamp = expectedId / 10;
        Assert.assertEquals(expectedTimestamp, event.getTimestamp());
        Assert.assertEquals("Testing " + expectedId, Charsets.UTF_8.decode(event.getBody()).toString());
        expectedId++;
        events.clear();
    }
    reader.close();
    // Reads four events every time, with a new reader.
    events.clear();
    reader = StreamDataFileReader.create(Locations.newInputSupplier(eventFile));
    int expectedSize = 4;
    while (reader.read(events, 4, 1, TimeUnit.SECONDS) >= 0) {
        Assert.assertEquals(expectedSize, events.size());
        expectedSize += 4;
        long position = reader.getPosition();
        reader.close();
        reader = StreamDataFileReader.createWithOffset(Locations.newInputSupplier(eventFile), Locations.newInputSupplier(indexFile), position);
    }
    // Verify all events are read
    Assert.assertEquals(1000, events.size());
    expectedId = 0;
    for (StreamEvent event : events) {
        long expectedTimestamp = expectedId / 10;
        Assert.assertEquals(expectedTimestamp, event.getTimestamp());
        Assert.assertEquals("Testing " + expectedId, Charsets.UTF_8.decode(event.getBody()).toString());
        expectedId++;
    }
}

Example 79 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class StreamDataFileTestBase method testIndex.

@Test
public void testIndex() throws Exception {
    Location dir = StreamFileTestUtils.createTempDir(getLocationFactory());
    Location eventFile = dir.getTempFile(".dat");
    Location indexFile = dir.getTempFile(".idx");
    // Write 1000 events with different timestamps, and create index for every 100 timestamps.
    StreamDataFileWriter writer = new StreamDataFileWriter(Locations.newOutputSupplier(eventFile), Locations.newOutputSupplier(indexFile), 100L);
    for (int i = 0; i < 1000; i++) {
        writer.append(StreamFileTestUtils.createEvent(1000 + i, "Testing " + i));
    }
    writer.close();
    // Read with index
    for (long ts : new long[] { 1050, 1110, 1200, 1290, 1301, 1400, 1500, 1600, 1898, 1900, 1999 }) {
        StreamDataFileReader reader = StreamDataFileReader.createByStartTime(Locations.newInputSupplier(eventFile), Locations.newInputSupplier(indexFile), ts);
        Queue<StreamEvent> events = Lists.newLinkedList();
        Assert.assertEquals(1, reader.read(events, 1, 1L, TimeUnit.MILLISECONDS));
        Assert.assertEquals(ts, events.poll().getTimestamp());
        reader.close();
    }
}

Example 80 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class StreamDataFileTestBase method testPosition.

@Test
public void testPosition() throws Exception {
    Location dir = StreamFileTestUtils.createTempDir(getLocationFactory());
    Location eventFile = dir.getTempFile(".dat");
    Location indexFile = dir.getTempFile(".idx");
    // Write 10 events with different timestamps. Index doesn't matter
    StreamDataFileWriter writer = new StreamDataFileWriter(Locations.newOutputSupplier(eventFile), Locations.newOutputSupplier(indexFile), 100L);
    for (int i = 0; i < 10; i++) {
        writer.append(StreamFileTestUtils.createEvent(i, "Testing " + i));
    }
    writer.close();
    // Read 4 events
    StreamDataFileReader reader = StreamDataFileReader.create(Locations.newInputSupplier(eventFile));
    List<StreamEvent> events = Lists.newArrayList();
    reader.read(events, 4, 1, TimeUnit.SECONDS);
    Assert.assertEquals(4, events.size());
    for (StreamEvent event : events) {
        Assert.assertEquals("Testing " + event.getTimestamp(), Charsets.UTF_8.decode(event.getBody()).toString());
    }
    long position = reader.getPosition();
    reader.close();
    // Open a new reader, read from the last position.
    reader = StreamDataFileReader.createWithOffset(Locations.newInputSupplier(eventFile), Locations.newInputSupplier(indexFile), position);
    events.clear();
    reader.read(events, 10, 1, TimeUnit.SECONDS);
    Assert.assertEquals(6, events.size());
    for (int i = 0; i < 6; i++) {
        StreamEvent event = events.get(i);
        Assert.assertEquals((long) (i + 4), event.getTimestamp());
        Assert.assertEquals("Testing " + event.getTimestamp(), Charsets.UTF_8.decode(event.getBody()).toString());
    }
}