Examples with StreamDataFileWriter co.cask.cdap.data.stream.StreamDataFileWriter used on opensource projects

Example 1 with StreamDataFileWriter

use of co.cask.cdap.data.stream.StreamDataFileWriter in project cdap by caskdata.

the class ConcurrentStreamWriterTestBase method generateFile.

private FileInfo generateFile(NamespacedLocationFactory locationFactory, int id, int events) throws IOException {
    NamespaceId dummyNs = new NamespaceId("dummy");
    Location eventLocation = locationFactory.get(dummyNs).append(UUID.randomUUID().toString());
    Location indexLocation = locationFactory.get(dummyNs).append(UUID.randomUUID().toString());
    StreamDataFileWriter writer = new StreamDataFileWriter(Locations.newOutputSupplier(eventLocation), Locations.newOutputSupplier(indexLocation), 1000L);
    for (int i = 0; i < events; i++) {
        writer.append(StreamFileTestUtils.createEvent(System.currentTimeMillis(), "Message " + i + " from " + id));
        if (i % 50 == 0) {
            writer.flush();
        }
    }
    writer.flush();
    return new FileInfo(eventLocation, indexLocation, writer, events);
}

Example 2 with StreamDataFileWriter

use of co.cask.cdap.data.stream.StreamDataFileWriter in project cdap by caskdata.

the class StreamFileSizeFetcherTest method testFetchSize.

@Test
public void testFetchSize() throws Exception {
    final String streamName = "testFetchSize";
    StreamId streamId = NamespaceId.DEFAULT.stream(streamName);
    final int nbEvents = 100;
    StreamAdmin streamAdmin = new TestStreamAdmin(namespacedLocationFactory, Long.MAX_VALUE, 1000);
    streamAdmin.create(streamId);
    StreamConfig config = streamAdmin.getConfig(streamId);
    try {
        StreamUtils.fetchStreamFilesSize(StreamUtils.createGenerationLocation(config.getLocation(), StreamUtils.getGeneration(config)));
        Assert.fail("No stream file created yet");
    } catch (IOException e) {
    // Expected
    }
    // Creates a stream file that has no event inside
    Location partitionLocation = StreamUtils.createPartitionLocation(config.getLocation(), 0, Long.MAX_VALUE);
    Location dataLocation = StreamUtils.createStreamLocation(partitionLocation, "writer", 0, StreamFileType.EVENT);
    Location idxLocation = StreamUtils.createStreamLocation(partitionLocation, "writer", 0, StreamFileType.INDEX);
    StreamDataFileWriter writer = new StreamDataFileWriter(Locations.newOutputSupplier(dataLocation), Locations.newOutputSupplier(idxLocation), 10000L);
    // Write 100 events to the stream
    for (int i = 0; i < nbEvents; i++) {
        writer.append(StreamFileTestUtils.createEvent(i, "foo"));
    }
    writer.close();
    long size = StreamUtils.fetchStreamFilesSize(StreamUtils.createGenerationLocation(config.getLocation(), StreamUtils.getGeneration(config)));
    Assert.assertTrue(size > 0);
    Assert.assertEquals(dataLocation.length(), size);
}