Examples with StreamFileOffset co.cask.cdap.data.stream.StreamFileOffset used on opensource projects

Example 6 with StreamFileOffset

use of co.cask.cdap.data.stream.StreamFileOffset in project cdap by caskdata.

the class AbstractStreamFileConsumerFactory method getFileOffsets.

protected void getFileOffsets(Location partitionLocation, Collection<? super StreamFileOffset> fileOffsets, int generation) throws IOException {
    // TODO: Support dynamic writer instances discovery
    // Current assume it won't change and is based on cConf
    int instances = cConf.getInt(Constants.Stream.CONTAINER_INSTANCES);
    String filePrefix = cConf.get(Constants.Stream.FILE_PREFIX);
    for (int i = 0; i < instances; i++) {
        // The actual file prefix is formed by file prefix in cConf + writer instance id
        String streamFilePrefix = filePrefix + '.' + i;
        Location eventLocation = StreamUtils.createStreamLocation(partitionLocation, streamFilePrefix, 0, StreamFileType.EVENT);
        fileOffsets.add(new StreamFileOffset(eventLocation, 0, generation));
    }
}

Example 7 with StreamFileOffset

use of co.cask.cdap.data.stream.StreamFileOffset in project cdap by caskdata.

the class StreamFetchHandler method createReader.

/**
 * Creates a {@link FileReader} that starts reading stream event from the given partition.
 */
private FileReader<StreamEventOffset, Iterable<StreamFileOffset>> createReader(StreamConfig streamConfig, long startTime) throws IOException {
    int generation = StreamUtils.getGeneration(streamConfig);
    Location startPartition = getStartPartitionLocation(streamConfig, startTime, generation);
    if (startPartition == null) {
        return createEmptyReader();
    }
    List<StreamFileOffset> fileOffsets = Lists.newArrayList();
    int instances = cConf.getInt(Constants.Stream.CONTAINER_INSTANCES);
    String filePrefix = cConf.get(Constants.Stream.FILE_PREFIX);
    for (int i = 0; i < instances; i++) {
        // The actual file prefix is formed by file prefix in cConf + writer instance id
        String streamFilePrefix = filePrefix + '.' + i;
        Location eventLocation = StreamUtils.createStreamLocation(startPartition, streamFilePrefix, 0, StreamFileType.EVENT);
        fileOffsets.add(new StreamFileOffset(eventLocation, 0, generation));
    }
    MultiLiveStreamFileReader reader = new MultiLiveStreamFileReader(streamConfig, fileOffsets);
    reader.initialize();
    return reader;
}