Examples with StreamConfig co.cask.cdap.data2.transaction.stream.StreamConfig used on opensource projects

Example 31 with StreamConfig

use of co.cask.cdap.data2.transaction.stream.StreamConfig in project cdap by caskdata.

the class HiveStreamInputFormat method getSplitFinder.

private StreamInputSplitFinder<InputSplit> getSplitFinder(JobConf conf) throws IOException {
    // first get the context we are in
    ContextManager.Context context = ContextManager.getContext(conf);
    Preconditions.checkNotNull(context);
    StreamConfig streamConfig = context.getStreamConfig(getStreamId(conf));
    // make sure we get the current generation so we don't read events that occurred before a truncate.
    Location streamPath = StreamUtils.createGenerationLocation(streamConfig.getLocation(), StreamUtils.getGeneration(streamConfig));
    StreamInputSplitFinder.Builder builder = StreamInputSplitFinder.builder(streamPath.toURI());
    // Get the Hive table path for the InputSplit created. It is just to satisfy hive. The InputFormat never uses it.
    JobContext jobContext = ShimLoader.getHadoopShims().newJobContext(Job.getInstance(conf));
    final Path[] tablePaths = FileInputFormat.getInputPaths(jobContext);
    return setupBuilder(conf, streamConfig, builder).build(new StreamInputSplitFactory<InputSplit>() {

        @Override
        public InputSplit createSplit(Path eventPath, Path indexPath, long startTime, long endTime, long start, long length, @Nullable String[] locations) {
            return new StreamInputSplit(tablePaths[0], eventPath, indexPath, startTime, endTime, start, length, locations);
        }
    });
}

Example 32 with StreamConfig

use of co.cask.cdap.data2.transaction.stream.StreamConfig in project cdap by caskdata.

the class StreamInputFormatProvider method getInputFormatConfiguration.

@Override
public Map<String, String> getInputFormatConfiguration() {
    try {
        StreamConfig streamConfig = streamAdmin.getConfig(streamId);
        Location streamPath = StreamUtils.createGenerationLocation(streamConfig.getLocation(), StreamUtils.getGeneration(streamConfig));
        Configuration hConf = new Configuration();
        hConf.clear();
        AbstractStreamInputFormat.setStreamId(hConf, streamId);
        AbstractStreamInputFormat.setTTL(hConf, streamConfig.getTTL());
        AbstractStreamInputFormat.setStreamPath(hConf, streamPath.toURI());
        AbstractStreamInputFormat.setTimeRange(hConf, streamInput.getStartTime(), streamInput.getEndTime());
        FormatSpecification formatSpec = streamInput.getBodyFormatSpec();
        if (formatSpec != null) {
            AbstractStreamInputFormat.setBodyFormatSpecification(hConf, formatSpec);
        } else {
            String decoderType = streamInput.getDecoderType();
            if (decoderType != null) {
                AbstractStreamInputFormat.setDecoderClassName(hConf, decoderType);
            }
        }
        return ConfigurationUtil.toMap(hConf);
    } catch (IOException e) {
        throw Throwables.propagate(e);
    }
}