Examples with FileFlusher io.divolte.server.filesinks.FileFlusher used on opensource projects

Example 1 with FileFlusher

use of io.divolte.server.filesinks.FileFlusher in project divolte-collector by divolte.

the class FileFlusherLocalHdfsTest method setupFlusher.

private void setupFlusher(final String rollEvery, final int recordCount) throws IOException {
    final Config config = ConfigFactory.parseMap(ImmutableMap.of("divolte.sinks.hdfs.file_strategy.roll_every", rollEvery, "divolte.sinks.hdfs.file_strategy.working_dir", tempInflightDir.toString(), "divolte.sinks.hdfs.file_strategy.publish_dir", tempPublishDir.toString())).withFallback(ConfigFactory.parseResources("hdfs-flusher-test.conf")).withFallback(ConfigFactory.parseResources("reference-test.conf"));
    final ValidatedConfiguration vc = new ValidatedConfiguration(() -> config);
    records = LongStream.range(0, recordCount).mapToObj((time) -> new GenericRecordBuilder(schema).set("ts", time).set("remoteHost", ARBITRARY_IP).build()).collect(Collectors.toList());
    flusher = new FileFlusher(vc.configuration().getSinkConfiguration("hdfs", FileSinkConfiguration.class).fileStrategy, HdfsFileManager.newFactory(vc, "hdfs", schema).create());
}