use of co.cask.cdap.filetailer.Pipe in project cdap-ingest by caskdata.
the class PollingListenerImpl method onFileCreate.
@Override
public void onFileCreate(File file) throws IOException {
LOG.info("File Added: {}", file.getAbsolutePath());
LOG.debug("Start configure pipe for file: {}", file.getAbsolutePath());
Pipe pipe = setupPipe(file);
LOG.debug("Pipe for file {} successfully configured", file.getAbsolutePath());
LOG.info("Start processing file: {}", file.getAbsolutePath());
pipe.startAsync();
}
use of co.cask.cdap.filetailer.Pipe in project cdap-ingest by caskdata.
the class PollingListenerImpl method setupPipe.
/**
* Pipe setup
*
* @throws IOException if can not setup pipe
*/
private Pipe setupPipe(File file) throws IOException {
PipeConfiguration pipeConfiguration = observerConf.getPipeConfiguration(file.getName());
FileTailerQueue queue = new FileTailerQueue(pipeConfiguration.getQueueSize());
StreamWriter writer = getStreamWriterForPipe(pipeConfiguration);
FileTailerStateProcessor stateProcessor = new FileTailerStateProcessorImpl(observerConf.getDaemonDir(), pipeConfiguration.getStateFile());
PipeListener pipeListener = new PipeListenerImpl(pipeConfiguration.getSourceConfiguration().getWorkDir(), file.getAbsolutePath(), observerConf.getDaemonDir() + "/" + pipeConfiguration.getStateFile());
Pipe pipe = new Pipe(new LogTailer(pipeConfiguration, queue, stateProcessor, metricsProcessor, pipeListener), new FileTailerSink(queue, writer, SinkStrategy.LOADBALANCE, stateProcessor, metricsProcessor, pipeListener, pipeConfiguration.getSinkConfiguration().getPackSize()));
pipeListener.setPipe(pipe);
return pipe;
}
Aggregations