use of co.cask.cdap.filetailer.state.FileTailerStateProcessorImpl in project cdap-ingest by caskdata.
the class SizeBasedRotationTest method fileRotationTest.
@Test
public void fileRotationTest() throws ConfigurationLoadingException, InterruptedException {
FileTailerQueue queue = new FileTailerQueue(QUEUE_SIZE);
PipeConfiguration flowConfig = TailerLogUtils.loadConfig();
FileTailerStateProcessor stateProcessor = new FileTailerStateProcessorImpl(flowConfig.getDaemonDir(), flowConfig.getStateFile());
FileTailerMetricsProcessor metricsProcessor = new FileTailerMetricsProcessor(flowConfig.getDaemonDir(), flowConfig.getStatisticsFile(), flowConfig.getStatisticsSleepInterval(), flowConfig.getPipeName(), flowConfig.getSourceConfiguration().getFileName());
LogTailer tailer = new LogTailer(TailerLogUtils.loadConfig(), queue, stateProcessor, metricsProcessor, null);
String filePath = flowConfig.getSourceConfiguration().getWorkDir().getAbsolutePath() + "/" + flowConfig.getSourceConfiguration().getFileName();
List<String> logList = new ArrayList<String>(ENTRY_WRITE_NUMBER);
RandomStringUtils randomUtils = new RandomStringUtils();
Logger logger = TailerLogUtils.getSizeLogger(filePath, LOG_FILE_SIZE);
tailer.startAsync();
for (int i = 0; i < ENTRY_WRITE_NUMBER; i++) {
String currLine = randomUtils.randomAlphanumeric(LINE_SIZE);
logger.debug(currLine);
logList.add(currLine);
if (i % 50 == 0) {
Thread.currentThread().sleep(300);
}
}
Thread.currentThread().sleep(SLEEP_TIME);
for (int i = 0; i < logList.size(); i++) {
Assert.assertEquals(true, queue.take().getEventData().contains(logList.get(i)));
}
tailer.stopAsync();
}
use of co.cask.cdap.filetailer.state.FileTailerStateProcessorImpl in project cdap-ingest by caskdata.
the class RunFromSaveStateTest method runFromSaveStateTest.
@Test
public void runFromSaveStateTest() throws Exception {
FileTailerQueue queue = new FileTailerQueue(QUEUE_SIZE);
PipeConfiguration flowConfig = TailerLogUtils.loadConfig();
FileTailerStateProcessor stateProcessor = new FileTailerStateProcessorImpl(flowConfig.getDaemonDir(), flowConfig.getStateFile());
FileTailerMetricsProcessor metricsProcessor = new FileTailerMetricsProcessor(flowConfig.getDaemonDir(), flowConfig.getStatisticsFile(), flowConfig.getStatisticsSleepInterval(), flowConfig.getPipeName(), flowConfig.getSourceConfiguration().getFileName());
String filePath = flowConfig.getSourceConfiguration().getWorkDir().getAbsolutePath() + "/" + flowConfig.getSourceConfiguration().getFileName();
List<String> logList = new ArrayList<String>(ENTRY_WRITE_NUMBER);
List<String> readLogList = new ArrayList<String>(ENTRY_WRITE_NUMBER);
Logger logger = TailerLogUtils.getSizeLogger(filePath, LOG_FILE_SIZE);
Field queueField = queue.getClass().getDeclaredField("queue");
queueField.setAccessible(true);
LinkedBlockingQueue<FileTailerEvent> intQueue = (LinkedBlockingQueue<FileTailerEvent>) queueField.get(queue);
write_log(ENTRY_WRITE_NUMBER, logger, logList);
LogTailer tailer = new LogTailer(TailerLogUtils.loadConfig(), queue, stateProcessor, metricsProcessor, null);
tailer.startAsync();
Thread.sleep(SLEEP_TIME);
tailer.stopAsync();
saveState(intQueue, queue, readLogList, stateProcessor);
write_log(ENTRY_WRITE_NUMBER, logger, logList);
tailer = new LogTailer(TailerLogUtils.loadConfig(), queue, stateProcessor, metricsProcessor, null);
tailer.startAsync();
Thread.sleep(SLEEP_TIME);
tailer.stopAsync();
saveState(intQueue, queue, readLogList, stateProcessor);
write_log(ENTRY_WRITE_NUMBER, logger, logList);
tailer = new LogTailer(TailerLogUtils.loadConfig(), queue, stateProcessor, metricsProcessor, null);
tailer.startAsync();
Thread.sleep(SLEEP_TIME);
tailer.stopAsync();
saveState(intQueue, queue, readLogList, stateProcessor);
for (int i = 0; i < logList.size(); i++) {
Assert.assertEquals(true, readLogList.get(i).contains(logList.get(i)));
}
}
use of co.cask.cdap.filetailer.state.FileTailerStateProcessorImpl in project cdap-ingest by caskdata.
the class TailerLogUtils method createTailer.
public static LogTailer createTailer(FileTailerQueue queue, PipeConfiguration flowConfig) throws ConfigurationLoadingException {
FileTailerStateProcessor stateProcessor = new FileTailerStateProcessorImpl(flowConfig.getDaemonDir(), flowConfig.getStateFile());
FileTailerMetricsProcessor metricsProcessor = new FileTailerMetricsProcessor(flowConfig.getDaemonDir(), flowConfig.getStatisticsFile(), flowConfig.getStatisticsSleepInterval(), flowConfig.getPipeName(), flowConfig.getSourceConfiguration().getFileName());
LogTailer tailer = new LogTailer(TailerLogUtils.loadConfig(), queue, stateProcessor, metricsProcessor, null);
return tailer;
}
use of co.cask.cdap.filetailer.state.FileTailerStateProcessorImpl in project cdap-ingest by caskdata.
the class FileTailerIT method mockMetricsProcessor.
private void mockMetricsProcessor(PipeManager manager) throws IOException, NoSuchMethodException, InvocationTargetException, IllegalAccessException, NoSuchFieldException {
List<Pipe> pipeList = new ArrayList<Pipe>();
StreamClient client = null;
StreamWriter writer = null;
try {
Method method1 = manager.getClass().getDeclaredMethod("getPipeConfigs");
method1.setAccessible(true);
List<PipeConfiguration> pipeConfList = (List<PipeConfiguration>) method1.invoke(manager);
for (PipeConfiguration pipeConf : pipeConfList) {
FileTailerQueue queue = new FileTailerQueue(pipeConf.getQueueSize());
client = pipeConf.getSinkConfiguration().getStreamClient();
String streamName = pipeConf.getSinkConfiguration().getStreamName();
Method method2 = manager.getClass().getDeclaredMethod("getStreamWriterForPipe", StreamClient.class, String.class);
method2.setAccessible(true);
writer = (StreamWriter) method2.invoke(manager, client, streamName);
FileTailerStateProcessor stateProcessor = new FileTailerStateProcessorImpl(pipeConf.getDaemonDir(), pipeConf.getStateFile());
FileTailerMetricsProcessor metricsProcessor = new FileTailerMetricsProcessor(pipeConf.getDaemonDir(), pipeConf.getStatisticsFile(), pipeConf.getStatisticsSleepInterval(), pipeConf.getPipeName(), pipeConf.getSourceConfiguration().getFileName()) {
@Override
public void onReadEventMetric(int eventSize) {
super.onReadEventMetric(eventSize);
read.incrementAndGet();
}
@Override
public void onIngestEventMetric(int latency) {
super.onIngestEventMetric(latency);
ingest.incrementAndGet();
}
};
pipeList.add(new Pipe(new LogTailer(pipeConf, queue, stateProcessor, metricsProcessor, null), new FileTailerSink(queue, writer, SinkStrategy.LOADBALANCE, stateProcessor, metricsProcessor, null, pipeConf.getSinkConfiguration().getPackSize()), metricsProcessor));
client = null;
writer = null;
}
Field field = manager.getClass().getDeclaredField("serviceManager");
field.setAccessible(true);
Field modifiersField = Field.class.getDeclaredField("modifiers");
modifiersField.setAccessible(true);
modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
field.set(manager, new ServiceManager(pipeList));
} finally {
if (client != null) {
client.close();
}
if (writer != null) {
writer.close();
}
}
}
use of co.cask.cdap.filetailer.state.FileTailerStateProcessorImpl in project cdap-ingest by caskdata.
the class PollingListenerImpl method setupPipe.
/**
* Pipe setup
*
* @throws IOException if can not setup pipe
*/
private Pipe setupPipe(File file) throws IOException {
PipeConfiguration pipeConfiguration = observerConf.getPipeConfiguration(file.getName());
FileTailerQueue queue = new FileTailerQueue(pipeConfiguration.getQueueSize());
StreamWriter writer = getStreamWriterForPipe(pipeConfiguration);
FileTailerStateProcessor stateProcessor = new FileTailerStateProcessorImpl(observerConf.getDaemonDir(), pipeConfiguration.getStateFile());
PipeListener pipeListener = new PipeListenerImpl(pipeConfiguration.getSourceConfiguration().getWorkDir(), file.getAbsolutePath(), observerConf.getDaemonDir() + "/" + pipeConfiguration.getStateFile());
Pipe pipe = new Pipe(new LogTailer(pipeConfiguration, queue, stateProcessor, metricsProcessor, pipeListener), new FileTailerSink(queue, writer, SinkStrategy.LOADBALANCE, stateProcessor, metricsProcessor, pipeListener, pipeConfiguration.getSinkConfiguration().getPackSize()));
pipeListener.setPipe(pipe);
return pipe;
}
Aggregations