Search in sources :

Example 1 with FileTailerMetricsProcessor

use of co.cask.cdap.filetailer.metrics.FileTailerMetricsProcessor in project cdap-ingest by caskdata.

the class FileTailerSinkTest method basicTestWithCustomPackSizeMultipleWriters.

@Test
public void basicTestWithCustomPackSizeMultipleWriters() throws Exception {
    FileTailerStateProcessor stateProcessor = Mockito.mock(FileTailerStateProcessor.class);
    FileTailerMetricsProcessor metricsProcessor = Mockito.mock(FileTailerMetricsProcessor.class);
    FileTailerQueue queue = new FileTailerQueue(DEFAULT_QUEUE_SIZE);
    final AtomicInteger count = new AtomicInteger(0);
    StreamWriter writers = getDummyConcurrentWriter(count);
    boolean success = false;
    FileTailerSink sink = new FileTailerSink(queue, writers, SinkStrategy.LOADBALANCE, stateProcessor, metricsProcessor, null, CUSTOM_PACK_SIZE);
    try {
        sink.startAsync();
        for (int i = 0; i < TEST_EVENTS_SIZE; i++) {
            queue.put(new FileTailerEvent(new FileTailerState("file", 0L, 42, 0L), "test", Charset.defaultCharset()));
        }
        int attempts = 10;
        while (attempts > 0) {
            attempts--;
            if (count.get() == TEST_EVENTS_SIZE) {
                success = true;
                break;
            }
            Thread.sleep(1000);
        }
    } finally {
        sink.stopAsync();
    }
    org.junit.Assert.assertTrue(success);
}
Also used : FileTailerState(co.cask.cdap.filetailer.state.FileTailerState) FileTailerQueue(co.cask.cdap.filetailer.queue.FileTailerQueue) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) FileTailerEvent(co.cask.cdap.filetailer.event.FileTailerEvent) StreamWriter(co.cask.cdap.client.StreamWriter) FileTailerStateProcessor(co.cask.cdap.filetailer.state.FileTailerStateProcessor) FileTailerMetricsProcessor(co.cask.cdap.filetailer.metrics.FileTailerMetricsProcessor) Test(org.junit.Test)

Example 2 with FileTailerMetricsProcessor

use of co.cask.cdap.filetailer.metrics.FileTailerMetricsProcessor in project cdap-ingest by caskdata.

the class FileTailerSinkTest method basicTestWithDefaultPackSize.

@Test
public void basicTestWithDefaultPackSize() throws Exception {
    FileTailerStateProcessor stateProcessor = Mockito.mock(FileTailerStateProcessor.class);
    FileTailerMetricsProcessor metricsProcessor = Mockito.mock(FileTailerMetricsProcessor.class);
    FileTailerQueue queue = new FileTailerQueue(DEFAULT_QUEUE_SIZE);
    StreamWriter writerMock = getDummyStreamWriter();
    FileTailerSink sink = new FileTailerSink(queue, writerMock, SinkStrategy.LOADBALANCE, stateProcessor, metricsProcessor, null);
    sink.startAsync();
    for (int i = 0; i < TEST_EVENTS_SIZE; i++) {
        queue.put(new FileTailerEvent(new FileTailerState("file", 0L, 42, 0L), "test", Charset.defaultCharset()));
    }
    Mockito.verify(writerMock, Mockito.timeout(10000).times(TEST_EVENTS_SIZE)).write("test", Charset.defaultCharset());
    sink.stopAsync();
}
Also used : FileTailerState(co.cask.cdap.filetailer.state.FileTailerState) FileTailerQueue(co.cask.cdap.filetailer.queue.FileTailerQueue) FileTailerEvent(co.cask.cdap.filetailer.event.FileTailerEvent) StreamWriter(co.cask.cdap.client.StreamWriter) FileTailerStateProcessor(co.cask.cdap.filetailer.state.FileTailerStateProcessor) FileTailerMetricsProcessor(co.cask.cdap.filetailer.metrics.FileTailerMetricsProcessor) Test(org.junit.Test)

Example 3 with FileTailerMetricsProcessor

use of co.cask.cdap.filetailer.metrics.FileTailerMetricsProcessor in project cdap-ingest by caskdata.

the class SizeBasedRotationTest method fileRotationTest.

@Test
public void fileRotationTest() throws ConfigurationLoadingException, InterruptedException {
    FileTailerQueue queue = new FileTailerQueue(QUEUE_SIZE);
    PipeConfiguration flowConfig = TailerLogUtils.loadConfig();
    FileTailerStateProcessor stateProcessor = new FileTailerStateProcessorImpl(flowConfig.getDaemonDir(), flowConfig.getStateFile());
    FileTailerMetricsProcessor metricsProcessor = new FileTailerMetricsProcessor(flowConfig.getDaemonDir(), flowConfig.getStatisticsFile(), flowConfig.getStatisticsSleepInterval(), flowConfig.getPipeName(), flowConfig.getSourceConfiguration().getFileName());
    LogTailer tailer = new LogTailer(TailerLogUtils.loadConfig(), queue, stateProcessor, metricsProcessor, null);
    String filePath = flowConfig.getSourceConfiguration().getWorkDir().getAbsolutePath() + "/" + flowConfig.getSourceConfiguration().getFileName();
    List<String> logList = new ArrayList<String>(ENTRY_WRITE_NUMBER);
    RandomStringUtils randomUtils = new RandomStringUtils();
    Logger logger = TailerLogUtils.getSizeLogger(filePath, LOG_FILE_SIZE);
    tailer.startAsync();
    for (int i = 0; i < ENTRY_WRITE_NUMBER; i++) {
        String currLine = randomUtils.randomAlphanumeric(LINE_SIZE);
        logger.debug(currLine);
        logList.add(currLine);
        if (i % 50 == 0) {
            Thread.currentThread().sleep(300);
        }
    }
    Thread.currentThread().sleep(SLEEP_TIME);
    for (int i = 0; i < logList.size(); i++) {
        Assert.assertEquals(true, queue.take().getEventData().contains(logList.get(i)));
    }
    tailer.stopAsync();
}
Also used : FileTailerStateProcessorImpl(co.cask.cdap.filetailer.state.FileTailerStateProcessorImpl) PipeConfiguration(co.cask.cdap.filetailer.config.PipeConfiguration) FileTailerQueue(co.cask.cdap.filetailer.queue.FileTailerQueue) ArrayList(java.util.ArrayList) FileTailerStateProcessor(co.cask.cdap.filetailer.state.FileTailerStateProcessor) Logger(ch.qos.logback.classic.Logger) FileTailerMetricsProcessor(co.cask.cdap.filetailer.metrics.FileTailerMetricsProcessor) RandomStringUtils(org.apache.commons.lang.RandomStringUtils) Test(org.junit.Test)

Example 4 with FileTailerMetricsProcessor

use of co.cask.cdap.filetailer.metrics.FileTailerMetricsProcessor in project cdap-ingest by caskdata.

the class TailerLogUtils method createTailer.

public static LogTailer createTailer(FileTailerQueue queue, PipeConfiguration flowConfig) throws ConfigurationLoadingException {
    FileTailerStateProcessor stateProcessor = new FileTailerStateProcessorImpl(flowConfig.getDaemonDir(), flowConfig.getStateFile());
    FileTailerMetricsProcessor metricsProcessor = new FileTailerMetricsProcessor(flowConfig.getDaemonDir(), flowConfig.getStatisticsFile(), flowConfig.getStatisticsSleepInterval(), flowConfig.getPipeName(), flowConfig.getSourceConfiguration().getFileName());
    LogTailer tailer = new LogTailer(TailerLogUtils.loadConfig(), queue, stateProcessor, metricsProcessor, null);
    return tailer;
}
Also used : FileTailerStateProcessorImpl(co.cask.cdap.filetailer.state.FileTailerStateProcessorImpl) FileTailerStateProcessor(co.cask.cdap.filetailer.state.FileTailerStateProcessor) FileTailerMetricsProcessor(co.cask.cdap.filetailer.metrics.FileTailerMetricsProcessor)

Example 5 with FileTailerMetricsProcessor

use of co.cask.cdap.filetailer.metrics.FileTailerMetricsProcessor in project cdap-ingest by caskdata.

the class RunFromSaveStateTest method runFromSaveStateTest.

@Test
public void runFromSaveStateTest() throws Exception {
    FileTailerQueue queue = new FileTailerQueue(QUEUE_SIZE);
    PipeConfiguration flowConfig = TailerLogUtils.loadConfig();
    FileTailerStateProcessor stateProcessor = new FileTailerStateProcessorImpl(flowConfig.getDaemonDir(), flowConfig.getStateFile());
    FileTailerMetricsProcessor metricsProcessor = new FileTailerMetricsProcessor(flowConfig.getDaemonDir(), flowConfig.getStatisticsFile(), flowConfig.getStatisticsSleepInterval(), flowConfig.getPipeName(), flowConfig.getSourceConfiguration().getFileName());
    String filePath = flowConfig.getSourceConfiguration().getWorkDir().getAbsolutePath() + "/" + flowConfig.getSourceConfiguration().getFileName();
    List<String> logList = new ArrayList<String>(ENTRY_WRITE_NUMBER);
    List<String> readLogList = new ArrayList<String>(ENTRY_WRITE_NUMBER);
    Logger logger = TailerLogUtils.getSizeLogger(filePath, LOG_FILE_SIZE);
    Field queueField = queue.getClass().getDeclaredField("queue");
    queueField.setAccessible(true);
    LinkedBlockingQueue<FileTailerEvent> intQueue = (LinkedBlockingQueue<FileTailerEvent>) queueField.get(queue);
    write_log(ENTRY_WRITE_NUMBER, logger, logList);
    LogTailer tailer = new LogTailer(TailerLogUtils.loadConfig(), queue, stateProcessor, metricsProcessor, null);
    tailer.startAsync();
    Thread.sleep(SLEEP_TIME);
    tailer.stopAsync();
    saveState(intQueue, queue, readLogList, stateProcessor);
    write_log(ENTRY_WRITE_NUMBER, logger, logList);
    tailer = new LogTailer(TailerLogUtils.loadConfig(), queue, stateProcessor, metricsProcessor, null);
    tailer.startAsync();
    Thread.sleep(SLEEP_TIME);
    tailer.stopAsync();
    saveState(intQueue, queue, readLogList, stateProcessor);
    write_log(ENTRY_WRITE_NUMBER, logger, logList);
    tailer = new LogTailer(TailerLogUtils.loadConfig(), queue, stateProcessor, metricsProcessor, null);
    tailer.startAsync();
    Thread.sleep(SLEEP_TIME);
    tailer.stopAsync();
    saveState(intQueue, queue, readLogList, stateProcessor);
    for (int i = 0; i < logList.size(); i++) {
        Assert.assertEquals(true, readLogList.get(i).contains(logList.get(i)));
    }
}
Also used : FileTailerEvent(co.cask.cdap.filetailer.event.FileTailerEvent) ArrayList(java.util.ArrayList) FileTailerStateProcessor(co.cask.cdap.filetailer.state.FileTailerStateProcessor) Logger(ch.qos.logback.classic.Logger) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) FileTailerMetricsProcessor(co.cask.cdap.filetailer.metrics.FileTailerMetricsProcessor) Field(java.lang.reflect.Field) FileTailerStateProcessorImpl(co.cask.cdap.filetailer.state.FileTailerStateProcessorImpl) PipeConfiguration(co.cask.cdap.filetailer.config.PipeConfiguration) FileTailerQueue(co.cask.cdap.filetailer.queue.FileTailerQueue) Test(org.junit.Test)

Aggregations

FileTailerMetricsProcessor (co.cask.cdap.filetailer.metrics.FileTailerMetricsProcessor)9 FileTailerStateProcessor (co.cask.cdap.filetailer.state.FileTailerStateProcessor)8 FileTailerQueue (co.cask.cdap.filetailer.queue.FileTailerQueue)7 Test (org.junit.Test)6 StreamWriter (co.cask.cdap.client.StreamWriter)5 FileTailerStateProcessorImpl (co.cask.cdap.filetailer.state.FileTailerStateProcessorImpl)5 PipeConfiguration (co.cask.cdap.filetailer.config.PipeConfiguration)4 FileTailerEvent (co.cask.cdap.filetailer.event.FileTailerEvent)4 FileTailerState (co.cask.cdap.filetailer.state.FileTailerState)3 Field (java.lang.reflect.Field)3 ArrayList (java.util.ArrayList)3 Logger (ch.qos.logback.classic.Logger)2 StreamClient (co.cask.cdap.client.StreamClient)2 FileTailerSink (co.cask.cdap.filetailer.sink.FileTailerSink)2 LogTailer (co.cask.cdap.filetailer.tailer.LogTailer)2 ServiceManager (com.google.common.util.concurrent.ServiceManager)2 ObserverConfiguration (co.cask.cdap.file.dropzone.config.ObserverConfiguration)1 PollingListener (co.cask.cdap.file.dropzone.polling.PollingListener)1 PollingListenerImpl (co.cask.cdap.file.dropzone.polling.PollingListenerImpl)1 PollingService (co.cask.cdap.file.dropzone.polling.PollingService)1