Search in sources :

Example 1 with FileTailerStateProcessorImpl

use of co.cask.cdap.filetailer.state.FileTailerStateProcessorImpl in project cdap-ingest by caskdata.

the class SizeBasedRotationTest method fileRotationTest.

@Test
public void fileRotationTest() throws ConfigurationLoadingException, InterruptedException {
    FileTailerQueue queue = new FileTailerQueue(QUEUE_SIZE);
    PipeConfiguration flowConfig = TailerLogUtils.loadConfig();
    FileTailerStateProcessor stateProcessor = new FileTailerStateProcessorImpl(flowConfig.getDaemonDir(), flowConfig.getStateFile());
    FileTailerMetricsProcessor metricsProcessor = new FileTailerMetricsProcessor(flowConfig.getDaemonDir(), flowConfig.getStatisticsFile(), flowConfig.getStatisticsSleepInterval(), flowConfig.getPipeName(), flowConfig.getSourceConfiguration().getFileName());
    LogTailer tailer = new LogTailer(TailerLogUtils.loadConfig(), queue, stateProcessor, metricsProcessor, null);
    String filePath = flowConfig.getSourceConfiguration().getWorkDir().getAbsolutePath() + "/" + flowConfig.getSourceConfiguration().getFileName();
    List<String> logList = new ArrayList<String>(ENTRY_WRITE_NUMBER);
    RandomStringUtils randomUtils = new RandomStringUtils();
    Logger logger = TailerLogUtils.getSizeLogger(filePath, LOG_FILE_SIZE);
    tailer.startAsync();
    for (int i = 0; i < ENTRY_WRITE_NUMBER; i++) {
        String currLine = randomUtils.randomAlphanumeric(LINE_SIZE);
        logger.debug(currLine);
        logList.add(currLine);
        if (i % 50 == 0) {
            Thread.currentThread().sleep(300);
        }
    }
    Thread.currentThread().sleep(SLEEP_TIME);
    for (int i = 0; i < logList.size(); i++) {
        Assert.assertEquals(true, queue.take().getEventData().contains(logList.get(i)));
    }
    tailer.stopAsync();
}
Also used : FileTailerStateProcessorImpl(co.cask.cdap.filetailer.state.FileTailerStateProcessorImpl) PipeConfiguration(co.cask.cdap.filetailer.config.PipeConfiguration) FileTailerQueue(co.cask.cdap.filetailer.queue.FileTailerQueue) ArrayList(java.util.ArrayList) FileTailerStateProcessor(co.cask.cdap.filetailer.state.FileTailerStateProcessor) Logger(ch.qos.logback.classic.Logger) FileTailerMetricsProcessor(co.cask.cdap.filetailer.metrics.FileTailerMetricsProcessor) RandomStringUtils(org.apache.commons.lang.RandomStringUtils) Test(org.junit.Test)

Example 2 with FileTailerStateProcessorImpl

use of co.cask.cdap.filetailer.state.FileTailerStateProcessorImpl in project cdap-ingest by caskdata.

the class RunFromSaveStateTest method runFromSaveStateTest.

@Test
public void runFromSaveStateTest() throws Exception {
    FileTailerQueue queue = new FileTailerQueue(QUEUE_SIZE);
    PipeConfiguration flowConfig = TailerLogUtils.loadConfig();
    FileTailerStateProcessor stateProcessor = new FileTailerStateProcessorImpl(flowConfig.getDaemonDir(), flowConfig.getStateFile());
    FileTailerMetricsProcessor metricsProcessor = new FileTailerMetricsProcessor(flowConfig.getDaemonDir(), flowConfig.getStatisticsFile(), flowConfig.getStatisticsSleepInterval(), flowConfig.getPipeName(), flowConfig.getSourceConfiguration().getFileName());
    String filePath = flowConfig.getSourceConfiguration().getWorkDir().getAbsolutePath() + "/" + flowConfig.getSourceConfiguration().getFileName();
    List<String> logList = new ArrayList<String>(ENTRY_WRITE_NUMBER);
    List<String> readLogList = new ArrayList<String>(ENTRY_WRITE_NUMBER);
    Logger logger = TailerLogUtils.getSizeLogger(filePath, LOG_FILE_SIZE);
    Field queueField = queue.getClass().getDeclaredField("queue");
    queueField.setAccessible(true);
    LinkedBlockingQueue<FileTailerEvent> intQueue = (LinkedBlockingQueue<FileTailerEvent>) queueField.get(queue);
    write_log(ENTRY_WRITE_NUMBER, logger, logList);
    LogTailer tailer = new LogTailer(TailerLogUtils.loadConfig(), queue, stateProcessor, metricsProcessor, null);
    tailer.startAsync();
    Thread.sleep(SLEEP_TIME);
    tailer.stopAsync();
    saveState(intQueue, queue, readLogList, stateProcessor);
    write_log(ENTRY_WRITE_NUMBER, logger, logList);
    tailer = new LogTailer(TailerLogUtils.loadConfig(), queue, stateProcessor, metricsProcessor, null);
    tailer.startAsync();
    Thread.sleep(SLEEP_TIME);
    tailer.stopAsync();
    saveState(intQueue, queue, readLogList, stateProcessor);
    write_log(ENTRY_WRITE_NUMBER, logger, logList);
    tailer = new LogTailer(TailerLogUtils.loadConfig(), queue, stateProcessor, metricsProcessor, null);
    tailer.startAsync();
    Thread.sleep(SLEEP_TIME);
    tailer.stopAsync();
    saveState(intQueue, queue, readLogList, stateProcessor);
    for (int i = 0; i < logList.size(); i++) {
        Assert.assertEquals(true, readLogList.get(i).contains(logList.get(i)));
    }
}
Also used : FileTailerEvent(co.cask.cdap.filetailer.event.FileTailerEvent) ArrayList(java.util.ArrayList) FileTailerStateProcessor(co.cask.cdap.filetailer.state.FileTailerStateProcessor) Logger(ch.qos.logback.classic.Logger) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) FileTailerMetricsProcessor(co.cask.cdap.filetailer.metrics.FileTailerMetricsProcessor) Field(java.lang.reflect.Field) FileTailerStateProcessorImpl(co.cask.cdap.filetailer.state.FileTailerStateProcessorImpl) PipeConfiguration(co.cask.cdap.filetailer.config.PipeConfiguration) FileTailerQueue(co.cask.cdap.filetailer.queue.FileTailerQueue) Test(org.junit.Test)

Example 3 with FileTailerStateProcessorImpl

use of co.cask.cdap.filetailer.state.FileTailerStateProcessorImpl in project cdap-ingest by caskdata.

the class TailerLogUtils method createTailer.

public static LogTailer createTailer(FileTailerQueue queue, PipeConfiguration flowConfig) throws ConfigurationLoadingException {
    FileTailerStateProcessor stateProcessor = new FileTailerStateProcessorImpl(flowConfig.getDaemonDir(), flowConfig.getStateFile());
    FileTailerMetricsProcessor metricsProcessor = new FileTailerMetricsProcessor(flowConfig.getDaemonDir(), flowConfig.getStatisticsFile(), flowConfig.getStatisticsSleepInterval(), flowConfig.getPipeName(), flowConfig.getSourceConfiguration().getFileName());
    LogTailer tailer = new LogTailer(TailerLogUtils.loadConfig(), queue, stateProcessor, metricsProcessor, null);
    return tailer;
}
Also used : FileTailerStateProcessorImpl(co.cask.cdap.filetailer.state.FileTailerStateProcessorImpl) FileTailerStateProcessor(co.cask.cdap.filetailer.state.FileTailerStateProcessor) FileTailerMetricsProcessor(co.cask.cdap.filetailer.metrics.FileTailerMetricsProcessor)

Example 4 with FileTailerStateProcessorImpl

use of co.cask.cdap.filetailer.state.FileTailerStateProcessorImpl in project cdap-ingest by caskdata.

the class FileTailerIT method mockMetricsProcessor.

private void mockMetricsProcessor(PipeManager manager) throws IOException, NoSuchMethodException, InvocationTargetException, IllegalAccessException, NoSuchFieldException {
    List<Pipe> pipeList = new ArrayList<Pipe>();
    StreamClient client = null;
    StreamWriter writer = null;
    try {
        Method method1 = manager.getClass().getDeclaredMethod("getPipeConfigs");
        method1.setAccessible(true);
        List<PipeConfiguration> pipeConfList = (List<PipeConfiguration>) method1.invoke(manager);
        for (PipeConfiguration pipeConf : pipeConfList) {
            FileTailerQueue queue = new FileTailerQueue(pipeConf.getQueueSize());
            client = pipeConf.getSinkConfiguration().getStreamClient();
            String streamName = pipeConf.getSinkConfiguration().getStreamName();
            Method method2 = manager.getClass().getDeclaredMethod("getStreamWriterForPipe", StreamClient.class, String.class);
            method2.setAccessible(true);
            writer = (StreamWriter) method2.invoke(manager, client, streamName);
            FileTailerStateProcessor stateProcessor = new FileTailerStateProcessorImpl(pipeConf.getDaemonDir(), pipeConf.getStateFile());
            FileTailerMetricsProcessor metricsProcessor = new FileTailerMetricsProcessor(pipeConf.getDaemonDir(), pipeConf.getStatisticsFile(), pipeConf.getStatisticsSleepInterval(), pipeConf.getPipeName(), pipeConf.getSourceConfiguration().getFileName()) {

                @Override
                public void onReadEventMetric(int eventSize) {
                    super.onReadEventMetric(eventSize);
                    read.incrementAndGet();
                }

                @Override
                public void onIngestEventMetric(int latency) {
                    super.onIngestEventMetric(latency);
                    ingest.incrementAndGet();
                }
            };
            pipeList.add(new Pipe(new LogTailer(pipeConf, queue, stateProcessor, metricsProcessor, null), new FileTailerSink(queue, writer, SinkStrategy.LOADBALANCE, stateProcessor, metricsProcessor, null, pipeConf.getSinkConfiguration().getPackSize()), metricsProcessor));
            client = null;
            writer = null;
        }
        Field field = manager.getClass().getDeclaredField("serviceManager");
        field.setAccessible(true);
        Field modifiersField = Field.class.getDeclaredField("modifiers");
        modifiersField.setAccessible(true);
        modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
        field.set(manager, new ServiceManager(pipeList));
    } finally {
        if (client != null) {
            client.close();
        }
        if (writer != null) {
            writer.close();
        }
    }
}
Also used : StreamWriter(co.cask.cdap.client.StreamWriter) ArrayList(java.util.ArrayList) Method(java.lang.reflect.Method) FileTailerStateProcessor(co.cask.cdap.filetailer.state.FileTailerStateProcessor) FileTailerMetricsProcessor(co.cask.cdap.filetailer.metrics.FileTailerMetricsProcessor) Field(java.lang.reflect.Field) FileTailerStateProcessorImpl(co.cask.cdap.filetailer.state.FileTailerStateProcessorImpl) PipeConfiguration(co.cask.cdap.filetailer.config.PipeConfiguration) FileTailerQueue(co.cask.cdap.filetailer.queue.FileTailerQueue) StreamClient(co.cask.cdap.client.StreamClient) LogTailer(co.cask.cdap.filetailer.tailer.LogTailer) FileTailerSink(co.cask.cdap.filetailer.sink.FileTailerSink) ServiceManager(com.google.common.util.concurrent.ServiceManager) ArrayList(java.util.ArrayList) List(java.util.List)

Example 5 with FileTailerStateProcessorImpl

use of co.cask.cdap.filetailer.state.FileTailerStateProcessorImpl in project cdap-ingest by caskdata.

the class PollingListenerImpl method setupPipe.

/**
 * Pipe setup
 *
 * @throws IOException if can not setup pipe
 */
private Pipe setupPipe(File file) throws IOException {
    PipeConfiguration pipeConfiguration = observerConf.getPipeConfiguration(file.getName());
    FileTailerQueue queue = new FileTailerQueue(pipeConfiguration.getQueueSize());
    StreamWriter writer = getStreamWriterForPipe(pipeConfiguration);
    FileTailerStateProcessor stateProcessor = new FileTailerStateProcessorImpl(observerConf.getDaemonDir(), pipeConfiguration.getStateFile());
    PipeListener pipeListener = new PipeListenerImpl(pipeConfiguration.getSourceConfiguration().getWorkDir(), file.getAbsolutePath(), observerConf.getDaemonDir() + "/" + pipeConfiguration.getStateFile());
    Pipe pipe = new Pipe(new LogTailer(pipeConfiguration, queue, stateProcessor, metricsProcessor, pipeListener), new FileTailerSink(queue, writer, SinkStrategy.LOADBALANCE, stateProcessor, metricsProcessor, pipeListener, pipeConfiguration.getSinkConfiguration().getPackSize()));
    pipeListener.setPipe(pipe);
    return pipe;
}
Also used : FileTailerStateProcessorImpl(co.cask.cdap.filetailer.state.FileTailerStateProcessorImpl) PipeConfiguration(co.cask.cdap.filetailer.config.PipeConfiguration) FileTailerQueue(co.cask.cdap.filetailer.queue.FileTailerQueue) LogTailer(co.cask.cdap.filetailer.tailer.LogTailer) FileTailerSink(co.cask.cdap.filetailer.sink.FileTailerSink) StreamWriter(co.cask.cdap.client.StreamWriter) PipeListener(co.cask.cdap.filetailer.PipeListener) Pipe(co.cask.cdap.filetailer.Pipe) FileTailerStateProcessor(co.cask.cdap.filetailer.state.FileTailerStateProcessor)

Aggregations

FileTailerStateProcessor (co.cask.cdap.filetailer.state.FileTailerStateProcessor)6 FileTailerStateProcessorImpl (co.cask.cdap.filetailer.state.FileTailerStateProcessorImpl)6 PipeConfiguration (co.cask.cdap.filetailer.config.PipeConfiguration)5 FileTailerMetricsProcessor (co.cask.cdap.filetailer.metrics.FileTailerMetricsProcessor)5 FileTailerQueue (co.cask.cdap.filetailer.queue.FileTailerQueue)5 StreamWriter (co.cask.cdap.client.StreamWriter)3 FileTailerSink (co.cask.cdap.filetailer.sink.FileTailerSink)3 LogTailer (co.cask.cdap.filetailer.tailer.LogTailer)3 ArrayList (java.util.ArrayList)3 Logger (ch.qos.logback.classic.Logger)2 StreamClient (co.cask.cdap.client.StreamClient)2 ServiceManager (com.google.common.util.concurrent.ServiceManager)2 Field (java.lang.reflect.Field)2 Test (org.junit.Test)2 Pipe (co.cask.cdap.filetailer.Pipe)1 PipeListener (co.cask.cdap.filetailer.PipeListener)1 ConfigurationLoadingException (co.cask.cdap.filetailer.config.exception.ConfigurationLoadingException)1 FileTailerEvent (co.cask.cdap.filetailer.event.FileTailerEvent)1 Method (java.lang.reflect.Method)1 List (java.util.List)1