Examples with FileTailerMetricsProcessor co.cask.cdap.filetailer.metrics.FileTailerMetricsProcessor used on opensource projects

Example 6 with FileTailerMetricsProcessor

use of co.cask.cdap.filetailer.metrics.FileTailerMetricsProcessor in project cdap-ingest by caskdata.

the class PipeManager method setupPipes.

/**
   * Pipes setup
   *
   * @throws IOException in case a client stream cannot be created
   */
private ServiceManager setupPipes() throws IOException {
    StreamClient client = null;
    StreamWriter writer = null;
    try {
        List<PipeConfiguration> pipeConfList = getPipeConfigs();
        for (PipeConfiguration pipeConf : pipeConfList) {
            FileTailerQueue queue = new FileTailerQueue(pipeConf.getQueueSize());
            client = pipeConf.getSinkConfiguration().getStreamClient();
            String streamName = pipeConf.getSinkConfiguration().getStreamName();
            writer = getStreamWriterForPipe(client, streamName);
            FileTailerStateProcessor stateProcessor = new FileTailerStateProcessorImpl(pipeConf.getDaemonDir(), pipeConf.getStateFile());
            FileTailerMetricsProcessor metricsProcessor = new FileTailerMetricsProcessor(pipeConf.getDaemonDir(), pipeConf.getStatisticsFile(), pipeConf.getStatisticsSleepInterval(), pipeConf.getPipeName(), pipeConf.getSourceConfiguration().getFileName());
            pipeList.add(new Pipe(new LogTailer(pipeConf, queue, stateProcessor, metricsProcessor, null), new FileTailerSink(queue, writer, SinkStrategy.LOADBALANCE, stateProcessor, metricsProcessor, null, pipeConf.getSinkConfiguration().getPackSize()), metricsProcessor));
            client = null;
            writer = null;
        }
        return new ServiceManager(pipeList);
    } catch (ConfigurationLoadingException e) {
        throw new ConfigurationLoadingException("Error during loading configuration from file: " + confFile.getAbsolutePath() + e.getMessage());
    } finally {
        if (client != null) {
            client.close();
        }
        if (writer != null) {
            writer.close();
        }
    }
}

Example 7 with FileTailerMetricsProcessor

use of co.cask.cdap.filetailer.metrics.FileTailerMetricsProcessor in project cdap-ingest by caskdata.

the class FileTailerSinkTest method basicTestWithCustomPackSize.

@Test
public void basicTestWithCustomPackSize() throws Exception {
    FileTailerStateProcessor stateProcessor = Mockito.mock(FileTailerStateProcessor.class);
    FileTailerMetricsProcessor metricsProcessor = Mockito.mock(FileTailerMetricsProcessor.class);
    FileTailerQueue queue = new FileTailerQueue(DEFAULT_QUEUE_SIZE);
    StreamWriter writerMock = getDummyStreamWriter();
    FileTailerSink sink = new FileTailerSink(queue, writerMock, SinkStrategy.LOADBALANCE, stateProcessor, metricsProcessor, null, CUSTOM_PACK_SIZE);
    try {
        sink.startAsync();
        for (int i = 0; i < TEST_EVENTS_SIZE; i++) {
            queue.put(new FileTailerEvent(new FileTailerState("file", 0L, 42, 0L), "test", Charset.defaultCharset()));
        }
        Mockito.verify(writerMock, Mockito.timeout(10000).times(TEST_EVENTS_SIZE)).write("test", Charset.defaultCharset());
    } finally {
        sink.stopAsync();
    }
}

Example 8 with FileTailerMetricsProcessor

use of co.cask.cdap.filetailer.metrics.FileTailerMetricsProcessor in project cdap-ingest by caskdata.

the class FileTailerIT method mockMetricsProcessor.

private void mockMetricsProcessor(PipeManager manager) throws IOException, NoSuchMethodException, InvocationTargetException, IllegalAccessException, NoSuchFieldException {
    List<Pipe> pipeList = new ArrayList<Pipe>();
    StreamClient client = null;
    StreamWriter writer = null;
    try {
        Method method1 = manager.getClass().getDeclaredMethod("getPipeConfigs");
        method1.setAccessible(true);
        List<PipeConfiguration> pipeConfList = (List<PipeConfiguration>) method1.invoke(manager);
        for (PipeConfiguration pipeConf : pipeConfList) {
            FileTailerQueue queue = new FileTailerQueue(pipeConf.getQueueSize());
            client = pipeConf.getSinkConfiguration().getStreamClient();
            String streamName = pipeConf.getSinkConfiguration().getStreamName();
            Method method2 = manager.getClass().getDeclaredMethod("getStreamWriterForPipe", StreamClient.class, String.class);
            method2.setAccessible(true);
            writer = (StreamWriter) method2.invoke(manager, client, streamName);
            FileTailerStateProcessor stateProcessor = new FileTailerStateProcessorImpl(pipeConf.getDaemonDir(), pipeConf.getStateFile());
            FileTailerMetricsProcessor metricsProcessor = new FileTailerMetricsProcessor(pipeConf.getDaemonDir(), pipeConf.getStatisticsFile(), pipeConf.getStatisticsSleepInterval(), pipeConf.getPipeName(), pipeConf.getSourceConfiguration().getFileName()) {

                @Override
                public void onReadEventMetric(int eventSize) {
                    super.onReadEventMetric(eventSize);
                    read.incrementAndGet();
                }

                @Override
                public void onIngestEventMetric(int latency) {
                    super.onIngestEventMetric(latency);
                    ingest.incrementAndGet();
                }
            };
            pipeList.add(new Pipe(new LogTailer(pipeConf, queue, stateProcessor, metricsProcessor, null), new FileTailerSink(queue, writer, SinkStrategy.LOADBALANCE, stateProcessor, metricsProcessor, null, pipeConf.getSinkConfiguration().getPackSize()), metricsProcessor));
            client = null;
            writer = null;
        }
        Field field = manager.getClass().getDeclaredField("serviceManager");
        field.setAccessible(true);
        Field modifiersField = Field.class.getDeclaredField("modifiers");
        modifiersField.setAccessible(true);
        modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
        field.set(manager, new ServiceManager(pipeList));
    } finally {
        if (client != null) {
            client.close();
        }
        if (writer != null) {
            writer.close();
        }
    }
}

Example 9 with FileTailerMetricsProcessor

use of co.cask.cdap.filetailer.metrics.FileTailerMetricsProcessor in project cdap-ingest by caskdata.

the class FileDropZoneIT method fileDropZoneBasicIT.

@Test
public void fileDropZoneBasicIT() throws Exception {
    final File configurationFile = getConfigFile();
    PollingServiceManager pollingServiceManager = new PollingServiceManager(configurationFile);
    pollingServiceManager.initManager();
    Field monitor = pollingServiceManager.getClass().getDeclaredField("monitor");
    monitor.setAccessible(true);
    PollingService myMonitor = (PollingService) monitor.get(pollingServiceManager);
    ObserverConfiguration observerConf = loadConfig(configurationFile);
    PollingListener myPollingListener = new PollingListenerImpl(myMonitor, observerConf);
    FileTailerMetricsProcessor metricsProcessor = getMetricsProcessor(observerConf);
    Field metricsProcessorField = myPollingListener.getClass().getDeclaredField("metricsProcessor");
    metricsProcessorField.setAccessible(true);
    metricsProcessorField.set(myPollingListener, metricsProcessor);
    myMonitor.registerDirMonitor(observerConf.getPipeConf().getSourceConfiguration().getWorkDir(), myPollingListener);
    createFile(observerConf.getPipeConf().getSourceConfiguration().getWorkDir().getAbsolutePath());
    long start = System.currentTimeMillis();
    pollingServiceManager.startMonitor();
    Thread.sleep(SLEEP_TIME);
    pollingServiceManager.stopMonitor();
    checkDeliveredEvents(start, System.currentTimeMillis());
    Assert.assertEquals(read.get(), ingest.get());
}