Examples with PipeConfiguration co.cask.cdap.filetailer.config.PipeConfiguration used on opensource projects

Example 1 with PipeConfiguration

use of co.cask.cdap.filetailer.config.PipeConfiguration in project cdap-ingest by caskdata.

the class ObserverConfigurationImplTest method getDaemonDirTest.

@Test
public void getDaemonDirTest() throws NoSuchFieldException, IllegalAccessException {
    Properties properties = Mockito.mock(Properties.class);
    PipeConfiguration pipeConfiguration = Mockito.mock(PipeConfiguration.class);
    Mockito.when(pipeConfiguration.getDaemonDir()).thenReturn(new File(OLD_DAEMON_DIR));
    ObserverConfiguration observerConfiguration = new ObserverConfigurationImpl("obs2", properties, "key");
    Field field = observerConfiguration.getClass().getDeclaredField("pipeConfiguration");
    field.setAccessible(true);
    Field modifiersField = Field.class.getDeclaredField("modifiers");
    modifiersField.setAccessible(true);
    modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
    field.set(observerConfiguration, pipeConfiguration);
    Assert.assertEquals(new File(NEW_DAEMON_DIR), observerConfiguration.getDaemonDir());
}

Example 2 with PipeConfiguration

use of co.cask.cdap.filetailer.config.PipeConfiguration in project cdap-ingest by caskdata.

the class SizeBasedRotationTest method fileRotationTest.

@Test
public void fileRotationTest() throws ConfigurationLoadingException, InterruptedException {
    FileTailerQueue queue = new FileTailerQueue(QUEUE_SIZE);
    PipeConfiguration flowConfig = TailerLogUtils.loadConfig();
    FileTailerStateProcessor stateProcessor = new FileTailerStateProcessorImpl(flowConfig.getDaemonDir(), flowConfig.getStateFile());
    FileTailerMetricsProcessor metricsProcessor = new FileTailerMetricsProcessor(flowConfig.getDaemonDir(), flowConfig.getStatisticsFile(), flowConfig.getStatisticsSleepInterval(), flowConfig.getPipeName(), flowConfig.getSourceConfiguration().getFileName());
    LogTailer tailer = new LogTailer(TailerLogUtils.loadConfig(), queue, stateProcessor, metricsProcessor, null);
    String filePath = flowConfig.getSourceConfiguration().getWorkDir().getAbsolutePath() + "/" + flowConfig.getSourceConfiguration().getFileName();
    List<String> logList = new ArrayList<String>(ENTRY_WRITE_NUMBER);
    RandomStringUtils randomUtils = new RandomStringUtils();
    Logger logger = TailerLogUtils.getSizeLogger(filePath, LOG_FILE_SIZE);
    tailer.startAsync();
    for (int i = 0; i < ENTRY_WRITE_NUMBER; i++) {
        String currLine = randomUtils.randomAlphanumeric(LINE_SIZE);
        logger.debug(currLine);
        logList.add(currLine);
        if (i % 50 == 0) {
            Thread.currentThread().sleep(300);
        }
    }
    Thread.currentThread().sleep(SLEEP_TIME);
    for (int i = 0; i < logList.size(); i++) {
        Assert.assertEquals(true, queue.take().getEventData().contains(logList.get(i)));
    }
    tailer.stopAsync();
}

Example 3 with PipeConfiguration

use of co.cask.cdap.filetailer.config.PipeConfiguration in project cdap-ingest by caskdata.

the class BaseTailerTest method baseReadingLogDirTest.

@Test
public void baseReadingLogDirTest() throws ConfigurationLoadingException, InterruptedException {
    FileTailerQueue queue = new FileTailerQueue(1);
    PipeConfiguration flowConfig = TailerLogUtils.loadConfig();
    LogTailer tailer = TailerLogUtils.createTailer(queue, flowConfig);
    String filePath = flowConfig.getSourceConfiguration().getWorkDir().getAbsolutePath() + "/" + flowConfig.getSourceConfiguration().getFileName();
    Logger logger = TailerLogUtils.getSizeLogger(filePath, LOG_FILE_SIZE);
    RandomStringUtils randomUtils = new RandomStringUtils();
    List<String> logList = new ArrayList<String>(ENTRY_NUMBER);
    for (int i = 0; i < ENTRY_NUMBER; i++) {
        String currLine = randomUtils.randomAlphanumeric(LINE_SIZE);
        logger.debug(currLine);
        logList.add(currLine);
    }
    tailer.startAsync();
    Thread.currentThread().sleep(SLEEP_TIME);
    for (String str : logList) {
        Assert.assertEquals(true, queue.take().getEventData().contains(str));
    }
    tailer.stopAsync();
    Thread.currentThread().sleep(1000);
}

Example 4 with PipeConfiguration

use of co.cask.cdap.filetailer.config.PipeConfiguration in project cdap-ingest by caskdata.

the class RunFromSaveStateTest method runFromSaveStateTest.

@Test
public void runFromSaveStateTest() throws Exception {
    FileTailerQueue queue = new FileTailerQueue(QUEUE_SIZE);
    PipeConfiguration flowConfig = TailerLogUtils.loadConfig();
    FileTailerStateProcessor stateProcessor = new FileTailerStateProcessorImpl(flowConfig.getDaemonDir(), flowConfig.getStateFile());
    FileTailerMetricsProcessor metricsProcessor = new FileTailerMetricsProcessor(flowConfig.getDaemonDir(), flowConfig.getStatisticsFile(), flowConfig.getStatisticsSleepInterval(), flowConfig.getPipeName(), flowConfig.getSourceConfiguration().getFileName());
    String filePath = flowConfig.getSourceConfiguration().getWorkDir().getAbsolutePath() + "/" + flowConfig.getSourceConfiguration().getFileName();
    List<String> logList = new ArrayList<String>(ENTRY_WRITE_NUMBER);
    List<String> readLogList = new ArrayList<String>(ENTRY_WRITE_NUMBER);
    Logger logger = TailerLogUtils.getSizeLogger(filePath, LOG_FILE_SIZE);
    Field queueField = queue.getClass().getDeclaredField("queue");
    queueField.setAccessible(true);
    LinkedBlockingQueue<FileTailerEvent> intQueue = (LinkedBlockingQueue<FileTailerEvent>) queueField.get(queue);
    write_log(ENTRY_WRITE_NUMBER, logger, logList);
    LogTailer tailer = new LogTailer(TailerLogUtils.loadConfig(), queue, stateProcessor, metricsProcessor, null);
    tailer.startAsync();
    Thread.sleep(SLEEP_TIME);
    tailer.stopAsync();
    saveState(intQueue, queue, readLogList, stateProcessor);
    write_log(ENTRY_WRITE_NUMBER, logger, logList);
    tailer = new LogTailer(TailerLogUtils.loadConfig(), queue, stateProcessor, metricsProcessor, null);
    tailer.startAsync();
    Thread.sleep(SLEEP_TIME);
    tailer.stopAsync();
    saveState(intQueue, queue, readLogList, stateProcessor);
    write_log(ENTRY_WRITE_NUMBER, logger, logList);
    tailer = new LogTailer(TailerLogUtils.loadConfig(), queue, stateProcessor, metricsProcessor, null);
    tailer.startAsync();
    Thread.sleep(SLEEP_TIME);
    tailer.stopAsync();
    saveState(intQueue, queue, readLogList, stateProcessor);
    for (int i = 0; i < logList.size(); i++) {
        Assert.assertEquals(true, readLogList.get(i).contains(logList.get(i)));
    }
}

Example 5 with PipeConfiguration

use of co.cask.cdap.filetailer.config.PipeConfiguration in project cdap-ingest by caskdata.

the class PipeManager method getPipeConfigs.

/**
   * Get pipes configuration
   *
   * @return the pipes configuration read from the configuration file
   * @throws ConfigurationLoadingException in case can not load configuration
   */
private List<PipeConfiguration> getPipeConfigs() throws ConfigurationLoadingException {
    ConfigurationLoader loader = new ConfigurationLoaderImpl();
    Configuration configuration = loader.load(confFile);
    return configuration.getPipeConfigurations();
}