Examples with PipeConfiguration co.cask.cdap.filetailer.config.PipeConfiguration used on opensource projects

Example 11 with PipeConfiguration

use of co.cask.cdap.filetailer.config.PipeConfiguration in project cdap-ingest by caskdata.

the class TailerLogUtils method loadConfig.

public static PipeConfiguration loadConfig() throws ConfigurationLoadingException {
    ConfigurationLoader loader = new ConfigurationLoaderImpl();
    Class<? extends Class> path1 = TailerLogUtils.class.getClass();
    String path = TailerLogUtils.class.getClassLoader().getResource("test4.properties").getFile();
    Configuration configuration = loader.load(new File(path));
    List<PipeConfiguration> flowConfig = configuration.getPipeConfigurations();
    return flowConfig.get(0);
}

Example 12 with PipeConfiguration

use of co.cask.cdap.filetailer.config.PipeConfiguration in project cdap-ingest by caskdata.

the class BaseTailerTest method fileTimeRotationTest.

@Test
public void fileTimeRotationTest() throws ConfigurationLoadingException, InterruptedException {
    FileTailerQueue queue = new FileTailerQueue(QUEUE_SIZE);
    PipeConfiguration flowConfig = TailerLogUtils.loadConfig();
    LogTailer tailer = TailerLogUtils.createTailer(queue, flowConfig);
    String filePath = flowConfig.getSourceConfiguration().getWorkDir().getAbsolutePath() + "/" + flowConfig.getSourceConfiguration().getFileName();
    Logger logger = TailerLogUtils.getTimeLogger(filePath);
    RandomStringUtils randomUtils = new RandomStringUtils();
    List<String> logList = new ArrayList<String>(ENTRY_NUMBER);
    tailer.startAsync();
    for (int i = 0; i < ENTRY_NUMBER; i++) {
        String currLine = randomUtils.randomAlphanumeric(LINE_SIZE);
        logger.debug(currLine);
        logList.add(currLine);
        Thread.currentThread().sleep(WRITING_INTERVAL);
    }
    Thread.currentThread().sleep(SLEEP_TIME);
    tailer.stopAsync();
    for (String str : logList) {
        Assert.assertEquals(true, queue.take().getEventData().contains(str));
    }
}

Example 13 with PipeConfiguration

use of co.cask.cdap.filetailer.config.PipeConfiguration in project cdap-ingest by caskdata.

the class FileTailerIT method deleteTestDir.

private void deleteTestDir() throws Exception {
    File configFile = getConfigFile();
    PipeConfiguration pipeConfig = loadConfig(configFile);
    File workDir = pipeConfig.getSourceConfiguration().getWorkDir();
    FileUtils.deleteDirectory(workDir);
    File daemonDir = pipeConfig.getDaemonDir();
    FileUtils.deleteDirectory(daemonDir);
}

Example 14 with PipeConfiguration

use of co.cask.cdap.filetailer.config.PipeConfiguration in project cdap-ingest by caskdata.

the class FileTailerIT method fileTailerNoLogsBeforeStartIT.

@Test
public void fileTailerNoLogsBeforeStartIT() throws Exception {
    File configFile = getConfigFile();
    PipeConfiguration pipeConfig = loadConfig(configFile);
    String logFilePath = pipeConfig.getSourceConfiguration().getWorkDir().getAbsolutePath() + "/" + pipeConfig.getSourceConfiguration().getFileName();
    Logger logger = getTimeLogger(logFilePath);
    PipeManager manager = new PipeManager(configFile);
    mockMetricsProcessor(manager);
    long startTime = System.currentTimeMillis();
    manager.startAsync();
    Thread.sleep(SLEEP_TIME);
    writeLogs(logger, ENTRY_NUMBER);
    Thread.sleep(SLEEP_TIME);
    logger.getAppender("File Tailer IT").stop();
    manager.stopAsync();
    Thread.sleep(SLEEP_TIME);
    Assert.assertEquals(read.get(), ingest.get());
    checkDeliveredEvents(ENTRY_NUMBER, startTime);
}

Example 15 with PipeConfiguration

use of co.cask.cdap.filetailer.config.PipeConfiguration in project cdap-ingest by caskdata.

the class FileTailerIT method mockMetricsProcessor.

private void mockMetricsProcessor(PipeManager manager) throws IOException, NoSuchMethodException, InvocationTargetException, IllegalAccessException, NoSuchFieldException {
    List<Pipe> pipeList = new ArrayList<Pipe>();
    StreamClient client = null;
    StreamWriter writer = null;
    try {
        Method method1 = manager.getClass().getDeclaredMethod("getPipeConfigs");
        method1.setAccessible(true);
        List<PipeConfiguration> pipeConfList = (List<PipeConfiguration>) method1.invoke(manager);
        for (PipeConfiguration pipeConf : pipeConfList) {
            FileTailerQueue queue = new FileTailerQueue(pipeConf.getQueueSize());
            client = pipeConf.getSinkConfiguration().getStreamClient();
            String streamName = pipeConf.getSinkConfiguration().getStreamName();
            Method method2 = manager.getClass().getDeclaredMethod("getStreamWriterForPipe", StreamClient.class, String.class);
            method2.setAccessible(true);
            writer = (StreamWriter) method2.invoke(manager, client, streamName);
            FileTailerStateProcessor stateProcessor = new FileTailerStateProcessorImpl(pipeConf.getDaemonDir(), pipeConf.getStateFile());
            FileTailerMetricsProcessor metricsProcessor = new FileTailerMetricsProcessor(pipeConf.getDaemonDir(), pipeConf.getStatisticsFile(), pipeConf.getStatisticsSleepInterval(), pipeConf.getPipeName(), pipeConf.getSourceConfiguration().getFileName()) {

                @Override
                public void onReadEventMetric(int eventSize) {
                    super.onReadEventMetric(eventSize);
                    read.incrementAndGet();
                }

                @Override
                public void onIngestEventMetric(int latency) {
                    super.onIngestEventMetric(latency);
                    ingest.incrementAndGet();
                }
            };
            pipeList.add(new Pipe(new LogTailer(pipeConf, queue, stateProcessor, metricsProcessor, null), new FileTailerSink(queue, writer, SinkStrategy.LOADBALANCE, stateProcessor, metricsProcessor, null, pipeConf.getSinkConfiguration().getPackSize()), metricsProcessor));
            client = null;
            writer = null;
        }
        Field field = manager.getClass().getDeclaredField("serviceManager");
        field.setAccessible(true);
        Field modifiersField = Field.class.getDeclaredField("modifiers");
        modifiersField.setAccessible(true);
        modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
        field.set(manager, new ServiceManager(pipeList));
    } finally {
        if (client != null) {
            client.close();
        }
        if (writer != null) {
            writer.close();
        }
    }
}