Examples with XLogFilter org.apache.oozie.util.XLogFilter used on opensource projects

Example 6 with XLogFilter

use of org.apache.oozie.util.XLogFilter in project oozie by apache.

the class TestXLogUserFilterParam method testCombination.

// Test multiple combination
public void testCombination() throws Exception {
    File log4jFile = new File(getTestCaseConfDir(), "test-log4j.properties");
    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    InputStream is = cl.getResourceAsStream("test-no-dash-log4j.properties");
    Properties log4jProps = new Properties();
    log4jProps.load(is);
    log4jProps.setProperty("log4j.appender.oozie.File", getTestCaseDir() + "/oozie.log");
    log4jProps.store(new FileOutputStream(log4jFile), "");
    setSystemProperty(XLogService.LOG4J_FILE, log4jFile.getName());
    new Services().init();
    XLogFilter.reset();
    Map<String, String[]> paramMap = new HashMap<String, String[]>();
    String param = "start=14-02-20 02:06:25,499;end=14-02-27 02:06:47,550;debug;loglevel=ERROR|WARN;recent=3m";
    paramMap.put(RestConstants.LOG_FILTER_OPTION, new String[] { param });
    XLogFilter filter = new XLogFilter(new XLogUserFilterParam(paramMap));
    // Param date will be overwritten by user param
    String out = doStreamLog(filter, new Date(), new Date());
    assertEquals(out.split(System.getProperty("line.separator")).length, 1);
    assertTrue(out.split(System.getProperty("line.separator"))[0].contains("Log start time = Tue Feb 27 02:03:47"));
    assertTrue(out.split(System.getProperty("line.separator"))[0].contains("Log end time = Tue Feb 27 02:06:47"));
}

Example 7 with XLogFilter

use of org.apache.oozie.util.XLogFilter in project oozie by apache.

the class TestXLogUserFilterParam method testStartEnd_startAbsolute.

// Test start and end date, both absolute
public void testStartEnd_startAbsolute() throws Exception {
    File log4jFile = new File(getTestCaseConfDir(), "test-log4j.properties");
    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    InputStream is = cl.getResourceAsStream("test-no-dash-log4j.properties");
    Properties log4jProps = new Properties();
    log4jProps.load(is);
    // prevent conflicts with other tests by changing the log file location
    log4jProps.setProperty("log4j.appender.oozie.File", getTestCaseDir() + "/oozie.log");
    log4jProps.store(new FileOutputStream(log4jFile), "");
    setSystemProperty(XLogService.LOG4J_FILE, log4jFile.getName());
    new Services().init();
    Map<String, String[]> paramMap = new HashMap<String, String[]>();
    String param = "start=14-03-20 02:06:25,499;end=4m;debug";
    paramMap.put(RestConstants.LOG_FILTER_OPTION, new String[] { param });
    XLogFilter filter = new XLogFilter(new XLogUserFilterParam(paramMap));
    // Param date will be overwritten by user param
    String out = doStreamLog(filter, dt.parse("14-01-20 02:06:25,499"), dt.parse("14-02-20 02:06:25,499"));
    assertEquals(out.split(System.getProperty("line.separator")).length, 1);
    assertTrue(out.split(System.getProperty("line.separator"))[0].contains("Log start time = Tue Mar 20 02:06:25"));
    assertTrue(out.split(System.getProperty("line.separator"))[0].contains("Log end time = Tue Mar 20 02:12:25"));
}

Example 8 with XLogFilter

use of org.apache.oozie.util.XLogFilter in project oozie by apache.

the class TestXLogUserFilterParam method testStartEnd_bothOffset.

// Test start and end date, both offset
public void testStartEnd_bothOffset() throws Exception {
    File log4jFile = new File(getTestCaseConfDir(), "test-log4j.properties");
    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    InputStream is = cl.getResourceAsStream("test-no-dash-log4j.properties");
    Properties log4jProps = new Properties();
    log4jProps.load(is);
    // prevent conflicts with other tests by changing the log file location
    log4jProps.setProperty("log4j.appender.oozie.File", getTestCaseDir() + "/oozie.log");
    log4jProps.store(new FileOutputStream(log4jFile), "");
    setSystemProperty(XLogService.LOG4J_FILE, log4jFile.getName());
    new Services().init();
    Map<String, String[]> paramMap = new HashMap<String, String[]>();
    String param = "start=3m;end=13m;debug";
    paramMap.put(RestConstants.LOG_FILTER_OPTION, new String[] { param });
    XLogFilter filter = new XLogFilter(new XLogUserFilterParam(paramMap));
    // Param date will be overwritten by user param
    String out = doStreamLog(filter, dt.parse("14-02-20 02:06:25,499"), new Date());
    assertEquals(out.split(System.getProperty("line.separator")).length, 1);
    assertTrue(out.split(System.getProperty("line.separator"))[0].contains("Log start time = Tue Feb 20 02:07:25"));
    assertTrue(out.split(System.getProperty("line.separator"))[0].contains("Log end time = Tue Feb 20 02:21:25"));
}

Example 9 with XLogFilter

use of org.apache.oozie.util.XLogFilter in project oozie by apache.

the class TestXLogUserFilterParam method testStartEnd_bothabsoulte.

// Test start and end date, both absolute
public void testStartEnd_bothabsoulte() throws Exception {
    File log4jFile = new File(getTestCaseConfDir(), "test-log4j.properties");
    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    InputStream is = cl.getResourceAsStream("test-no-dash-log4j.properties");
    Properties log4jProps = new Properties();
    log4jProps.load(is);
    // prevent conflicts with other tests by changing the log file location
    log4jProps.setProperty("log4j.appender.oozie.File", getTestCaseDir() + "/oozie.log");
    log4jProps.store(new FileOutputStream(log4jFile), "");
    setSystemProperty(XLogService.LOG4J_FILE, log4jFile.getName());
    new Services().init();
    Map<String, String[]> paramMap = new HashMap<String, String[]>();
    String param = "start=14-03-20 02:06:25,499;end=14-03-20 02:10:25,499;debug";
    paramMap.put(RestConstants.LOG_FILTER_OPTION, new String[] { param });
    XLogFilter filter = new XLogFilter(new XLogUserFilterParam(paramMap));
    // Param date will be overwritten by user param
    String out = doStreamLog(filter, dt.parse("14-01-20 02:06:25,499"), dt.parse("14-02-20 02:06:25,499"));
    assertEquals(out.split(System.getProperty("line.separator")).length, 1);
    assertTrue(out.split(System.getProperty("line.separator"))[0].contains("Log start time = Tue Mar 20 02:06:25"));
    assertTrue(out.split(System.getProperty("line.separator"))[0].contains("Log end time = Tue Mar 20 02:10:25"));
    paramMap = new HashMap<String, String[]>();
    param = "start=14-03-20 02:06:25;end=14-03-20 02:10:25;debug";
    paramMap.put(RestConstants.LOG_FILTER_OPTION, new String[] { param });
    filter = new XLogFilter(new XLogUserFilterParam(paramMap));
    // Param date will be overwritten by user param
    out = doStreamLog(filter, dt.parse("14-01-20 02:06:25,499"), dt.parse("14-02-20 02:06:25,499"));
    assertEquals(out.split(System.getProperty("line.separator")).length, 1);
    assertTrue(out.split(System.getProperty("line.separator"))[0].contains("Log start time = Tue Mar 20 02:06:25"));
    assertTrue(out.split(System.getProperty("line.separator"))[0].contains("Log end time = Tue Mar 20 02:10:25"));
}

Example 10 with XLogFilter

use of org.apache.oozie.util.XLogFilter in project oozie by apache.

the class TestCoordinatorEngineStreamLog method testCoordLogStreaming.

public void testCoordLogStreaming() throws Exception {
    services.setService(DummyXLogStreamingService.class);
    new DagXLogInfoService().init(services);
    CoordinatorEngine ce = createCoordinatorEngine();
    final String jobId = createJobs(6);
    CoordinatorJobBean cjb = ce.getCoordJob(jobId);
    Date createdDate = cjb.getCreatedTime();
    Date endDate = cjb.getEndTime();
    assertTrue(endDate.after(createdDate));
    List<CoordinatorAction> list = cjb.getActions();
    Collections.sort(list, new Comparator<CoordinatorAction>() {

        public int compare(CoordinatorAction a, CoordinatorAction b) {
            return a.getId().compareTo(b.getId());
        }
    });
    // Test 1.to test if fields are injected
    ce.streamLog(jobId, new StringWriter(), new HashMap<String, String[]>());
    DummyXLogStreamingService service = (DummyXLogStreamingService) services.get(XLogStreamingService.class);
    XLogFilter filter = service.filter;
    assertEquals(filter.getFilterParams().get(DagXLogInfoService.JOB), jobId);
    assertEquals(cjb.getCreatedTime(), service.startTime);
    assertEquals(cjb.getLastModifiedTime(), service.endTime);
    // Test2
    // * Test method org.apache.oozie.CoordinatorEngine.streamLog(String,
    // String,
    // * String, Writer) with null 2nd and 3rd arguments.
    // */
    ce.streamLog(jobId, null, null, new StringWriter(), new HashMap<String, String[]>());
    service = (DummyXLogStreamingService) services.get(XLogStreamingService.class);
    filter = service.filter;
    assertEquals(filter.getFilterParams().get(DagXLogInfoService.JOB), jobId);
    // Test 3
    // * Test method org.apache.oozie.CoordinatorEngine.streamLog(String,
    // String,
    // * String, Writer) with RestConstants.JOB_LOG_ACTION and non-null 2nd
    // * argument.
    ce.streamLog(jobId, "1, 3-4, 6", RestConstants.JOB_LOG_ACTION, new StringWriter(), new HashMap<String, String[]>());
    service = (DummyXLogStreamingService) services.get(XLogStreamingService.class);
    filter = service.filter;
    assertEquals(jobId, filter.getFilterParams().get(DagXLogInfoService.JOB));
    assertEquals("(" + jobId + "@1|" + jobId + "@3|" + jobId + "@4|" + jobId + "@6)", filter.getFilterParams().get(DagXLogInfoService.ACTION));
    // Test 4. testing with date range
    long middle = (createdDate.getTime() + endDate.getTime()) / 2;
    Date middleDate = new Date(middle);
    ce.streamLog(jobId, DateUtils.formatDateOozieTZ(createdDate) + "::" + DateUtils.formatDateOozieTZ(middleDate) + "," + DateUtils.formatDateOozieTZ(middleDate) + "::" + DateUtils.formatDateOozieTZ(endDate), RestConstants.JOB_LOG_DATE, new StringWriter(), new HashMap<String, String[]>());
    service = (DummyXLogStreamingService) services.get(XLogStreamingService.class);
    filter = service.filter;
    assertEquals(jobId, filter.getFilterParams().get(DagXLogInfoService.JOB));
    final String action = filter.getFilterParams().get(DagXLogInfoService.ACTION);
    assertEquals("(" + jobId + "@1|" + jobId + "@2|" + jobId + "@3|" + jobId + "@4|" + jobId + "@5|" + jobId + "@6)", action);
    // Test 5 testing with action list range
    ce.streamLog(jobId, "2-4", RestConstants.JOB_LOG_ACTION, new StringWriter(), new HashMap<String, String[]>());
    service = (DummyXLogStreamingService) services.get(XLogStreamingService.class);
    assertEquals(list.get(1).getCreatedTime(), service.startTime);
    assertEquals(list.get(3).getLastModifiedTime(), service.endTime);
    // Test 6, testing with 1 action list
    ce.streamLog(jobId, "5", RestConstants.JOB_LOG_ACTION, new StringWriter(), new HashMap<String, String[]>());
    service = (DummyXLogStreamingService) services.get(XLogStreamingService.class);
    assertEquals(list.get(4).getCreatedTime(), service.startTime);
    assertEquals(list.get(4).getLastModifiedTime(), service.endTime);
    // Test 7, testing with 1 action list + range
    ce.streamLog(jobId, "1,2-4,5", RestConstants.JOB_LOG_ACTION, new StringWriter(), new HashMap<String, String[]>());
    service = (DummyXLogStreamingService) services.get(XLogStreamingService.class);
    assertEquals(list.get(0).getCreatedTime(), service.startTime);
    assertEquals(list.get(4).getLastModifiedTime(), service.endTime);
    // Test 8, testing with out order range
    ce.streamLog(jobId, "5,3-4,1", RestConstants.JOB_LOG_ACTION, new StringWriter(), new HashMap<String, String[]>());
    service = (DummyXLogStreamingService) services.get(XLogStreamingService.class);
    assertEquals(list.get(0).getCreatedTime(), service.startTime);
    assertEquals(list.get(4).getLastModifiedTime(), service.endTime);
    // Test 9, testing with date range
    ce.streamLog(jobId, DateUtils.formatDateOozieTZ(list.get(1).getCreatedTime()) + "::" + DateUtils.formatDateOozieTZ(list.get(4).getLastModifiedTime()) + ",", RestConstants.JOB_LOG_DATE, new StringWriter(), new HashMap<String, String[]>());
    service = (DummyXLogStreamingService) services.get(XLogStreamingService.class);
    assertEquals(list.get(1).getCreatedTime().toString(), service.startTime.toString());
    assertEquals(list.get(4).getLastModifiedTime().toString(), service.endTime.toString());
    // Test 10, testing with multiple date range
    ce.streamLog(jobId, DateUtils.formatDateOozieTZ(list.get(1).getCreatedTime()) + "::" + DateUtils.formatDateOozieTZ(list.get(2).getLastModifiedTime()) + "," + DateUtils.formatDateOozieTZ(list.get(3).getCreatedTime()) + "::" + DateUtils.formatDateOozieTZ(list.get(5).getLastModifiedTime()), RestConstants.JOB_LOG_DATE, new StringWriter(), new HashMap<String, String[]>());
    service = (DummyXLogStreamingService) services.get(XLogStreamingService.class);
    assertEquals(list.get(1).getCreatedTime().toString(), service.startTime.toString());
    assertEquals(list.get(5).getLastModifiedTime().toString(), service.endTime.toString());
    // Test 11, testing -scope option with Max Count
    Services.get().getConf().setInt(CoordinatorEngine.COORD_ACTIONS_LOG_MAX_COUNT, 1);
    ce = createCoordinatorEngine();
    try {
        ce.streamLog(jobId, "1-3", RestConstants.JOB_LOG_ACTION, new StringWriter(), new HashMap<String, String[]>());
    } catch (XException e) {
        assertEquals(e.getErrorCode(), ErrorCode.E0302);
        assertTrue(e.getMessage().indexOf("Retrieving log of too many coordinator actions") != -1);
    }
    // Test 12, testing -date option with Max Count
    try {
        ce.streamLog(jobId, DateUtils.formatDateOozieTZ(createdDate) + "::" + DateUtils.formatDateOozieTZ(endDate), RestConstants.JOB_LOG_DATE, new StringWriter(), new HashMap<String, String[]>());
    } catch (XException e) {
        assertEquals(e.getErrorCode(), ErrorCode.E0302);
        assertTrue(e.getMessage().indexOf("Retrieving log of too many coordinator actions") != -1);
    }
}