Examples with XLogUserFilterParam org.apache.oozie.util.XLogUserFilterParam used on opensource projects

Example 6 with XLogUserFilterParam

use of org.apache.oozie.util.XLogUserFilterParam in project oozie by apache.

the class TestXLogUserFilterParam method testStartEnd_bothOffset.

// Test start and end date, both offset
public void testStartEnd_bothOffset() throws Exception {
    File log4jFile = new File(getTestCaseConfDir(), "test-log4j.properties");
    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    InputStream is = cl.getResourceAsStream("test-no-dash-log4j.properties");
    Properties log4jProps = new Properties();
    log4jProps.load(is);
    // prevent conflicts with other tests by changing the log file location
    log4jProps.setProperty("log4j.appender.oozie.File", getTestCaseDir() + "/oozie.log");
    log4jProps.store(new FileOutputStream(log4jFile), "");
    setSystemProperty(XLogService.LOG4J_FILE, log4jFile.getName());
    new Services().init();
    Map<String, String[]> paramMap = new HashMap<String, String[]>();
    String param = "start=3m;end=13m;debug";
    paramMap.put(RestConstants.LOG_FILTER_OPTION, new String[] { param });
    XLogFilter filter = new XLogFilter(new XLogUserFilterParam(paramMap));
    // Param date will be overwritten by user param
    String out = doStreamLog(filter, dt.parse("14-02-20 02:06:25,499"), new Date());
    assertEquals(out.split(System.getProperty("line.separator")).length, 1);
    assertTrue(out.split(System.getProperty("line.separator"))[0].contains("Log start time = Tue Feb 20 02:07:25"));
    assertTrue(out.split(System.getProperty("line.separator"))[0].contains("Log end time = Tue Feb 20 02:21:25"));
}

Example 7 with XLogUserFilterParam

use of org.apache.oozie.util.XLogUserFilterParam in project oozie by apache.

the class TestXLogUserFilterParam method testStartEnd_bothabsoulte.

// Test start and end date, both absolute
public void testStartEnd_bothabsoulte() throws Exception {
    File log4jFile = new File(getTestCaseConfDir(), "test-log4j.properties");
    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    InputStream is = cl.getResourceAsStream("test-no-dash-log4j.properties");
    Properties log4jProps = new Properties();
    log4jProps.load(is);
    // prevent conflicts with other tests by changing the log file location
    log4jProps.setProperty("log4j.appender.oozie.File", getTestCaseDir() + "/oozie.log");
    log4jProps.store(new FileOutputStream(log4jFile), "");
    setSystemProperty(XLogService.LOG4J_FILE, log4jFile.getName());
    new Services().init();
    Map<String, String[]> paramMap = new HashMap<String, String[]>();
    String param = "start=14-03-20 02:06:25,499;end=14-03-20 02:10:25,499;debug";
    paramMap.put(RestConstants.LOG_FILTER_OPTION, new String[] { param });
    XLogFilter filter = new XLogFilter(new XLogUserFilterParam(paramMap));
    // Param date will be overwritten by user param
    String out = doStreamLog(filter, dt.parse("14-01-20 02:06:25,499"), dt.parse("14-02-20 02:06:25,499"));
    assertEquals(out.split(System.getProperty("line.separator")).length, 1);
    assertTrue(out.split(System.getProperty("line.separator"))[0].contains("Log start time = Tue Mar 20 02:06:25"));
    assertTrue(out.split(System.getProperty("line.separator"))[0].contains("Log end time = Tue Mar 20 02:10:25"));
    paramMap = new HashMap<String, String[]>();
    param = "start=14-03-20 02:06:25;end=14-03-20 02:10:25;debug";
    paramMap.put(RestConstants.LOG_FILTER_OPTION, new String[] { param });
    filter = new XLogFilter(new XLogUserFilterParam(paramMap));
    // Param date will be overwritten by user param
    out = doStreamLog(filter, dt.parse("14-01-20 02:06:25,499"), dt.parse("14-02-20 02:06:25,499"));
    assertEquals(out.split(System.getProperty("line.separator")).length, 1);
    assertTrue(out.split(System.getProperty("line.separator"))[0].contains("Log start time = Tue Mar 20 02:06:25"));
    assertTrue(out.split(System.getProperty("line.separator"))[0].contains("Log end time = Tue Mar 20 02:10:25"));
}

Example 8 with XLogUserFilterParam

use of org.apache.oozie.util.XLogUserFilterParam in project oozie by apache.

the class CoordinatorEngine method streamLog.

/**
 * Add list of actions to the filter based on conditions
 *
 * @param jobId Job Id
 * @param logRetrievalScope Value for the retrieval type
 * @param logRetrievalType Based on which filter criteria the log is retrieved
 * @param writer writer to stream the log to
 * @param requestParameters additional parameters from the request
 * @throws IOException in case of IO error
 * @throws BaseEngineException if there is an error during streaming
 * @throws CommandException if a parameter could not be parsed
 */
public void streamLog(String jobId, String logRetrievalScope, String logRetrievalType, Writer writer, Map<String, String[]> requestParameters) throws IOException, BaseEngineException, CommandException {
    Date startTime = null;
    Date endTime = null;
    XLogFilter filter = new XLogFilter(new XLogUserFilterParam(requestParameters));
    filter.setParameter(DagXLogInfoService.JOB, jobId);
    if (logRetrievalScope != null && logRetrievalType != null) {
        // if coordinator action logs are to be retrieved based on action id range
        if (logRetrievalType.equals(RestConstants.JOB_LOG_ACTION)) {
            // Use set implementation that maintains order or elements to achieve reproducibility:
            Set<String> actionSet = new LinkedHashSet<String>();
            String[] list = logRetrievalScope.split(",");
            for (String s : list) {
                s = s.trim();
                if (s.contains("-")) {
                    String[] range = s.split("-");
                    if (range.length != 2) {
                        throw new CommandException(ErrorCode.E0302, "format is wrong for action's range '" + s + "'");
                    }
                    int start;
                    int end;
                    try {
                        start = Integer.parseInt(range[0].trim());
                    } catch (NumberFormatException ne) {
                        throw new CommandException(ErrorCode.E0302, "could not parse " + range[0].trim() + "into an integer", ne);
                    }
                    try {
                        end = Integer.parseInt(range[1].trim());
                    } catch (NumberFormatException ne) {
                        throw new CommandException(ErrorCode.E0302, "could not parse " + range[1].trim() + "into an integer", ne);
                    }
                    if (start > end) {
                        throw new CommandException(ErrorCode.E0302, "format is wrong for action's range '" + s + "'");
                    }
                    for (int i = start; i <= end; i++) {
                        actionSet.add(jobId + "@" + i);
                    }
                } else {
                    try {
                        Integer.parseInt(s);
                    } catch (NumberFormatException ne) {
                        throw new CommandException(ErrorCode.E0302, "format is wrong for action id'" + s + "'. Integer only.");
                    }
                    actionSet.add(jobId + "@" + s);
                }
            }
            if (actionSet.size() >= maxNumActionsForLog) {
                throw new CommandException(ErrorCode.E0302, "Retrieving log of too many coordinator actions. Max count is " + maxNumActionsForLog + " actions");
            }
            Iterator<String> actionsIterator = actionSet.iterator();
            StringBuilder orSeparatedActions = new StringBuilder("");
            boolean orRequired = false;
            while (actionsIterator.hasNext()) {
                if (orRequired) {
                    orSeparatedActions.append("|");
                }
                orSeparatedActions.append(actionsIterator.next().toString());
                orRequired = true;
            }
            if (actionSet.size() > 1 && orRequired) {
                orSeparatedActions.insert(0, "(");
                orSeparatedActions.append(")");
            }
            filter.setParameter(DagXLogInfoService.ACTION, orSeparatedActions.toString());
            if (actionSet != null && actionSet.size() == 1) {
                CoordinatorActionBean actionBean = getCoordAction(actionSet.iterator().next());
                startTime = actionBean.getCreatedTime();
                endTime = actionBean.getStatus().equals(CoordinatorAction.Status.RUNNING) ? new Date() : actionBean.getLastModifiedTime();
                filter.setActionList(true);
            } else if (actionSet != null && actionSet.size() > 0) {
                List<String> tempList = new ArrayList<String>(actionSet);
                Collections.sort(tempList, new Comparator<String>() {

                    public int compare(String a, String b) {
                        return Integer.valueOf(a.substring(a.lastIndexOf("@") + 1)).compareTo(Integer.valueOf(b.substring(b.lastIndexOf("@") + 1)));
                    }
                });
                startTime = getCoordAction(tempList.get(0)).getCreatedTime();
                endTime = CoordActionsInDateRange.getCoordActionsLastModifiedDate(jobId, tempList.get(0), tempList.get(tempList.size() - 1));
                filter.setActionList(true);
            }
        }
        // this block gets the corresponding list of coordinator actions to be used by the log filter
        if (logRetrievalType.equalsIgnoreCase(RestConstants.JOB_LOG_DATE)) {
            List<String> coordActionIdList = null;
            try {
                coordActionIdList = CoordActionsInDateRange.getCoordActionIdsFromDates(jobId, logRetrievalScope);
            } catch (XException xe) {
                throw new CommandException(ErrorCode.E0302, "Error in date range for coordinator actions", xe);
            }
            if (coordActionIdList.size() >= maxNumActionsForLog) {
                throw new CommandException(ErrorCode.E0302, "Retrieving log of too many coordinator actions. Max count is " + maxNumActionsForLog + " actions");
            }
            StringBuilder orSeparatedActions = new StringBuilder("");
            boolean orRequired = false;
            for (String coordActionId : coordActionIdList) {
                if (orRequired) {
                    orSeparatedActions.append("|");
                }
                orSeparatedActions.append(coordActionId);
                orRequired = true;
            }
            if (coordActionIdList.size() > 1 && orRequired) {
                orSeparatedActions.insert(0, "(");
                orSeparatedActions.append(")");
            }
            filter.setParameter(DagXLogInfoService.ACTION, orSeparatedActions.toString());
            if (coordActionIdList != null && coordActionIdList.size() == 1) {
                CoordinatorActionBean actionBean = getCoordAction(coordActionIdList.get(0));
                startTime = actionBean.getCreatedTime();
                endTime = actionBean.getStatus().equals(CoordinatorAction.Status.RUNNING) ? new Date() : actionBean.getLastModifiedTime();
                filter.setActionList(true);
            } else if (coordActionIdList != null && coordActionIdList.size() > 0) {
                Collections.sort(coordActionIdList, new Comparator<String>() {

                    public int compare(String a, String b) {
                        return Integer.valueOf(a.substring(a.lastIndexOf("@") + 1)).compareTo(Integer.valueOf(b.substring(b.lastIndexOf("@") + 1)));
                    }
                });
                startTime = getCoordAction(coordActionIdList.get(0)).getCreatedTime();
                endTime = CoordActionsInDateRange.getCoordActionsLastModifiedDate(jobId, coordActionIdList.get(0), coordActionIdList.get(coordActionIdList.size() - 1));
                filter.setActionList(true);
            }
        }
    }
    if (startTime == null || endTime == null) {
        CoordinatorJobBean job = getCoordJobWithNoActionInfo(jobId);
        if (startTime == null) {
            startTime = job.getCreatedTime();
        }
        if (endTime == null) {
            if (job.isTerminalStatus()) {
                endTime = job.getLastModifiedTime();
            }
            if (endTime == null) {
                endTime = new Date();
            }
        }
    }
    Services.get().get(XLogStreamingService.class).streamLog(new XLogStreamer(filter, requestParameters), startTime, endTime, writer);
}

Example 9 with XLogUserFilterParam

use of org.apache.oozie.util.XLogUserFilterParam in project oozie by apache.

the class TestXLogStreamingService method testNoDashInConversionPattern.

public void testNoDashInConversionPattern() throws Exception {
    setupXLog();
    XLogFilter xf = new XLogFilter(new XLogUserFilterParam(null));
    xf.setParameter("USER", "oozie");
    xf.setLogLevel("DEBUG|INFO");
    // Previously, a dash ("-") was always required somewhere in a line in order for that line to pass the filter; this test
    // checks that this condition is no longer required for log streaming to work
    File log4jFile = new File(getTestCaseConfDir(), "test-log4j.properties");
    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    InputStream is = cl.getResourceAsStream("test-no-dash-log4j.properties");
    Properties log4jProps = new Properties();
    log4jProps.load(is);
    // prevent conflicts with other tests by changing the log file location
    log4jProps.setProperty("log4j.appender.oozie.File", getTestCaseDir() + "/oozie.log");
    log4jProps.store(new FileOutputStream(log4jFile), "");
    setSystemProperty(XLogService.LOG4J_FILE, log4jFile.getName());
    try {
        new Services().init();
        assertFalse(doStreamDisabledCheck());
        LogFactory.getLog("a").info("2009-06-24 02:43:14,505 INFO _L1_:317 - SERVER[foo] USER[oozie] GROUP[oozie] TOKEN[-] " + "APP[-] JOB[-] ACTION[-] Released Lock");
        LogFactory.getLog("a").info("2009-06-24 02:43:14,505 INFO _L2_:317 - SERVER[foo] USER[blah] GROUP[oozie] TOKEN[-]" + "APP[-] JOB[-] ACTION[-] Released Lock");
        LogFactory.getLog("a").info("2009-06-24 02:43:14,505 INFO _L3_:317 SERVER[foo] USER[oozie] GROUP[oozie] TOKEN[-] APP[-]" + " JOB[-] ACTION[-] Released Lock");
        LogFactory.getLog("a").info("2009-06-24 02:43:14,505 INFO _L4_:317 SERVER[foo] USER[blah] GROUP[oozie] TOKEN[-] APP[-] " + "JOB[-] ACTION[-] Released Lock");
        String out = doStreamLog(xf);
        String[] outArr = out.split("\n");
        // Lines 2 and 4 are filtered out because they have the wrong user
        assertEquals(2, outArr.length);
        assertTrue(outArr[0].contains("_L1_"));
        assertFalse(out.contains("_L2_"));
        assertTrue(outArr[1].contains("_L3_"));
        assertFalse(out.contains("_L4_"));
    } finally {
        Services.get().destroy();
    }
}

Example 10 with XLogUserFilterParam

use of org.apache.oozie.util.XLogUserFilterParam in project oozie by apache.

the class TestXLogUserFilterParam method testsearchText_logLevel.

// Test text search with log level
public void testsearchText_logLevel() throws Exception {
    setLogFile();
    String param = XLogUserFilterParam.SEARCH_TEXT + "=substitution;" + XLogUserFilterParam.LOG_LEVEL + "=DEBUG";
    Map<String, String[]> paramMap = new HashMap<String, String[]>();
    paramMap.put(RestConstants.LOG_FILTER_OPTION, new String[] { param });
    XLogFilter filter = new XLogFilter(new XLogUserFilterParam(paramMap));
    String out = doStreamLog(filter);
    String[] lines = out.split(System.getProperty("line.separator"));
    assertEquals(lines.length, 1);
    assertTrue(lines[0].contains("2014-02-27 02:06:47,499 DEBUG CoordActionStartXCommand:536 [pool-2-thread-236] - USER[-]"));
    assertTrue(lines[0].contains("E0803: IO error, Variable substitution depth too large: 20 ${dniInputDir}"));
}