Examples with ActionDependency org.apache.oozie.dependency.ActionDependency used on opensource projects

Example 6 with ActionDependency

use of org.apache.oozie.dependency.ActionDependency in project oozie by apache.

the class CoordActionMissingDependenciesXCommand method execute.

@Override
protected List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> execute() throws CommandException {
    List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> inputDependenciesListPair = new ArrayList<Pair<CoordinatorActionBean, Map<String, ActionDependency>>>();
    try {
        for (CoordinatorActionBean coordAction : coordActions) {
            CoordInputDependency coordPullInputDependency = coordAction.getPullInputDependencies();
            CoordInputDependency coordPushInputDependency = coordAction.getPushInputDependencies();
            Map<String, ActionDependency> dependencyMap = new HashMap<String, ActionDependency>();
            dependencyMap.putAll(coordPullInputDependency.getMissingDependencies(coordAction));
            dependencyMap.putAll(coordPushInputDependency.getMissingDependencies(coordAction));
            inputDependenciesListPair.add(new Pair<CoordinatorActionBean, Map<String, ActionDependency>>(coordAction, dependencyMap));
        }
    } catch (Exception e) {
        throw new CommandException(ErrorCode.E1028, e.getMessage(), e);
    }
    return inputDependenciesListPair;
}

Example 7 with ActionDependency

use of org.apache.oozie.dependency.ActionDependency in project oozie by apache.

the class CoordOldInputDependency method getMissingDependencies.

public Map<String, ActionDependency> getMissingDependencies(CoordinatorActionBean coordAction) throws CommandException, IOException, JDOMException {
    Map<String, ActionDependency> dependenciesMap = null;
    try {
        dependenciesMap = getDependency(coordAction);
    } catch (URIHandlerException e) {
        throw new IOException(e);
    }
    StringBuilder nonExistList = new StringBuilder();
    StringBuilder nonResolvedList = new StringBuilder();
    CoordCommandUtils.getResolvedList(getMissingDependencies(), nonExistList, nonResolvedList);
    Set<String> missingSets = new HashSet<String>(Arrays.asList(nonExistList.toString().split(CoordELFunctions.INSTANCE_SEPARATOR)));
    missingSets.addAll(Arrays.asList(nonResolvedList.toString().split(CoordCommandUtils.RESOLVED_UNRESOLVED_SEPARATOR)));
    for (Iterator<Map.Entry<String, ActionDependency>> it = dependenciesMap.entrySet().iterator(); it.hasNext(); ) {
        Map.Entry<String, ActionDependency> entry = it.next();
        ActionDependency dependency = entry.getValue();
        dependency.getMissingDependencies().retainAll(missingSets);
        if (dependency.getUriTemplate() != null) {
            for (int i = 0; i < dependency.getMissingDependencies().size(); i++) {
                if (dependency.getMissingDependencies().get(i).trim().startsWith("${coord:")) {
                    dependency.getMissingDependencies().set(i, dependency.getMissingDependencies().get(i) + " -> " + dependency.getUriTemplate());
                }
            }
        }
        if (dependenciesMap.get(entry.getKey()).getMissingDependencies().isEmpty()) {
            it.remove();
        }
    }
    return dependenciesMap;
}

Example 8 with ActionDependency

use of org.apache.oozie.dependency.ActionDependency in project oozie by apache.

the class V2JobServlet method getCoordActionMissingDependencies.

@SuppressWarnings("unchecked")
@Override
protected JSONObject getCoordActionMissingDependencies(HttpServletRequest request, HttpServletResponse response) throws XServletException, IOException {
    String jobId = getResourceName(request);
    String actions = request.getParameter(RestConstants.JOB_COORD_SCOPE_ACTION_LIST);
    String dates = request.getParameter(RestConstants.JOB_COORD_SCOPE_DATE);
    try {
        List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> dependenciesList = Services.get().get(CoordinatorEngineService.class).getCoordinatorEngine(getUser(request)).getCoordActionMissingDependencies(jobId, actions, dates);
        JSONArray dependenciesArray = new JSONArray();
        for (Pair<CoordinatorActionBean, Map<String, ActionDependency>> dependencies : dependenciesList) {
            JSONObject json = new JSONObject();
            JSONArray parentJsonArray = new JSONArray();
            for (String key : dependencies.getSecond().keySet()) {
                JSONObject dependencyList = new JSONObject();
                JSONArray jsonArray = new JSONArray();
                jsonArray.addAll(dependencies.getSecond().get(key).getMissingDependencies());
                dependencyList.put(JsonTags.COORDINATOR_ACTION_MISSING_DEPS, jsonArray);
                dependencyList.put(JsonTags.COORDINATOR_ACTION_DATASET, key);
                parentJsonArray.add(dependencyList);
            }
            json.put(JsonTags.COORD_ACTION_FIRST_MISSING_DEPENDENCIES, CoordCommandUtils.getFirstMissingDependency(dependencies.getFirst()));
            json.put(JsonTags.COORDINATOR_ACTION_ID, dependencies.getFirst().getActionNumber());
            json.put(JsonTags.COORDINATOR_ACTION_DATASETS, parentJsonArray);
            dependenciesArray.add(json);
        }
        JSONObject jsonObject = new JSONObject();
        jsonObject.put(JsonTags.COORD_ACTION_MISSING_DEPENDENCIES, dependenciesArray);
        return jsonObject;
    } catch (CommandException e) {
        throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, e);
    }
}

Example 9 with ActionDependency

use of org.apache.oozie.dependency.ActionDependency in project oozie by apache.

the class TestCoordActionMissingDependenciesXCommand method testCoordActionInputLogicMissing.

public void testCoordActionInputLogicMissing() throws Exception {
    createTestTables();
    Configuration conf = TestCoordInputLogicPush.getConfForCombine("file://" + getTestCaseDir(), "hcat://" + getMetastoreAuthority());
    conf.set("initial_instance_a", "2014-10-07T00:00Z");
    conf.set("initial_instance_b", "2014-10-07T00:00Z");
    String inputLogic = // @formatter:off
    "<and name=\"test\">" + "<data-in dataset=\"A\" />" + "<data-in dataset=\"B\" />" + "</and>";
    // @formatter:on
    String jobId = TestCoordInputLogicPush.submitCoord(getTestCaseDir(), "coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.CURRENT_SINGLE, TEST_TYPE.CURRENT_SINGLE, TEST_TYPE.CURRENT_RANGE, TEST_TYPE.LATEST_RANGE);
    new CoordMaterializeTransitionXCommand(jobId, 3600).call();
    List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call();
    Map<String, ActionDependency> dependencyMap = data.get(0).getSecond();
    assertEquals(6, dependencyMap.size());
    assertNull(CoordCommandUtils.getFirstMissingDependency(data.get(0).getFirst()));
    createTestCaseSubDir("input-data/b/2014/10/08/_SUCCESS".split("/"));
    addPartition(DB_A, TABLE, "dt=20141008;country=usa");
    new CoordActionInputCheckXCommand(jobId + "@1", jobId).call();
    new CoordPushDependencyCheckXCommand(jobId + "@1").call();
    new CoordActionInputCheckXCommand(jobId + "@1", jobId).call();
    data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call();
    dependencyMap = data.get(0).getSecond();
    assertEquals(4, dependencyMap.size());
    assertNull(dependencyMap.get("A"));
    assertNull(dependencyMap.get("B"));
}

Example 10 with ActionDependency

use of org.apache.oozie.dependency.ActionDependency in project oozie by apache.

the class TestCoordActionMissingDependenciesXCommand method testCoordActionPullDependencyMissing.

public void testCoordActionPullDependencyMissing() throws Exception {
    Configuration conf = new XConfiguration();
    File appPathFile = new File(getTestCaseDir(), "coordinator.xml");
    // CASE 1: Failure case i.e. multiple data-in instances
    Reader reader = IOUtils.getResourceAsReader("coord-multiple-output-instance5.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "coordinator.xml"));
    IOUtils.copyCharStream(reader, writer);
    conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString());
    conf.set(OozieClient.USER_NAME, getTestUser());
    conf.set("data_set_a", "file://" + getTestCaseDir() + "/input-data/a/${YEAR}/${DAY}");
    conf.set("data_set_b", "file://" + getTestCaseDir() + "/input-data/b/${YEAR}/${DAY}");
    conf.set("data_set_c", "file://" + getTestCaseDir() + "/input-data/c/${YEAR}/${DAY}");
    conf.set("data_set_d", "file://" + getTestCaseDir() + "/input-data/d/${YEAR}/${DAY}");
    conf.set("data_set_e", "file://" + getTestCaseDir() + "/input-data/e/${YEAR}/${DAY}");
    CoordSubmitXCommand sc = new CoordSubmitXCommand(conf);
    String jobId = sc.call();
    new CoordMaterializeTransitionXCommand(jobId, 3600).call();
    List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call();
    Map<String, ActionDependency> dependencyMap = data.get(0).getSecond();
    assertEquals(6, dependencyMap.size());
    assertEquals(1, dependencyMap.get("A").getMissingDependencies().size());
    assertEquals(6, dependencyMap.get("B").getMissingDependencies().size());
    assertEquals(1, dependencyMap.get("C").getMissingDependencies().size());
    assertEquals(1, dependencyMap.get("D").getMissingDependencies().size());
    assertEquals(6, dependencyMap.get("E").getMissingDependencies().size());
    createTestCaseSubDir("input-data/a/2009/01/_SUCCESS".split("/"));
    createTestCaseSubDir("input-data/b/2009/01/_SUCCESS".split("/"));
    new CoordActionInputCheckXCommand(jobId + "@1", jobId).call();
    data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call();
    dependencyMap = data.get(0).getSecond();
    assertEquals(5, dependencyMap.size());
    assertNull(dependencyMap.get("A"));
    assertEquals(5, dependencyMap.get("B").getMissingDependencies().size());
    assertTrue(dependencyMap.get("B").getMissingDependencies().contains("file://" + getTestCaseDir() + "/input-data/b/2009/31/_SUCCESS"));
    assertTrue(dependencyMap.get("B").getMissingDependencies().contains("file://" + getTestCaseDir() + "/input-data/b/2009/30/_SUCCESS"));
    assertTrue(dependencyMap.get("B").getMissingDependencies().contains("file://" + getTestCaseDir() + "/input-data/b/2009/29/_SUCCESS"));
    assertTrue(dependencyMap.get("B").getMissingDependencies().contains("file://" + getTestCaseDir() + "/input-data/b/2009/28/_SUCCESS"));
    assertTrue(dependencyMap.get("B").getMissingDependencies().contains("file://" + getTestCaseDir() + "/input-data/b/2009/27/_SUCCESS"));
    assertEquals(1, dependencyMap.get("C").getMissingDependencies().size());
    assertEquals(1, dependencyMap.get("D").getMissingDependencies().size());
    assertEquals(dependencyMap.get("D").getMissingDependencies().get(0), "file://" + getTestCaseDir() + "/input-data/d/2009/01");
    assertEquals(6, dependencyMap.get("E").getMissingDependencies().size());
    assertEquals(2, dependencyMap.get("F").getMissingDependencies().size());
    assertEquals(dependencyMap.get("F").getMissingDependencies().get(0), "file://" + getTestCaseDir() + "/input-data/e/2009/01/_SUCCESS");
    assertEquals(dependencyMap.get("F").getMissingDependencies().get(1), "${coord:latest(0)} -> file://" + getTestCaseDir() + "/input-data/e/${YEAR}/${DAY}");
}