Examples with Pair org.apache.oozie.util.Pair used on opensource projects

Example 6 with Pair

use of org.apache.oozie.util.Pair in project oozie by apache.

the class CoordActionMissingDependenciesXCommand method execute.

@Override
protected List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> execute() throws CommandException {
    List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> inputDependenciesListPair = new ArrayList<Pair<CoordinatorActionBean, Map<String, ActionDependency>>>();
    try {
        for (CoordinatorActionBean coordAction : coordActions) {
            CoordInputDependency coordPullInputDependency = coordAction.getPullInputDependencies();
            CoordInputDependency coordPushInputDependency = coordAction.getPushInputDependencies();
            Map<String, ActionDependency> dependencyMap = new HashMap<String, ActionDependency>();
            dependencyMap.putAll(coordPullInputDependency.getMissingDependencies(coordAction));
            dependencyMap.putAll(coordPushInputDependency.getMissingDependencies(coordAction));
            inputDependenciesListPair.add(new Pair<CoordinatorActionBean, Map<String, ActionDependency>>(coordAction, dependencyMap));
        }
    } catch (Exception e) {
        throw new CommandException(ErrorCode.E1028, e.getMessage(), e);
    }
    return inputDependenciesListPair;
}

Example 7 with Pair

use of org.apache.oozie.util.Pair in project oozie by apache.

the class V2JobServlet method getCoordActionMissingDependencies.

@SuppressWarnings("unchecked")
@Override
protected JSONObject getCoordActionMissingDependencies(HttpServletRequest request, HttpServletResponse response) throws XServletException, IOException {
    String jobId = getResourceName(request);
    String actions = request.getParameter(RestConstants.JOB_COORD_SCOPE_ACTION_LIST);
    String dates = request.getParameter(RestConstants.JOB_COORD_SCOPE_DATE);
    try {
        List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> dependenciesList = Services.get().get(CoordinatorEngineService.class).getCoordinatorEngine(getUser(request)).getCoordActionMissingDependencies(jobId, actions, dates);
        JSONArray dependenciesArray = new JSONArray();
        for (Pair<CoordinatorActionBean, Map<String, ActionDependency>> dependencies : dependenciesList) {
            JSONObject json = new JSONObject();
            JSONArray parentJsonArray = new JSONArray();
            for (String key : dependencies.getSecond().keySet()) {
                JSONObject dependencyList = new JSONObject();
                JSONArray jsonArray = new JSONArray();
                jsonArray.addAll(dependencies.getSecond().get(key).getMissingDependencies());
                dependencyList.put(JsonTags.COORDINATOR_ACTION_MISSING_DEPS, jsonArray);
                dependencyList.put(JsonTags.COORDINATOR_ACTION_DATASET, key);
                parentJsonArray.add(dependencyList);
            }
            json.put(JsonTags.COORD_ACTION_FIRST_MISSING_DEPENDENCIES, CoordCommandUtils.getFirstMissingDependency(dependencies.getFirst()));
            json.put(JsonTags.COORDINATOR_ACTION_ID, dependencies.getFirst().getActionNumber());
            json.put(JsonTags.COORDINATOR_ACTION_DATASETS, parentJsonArray);
            dependenciesArray.add(json);
        }
        JSONObject jsonObject = new JSONObject();
        jsonObject.put(JsonTags.COORD_ACTION_MISSING_DEPENDENCIES, dependenciesArray);
        return jsonObject;
    } catch (CommandException e) {
        throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, e);
    }
}

Example 8 with Pair

use of org.apache.oozie.util.Pair in project oozie by apache.

the class TestCoordActionMissingDependenciesXCommand method testCoordActionInputLogicMissing.

public void testCoordActionInputLogicMissing() throws Exception {
    createTestTables();
    Configuration conf = TestCoordInputLogicPush.getConfForCombine("file://" + getTestCaseDir(), "hcat://" + getMetastoreAuthority());
    conf.set("initial_instance_a", "2014-10-07T00:00Z");
    conf.set("initial_instance_b", "2014-10-07T00:00Z");
    String inputLogic = // @formatter:off
    "<and name=\"test\">" + "<data-in dataset=\"A\" />" + "<data-in dataset=\"B\" />" + "</and>";
    // @formatter:on
    String jobId = TestCoordInputLogicPush.submitCoord(getTestCaseDir(), "coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.CURRENT_SINGLE, TEST_TYPE.CURRENT_SINGLE, TEST_TYPE.CURRENT_RANGE, TEST_TYPE.LATEST_RANGE);
    new CoordMaterializeTransitionXCommand(jobId, 3600).call();
    List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call();
    Map<String, ActionDependency> dependencyMap = data.get(0).getSecond();
    assertEquals(6, dependencyMap.size());
    assertNull(CoordCommandUtils.getFirstMissingDependency(data.get(0).getFirst()));
    createTestCaseSubDir("input-data/b/2014/10/08/_SUCCESS".split("/"));
    addPartition(DB_A, TABLE, "dt=20141008;country=usa");
    new CoordActionInputCheckXCommand(jobId + "@1", jobId).call();
    new CoordPushDependencyCheckXCommand(jobId + "@1").call();
    new CoordActionInputCheckXCommand(jobId + "@1", jobId).call();
    data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call();
    dependencyMap = data.get(0).getSecond();
    assertEquals(4, dependencyMap.size());
    assertNull(dependencyMap.get("A"));
    assertNull(dependencyMap.get("B"));
}

Example 9 with Pair

use of org.apache.oozie.util.Pair in project oozie by apache.

the class TestCoordActionMissingDependenciesXCommand method testCoordActionPullDependencyMissing.

public void testCoordActionPullDependencyMissing() throws Exception {
    Configuration conf = new XConfiguration();
    File appPathFile = new File(getTestCaseDir(), "coordinator.xml");
    // CASE 1: Failure case i.e. multiple data-in instances
    Reader reader = IOUtils.getResourceAsReader("coord-multiple-output-instance5.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "coordinator.xml"));
    IOUtils.copyCharStream(reader, writer);
    conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString());
    conf.set(OozieClient.USER_NAME, getTestUser());
    conf.set("data_set_a", "file://" + getTestCaseDir() + "/input-data/a/${YEAR}/${DAY}");
    conf.set("data_set_b", "file://" + getTestCaseDir() + "/input-data/b/${YEAR}/${DAY}");
    conf.set("data_set_c", "file://" + getTestCaseDir() + "/input-data/c/${YEAR}/${DAY}");
    conf.set("data_set_d", "file://" + getTestCaseDir() + "/input-data/d/${YEAR}/${DAY}");
    conf.set("data_set_e", "file://" + getTestCaseDir() + "/input-data/e/${YEAR}/${DAY}");
    CoordSubmitXCommand sc = new CoordSubmitXCommand(conf);
    String jobId = sc.call();
    new CoordMaterializeTransitionXCommand(jobId, 3600).call();
    List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call();
    Map<String, ActionDependency> dependencyMap = data.get(0).getSecond();
    assertEquals(6, dependencyMap.size());
    assertEquals(1, dependencyMap.get("A").getMissingDependencies().size());
    assertEquals(6, dependencyMap.get("B").getMissingDependencies().size());
    assertEquals(1, dependencyMap.get("C").getMissingDependencies().size());
    assertEquals(1, dependencyMap.get("D").getMissingDependencies().size());
    assertEquals(6, dependencyMap.get("E").getMissingDependencies().size());
    createTestCaseSubDir("input-data/a/2009/01/_SUCCESS".split("/"));
    createTestCaseSubDir("input-data/b/2009/01/_SUCCESS".split("/"));
    new CoordActionInputCheckXCommand(jobId + "@1", jobId).call();
    data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call();
    dependencyMap = data.get(0).getSecond();
    assertEquals(5, dependencyMap.size());
    assertNull(dependencyMap.get("A"));
    assertEquals(5, dependencyMap.get("B").getMissingDependencies().size());
    assertTrue(dependencyMap.get("B").getMissingDependencies().contains("file://" + getTestCaseDir() + "/input-data/b/2009/31/_SUCCESS"));
    assertTrue(dependencyMap.get("B").getMissingDependencies().contains("file://" + getTestCaseDir() + "/input-data/b/2009/30/_SUCCESS"));
    assertTrue(dependencyMap.get("B").getMissingDependencies().contains("file://" + getTestCaseDir() + "/input-data/b/2009/29/_SUCCESS"));
    assertTrue(dependencyMap.get("B").getMissingDependencies().contains("file://" + getTestCaseDir() + "/input-data/b/2009/28/_SUCCESS"));
    assertTrue(dependencyMap.get("B").getMissingDependencies().contains("file://" + getTestCaseDir() + "/input-data/b/2009/27/_SUCCESS"));
    assertEquals(1, dependencyMap.get("C").getMissingDependencies().size());
    assertEquals(1, dependencyMap.get("D").getMissingDependencies().size());
    assertEquals(dependencyMap.get("D").getMissingDependencies().get(0), "file://" + getTestCaseDir() + "/input-data/d/2009/01");
    assertEquals(6, dependencyMap.get("E").getMissingDependencies().size());
    assertEquals(2, dependencyMap.get("F").getMissingDependencies().size());
    assertEquals(dependencyMap.get("F").getMissingDependencies().get(0), "file://" + getTestCaseDir() + "/input-data/e/2009/01/_SUCCESS");
    assertEquals(dependencyMap.get("F").getMissingDependencies().get(1), "${coord:latest(0)} -> file://" + getTestCaseDir() + "/input-data/e/${YEAR}/${DAY}");
}

Example 10 with Pair

use of org.apache.oozie.util.Pair in project oozie by apache.

the class TestCoordActionMissingDependenciesXCommand method testCoordActionPushDependencyMissing.

public void testCoordActionPushDependencyMissing() throws Exception {
    createTestTables();
    Configuration conf = new XConfiguration();
    File appPathFile = new File(getTestCaseDir(), "coordinator.xml");
    Reader reader = IOUtils.getResourceAsReader("coord-multiple-output-instance5.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "coordinator.xml"));
    IOUtils.copyCharStream(reader, writer);
    conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString());
    conf.set(OozieClient.USER_NAME, getTestUser());
    String datasetSuffix = "/" + TABLE + "/dt=${YEAR}${DAY};country=usa";
    String datasetPrefix = "hcat://" + getMetastoreAuthority() + "/";
    conf.set("data_set_a", datasetPrefix.toString() + DB_A + datasetSuffix);
    conf.set("data_set_b", datasetPrefix.toString() + DB_B + datasetSuffix);
    conf.set("data_set_c", datasetPrefix.toString() + DB_C + datasetSuffix);
    conf.set("data_set_d", datasetPrefix.toString() + DB_D + datasetSuffix);
    conf.set("data_set_e", datasetPrefix.toString() + DB_E + datasetSuffix);
    CoordSubmitXCommand sc = new CoordSubmitXCommand(conf);
    String jobId = sc.call();
    new CoordMaterializeTransitionXCommand(jobId, 3600).call();
    List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call();
    assertEquals(datasetPrefix + "db_a/table1/dt=200901;country=usa", CoordCommandUtils.getFirstMissingDependency(data.get(0).getFirst()));
    Map<String, ActionDependency> dependencyMap = data.get(0).getSecond();
    assertEquals(6, dependencyMap.size());
    assertEquals(1, dependencyMap.get("A").getMissingDependencies().size());
    assertEquals(6, dependencyMap.get("B").getMissingDependencies().size());
    assertEquals(1, dependencyMap.get("C").getMissingDependencies().size());
    assertEquals("${coord:latestRange(-5,0)} -> " + datasetPrefix.toString() + DB_C + datasetSuffix, dependencyMap.get("C").getMissingDependencies().get(0));
    assertEquals(1, dependencyMap.get("D").getMissingDependencies().size());
    assertEquals(6, dependencyMap.get("E").getMissingDependencies().size());
    addPartition(DB_A, TABLE, "dt=200901;country=usa");
    new CoordPushDependencyCheckXCommand(jobId + "@1").call();
    data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call();
    dependencyMap = data.get(0).getSecond();
    assertEquals(5, dependencyMap.size());
    assertNull(dependencyMap.get("A"));
    addPartition(DB_B, TABLE, "dt=200901;country=usa");
    addPartition(DB_B, TABLE, "dt=200931;country=usa");
    addPartition(DB_B, TABLE, "dt=200930;country=usa");
    addPartition(DB_B, TABLE, "dt=200929;country=usa");
    addPartition(DB_B, TABLE, "dt=200928;country=usa");
    addPartition(DB_B, TABLE, "dt=200927;country=usa");
    new CoordPushDependencyCheckXCommand(jobId + "@1").call();
    data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call();
    dependencyMap = data.get(0).getSecond();
    assertEquals(4, dependencyMap.size());
    assertNull(dependencyMap.get("B"));
}