Examples with Pair org.apache.oozie.util.Pair used on opensource projects

Example 11 with Pair

use of org.apache.oozie.util.Pair in project oozie by apache.

the class TestCoordActionMissingDependenciesXCommand method testCoordActionPullPushDependencyMissing.

public void testCoordActionPullPushDependencyMissing() throws Exception {
    createTestTables();
    Configuration conf = new XConfiguration();
    File appPathFile = new File(getTestCaseDir(), "coordinator.xml");
    Reader reader = IOUtils.getResourceAsReader("coord-multiple-output-instance5.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "coordinator.xml"));
    IOUtils.copyCharStream(reader, writer);
    conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString());
    conf.set(OozieClient.USER_NAME, getTestUser());
    String datasetPrefix = "/" + TABLE + "/dt=${YEAR}${DAY};country=usa";
    String datasetSuffix = "hcat://" + getMetastoreAuthority() + "/";
    conf.set("data_set_a", datasetSuffix.toString() + DB_A + datasetPrefix);
    conf.set("data_set_b", datasetSuffix.toString() + DB_B + datasetPrefix);
    conf.set("data_set_c", datasetSuffix.toString() + DB_C + datasetPrefix);
    conf.set("data_set_d", "file://" + getTestCaseDir() + "/input-data/d/${YEAR}/${DAY}");
    conf.set("data_set_e", "file://" + getTestCaseDir() + "/input-data/e/${YEAR}/${DAY}");
    CoordSubmitXCommand sc = new CoordSubmitXCommand(conf);
    String jobId = sc.call();
    new CoordMaterializeTransitionXCommand(jobId, 3600).call();
    List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call();
    assertEquals("file://" + getTestCaseDir() + "/input-data/d/2009/01", CoordCommandUtils.getFirstMissingDependency(data.get(0).getFirst()));
    Map<String, ActionDependency> dependencyMap = data.get(0).getSecond();
    assertEquals(6, dependencyMap.size());
    assertEquals(1, dependencyMap.get("A").getMissingDependencies().size());
    assertEquals(6, dependencyMap.get("B").getMissingDependencies().size());
    assertEquals(1, dependencyMap.get("C").getMissingDependencies().size());
    assertEquals(1, dependencyMap.get("D").getMissingDependencies().size());
    assertEquals(6, dependencyMap.get("E").getMissingDependencies().size());
    addPartition(DB_A, TABLE, "dt=200901;country=usa");
    addPartition(DB_B, TABLE, "dt=200901;country=usa");
    addPartition(DB_B, TABLE, "dt=200931;country=usa");
    addPartition(DB_B, TABLE, "dt=200930;country=usa");
    addPartition(DB_B, TABLE, "dt=200929;country=usa");
    addPartition(DB_B, TABLE, "dt=200928;country=usa");
    addPartition(DB_B, TABLE, "dt=200927;country=usa");
    createTestCaseSubDir("input-data/d/2009/01".split("/"));
    new CoordActionInputCheckXCommand(jobId + "@1", jobId).call();
    new CoordPushDependencyCheckXCommand(jobId + "@1").call();
    data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call();
    dependencyMap = data.get(0).getSecond();
    assertEquals(3, dependencyMap.size());
    assertNull(dependencyMap.get("B"));
    assertNull(dependencyMap.get("D"));
}

Example 12 with Pair

use of org.apache.oozie.util.Pair in project oozie by apache.

the class TestCoordUtils method testGetWhereClause.

public void testGetWhereClause() throws Exception {
    Map<Pair<String, CoordinatorEngine.FILTER_COMPARATORS>, List<Object>> filterMap = new HashMap<Pair<String, CoordinatorEngine.FILTER_COMPARATORS>, List<Object>>();
    final Pair<String, CoordinatorEngine.FILTER_COMPARATORS> STATUS_FILTER = Pair.of(OozieClient.FILTER_STATUS, CoordinatorEngine.FILTER_COMPARATORS.EQUALS);
    List<Object> positiveFilter = new ArrayList<Object>();
    positiveFilter.add("RUNNING");
    positiveFilter.add("KILLED");
    filterMap.put(STATUS_FILTER, positiveFilter);
    JPAService jpaService = Services.get().get(JPAService.class);
    EntityManager em = jpaService.getEntityManager();
    Query q = em.createNamedQuery("GET_COORD_ACTIONS_COUNT_BY_JOBID");
    String query = q.toString();
    StringBuilder sbTotal = new StringBuilder(query);
    // Get the 'where' clause for status filters
    StringBuilder statusClause = new StringBuilder();
    Map<String, Object> params = CoordUtils.getWhereClause(statusClause, filterMap);
    sbTotal.insert(sbTotal.length(), statusClause);
    assertTrue(sbTotal.toString().contains("and a.statusStr IN (:p1, :p2)"));
    assertEquals(params.get("p1"), "RUNNING");
    assertEquals(params.get("p2"), "KILLED");
}

Example 13 with Pair

use of org.apache.oozie.util.Pair in project oozie by apache.

the class TestCoordJobGetActionsSubsetJPAExecutor method testGetActionsWithNominalTimeFilter.

public void testGetActionsWithNominalTimeFilter() throws Exception {
    CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, false, false);
    String[] timeStr = { "2009-02-01T00:00Z", "2009-02-01T05:00Z", "2009-02-01T10:00Z", "2009-02-01T15:00Z" };
    Date[] ntime = { getSqlTime(timeStr[0]), getSqlTime(timeStr[1]), getSqlTime(timeStr[2]), getSqlTime(timeStr[3]) };
    List<String> actionIds = new ArrayList<String>(timeStr.length);
    int startAction = 5;
    for (Date time : ntime) {
        CoordinatorActionBean action = addRecordToCoordActionTable(job.getId(), startAction++, Status.WAITING, "coord-action-get.xml", 0, time);
        actionIds.add(action.getId());
    }
    JPAService jpaService = Services.get().get(JPAService.class);
    assertNotNull(jpaService);
    // Filter with nominalTime >=, asc
    Map<Pair<String, FILTER_COMPARATORS>, List<Object>> filterMap = new HashMap<Pair<String, FILTER_COMPARATORS>, List<Object>>();
    filterMap.put(Pair.of(OozieClient.FILTER_NOMINAL_TIME, FILTER_COMPARATORS.GREATER_EQUAL), getList(ntime[2]));
    CoordJobGetActionsSubsetJPAExecutor actionGetCmd = new CoordJobGetActionsSubsetJPAExecutor(job.getId(), filterMap, 1, 10, false);
    List<CoordinatorActionBean> actions = jpaService.execute(actionGetCmd);
    Assert.assertEquals(2, actions.size());
    Assert.assertEquals(actionIds.get(2), actions.get(0).getId());
    Assert.assertEquals(actionIds.get(3), actions.get(1).getId());
    // Filter with nominalTime >=, desc
    actionGetCmd = new CoordJobGetActionsSubsetJPAExecutor(job.getId(), filterMap, 1, 10, true);
    actions = jpaService.execute(actionGetCmd);
    Assert.assertEquals(2, actions.size());
    Assert.assertEquals(actionIds.get(3), actions.get(0).getId());
    Assert.assertEquals(actionIds.get(2), actions.get(1).getId());
    // Filter with nominalTime <, asc
    filterMap.clear();
    filterMap.put(Pair.of(OozieClient.FILTER_NOMINAL_TIME, FILTER_COMPARATORS.LESSTHAN), getList(ntime[2]));
    actionGetCmd = new CoordJobGetActionsSubsetJPAExecutor(job.getId(), filterMap, 1, 10, false);
    actions = jpaService.execute(actionGetCmd);
    Assert.assertEquals(2, actions.size());
    Assert.assertEquals(actionIds.get(0), actions.get(0).getId());
    Assert.assertEquals(actionIds.get(1), actions.get(1).getId());
    // Filter with nominalTime <, desc
    actionGetCmd = new CoordJobGetActionsSubsetJPAExecutor(job.getId(), filterMap, 1, 10, true);
    actions = jpaService.execute(actionGetCmd);
    Assert.assertEquals(2, actions.size());
    Assert.assertEquals(actionIds.get(1), actions.get(0).getId());
    Assert.assertEquals(actionIds.get(0), actions.get(1).getId());
    // Filter with nominalTime >=, nominalTime <, asc
    filterMap.put(Pair.of(OozieClient.FILTER_NOMINAL_TIME, FILTER_COMPARATORS.GREATER_EQUAL), getList(ntime[1]));
    filterMap.put(Pair.of(OozieClient.FILTER_NOMINAL_TIME, FILTER_COMPARATORS.LESSTHAN), getList(ntime[3]));
    actionGetCmd = new CoordJobGetActionsSubsetJPAExecutor(job.getId(), filterMap, 1, 10, false);
    actions = jpaService.execute(actionGetCmd);
    Assert.assertEquals(2, actions.size());
    Assert.assertEquals(actionIds.get(1), actions.get(0).getId());
    Assert.assertEquals(actionIds.get(2), actions.get(1).getId());
    // Filter with nominalTime >=, nominalTime <, desc
    actionGetCmd = new CoordJobGetActionsSubsetJPAExecutor(job.getId(), filterMap, 1, 10, true);
    actions = jpaService.execute(actionGetCmd);
    Assert.assertEquals(2, actions.size());
    Assert.assertEquals(actionIds.get(2), actions.get(0).getId());
    Assert.assertEquals(actionIds.get(1), actions.get(1).getId());
    // Filter with nominalTime >=, nominalTime <, desc, offset
    actionGetCmd = new CoordJobGetActionsSubsetJPAExecutor(job.getId(), filterMap, 2, 10, true);
    actions = jpaService.execute(actionGetCmd);
    Assert.assertEquals(1, actions.size());
    Assert.assertEquals(actionIds.get(1), actions.get(0).getId());
    // Filter with nominalTime >=, nominalTime <, asc, offset
    actionGetCmd = new CoordJobGetActionsSubsetJPAExecutor(job.getId(), filterMap, 2, 10, false);
    actions = jpaService.execute(actionGetCmd);
    Assert.assertEquals(1, actions.size());
    Assert.assertEquals(actionIds.get(2), actions.get(0).getId());
    // Filter with nominalTime >=, nominalTime <, desc, len
    actionGetCmd = new CoordJobGetActionsSubsetJPAExecutor(job.getId(), filterMap, 1, 2, true);
    actions = jpaService.execute(actionGetCmd);
    Assert.assertEquals(2, actions.size());
    Assert.assertEquals(actionIds.get(2), actions.get(0).getId());
    Assert.assertEquals(actionIds.get(1), actions.get(1).getId());
    // Filter with nominalTime >=, nominalTime <, asc, len
    actionGetCmd = new CoordJobGetActionsSubsetJPAExecutor(job.getId(), filterMap, 1, 2, false);
    actions = jpaService.execute(actionGetCmd);
    Assert.assertEquals(2, actions.size());
    Assert.assertEquals(actionIds.get(1), actions.get(0).getId());
    Assert.assertEquals(actionIds.get(2), actions.get(1).getId());
    // Filter with nominalTime >=, nominalTime <, asc, offset, len
    filterMap.put(Pair.of(OozieClient.FILTER_NOMINAL_TIME, FILTER_COMPARATORS.LESSTHAN), getList(getSqlTime("2009-02-01T23:00Z")));
    actionGetCmd = new CoordJobGetActionsSubsetJPAExecutor(job.getId(), filterMap, 2, 2, false);
    actions = jpaService.execute(actionGetCmd);
    Assert.assertEquals(2, actions.size());
    Assert.assertEquals(actionIds.get(2), actions.get(0).getId());
    Assert.assertEquals(actionIds.get(3), actions.get(1).getId());
    // Filter with nominalTime >=, nominalTime <, desc, offset, len
    actionGetCmd = new CoordJobGetActionsSubsetJPAExecutor(job.getId(), filterMap, 2, 2, true);
    actions = jpaService.execute(actionGetCmd);
    Assert.assertEquals(2, actions.size());
    Assert.assertEquals(actionIds.get(2), actions.get(0).getId());
    Assert.assertEquals(actionIds.get(1), actions.get(1).getId());
    // Filter with nominalTime and status
    filterMap.put(Pair.of(OozieClient.FILTER_STATUS, FILTER_COMPARATORS.EQUALS), getList(Status.SUCCEEDED.name()));
    actionGetCmd = new CoordJobGetActionsSubsetJPAExecutor(job.getId(), filterMap, 1, 10, true);
    actions = jpaService.execute(actionGetCmd);
    Assert.assertEquals(0, actions.size());
}

Example 14 with Pair

use of org.apache.oozie.util.Pair in project oozie by apache.

the class TestCoordActionsCountForJobIdJPAExecutor method testGetActionsCount.

public void testGetActionsCount() throws Exception {
    CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, false, false);
    String[] timeStr = { "2009-02-01T00:00Z", "2009-02-01T05:00Z", "2009-02-01T10:00Z", "2009-02-01T15:00Z" };
    Date[] ntime = { getSqlTime(timeStr[0]), getSqlTime(timeStr[1]), getSqlTime(timeStr[2]), getSqlTime(timeStr[3]) };
    List<String> actionIds = new ArrayList<String>(timeStr.length);
    int startAction = 5;
    for (Date time : ntime) {
        CoordinatorActionBean action = addRecordToCoordActionTable(job.getId(), startAction++, CoordinatorAction.Status.WAITING, "coord-action-get.xml", 0, time);
        actionIds.add(action.getId());
    }
    JPAService jpaService = Services.get().get(JPAService.class);
    assertNotNull(jpaService);
    // Without Filters
    Map<Pair<String, CoordinatorEngine.FILTER_COMPARATORS>, List<Object>> filterMap = new HashMap<Pair<String, CoordinatorEngine.FILTER_COMPARATORS>, List<Object>>();
    CoordActionsCountForJobIdJPAExecutor actionGetCountCmd = new CoordActionsCountForJobIdJPAExecutor(job.getId(), filterMap);
    int actionCount = jpaService.execute(actionGetCountCmd);
    Assert.assertEquals(4, actionCount);
    // With Filter
    filterMap.put(Pair.of(OozieClient.FILTER_NOMINAL_TIME, CoordinatorEngine.FILTER_COMPARATORS.GREATER_EQUAL), getList(ntime[2]));
    actionGetCountCmd = new CoordActionsCountForJobIdJPAExecutor(job.getId(), filterMap);
    actionCount = jpaService.execute(actionGetCountCmd);
    Assert.assertEquals(2, actionCount);
}

Example 15 with Pair

use of org.apache.oozie.util.Pair in project oozie by apache.

the class DummySLACalculatorMemory method testSLAAlertCommandWithHA.

public void testSLAAlertCommandWithHA() throws Exception {
    // Test SLA ALERT commands in HA mode.
    // slaCalcMem1 is for server 1 and slaCalcMem2 is for server2
    String id = "0000001-130521183438837-oozie-test-C@1";
    // 2 hrs passed
    Date expectedStartTS = new Date(System.currentTimeMillis() - 2 * 3600 * 1000);
    // 1 hour ahead
    Date expectedEndTS1 = new Date(System.currentTimeMillis() + 1 * 3600 * 1000);
    // Coord Action of jobs 1-4 not started yet
    createDBEntry(id, expectedStartTS, expectedEndTS1);
    SLAService slas = Services.get().get(SLAService.class);
    SLACalculatorMemory slaCalcMem1 = (SLACalculatorMemory) slas.getSLACalculator();
    slaCalcMem1.init(Services.get().get(ConfigurationService.class).getConf());
    slaCalcMem1.updateAllSlaStatus();
    List<String> idList = new ArrayList<String>();
    idList.add(id);
    slaCalcMem1.disableAlert(idList);
    assertTrue(slaCalcMem1.get(id).getSLAConfigMap().containsKey(OozieClient.SLA_DISABLE_ALERT));
    DummyZKOozie dummyOozie_1 = null;
    try {
        // start another dummy oozie instance (dummy sla and event handler services)
        dummyOozie_1 = new DummyZKOozie("a", "http://blah");
        DummySLACalculatorMemory slaCalcMem2 = new DummySLACalculatorMemory();
        EventHandlerService dummyEhs = new EventHandlerService();
        slaCalcMem2.setEventHandlerService(dummyEhs);
        // So that job sla updated doesn't run automatically
        Services.get().get(ConfigurationService.class).getConf().setInt(SLAService.CONF_SLA_CHECK_INTERVAL, 100000);
        Services.get().get(ConfigurationService.class).getConf().setInt(SLAService.CONF_SLA_CHECK_INITIAL_DELAY, 100000);
        dummyEhs.init(Services.get());
        slaCalcMem2.init(Services.get().get(ConfigurationService.class).getConf());
        slaCalcMem2.updateAllSlaStatus();
        assertTrue(slaCalcMem2.get(id).getSLAConfigMap().containsKey(OozieClient.SLA_DISABLE_ALERT));
        String newParams = RestConstants.SLA_MAX_DURATION + "=5";
        List<Pair<String, Map<String, String>>> jobIdsSLAPair = new ArrayList<Pair<String, Map<String, String>>>();
        jobIdsSLAPair.add(new Pair<String, Map<String, String>>(id, JobUtils.parseChangeValue(newParams)));
        slaCalcMem1.changeDefinition(jobIdsSLAPair);
        assertEquals(slaCalcMem1.get(id).getExpectedDuration(), 5 * 60 * 1000);
        // Before update, default is 10.
        assertEquals(slaCalcMem2.get(id).getExpectedDuration(), 10 * 60 * 1000);
        slaCalcMem2.updateAllSlaStatus();
        assertEquals(slaCalcMem2.get(id).getExpectedDuration(), 5 * 60 * 1000);
        newParams = RestConstants.SLA_MAX_DURATION + "=15";
        jobIdsSLAPair.clear();
        jobIdsSLAPair.add(new Pair<String, Map<String, String>>(id, JobUtils.parseChangeValue(newParams)));
        slaCalcMem1.changeDefinition(jobIdsSLAPair);
        // Before update
        assertEquals(slaCalcMem2.get(id).getExpectedDuration(), 5 * 60 * 1000);
        slaCalcMem2.updateAllSlaStatus();
        assertEquals(slaCalcMem2.get(id).getExpectedDuration(), 15 * 60 * 1000);
    } finally {
        if (dummyOozie_1 != null) {
            dummyOozie_1.teardown();
        }
    }
}