Examples with ActionDependency org.apache.oozie.dependency.ActionDependency used on opensource projects

Example 11 with ActionDependency

use of org.apache.oozie.dependency.ActionDependency in project oozie by apache.

the class TestCoordActionMissingDependenciesXCommand method testCoordActionPushDependencyMissing.

public void testCoordActionPushDependencyMissing() throws Exception {
    createTestTables();
    Configuration conf = new XConfiguration();
    File appPathFile = new File(getTestCaseDir(), "coordinator.xml");
    Reader reader = IOUtils.getResourceAsReader("coord-multiple-output-instance5.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "coordinator.xml"));
    IOUtils.copyCharStream(reader, writer);
    conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString());
    conf.set(OozieClient.USER_NAME, getTestUser());
    String datasetSuffix = "/" + TABLE + "/dt=${YEAR}${DAY};country=usa";
    String datasetPrefix = "hcat://" + getMetastoreAuthority() + "/";
    conf.set("data_set_a", datasetPrefix.toString() + DB_A + datasetSuffix);
    conf.set("data_set_b", datasetPrefix.toString() + DB_B + datasetSuffix);
    conf.set("data_set_c", datasetPrefix.toString() + DB_C + datasetSuffix);
    conf.set("data_set_d", datasetPrefix.toString() + DB_D + datasetSuffix);
    conf.set("data_set_e", datasetPrefix.toString() + DB_E + datasetSuffix);
    CoordSubmitXCommand sc = new CoordSubmitXCommand(conf);
    String jobId = sc.call();
    new CoordMaterializeTransitionXCommand(jobId, 3600).call();
    List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call();
    assertEquals(datasetPrefix + "db_a/table1/dt=200901;country=usa", CoordCommandUtils.getFirstMissingDependency(data.get(0).getFirst()));
    Map<String, ActionDependency> dependencyMap = data.get(0).getSecond();
    assertEquals(6, dependencyMap.size());
    assertEquals(1, dependencyMap.get("A").getMissingDependencies().size());
    assertEquals(6, dependencyMap.get("B").getMissingDependencies().size());
    assertEquals(1, dependencyMap.get("C").getMissingDependencies().size());
    assertEquals("${coord:latestRange(-5,0)} -> " + datasetPrefix.toString() + DB_C + datasetSuffix, dependencyMap.get("C").getMissingDependencies().get(0));
    assertEquals(1, dependencyMap.get("D").getMissingDependencies().size());
    assertEquals(6, dependencyMap.get("E").getMissingDependencies().size());
    addPartition(DB_A, TABLE, "dt=200901;country=usa");
    new CoordPushDependencyCheckXCommand(jobId + "@1").call();
    data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call();
    dependencyMap = data.get(0).getSecond();
    assertEquals(5, dependencyMap.size());
    assertNull(dependencyMap.get("A"));
    addPartition(DB_B, TABLE, "dt=200901;country=usa");
    addPartition(DB_B, TABLE, "dt=200931;country=usa");
    addPartition(DB_B, TABLE, "dt=200930;country=usa");
    addPartition(DB_B, TABLE, "dt=200929;country=usa");
    addPartition(DB_B, TABLE, "dt=200928;country=usa");
    addPartition(DB_B, TABLE, "dt=200927;country=usa");
    new CoordPushDependencyCheckXCommand(jobId + "@1").call();
    data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call();
    dependencyMap = data.get(0).getSecond();
    assertEquals(4, dependencyMap.size());
    assertNull(dependencyMap.get("B"));
}

Example 12 with ActionDependency

use of org.apache.oozie.dependency.ActionDependency in project oozie by apache.

the class TestCoordActionMissingDependenciesXCommand method testCoordActionPullPushDependencyMissing.

public void testCoordActionPullPushDependencyMissing() throws Exception {
    createTestTables();
    Configuration conf = new XConfiguration();
    File appPathFile = new File(getTestCaseDir(), "coordinator.xml");
    Reader reader = IOUtils.getResourceAsReader("coord-multiple-output-instance5.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "coordinator.xml"));
    IOUtils.copyCharStream(reader, writer);
    conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString());
    conf.set(OozieClient.USER_NAME, getTestUser());
    String datasetPrefix = "/" + TABLE + "/dt=${YEAR}${DAY};country=usa";
    String datasetSuffix = "hcat://" + getMetastoreAuthority() + "/";
    conf.set("data_set_a", datasetSuffix.toString() + DB_A + datasetPrefix);
    conf.set("data_set_b", datasetSuffix.toString() + DB_B + datasetPrefix);
    conf.set("data_set_c", datasetSuffix.toString() + DB_C + datasetPrefix);
    conf.set("data_set_d", "file://" + getTestCaseDir() + "/input-data/d/${YEAR}/${DAY}");
    conf.set("data_set_e", "file://" + getTestCaseDir() + "/input-data/e/${YEAR}/${DAY}");
    CoordSubmitXCommand sc = new CoordSubmitXCommand(conf);
    String jobId = sc.call();
    new CoordMaterializeTransitionXCommand(jobId, 3600).call();
    List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call();
    assertEquals("file://" + getTestCaseDir() + "/input-data/d/2009/01", CoordCommandUtils.getFirstMissingDependency(data.get(0).getFirst()));
    Map<String, ActionDependency> dependencyMap = data.get(0).getSecond();
    assertEquals(6, dependencyMap.size());
    assertEquals(1, dependencyMap.get("A").getMissingDependencies().size());
    assertEquals(6, dependencyMap.get("B").getMissingDependencies().size());
    assertEquals(1, dependencyMap.get("C").getMissingDependencies().size());
    assertEquals(1, dependencyMap.get("D").getMissingDependencies().size());
    assertEquals(6, dependencyMap.get("E").getMissingDependencies().size());
    addPartition(DB_A, TABLE, "dt=200901;country=usa");
    addPartition(DB_B, TABLE, "dt=200901;country=usa");
    addPartition(DB_B, TABLE, "dt=200931;country=usa");
    addPartition(DB_B, TABLE, "dt=200930;country=usa");
    addPartition(DB_B, TABLE, "dt=200929;country=usa");
    addPartition(DB_B, TABLE, "dt=200928;country=usa");
    addPartition(DB_B, TABLE, "dt=200927;country=usa");
    createTestCaseSubDir("input-data/d/2009/01".split("/"));
    new CoordActionInputCheckXCommand(jobId + "@1", jobId).call();
    new CoordPushDependencyCheckXCommand(jobId + "@1").call();
    data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call();
    dependencyMap = data.get(0).getSecond();
    assertEquals(3, dependencyMap.size());
    assertNull(dependencyMap.get("B"));
    assertNull(dependencyMap.get("D"));
}