Examples with HCatURI org.apache.oozie.util.HCatURI used on opensource projects

Example 26 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class HCatELFunctions method ph3_coord_dataInPartitions.

/**
 * Used to specify the entire HCat partition defining input for workflow job. <p> Look for two evaluator-level
 * variables <p> A) .datain.&lt;DATAIN_NAME&gt; B) .datain.&lt;DATAIN_NAME&gt;.unresolved <p> A defines the data-in HCat URI.
 * <p> B defines whether there are any unresolved EL-function (i.e latest) <p> If there are something unresolved,
 * this function will echo back the original function <p> otherwise it sends the partition.
 *
 * @param dataInName : DataIn name
 * @param type : for action type: hive-export
 */
public static String ph3_coord_dataInPartitions(String dataInName, String type) {
    ELEvaluator eval = ELEvaluator.getCurrent();
    String uri = (String) eval.getVariable(".datain." + dataInName);
    Boolean unresolved = (Boolean) eval.getVariable(".datain." + dataInName + ".unresolved");
    if (unresolved != null && unresolved.booleanValue() == true) {
        return "${coord:dataInPartitions('" + dataInName + "', '" + type + "')}";
    }
    String partitionValue = null;
    if (uri != null) {
        if (type.equals("hive-export")) {
            String[] uriList = HCatURIParser.splitHCatUris(uri, HCAT_URI_PATTERN);
            if (uriList.length > 1) {
                throw new RuntimeException("Multiple partitions not supported for hive-export type. Dataset name: " + dataInName + " URI: " + uri);
            }
            try {
                partitionValue = new HCatURI(uri).toPartitionValueString(type);
            } catch (URISyntaxException e) {
                throw new RuntimeException("Parsing exception for HCatURI " + uri, e);
            }
        } else {
            throw new RuntimeException("Unsupported type: " + type + " dataset name: " + dataInName);
        }
    } else {
        XLog.getLog(HCatELFunctions.class).warn("URI is null");
        return null;
    }
    return partitionValue;
}

Example 27 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class HCatELFunctions method ph3_coord_dataOutPartitions.

/**
 * Used to specify the entire HCat partition defining output for workflow job.<p> Look for two evaluator-level
 * variables <p> A) .dataout.&lt;DATAOUT_NAME&gt; B) .dataout.&lt;DATAOUT_NAME&gt;.unresolved <p> A defines the data-out
 * HCat URI. <p> B defines whether there are any unresolved EL-function (i.e latest) <p> If there are something
 * unresolved, this function will echo back the original function <p> otherwise it sends the partition.
 *
 * @param dataOutName : DataOut name
 */
public static String ph3_coord_dataOutPartitions(String dataOutName) {
    ELEvaluator eval = ELEvaluator.getCurrent();
    String uri = (String) eval.getVariable(".dataout." + dataOutName);
    Boolean unresolved = (Boolean) eval.getVariable(".dataout." + dataOutName + ".unresolved");
    if (unresolved != null && unresolved.booleanValue() == true) {
        return "${coord:dataOutPartitions('" + dataOutName + "')}";
    }
    try {
        return new HCatURI(uri).toPartitionString();
    } catch (URISyntaxException e) {
        throw new RuntimeException("Parsing exception for HCatURI " + uri + ". details: " + e);
    }
}

Example 28 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class TestCoordPushDependencyCheckXCommand method testTimeOutWithUnresolvedMissingDependencies.

@Test
public void testTimeOutWithUnresolvedMissingDependencies() throws Exception {
    String db = "default";
    String table = "tablename";
    String newHCatDependency1 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=brazil";
    String newHCatDependency2 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=usa";
    String newHCatDependency3 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=uk";
    String newHCatDependency = newHCatDependency1 + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency2;
    populateTable(db, table);
    String actionId = addInitRecords(newHCatDependency);
    checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
    new CoordPushDependencyCheckXCommand(actionId, true).call();
    checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.WAITING);
    PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    assertTrue(pdms.getWaitingActions(new HCatURI(newHCatDependency1)).contains(actionId));
    assertTrue(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
    // Timeout is 10 mins. Change action created time to before 12 min to make the action
    // timeout.
    long timeOutCreationTime = System.currentTimeMillis() - (12 * 60 * 1000);
    setCoordActionCreationTime(actionId, timeOutCreationTime);
    // Set some missing dependency. Instead of latest or future just setting a current one for testing as
    // we are only interested in ensuring CoordActionInputCheckXCommand is run
    setMissingDependencies(actionId, newHCatDependency + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency3);
    addPartition(db, table, "dt=20120430;country=brazil");
    checkDependencies(actionId, newHCatDependency + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency3, newHCatDependency1);
    new CoordPushDependencyCheckXCommand(actionId).call();
    // Somehow with hive 0.10 it takes 1 second more.
    Thread.sleep(1300);
    checkDependencies(actionId, newHCatDependency3, "");
    assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency1)));
    assertFalse(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
}

Example 29 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class TestCoordPushDependencyCheckXCommand method testTimeOut.

@Test
public void testTimeOut() throws Exception {
    String db = "default";
    String table = "tablename";
    String newHCatDependency1 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=brazil";
    String newHCatDependency2 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=usa";
    String newHCatDependency = newHCatDependency1 + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency2;
    populateTable(db, table);
    String actionId = addInitRecords(newHCatDependency);
    checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
    new CoordPushDependencyCheckXCommand(actionId, true).call();
    checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.WAITING);
    PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    assertTrue(pdms.getWaitingActions(new HCatURI(newHCatDependency1)).contains(actionId));
    assertTrue(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
    // Timeout is 10 mins. Change action created time to before 12 min to make the action
    // timeout.
    long timeOutCreationTime = System.currentTimeMillis() - (12 * 60 * 1000);
    setCoordActionCreationTime(actionId, timeOutCreationTime);
    new CoordPushDependencyCheckXCommand(actionId).call();
    Thread.sleep(100);
    // Check for timeout status and unregistered missing dependencies
    checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.TIMEDOUT);
    assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency1)));
    assertFalse(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
}

Example 30 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class TestCoordActionUpdatePushMissingDependency method testUpdateCoordTableAdvanced.

@Test
public void testUpdateCoordTableAdvanced() throws Exception {
    String newHCatDependency1 = "hcat://hcat.server.com:5080/mydb/clicks/datastamp=11;region=us";
    String newHCatDependency2 = "hcat://hcat.server.com:5080/mydb/clicks/datastamp=12;region=us";
    HCatURI hcatUri1 = new HCatURI(newHCatDependency1);
    HCatURI hcatUri2 = new HCatURI(newHCatDependency2);
    String fullDeps = newHCatDependency1 + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency2;
    String actionId = addInitRecords(fullDeps);
    checkCoordAction(actionId, fullDeps, CoordinatorAction.Status.WAITING);
    PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
    pdms.addMissingDependency(hcatUri1, actionId);
    pdms.addMissingDependency(hcatUri2, actionId);
    pdms.partitionAvailable("hcat.server.com:5080", "mydb", "clicks", getPartitionMap("src=search;datastamp=12;region=us"));
    Collection<String> availableURIs = pdms.getAvailableDependencyURIs(actionId);
    assertEquals(1, availableURIs.size());
    assertTrue(availableURIs.contains(newHCatDependency2));
    new CoordActionUpdatePushMissingDependency(actionId).call();
    checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.WAITING);
    // second partition available
    pdms.partitionAvailable("hcat.server.com:5080", "mydb", "clicks", getPartitionMap("src=search;datastamp=11;region=us"));
    availableURIs = pdms.getAvailableDependencyURIs(actionId);
    assertEquals(1, availableURIs.size());
    assertTrue(availableURIs.contains(newHCatDependency1));
    new CoordActionUpdatePushMissingDependency(actionId).call();
    checkCoordAction(actionId, "", CoordinatorAction.Status.READY);
    assertNull(pdms.getAvailableDependencyURIs(actionId));
}