Examples with HCatURI org.apache.oozie.util.HCatURI used on opensource projects

Example 21 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class SimpleHCatDependencyCache method getWaitingActions.

@Override
public Collection<String> getWaitingActions(HCatURI hcatURI) {
    String tableKey = canonicalizeHostname(hcatURI.getServer()) + DELIMITER + hcatURI.getDb() + DELIMITER + hcatURI.getTable();
    SortedPKV sortedPKV = new SortedPKV(hcatURI.getPartitionMap());
    String partKey = sortedPKV.getPartKeys();
    String partVal = sortedPKV.getPartVals();
    Map<String, Map<String, Collection<WaitingAction>>> partKeyPatterns = missingDeps.get(tableKey);
    if (partKeyPatterns == null) {
        return null;
    }
    Map<String, Collection<WaitingAction>> partValues = partKeyPatterns.get(partKey);
    if (partValues == null) {
        return null;
    }
    Collection<WaitingAction> waitingActions = partValues.get(partVal);
    if (waitingActions == null) {
        return null;
    }
    Collection<String> actionIDs = new ArrayList<String>();
    URI uri = hcatURI.getURI();
    String uriString = null;
    try {
        uriString = new URI(uri.getScheme(), canonicalizeHostname(uri.getAuthority()), uri.getPath(), uri.getQuery(), uri.getFragment()).toString();
    } catch (URISyntaxException e) {
        uriString = hcatURI.toURIString();
    }
    for (WaitingAction action : waitingActions) {
        if (action.getDependencyURI().equals(uriString)) {
            actionIDs.add(action.getActionID());
        }
    }
    return actionIDs;
}

Example 22 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class SimpleHCatDependencyCache method removePartitions.

private HCatURI removePartitions(String coordActionId, Collection<String> partKeys, Map<String, Map<String, Collection<WaitingAction>>> partKeyPatterns) {
    HCatURI hcatUri = null;
    for (String partKey : partKeys) {
        Map<String, Collection<WaitingAction>> partValues = partKeyPatterns.get(partKey);
        Iterator<String> partValItr = partValues.keySet().iterator();
        while (partValItr.hasNext()) {
            String partVal = partValItr.next();
            Collection<WaitingAction> waitingActions = partValues.get(partVal);
            if (waitingActions != null) {
                Iterator<WaitingAction> waitItr = waitingActions.iterator();
                while (waitItr.hasNext()) {
                    WaitingAction waction = waitItr.next();
                    if (coordActionId.contains(waction.getActionID())) {
                        waitItr.remove();
                        if (hcatUri == null) {
                            try {
                                hcatUri = new HCatURI(waction.getDependencyURI());
                            } catch (URISyntaxException e) {
                                continue;
                            }
                        }
                    }
                }
            }
            // delete partition value with no waiting actions
            if (waitingActions.size() == 0) {
                partValItr.remove();
            }
        }
        if (partValues.size() == 0) {
            partKeyPatterns.remove(partKey);
        }
    }
    return hcatUri;
}

Example 23 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class HCatELFunctions method ph3_coord_dataInPartitionMax.

/**
 * Used to specify the MINIMUM value of an HCat partition which is input dependency for workflow job.
 * <p> Look for two evaluator-level
 * variables <p> A) .datain.&lt;DATAIN_NAME&gt; B) .datain.&lt;DATAIN_NAME&gt;.unresolved <p> A defines the current list of
 * HCat URIs. <p> B defines whether there are any unresolved EL-function (i.e latest) <p> If there are something
 * unresolved, this function will echo back the original function <p> otherwise it sends the min partition value.
 *
 * @param dataInName : Datain name
 * @param partitionName : Specific partition name whose MIN value is wanted
 */
public static String ph3_coord_dataInPartitionMax(String dataInName, String partitionName) {
    ELEvaluator eval = ELEvaluator.getCurrent();
    String uris = (String) eval.getVariable(".datain." + dataInName);
    Boolean unresolved = (Boolean) eval.getVariable(".datain." + dataInName + ".unresolved");
    if (unresolved != null && unresolved.booleanValue() == true) {
        return "${coord:dataInPartitionMin('" + dataInName + "', '" + partitionName + "')}";
    }
    String maxPartition = null;
    if (uris != null) {
        String[] uriList = HCatURIParser.splitHCatUris(uris, HCAT_URI_PATTERN);
        // get the partition values list and find minimum
        try {
            // initialize minValue with first partition value
            maxPartition = new HCatURI(uriList[0]).getPartitionValue(partitionName);
            if (maxPartition == null || maxPartition.isEmpty()) {
                throw new RuntimeException("No value in data-in uri for partition key: " + partitionName);
            }
            for (int i = 1; i < uriList.length; i++) {
                String value = new HCatURI(uriList[i]).getPartitionValue(partitionName);
                if (value.compareTo(maxPartition) > 0) {
                    maxPartition = value;
                }
            }
        } catch (URISyntaxException urie) {
            throw new RuntimeException("HCat URI can't be parsed " + urie);
        }
    } else {
        XLog.getLog(HCatELFunctions.class).warn("URI is null");
        return null;
    }
    return maxPartition;
}

Example 24 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class HCatELFunctions method ph3_coord_dataInPartitionMin.

/**
 * Used to specify the MAXIMUM value of an HCat partition which is input dependency for workflow job.
 * <p> Look for two evaluator-level
 * variables <p> A) .datain.&lt;DATAIN_NAME&gt; B) .datain.&lt;DATAIN_NAME&gt;.unresolved <p> A defines the current list of
 * HCat URIs. <p> B defines whether there are any unresolved EL-function (i.e latest) <p> If there are something
 * unresolved, this function will echo back the original function <p> otherwise it sends the max partition value.
 *
 * @param dataInName : Datain name
 * @param partitionName : Specific partition name whose MAX value is wanted
 */
public static String ph3_coord_dataInPartitionMin(String dataInName, String partitionName) {
    ELEvaluator eval = ELEvaluator.getCurrent();
    String uris = (String) eval.getVariable(".datain." + dataInName);
    Boolean unresolved = (Boolean) eval.getVariable(".datain." + dataInName + ".unresolved");
    if (unresolved != null && unresolved.booleanValue() == true) {
        return "${coord:dataInPartitionMin('" + dataInName + "', '" + partitionName + "')}";
    }
    String minPartition = null;
    if (uris != null) {
        String[] uriList = HCatURIParser.splitHCatUris(uris, HCAT_URI_PATTERN);
        // get the partition values list and find minimum
        try {
            // initialize minValue with first partition value
            minPartition = new HCatURI(uriList[0]).getPartitionValue(partitionName);
            if (minPartition == null || minPartition.isEmpty()) {
                throw new RuntimeException("No value in data-in uri for partition key: " + partitionName);
            }
            for (int i = 1; i < uriList.length; i++) {
                String value = new HCatURI(uriList[i]).getPartitionValue(partitionName);
                if (value.compareTo(minPartition) < 0) {
                    // sticking to string comparison since some numerical date
                    // values can also contain letters e.g. 20120101T0300Z (UTC)
                    minPartition = value;
                }
            }
        } catch (URISyntaxException urie) {
            throw new RuntimeException("HCat URI can't be parsed " + urie);
        }
    } else {
        XLog.getLog(HCatELFunctions.class).warn("URI is null");
        return null;
    }
    return minPartition;
}

Example 25 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class HCatELFunctions method ph3_coord_dataOutPartitionValue.

/**
 * Used to specify the HCat partition's value defining output for workflow job.<p> Look for two evaluator-level
 * variables <p> A) .dataout.&lt;DATAOUT_NAME&gt; B) .dataout.&lt;DATAOUT_NAME&gt;.unresolved <p> A defines the current list of
 * HCat URIs. <p> B defines whether there are any unresolved EL-function (i.e latest) <p> If there are something
 * unresolved, this function will echo back the original function <p> otherwise it sends the partition value.
 *
 * @param dataOutName : Dataout name
 * @param partitionName : Specific partition name whose value is wanted
 */
public static String ph3_coord_dataOutPartitionValue(String dataOutName, String partitionName) {
    ELEvaluator eval = ELEvaluator.getCurrent();
    String uri = (String) eval.getVariable(".dataout." + dataOutName);
    Boolean unresolved = (Boolean) eval.getVariable(".dataout." + dataOutName + ".unresolved");
    if (unresolved != null && unresolved.booleanValue() == true) {
        return "${coord:dataOutPartitionValue('" + dataOutName + "', '" + partitionName + "')}";
    }
    try {
        HCatURI hcatUri = new HCatURI(uri);
        return hcatUri.getPartitionValue(partitionName);
    } catch (URISyntaxException urie) {
        XLog.getLog(HCatELFunctions.class).warn("Exception with uriTemplate [{0}]. Reason [{1}]: ", uri, urie);
        throw new RuntimeException("HCat URI can't be parsed " + urie);
    }
}