Examples with HCatAccessorService org.apache.oozie.service.HCatAccessorService used on opensource projects

Example 6 with HCatAccessorService

use of org.apache.oozie.service.HCatAccessorService in project oozie by apache.

the class HCatURIHandler method registerForNotification.

@Override
public void registerForNotification(URI uri, Configuration conf, String user, String actionID) throws URIHandlerException {
    HCatURI hcatURI;
    try {
        hcatURI = new HCatURI(uri);
    } catch (URISyntaxException e) {
        throw new URIHandlerException(ErrorCode.E0906, uri, e);
    }
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    if (!hcatService.isRegisteredForNotification(hcatURI)) {
        HCatClient client = getHCatClient(uri, conf);
        try {
            String topic = client.getMessageBusTopicName(hcatURI.getDb(), hcatURI.getTable());
            if (topic == null) {
                return;
            }
            hcatService.registerForNotification(hcatURI, topic, new HCatMessageHandler(uri.getAuthority()));
        } catch (HCatException e) {
            throw new HCatAccessorException(ErrorCode.E1501, e);
        } finally {
            closeQuietly(client, null, true);
        }
    }
    PartitionDependencyManagerService pdmService = Services.get().get(PartitionDependencyManagerService.class);
    pdmService.addMissingDependency(hcatURI, actionID);
}

Example 7 with HCatAccessorService

use of org.apache.oozie.service.HCatAccessorService in project oozie by apache.

the class EhcacheHCatDependencyCache method unregisterFromNotifications.

private void unregisterFromNotifications(String server, String db, String table) {
    // Close JMS session. Stop listening on topic
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    hcatService.unregisterFromNotification(server, db, table);
}

Example 8 with HCatAccessorService

use of org.apache.oozie.service.HCatAccessorService in project oozie by apache.

the class SimpleHCatDependencyCache method removeMissingDependency.

@Override
public boolean removeMissingDependency(HCatURI hcatURI, String actionID) {
    String tableKey = canonicalizeHostname(hcatURI.getServer()) + DELIMITER + hcatURI.getDb() + DELIMITER + hcatURI.getTable();
    SortedPKV sortedPKV = new SortedPKV(hcatURI.getPartitionMap());
    String partKey = sortedPKV.getPartKeys();
    String partVal = sortedPKV.getPartVals();
    Map<String, Map<String, Collection<WaitingAction>>> partKeyPatterns = missingDeps.get(tableKey);
    if (partKeyPatterns == null) {
        LOG.warn("Remove missing dependency - Missing table entry - uri={0}, actionID={1}", hcatURI.toURIString(), actionID);
        return false;
    }
    ConcurrentMap<String, Collection<String>> partitionMap = actionPartitionMap.get(actionID);
    if (partitionMap != null) {
        synchronized (partitionMap) {
            Collection<String> partKeys = partitionMap.get(tableKey);
            if (partKeys != null) {
                partKeys.remove(partKey);
            }
            if (partKeys.size() == 0) {
                partitionMap.remove(tableKey);
            }
            if (partitionMap.size() == 0) {
                actionPartitionMap.remove(actionID);
            }
        }
    }
    synchronized (partKeyPatterns) {
        Map<String, Collection<WaitingAction>> partValues = partKeyPatterns.get(partKey);
        if (partValues == null) {
            LOG.warn("Remove missing dependency - Missing partition pattern - uri={0}, actionID={1}", hcatURI.toURIString(), actionID);
            return false;
        }
        Collection<WaitingAction> waitingActions = partValues.get(partVal);
        if (waitingActions == null) {
            LOG.warn("Remove missing dependency - Missing partition value - uri={0}, actionID={1}", hcatURI.toURIString(), actionID);
            return false;
        }
        boolean removed = waitingActions.remove(new WaitingAction(actionID, hcatURI.toURIString()));
        if (!removed) {
            LOG.warn("Remove missing dependency - Missing action ID - uri={0}, actionID={1}", hcatURI.toURIString(), actionID);
        }
        if (waitingActions.isEmpty()) {
            partValues.remove(partVal);
            if (partValues.isEmpty()) {
                partKeyPatterns.remove(partKey);
            }
            if (partKeyPatterns.isEmpty()) {
                missingDeps.remove(tableKey);
                // Close JMS session. Stop listening on topic
                HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
                hcatService.unregisterFromNotification(hcatURI);
            }
        }
        return removed;
    }
}

Example 9 with HCatAccessorService

use of org.apache.oozie.service.HCatAccessorService in project oozie by apache.

the class SimpleHCatDependencyCache method removeNonWaitingCoordActions.

@Override
public void removeNonWaitingCoordActions(Set<String> coordActions) {
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    for (String coordActionId : coordActions) {
        LOG.info("Removing non waiting coord action {0} from partition dependency map", coordActionId);
        synchronized (actionPartitionMap) {
            Map<String, Collection<String>> partitionMap = actionPartitionMap.get(coordActionId);
            if (partitionMap != null) {
                Iterator<String> tableItr = partitionMap.keySet().iterator();
                while (tableItr.hasNext()) {
                    String tableKey = tableItr.next();
                    HCatURI hcatUri = null;
                    Map<String, Map<String, Collection<WaitingAction>>> partKeyPatterns = missingDeps.get(tableKey);
                    if (partKeyPatterns != null) {
                        synchronized (partKeyPatterns) {
                            Collection<String> partKeys = partitionMap.get(tableKey);
                            if (partKeys != null) {
                                hcatUri = removePartitions(coordActionId, partKeys, partKeyPatterns);
                            }
                        }
                        if (partKeyPatterns.size() == 0) {
                            tableItr.remove();
                            if (hcatUri != null) {
                                // Close JMS session. Stop listening on topic
                                hcatService.unregisterFromNotification(hcatUri);
                            }
                        }
                    }
                }
            }
            actionPartitionMap.remove(coordActionId);
        }
    }
}

Example 10 with HCatAccessorService

use of org.apache.oozie.service.HCatAccessorService in project oozie by apache.

the class SimpleHCatDependencyCache method markDependencyAvailable.

@Override
public Collection<String> markDependencyAvailable(String server, String db, String table, Map<String, String> partitions) {
    String tableKey = canonicalizeHostname(server) + DELIMITER + db + DELIMITER + table;
    Map<String, Map<String, Collection<WaitingAction>>> partKeyPatterns = missingDeps.get(tableKey);
    if (partKeyPatterns == null) {
        LOG.warn("Got partition available notification for " + tableKey + ". Unexpected and should not be listening to topic. Unregistering topic");
        HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
        hcatService.unregisterFromNotification(server, db, table);
        return null;
    }
    Collection<String> actionsWithAvailDep = new HashSet<String>();
    List<String> partKeysToRemove = new ArrayList<String>();
    StringBuilder partValSB = new StringBuilder();
    synchronized (partKeyPatterns) {
        // for the matching value in the dependency map, get the waiting actions.
        for (Entry<String, Map<String, Collection<WaitingAction>>> entry : partKeyPatterns.entrySet()) {
            String[] partKeys = entry.getKey().split(DELIMITER);
            partValSB.setLength(0);
            for (String key : partKeys) {
                partValSB.append(partitions.get(key)).append(DELIMITER);
            }
            partValSB.setLength(partValSB.length() - 1);
            Map<String, Collection<WaitingAction>> partValues = entry.getValue();
            String partVal = partValSB.toString();
            Collection<WaitingAction> wActions = entry.getValue().get(partVal);
            if (wActions == null)
                continue;
            for (WaitingAction wAction : wActions) {
                String actionID = wAction.getActionID();
                actionsWithAvailDep.add(actionID);
                Collection<String> depURIs = availableDeps.get(actionID);
                if (depURIs == null) {
                    depURIs = new ArrayList<String>();
                    Collection<String> existing = availableDeps.putIfAbsent(actionID, depURIs);
                    if (existing != null) {
                        depURIs = existing;
                    }
                }
                synchronized (depURIs) {
                    depURIs.add(wAction.getDependencyURI());
                    availableDeps.put(actionID, depURIs);
                }
            }
            partValues.remove(partVal);
            if (partValues.isEmpty()) {
                partKeysToRemove.add(entry.getKey());
            }
        }
        for (String partKey : partKeysToRemove) {
            partKeyPatterns.remove(partKey);
        }
        if (partKeyPatterns.isEmpty()) {
            missingDeps.remove(tableKey);
            // Close JMS session. Stop listening on topic
            HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
            hcatService.unregisterFromNotification(server, db, table);
        }
    }
    return actionsWithAvailDep;
}