Examples with PartitionDependencyManagerService org.apache.oozie.service.PartitionDependencyManagerService used on opensource projects

Example 6 with PartitionDependencyManagerService

use of org.apache.oozie.service.PartitionDependencyManagerService in project oozie by apache.

the class HCatURIHandler method registerForNotification.

@Override
public void registerForNotification(URI uri, Configuration conf, String user, String actionID) throws URIHandlerException {
    HCatURI hcatURI;
    try {
        hcatURI = new HCatURI(uri);
    } catch (URISyntaxException e) {
        throw new URIHandlerException(ErrorCode.E0906, uri, e);
    }
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    if (!hcatService.isRegisteredForNotification(hcatURI)) {
        HCatClient client = getHCatClient(uri, conf);
        try {
            String topic = client.getMessageBusTopicName(hcatURI.getDb(), hcatURI.getTable());
            if (topic == null) {
                return;
            }
            hcatService.registerForNotification(hcatURI, topic, new HCatMessageHandler(uri.getAuthority()));
        } catch (HCatException e) {
            throw new HCatAccessorException(ErrorCode.E1501, e);
        } finally {
            closeQuietly(client, null, true);
        }
    }
    PartitionDependencyManagerService pdmService = Services.get().get(PartitionDependencyManagerService.class);
    pdmService.addMissingDependency(hcatURI, actionID);
}

Example 7 with PartitionDependencyManagerService

use of org.apache.oozie.service.PartitionDependencyManagerService in project oozie by apache.

the class CoordActionUpdatePushMissingDependency method execute.

@Override
protected Void execute() throws CommandException {
    CoordInputDependency coordPushInputDependency = coordAction.getPushInputDependencies();
    CoordInputDependency coordPullInputDependency = coordAction.getPullInputDependencies();
    LOG.info("STARTED for Action id [{0}]", actionId);
    if (coordPushInputDependency.isDependencyMet()) {
        LOG.info("Nothing to check. Empty push missing dependency");
    } else {
        PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
        Collection<String> availDepList = pdms.getAvailableDependencyURIs(actionId);
        if (availDepList == null || availDepList.size() == 0) {
            LOG.info("There are no available dependencies");
            if (isTimeout()) {
                // Poll and check as one last try
                queue(new CoordPushDependencyCheckXCommand(coordAction.getId()), 100);
            }
        } else {
            String pushMissingDependencies = coordPushInputDependency.getMissingDependencies().toString();
            LOG.debug("Updating with available uris = [{0}] where missing uris = [{1}]", pushMissingDependencies);
            String[] missingDependenciesArray = DependencyChecker.dependenciesAsArray(pushMissingDependencies);
            coordPushInputDependency.addToAvailableDependencies(availDepList);
            boolean isChangeInDependency = true;
            if (coordPushInputDependency.isDependencyMet()) {
                // All push-based dependencies are available
                onAllPushDependenciesAvailable(coordPullInputDependency.isDependencyMet());
            } else {
                if (coordPushInputDependency.getMissingDependenciesAsList().size() == missingDependenciesArray.length) {
                    isChangeInDependency = false;
                }
                if (isTimeout()) {
                    // Poll and check as one last try
                    queue(new CoordPushDependencyCheckXCommand(coordAction.getId()), 100);
                }
            }
            updateCoordAction(coordAction, isChangeInDependency);
            removeAvailableDependencies(pdms, availDepList);
            LOG.info("ENDED for Action id [{0}]", actionId);
        }
    }
    return null;
}

Example 8 with PartitionDependencyManagerService

use of org.apache.oozie.service.PartitionDependencyManagerService in project oozie by apache.

the class HCatURIHandler method unregisterFromNotification.

@Override
public boolean unregisterFromNotification(URI uri, String actionID) {
    HCatURI hcatURI;
    try {
        hcatURI = new HCatURI(uri);
    } catch (URISyntaxException e) {
        // Unexpected at this point
        throw new RuntimeException(e);
    }
    PartitionDependencyManagerService pdmService = Services.get().get(PartitionDependencyManagerService.class);
    return pdmService.removeMissingDependency(hcatURI, actionID);
}

Example 9 with PartitionDependencyManagerService

use of org.apache.oozie.service.PartitionDependencyManagerService in project oozie by apache.

the class TestCoordPushDependencyCheckXCommand method testTimeOutWithUnresolvedMissingDependencies.

@Test
public void testTimeOutWithUnresolvedMissingDependencies() throws Exception {
    String db = "default";
    String table = "tablename";
    String newHCatDependency1 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=brazil";
    String newHCatDependency2 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=usa";
    String newHCatDependency3 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=uk";
    String newHCatDependency = newHCatDependency1 + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency2;
    populateTable(db, table);
    String actionId = addInitRecords(newHCatDependency);
    checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
    new CoordPushDependencyCheckXCommand(actionId, true).call();
    checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.WAITING);
    PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    assertTrue(pdms.getWaitingActions(new HCatURI(newHCatDependency1)).contains(actionId));
    assertTrue(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
    // Timeout is 10 mins. Change action created time to before 12 min to make the action
    // timeout.
    long timeOutCreationTime = System.currentTimeMillis() - (12 * 60 * 1000);
    setCoordActionCreationTime(actionId, timeOutCreationTime);
    // Set some missing dependency. Instead of latest or future just setting a current one for testing as
    // we are only interested in ensuring CoordActionInputCheckXCommand is run
    setMissingDependencies(actionId, newHCatDependency + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency3);
    addPartition(db, table, "dt=20120430;country=brazil");
    checkDependencies(actionId, newHCatDependency + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency3, newHCatDependency1);
    new CoordPushDependencyCheckXCommand(actionId).call();
    // Somehow with hive 0.10 it takes 1 second more.
    Thread.sleep(1300);
    checkDependencies(actionId, newHCatDependency3, "");
    assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency1)));
    assertFalse(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
}

Example 10 with PartitionDependencyManagerService

use of org.apache.oozie.service.PartitionDependencyManagerService in project oozie by apache.

the class TestCoordPushDependencyCheckXCommand method testTimeOut.

@Test
public void testTimeOut() throws Exception {
    String db = "default";
    String table = "tablename";
    String newHCatDependency1 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=brazil";
    String newHCatDependency2 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=usa";
    String newHCatDependency = newHCatDependency1 + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency2;
    populateTable(db, table);
    String actionId = addInitRecords(newHCatDependency);
    checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
    new CoordPushDependencyCheckXCommand(actionId, true).call();
    checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.WAITING);
    PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    assertTrue(pdms.getWaitingActions(new HCatURI(newHCatDependency1)).contains(actionId));
    assertTrue(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
    // Timeout is 10 mins. Change action created time to before 12 min to make the action
    // timeout.
    long timeOutCreationTime = System.currentTimeMillis() - (12 * 60 * 1000);
    setCoordActionCreationTime(actionId, timeOutCreationTime);
    new CoordPushDependencyCheckXCommand(actionId).call();
    Thread.sleep(100);
    // Check for timeout status and unregistered missing dependencies
    checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.TIMEDOUT);
    assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency1)));
    assertFalse(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
}