Examples with HCatURI org.apache.oozie.util.HCatURI used on opensource projects

Example 6 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class TestCoordKillXCommand method testCoordKillRemovePushMissingDeps.

public void testCoordKillRemovePushMissingDeps() throws Exception {
    try {
        services.destroy();
        services = super.setupServicesForHCatalog();
        services.init();
        String db = "default";
        String table = "tablename";
        String server = "hcatserver";
        String newHCatDependency1 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=brazil";
        String newHCatDependency2 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=usa";
        String pushMissingDeps = newHCatDependency1 + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency2;
        PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
        CoordinatorJobBean job = addRecordToCoordJobTableForWaiting("coord-job-for-action-input-check.xml", CoordinatorJob.Status.RUNNING, false, true);
        CoordinatorActionBean action1 = addRecordToCoordActionTableForWaiting(job.getId(), 1, CoordinatorAction.Status.WAITING, "coord-action-for-action-input-check.xml", null, pushMissingDeps, "Z");
        String newHCatDependency3 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=russia";
        CoordinatorActionBean action2 = addRecordToCoordActionTableForWaiting(job.getId(), 2, CoordinatorAction.Status.WAITING, "coord-action-for-action-input-check.xml", null, newHCatDependency3, "Z");
        HCatURI hcatURI1, hcatURI2, hcatURI3;
        hcatURI1 = new HCatURI(newHCatDependency1);
        hcatURI2 = new HCatURI(newHCatDependency2);
        hcatURI3 = new HCatURI(newHCatDependency3);
        pdms.addMissingDependency(hcatURI1, action1.getId());
        pdms.addMissingDependency(hcatURI2, action1.getId());
        pdms.addMissingDependency(hcatURI3, action2.getId());
        assertTrue(pdms.getWaitingActions(new HCatURI(newHCatDependency1)).contains(action1.getId()));
        assertTrue(pdms.getWaitingActions(new HCatURI(newHCatDependency2)).contains(action1.getId()));
        assertTrue(pdms.getWaitingActions(new HCatURI(newHCatDependency3)).contains(action2.getId()));
        new CoordKillXCommand(job.getId()).call();
        assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency1)));
        assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency2)));
        assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency3)));
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 7 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class TestCoordPushDependencyCheckXCommand method testUpdateCoordTableMultipleDepsV3.

@Test
public void testUpdateCoordTableMultipleDepsV3() throws Exception {
    // Test for two dependencies : one of them is already existing in the
    // hcat server. Other one is not.
    // Expected to see only first action in WAITING as we check for all dependencies.
    // Later make the other partition also available. action is expected to
    // be READY
    String db = "default";
    String table = "tablename";
    String newHCatDependency1 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=brazil";
    String newHCatDependency2 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=usa";
    String newHCatDependency = newHCatDependency1 + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency2;
    populateTable(db, table);
    String actionId = addInitRecords(newHCatDependency);
    checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
    // Checks for all missing dependencies
    new CoordPushDependencyCheckXCommand(actionId, true).call();
    checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.WAITING);
    PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    assertTrue(pdms.getWaitingActions(new HCatURI(newHCatDependency1)).contains(actionId));
    assertTrue(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
    assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency2)));
    // Make first dependency available
    addPartition(db, table, "dt=20120430;country=brazil");
    new CoordPushDependencyCheckXCommand(actionId).call();
    checkCoordAction(actionId, "", CoordinatorAction.Status.READY);
    assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency1)));
    assertFalse(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
}

Example 8 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class TestCoordPushDependencyCheckXCommand method testTimeOutWithException2.

@Test
public void testTimeOutWithException2() throws Exception {
    // Test timeout when table containing missing dependencies is dropped in between
    String db = "default";
    String table = "tablename";
    String newHCatDependency1 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=brazil";
    String newHCatDependency2 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=usa";
    String newHCatDependency = newHCatDependency1 + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency2;
    populateTable(db, table);
    String actionId = addInitRecords(newHCatDependency);
    checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
    new CoordPushDependencyCheckXCommand(actionId, true).call();
    checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.WAITING);
    PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    assertTrue(pdms.getWaitingActions(new HCatURI(newHCatDependency1)).contains(actionId));
    assertTrue(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
    // Timeout is 10 mins. Change action created time to before 12 min to make the action
    // timeout.
    long timeOutCreationTime = System.currentTimeMillis() - (12 * 60 * 1000);
    setCoordActionCreationTime(actionId, timeOutCreationTime);
    dropTable(db, table, true);
    try {
        new CoordPushDependencyCheckXCommand(actionId).call();
        fail();
    } catch (Exception e) {
        assertTrue(e.getMessage().contains("NoSuchObjectException"));
    }
    Thread.sleep(100);
    // Check for timeout status and unregistered missing dependencies
    checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.TIMEDOUT);
    assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency1)));
    assertFalse(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
}

Example 9 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class TestCoordPushDependencyCheckXCommand method testTimeOutWithException1.

@Test
public void testTimeOutWithException1() throws Exception {
    // Test timeout when missing dependencies are from a non existing table
    String newHCatDependency1 = "hcat://" + server + "/nodb/notable/dt=20120430;country=brazil";
    String newHCatDependency2 = "hcat://" + server + "/nodb/notable/dt=20120430;country=usa";
    String newHCatDependency = newHCatDependency1 + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency2;
    String actionId = addInitRecords(newHCatDependency);
    checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
    try {
        new CoordPushDependencyCheckXCommand(actionId, true).call();
        fail();
    } catch (Exception e) {
        assertTrue(e.getMessage().contains("NoSuchObjectException"));
    }
    checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
    PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency1)));
    assertFalse(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
    // Timeout is 10 mins. Change action created time to before 12 min to make the action
    // timeout.
    long timeOutCreationTime = System.currentTimeMillis() - (12 * 60 * 1000);
    setCoordActionCreationTime(actionId, timeOutCreationTime);
    try {
        new CoordPushDependencyCheckXCommand(actionId).call();
        fail();
    } catch (Exception e) {
        assertTrue(e.getMessage().contains("NoSuchObjectException"));
    }
    Thread.sleep(100);
    // Check for timeout status and unregistered missing dependencies
    checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.TIMEDOUT);
}

Example 10 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class HCatURIHandler method registerForNotification.

@Override
public void registerForNotification(URI uri, Configuration conf, String user, String actionID) throws URIHandlerException {
    HCatURI hcatURI;
    try {
        hcatURI = new HCatURI(uri);
    } catch (URISyntaxException e) {
        throw new URIHandlerException(ErrorCode.E0906, uri, e);
    }
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    if (!hcatService.isRegisteredForNotification(hcatURI)) {
        HCatClient client = getHCatClient(uri, conf);
        try {
            String topic = client.getMessageBusTopicName(hcatURI.getDb(), hcatURI.getTable());
            if (topic == null) {
                return;
            }
            hcatService.registerForNotification(hcatURI, topic, new HCatMessageHandler(uri.getAuthority()));
        } catch (HCatException e) {
            throw new HCatAccessorException(ErrorCode.E1501, e);
        } finally {
            closeQuietly(client, null, true);
        }
    }
    PartitionDependencyManagerService pdmService = Services.get().get(PartitionDependencyManagerService.class);
    pdmService.addMissingDependency(hcatURI, actionID);
}