Examples with HCatAccessorService org.apache.oozie.service.HCatAccessorService used on opensource projects

Example 11 with HCatAccessorService

use of org.apache.oozie.service.HCatAccessorService in project oozie by apache.

the class HCatCredentials method getProperty.

/**
 * Returns the value for the oozieConfName if its present in prop map else
 * value of hiveConfName. It will also check HCatAccessorService and
 * HiveConf for hiveConfName.
 *
 * @param prop
 * @param oozieConfName
 * @param hiveConfName
 * @return value for the oozieConfName if its present else value of
 *         hiveConfName. If both are absent then returns null.
 */
private String getProperty(HashMap<String, String> prop, String oozieConfName, String hiveConfName) {
    String value = prop.get(oozieConfName) == null ? prop.get(hiveConfName) : prop.get(oozieConfName);
    if (value == null || value.isEmpty()) {
        HCatAccessorService hCatService = Services.get().get(HCatAccessorService.class);
        Configuration hCatConf = hCatService.getHCatConf();
        if (hCatConf != null) {
            value = hCatConf.get(hiveConfName);
        }
    }
    if (value == null || value.isEmpty()) {
        value = hiveConf.get(hiveConfName);
    }
    return value;
}

Example 12 with HCatAccessorService

use of org.apache.oozie.service.HCatAccessorService in project oozie by apache.

the class HCatURIHandler method getDependencyType.

@Override
public DependencyType getDependencyType(URI uri) throws URIHandlerException {
    DependencyType depType = DependencyType.PULL;
    // Not initializing in constructor as this will be part of oozie.services.ext
    // and will be initialized after URIHandlerService
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    if (hcatService != null) {
        depType = dependencyTypes.get(uri.getAuthority());
        if (depType == null) {
            depType = hcatService.isKnownPublisher(uri) ? DependencyType.PUSH : DependencyType.PULL;
            dependencyTypes.put(uri.getAuthority(), depType);
        }
    }
    return depType;
}

Example 13 with HCatAccessorService

use of org.apache.oozie.service.HCatAccessorService in project oozie by apache.

the class TestCoordPushDependencyCheckXCommand method testTimeOutWithUnresolvedMissingDependencies.

@Test
public void testTimeOutWithUnresolvedMissingDependencies() throws Exception {
    String db = "default";
    String table = "tablename";
    String newHCatDependency1 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=brazil";
    String newHCatDependency2 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=usa";
    String newHCatDependency3 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=uk";
    String newHCatDependency = newHCatDependency1 + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency2;
    populateTable(db, table);
    String actionId = addInitRecords(newHCatDependency);
    checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
    new CoordPushDependencyCheckXCommand(actionId, true).call();
    checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.WAITING);
    PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    assertTrue(pdms.getWaitingActions(new HCatURI(newHCatDependency1)).contains(actionId));
    assertTrue(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
    // Timeout is 10 mins. Change action created time to before 12 min to make the action
    // timeout.
    long timeOutCreationTime = System.currentTimeMillis() - (12 * 60 * 1000);
    setCoordActionCreationTime(actionId, timeOutCreationTime);
    // Set some missing dependency. Instead of latest or future just setting a current one for testing as
    // we are only interested in ensuring CoordActionInputCheckXCommand is run
    setMissingDependencies(actionId, newHCatDependency + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency3);
    addPartition(db, table, "dt=20120430;country=brazil");
    checkDependencies(actionId, newHCatDependency + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency3, newHCatDependency1);
    new CoordPushDependencyCheckXCommand(actionId).call();
    // Somehow with hive 0.10 it takes 1 second more.
    Thread.sleep(1300);
    checkDependencies(actionId, newHCatDependency3, "");
    assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency1)));
    assertFalse(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
}

Example 14 with HCatAccessorService

use of org.apache.oozie.service.HCatAccessorService in project oozie by apache.

the class TestCoordPushDependencyCheckXCommand method testTimeOut.

@Test
public void testTimeOut() throws Exception {
    String db = "default";
    String table = "tablename";
    String newHCatDependency1 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=brazil";
    String newHCatDependency2 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=usa";
    String newHCatDependency = newHCatDependency1 + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency2;
    populateTable(db, table);
    String actionId = addInitRecords(newHCatDependency);
    checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
    new CoordPushDependencyCheckXCommand(actionId, true).call();
    checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.WAITING);
    PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    assertTrue(pdms.getWaitingActions(new HCatURI(newHCatDependency1)).contains(actionId));
    assertTrue(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
    // Timeout is 10 mins. Change action created time to before 12 min to make the action
    // timeout.
    long timeOutCreationTime = System.currentTimeMillis() - (12 * 60 * 1000);
    setCoordActionCreationTime(actionId, timeOutCreationTime);
    new CoordPushDependencyCheckXCommand(actionId).call();
    Thread.sleep(100);
    // Check for timeout status and unregistered missing dependencies
    checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.TIMEDOUT);
    assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency1)));
    assertFalse(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
}