Examples with HCatURI org.apache.oozie.util.HCatURI used on opensource projects

Example 31 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class TestCoordActionUpdatePushMissingDependency method testUpdateCoordTableBasic.

@Test
public void testUpdateCoordTableBasic() throws Exception {
    String newHCatDependency = "hcat://hcat.server.com:5080/mydb/clicks/datastamp=12;region=us";
    HCatURI hcatUri = new HCatURI(newHCatDependency);
    String actionId = addInitRecords(newHCatDependency);
    checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
    PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
    pdms.addMissingDependency(hcatUri, actionId);
    pdms.partitionAvailable("hcat.server.com:5080", "mydb", "clicks", getPartitionMap("src=search;datastamp=12;region=us"));
    Collection<String> availableURIs = pdms.getAvailableDependencyURIs(actionId);
    assertEquals(availableURIs.size(), 1);
    assertTrue(availableURIs.contains(newHCatDependency));
    new CoordActionUpdatePushMissingDependency(actionId).call();
    checkCoordAction(actionId, "", CoordinatorAction.Status.READY);
    assertNull(pdms.getAvailableDependencyURIs(actionId));
}

Example 32 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class TestHAPartitionDependencyManagerService method testDependencyCacheWithHA.

public void testDependencyCacheWithHA() throws Exception {
    db = "default";
    table1 = "mytbl";
    table2 = "mytb2";
    part1 = "dt=20120101;country=us";
    part2 = "dt=20120102;country=us";
    part3 = "dt=20120103;country=us";
    String newHCatDependency1 = "hcat://" + server + "/" + db + "/" + table1 + "/" + part1;
    String newHCatDependency2 = "hcat://" + server + "/" + db + "/" + table1 + "/" + part2;
    String newHCatDependency3 = "hcat://" + server + "/" + db + "/" + table2 + "/" + part3;
    HCatURI dep1 = new HCatURI(newHCatDependency1);
    HCatURI dep2 = new HCatURI(newHCatDependency2);
    HCatURI dep3 = new HCatURI(newHCatDependency3);
    // create db, table and partitions
    populateTable();
    String actionId1 = addInitRecords(newHCatDependency1);
    String actionId2 = addInitRecords(newHCatDependency2);
    String actionId3 = addInitRecords(newHCatDependency3);
    // Assume dependency cache on dummy server with missing push dependencies registered
    PartitionDependencyManagerService dummyPdms = new PartitionDependencyManagerService();
    PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
    dummyPdms.init(Services.get());
    dummyPdms.addMissingDependency(dep1, actionId1);
    dummyPdms.addMissingDependency(dep2, actionId2);
    dummyPdms.addMissingDependency(dep3, actionId3);
    Collection<String> waitingActions = (Collection<String>) dummyPdms.getWaitingActions(dep1);
    assertEquals(1, waitingActions.size());
    waitingActions = (Collection<String>) dummyPdms.getWaitingActions(dep2);
    assertEquals(1, waitingActions.size());
    waitingActions = (Collection<String>) dummyPdms.getWaitingActions(dep3);
    assertEquals(1, waitingActions.size());
    // Dependency cache on living server doesn't have these partitions registered at this point
    waitingActions = (Collection<String>) pdms.getWaitingActions(dep1);
    assertNull(waitingActions);
    waitingActions = (Collection<String>) pdms.getWaitingActions(dep2);
    assertNull(waitingActions);
    waitingActions = (Collection<String>) pdms.getWaitingActions(dep3);
    assertNull(waitingActions);
    // Assume dummy server is down, and recovery service on living server pick up these jobs
    dummyPdms.destroy();
    Runnable recoveryRunnable = new RecoveryRunnable(60, 0, 60);
    recoveryRunnable.run();
    waitFor(30 * 1000, new Predicate() {

        public boolean evaluate() throws Exception {
            Collection<String> waitingActions;
            PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
            HCatURI dep1 = new HCatURI("hcat://" + server + "/" + db + "/" + table1 + "/" + part1);
            HCatURI dep2 = new HCatURI("hcat://" + server + "/" + db + "/" + table1 + "/" + part2);
            HCatURI dep3 = new HCatURI("hcat://" + server + "/" + db + "/" + table2 + "/" + part3);
            waitingActions = pdms.getWaitingActions(dep1);
            if (waitingActions == null) {
                return false;
            }
            waitingActions = pdms.getWaitingActions(dep2);
            if (waitingActions == null) {
                return false;
            }
            waitingActions = pdms.getWaitingActions(dep3);
            if (waitingActions == null) {
                return false;
            }
            return true;
        }
    });
    // Dependency cache on living server has missing partitions added
    waitingActions = (Collection<String>) pdms.getWaitingActions(dep1);
    assertEquals(1, waitingActions.size());
    assertTrue(waitingActions.contains(actionId1));
    waitingActions = (Collection<String>) pdms.getWaitingActions(dep2);
    assertEquals(1, waitingActions.size());
    assertTrue(waitingActions.contains(actionId2));
    waitingActions = (Collection<String>) pdms.getWaitingActions(dep3);
    assertEquals(1, waitingActions.size());
    assertTrue(waitingActions.contains(actionId3));
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    // mytbl and mytb2 registered to topic map to receive notification
    assertTrue(hcatService.isRegisteredForNotification(dep1));
    assertTrue(hcatService.isRegisteredForNotification(dep2));
    assertTrue(hcatService.isRegisteredForNotification(dep3));
}

Example 33 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class TestPartitionDependencyManagerEhcache method testEvictionOnTimeToIdle.

@Test
public void testEvictionOnTimeToIdle() throws Exception {
    setupServices("testevictionontimetoidle");
    PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
    int numItems = 50;
    long startTime = System.currentTimeMillis();
    for (int i = 0; i < numItems; i++) {
        HCatURI dep = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + i);
        pdms.addMissingDependency(dep, "" + i);
    }
    verifyWaitingAction(pdms, numItems);
    LOG.info("Time taken to insert and retrive " + numItems + " items is " + (System.currentTimeMillis() - startTime));
    // timeToIdleSeconds is 1
    Thread.sleep(1100);
    for (int i = 0; i < numItems; i++) {
        assertNull(pdms.getWaitingActions(new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + "" + i)));
    }
}

Example 34 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class TestPartitionDependencyManagerEhcache method testEvictionOnTimeToLive.

@Test
public void testEvictionOnTimeToLive() throws Exception {
    setupServices("testevictionontimetolive");
    PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
    int numItems = 50;
    long startTime = System.currentTimeMillis();
    for (int i = 0; i < numItems; i++) {
        HCatURI dep = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + i);
        pdms.addMissingDependency(dep, "" + i);
    }
    verifyWaitingAction(pdms, numItems);
    LOG.info("Time taken to insert and retrive " + numItems + " items is " + (System.currentTimeMillis() - startTime));
    // timeToLiveSeconds is 1
    Thread.sleep(1100);
    for (int i = 0; i < numItems; i++) {
        assertNull(pdms.getWaitingActions(new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + "" + i)));
    }
}

Example 35 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class EhcacheHCatDependencyCacheExtended method assertSpeedAndMemory.

protected void assertSpeedAndMemory(int numItems, int insertTimeinMillis, int retrievalTimeinMillis, long memIncreaseAfterInsert, long memIncreaseAfterInsertAndGC) throws Exception {
    PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
    System.gc();
    MemoryMXBean mb = ManagementFactory.getMemoryMXBean();
    long usedMemBeforeInsert = mb.getHeapMemoryUsage().getUsed();
    long startTime = System.currentTimeMillis();
    for (int i = 0; i < numItems; i++) {
        HCatURI dep = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + i);
        pdms.addMissingDependency(dep, "" + i);
    }
    long usedMemAfterInsert = mb.getHeapMemoryUsage().getUsed();
    long endTime = System.currentTimeMillis();
    LOG.info("Time taken to insert " + numItems + " items is " + (endTime - startTime));
    assertTrue((endTime - startTime) < insertTimeinMillis);
    LOG.info("Memory before and after insert: " + usedMemBeforeInsert + "," + usedMemAfterInsert);
    verifyWaitingAction(pdms, numItems);
    LOG.info("Time taken to retrieve " + numItems + " items is " + (System.currentTimeMillis() - endTime));
    assertTrue((System.currentTimeMillis() - endTime) < retrievalTimeinMillis);
    long usedMemAfterRetrieval = mb.getHeapMemoryUsage().getUsed();
    System.gc();
    long usedMemAfterGC = mb.getHeapMemoryUsage().getUsed();
    LOG.info("Memory before insert = " + usedMemBeforeInsert);
    LOG.info("Memory after insert = " + usedMemAfterInsert);
    LOG.info("Memory after retrieval = " + usedMemAfterRetrieval);
    LOG.info("Memory after GC = " + usedMemAfterGC);
// Commenting out as memory assertion is not reliable when running the full suite of tests.
// assertTrue((usedMemAfterInsert - usedMemBeforeInsert) < memIncreaseAfterInsert);
// assertTrue((usedMemAfterGC - usedMemBeforeInsert) < memIncreaseAfterInsertAndGC);
}