Examples with HCatURI org.apache.oozie.util.HCatURI used on opensource projects

Example 36 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class EhcacheHCatDependencyCacheExtended method testHCatCanonicalHostName.

@Test
public void testHCatCanonicalHostName() throws Exception {
    ConfigurationService.setBoolean(SimpleHCatDependencyCache.USE_CANONICAL_HOSTNAME, true);
    ConfigurationService.set(PartitionDependencyManagerService.CACHE_MANAGER_IMPL, SimpleHCatDependencyCacheExtended.class.getName());
    services.init();
    // Test all APIs related to dependency caching
    String actionId1 = "1";
    String server1 = "hcat-server1-A:5080";
    String server2 = "hcat-server1-B:5080";
    String db = "mydb";
    String table1 = "mytbl1";
    HCatURI dep1 = new HCatURI(new URI("hcat://" + server1 + "/" + db + "/" + table1 + "/dt=20120101;country=us"));
    HCatURI dep2 = new HCatURI(new URI("hcat://" + server2 + "/" + db + "/" + table1 + "/dt=20120101;country=us"));
    PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
    addMissingDependencyAndRegister(dep1, actionId1, pdms);
    assertTrue(pdms.getWaitingActions(dep1).contains(actionId1));
    assertTrue(pdms.getWaitingActions(dep2).contains(actionId1));
    ConfigurationService.set(PartitionDependencyManagerService.CACHE_MANAGER_IMPL, EhcacheHCatDependencyCacheExtended.class.getName());
    services.init();
    pdms = Services.get().get(PartitionDependencyManagerService.class);
    addMissingDependencyAndRegister(dep1, actionId1, pdms);
    assertTrue(pdms.getWaitingActions(dep1).contains(actionId1));
    assertTrue(pdms.getWaitingActions(dep2).contains(actionId1));
}

Example 37 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class EhcacheHCatDependencyCacheExtended method verifyWaitingAction.

protected void verifyWaitingAction(PartitionDependencyManagerService pdms, int numItems) throws URISyntaxException {
    for (int i = 0; i < numItems; i++) {
        String actionID = "" + i;
        HCatURI dep = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + actionID);
        Collection<String> waitingActions = pdms.getWaitingActions(dep);
        assertNotNull(dep.toURIString() + " is missing in cache", waitingActions);
        assertTrue(dep.toURIString() + " is missing in cache", waitingActions.contains(actionID));
    }
}

Example 38 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class EhcacheHCatDependencyCacheExtended method testPartitionDependency.

@Test
public void testPartitionDependency() throws Exception {
    // Test all APIs related to dependency caching
    String actionId1 = "1234465451";
    String actionId2 = "1234465452";
    String actionId3 = "1234465453";
    String actionId4 = "1234465454";
    String server1 = "hcat-server1.domain.com:5080";
    String server2 = "hcat-server2.domain.com:5080";
    String db = "mydb";
    String table1 = "mytbl1";
    String table2 = "mytbl2";
    // add partition as missing
    HCatURI dep1 = new HCatURI("hcat://hcat-server1.domain.com:5080/mydb/mytbl1/dt=20120101;country=us");
    HCatURI dep2 = new HCatURI("hcat://hcat-server1.domain.com:5080/mydb/mytbl1/country=us;dt=20120101");
    HCatURI dep3 = new HCatURI("hcat://hcat-server2.domain.com:5080/mydb/mytbl2/dt=20120102;country=us");
    HCatURI dep4 = new HCatURI("hcat://hcat-server2.domain.com:5080/mydb/mytbl2/dt=20120102;country=us;state=CA");
    PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
    addMissingDependencyAndRegister(dep1, actionId1, pdms);
    addMissingDependencyAndRegister(dep2, actionId1, pdms);
    addMissingDependencyAndRegister(dep2, actionId2, pdms);
    addMissingDependencyAndRegister(dep2, actionId3, pdms);
    addMissingDependencyAndRegister(dep3, actionId3, pdms);
    addMissingDependencyAndRegister(dep4, actionId4, pdms);
    // Add duplicates. RecoveryService will add duplicates
    addMissingDependencyAndRegister(dep4, actionId4, pdms);
    addMissingDependencyAndRegister(dep4, actionId4, pdms);
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    JMSAccessorService jmsService = Services.get().get(JMSAccessorService.class);
    JMSConnectionInfo connInfo = hcatService.getJMSConnectionInfo(dep1.getURI());
    // server1,db,table1
    assertTrue(hcatService.isRegisteredForNotification(dep1));
    // server2,db,table2
    assertTrue(hcatService.isRegisteredForNotification(dep3));
    assertTrue(jmsService.isListeningToTopic(connInfo, dep1.getDb() + "." + dep1.getTable()));
    assertTrue(jmsService.isListeningToTopic(connInfo, dep3.getDb() + "." + dep3.getTable()));
    assertTrue(pdms.getWaitingActions(dep1).contains(actionId1));
    assertTrue(pdms.getWaitingActions(dep2).contains(actionId1));
    assertTrue(pdms.getWaitingActions(dep2).contains(actionId2));
    assertTrue(pdms.getWaitingActions(dep2).contains(actionId2));
    assertTrue(pdms.getWaitingActions(dep3).contains(actionId3));
    assertTrue(pdms.getWaitingActions(dep4).contains(actionId4));
    // Should not contain duplicates
    assertEquals(1, pdms.getWaitingActions(dep4).size());
    pdms.removeMissingDependency(dep2, actionId1);
    assertTrue(pdms.getWaitingActions(dep1).contains(actionId1));
    assertEquals(2, pdms.getWaitingActions(dep2).size());
    assertTrue(!pdms.getWaitingActions(dep2).contains(actionId1));
    assertNull(pdms.getAvailableDependencyURIs(actionId1));
    pdms.partitionAvailable(server2, db, table2, getPartitionMap("dt=20120102;country=us;state=NY"));
    assertNull(pdms.getWaitingActions(dep3));
    assertTrue(pdms.getAvailableDependencyURIs(actionId3).contains(dep3.getURI().toString()));
    pdms.partitionAvailable(server2, db, table2, getPartitionMap("dt=20120102;country=us;state=CA"));
    assertNull(pdms.getWaitingActions(dep4));
    assertTrue(pdms.getAvailableDependencyURIs(actionId4).contains(dep4.getURI().toString()));
    pdms.partitionAvailable(server1, db, table1, getPartitionMap("dt=20120101;country=us"));
    assertNull(pdms.getWaitingActions(dep1));
    assertNull(pdms.getWaitingActions(dep2));
    assertTrue(pdms.getAvailableDependencyURIs(actionId2).contains(dep2.getURI().toString()));
    assertTrue(pdms.getAvailableDependencyURIs(actionId3).contains(dep2.getURI().toString()));
    assertTrue(pdms.getAvailableDependencyURIs(actionId3).contains(dep3.getURI().toString()));
    assertTrue(pdms.removeAvailableDependencyURIs(actionId3, pdms.getAvailableDependencyURIs(actionId3)));
    assertNull(pdms.getAvailableDependencyURIs(actionId3));
    // server1,db,table1
    assertFalse(hcatService.isRegisteredForNotification(dep1));
    // server2,db,table2
    assertFalse(hcatService.isRegisteredForNotification(dep3));
    assertFalse(jmsService.isListeningToTopic(connInfo, dep1.getDb() + "." + dep1.getTable()));
    assertFalse(jmsService.isListeningToTopic(connInfo, dep3.getDb() + "." + dep3.getTable()));
}