Examples with HCatURI org.apache.oozie.util.HCatURI used on opensource projects

Example 1 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class TestHCatMessageHandler method testCacheUpdateByMessage.

/**
 * Test that message is processed to update the dependency map and mark
 * partition as available
 */
@Test
public void testCacheUpdateByMessage() {
    try {
        String actionId1 = "1234465451";
        String actionId2 = "1234465452";
        String actionId3 = "1234465453";
        String actionId4 = "1234465454";
        // add partition as missing
        HCatURI dep1 = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/dt=20120101;country=us");
        HCatURI dep2 = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/country=us;dt=20120101");
        HCatURI dep3 = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/dt=20120102;country=us");
        HCatURI dep4 = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/dt=20120102;country=us;state=CA");
        PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
        pdms.addMissingDependency(dep1, actionId1);
        pdms.addMissingDependency(dep2, actionId2);
        pdms.addMissingDependency(dep3, actionId3);
        pdms.addMissingDependency(dep4, actionId4);
        assertTrue(pdms.getWaitingActions(dep1).contains(actionId1));
        assertTrue(pdms.getWaitingActions(dep2).contains(actionId2));
        assertTrue(pdms.getWaitingActions(dep3).contains(actionId3));
        assertTrue(pdms.getWaitingActions(dep4).contains(actionId4));
        // construct message
        List<Map<String, String>> partitionsList = new ArrayList<Map<String, String>>();
        partitionsList.add(getPartitionMap("dt=20120101;country=us;state=CA"));
        partitionsList.add(getPartitionMap("dt=20120101;country=us;state=NY"));
        JSONAddPartitionMessage jsonMsg = new JSONAddPartitionMessage("thrift://" + dep1.getServer(), "", dep1.getDb(), dep1.getTable(), partitionsList, System.currentTimeMillis());
        Message msg = session.createTextMessage(jsonMsg.toString());
        msg.setStringProperty(HCatConstants.HCAT_EVENT, HCatEventMessage.EventType.ADD_PARTITION.toString());
        // test message processing
        HCatMessageHandler hcatHandler = new HCatMessageHandler("hcat.server.com:5080");
        hcatHandler.process(msg);
        assertNull(pdms.getWaitingActions(dep1));
        assertNull(pdms.getWaitingActions(dep2));
        assertTrue(pdms.getWaitingActions(dep3).contains(actionId3));
        assertTrue(pdms.getWaitingActions(dep4).contains(actionId4));
        // test message processing through JMS notification listener
        partitionsList.clear();
        partitionsList.add(getPartitionMap("dt=20120102;country=us;state=CA"));
        partitionsList.add(getPartitionMap("dt=20120102;country=us;state=NY"));
        jsonMsg = new JSONAddPartitionMessage("thrift://" + dep1.getServer(), "", dep1.getDb(), dep1.getTable(), partitionsList, System.currentTimeMillis());
        msg = session.createTextMessage(jsonMsg.toString());
        msg.setStringProperty(HCatConstants.HCAT_EVENT, HCatEventMessage.EventType.ADD_PARTITION.toString());
        HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
        hcatService.registerForNotification(dep1, "hcat.topic1", hcatHandler);
        Topic topic = session.createTopic("hcat.topic1");
        MessageProducer producer = session.createProducer(topic);
        producer.send(msg);
        Thread.sleep(500);
        assertNull(pdms.getWaitingActions(dep3));
        assertNull(pdms.getWaitingActions(dep4));
        assertTrue(pdms.getAvailableDependencyURIs(actionId1).contains(dep1.getURI().toString()));
        assertTrue(pdms.getAvailableDependencyURIs(actionId2).contains(dep2.getURI().toString()));
        assertTrue(pdms.getAvailableDependencyURIs(actionId3).contains(dep3.getURI().toString()));
        assertTrue(pdms.getAvailableDependencyURIs(actionId4).contains(dep4.getURI().toString()));
    } catch (Exception e) {
        e.printStackTrace();
        fail("Exception: " + e.getMessage());
    }
}

Example 2 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class TestRecoveryService method testCoordActionRecoveryServiceForWaitingRegisterPartition.

public void testCoordActionRecoveryServiceForWaitingRegisterPartition() throws Exception {
    services.destroy();
    services = super.setupServicesForHCatalog();
    services.getConf().set(URIHandlerService.URI_HANDLERS, FSURIHandler.class.getName() + "," + HCatURIHandler.class.getName());
    services.getConf().setLong(RecoveryService.CONF_PUSH_DEPENDENCY_INTERVAL, 1);
    services.init();
    String db = "default";
    String table = "tablename";
    // dep1 is not available and dep2 is available
    String newHCatDependency1 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=brazil";
    String newHCatDependency2 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=usa";
    String newHCatDependency = newHCatDependency1 + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency2;
    HCatAccessorService hcatService = services.get(HCatAccessorService.class);
    JMSAccessorService jmsService = services.get(JMSAccessorService.class);
    PartitionDependencyManagerService pdms = services.get(PartitionDependencyManagerService.class);
    assertFalse(jmsService.isListeningToTopic(hcatService.getJMSConnectionInfo(new URI(newHCatDependency1)), db + "." + table));
    populateTable(db, table);
    String actionId = addInitRecords(newHCatDependency);
    CoordinatorAction ca = checkCoordActionDependencies(actionId, newHCatDependency);
    assertEquals(CoordinatorAction.Status.WAITING, ca.getStatus());
    // Register the missing dependencies to PDMS assuming CoordPushDependencyCheckCommand did this.
    pdms.addMissingDependency(new HCatURI(newHCatDependency1), actionId);
    pdms.addMissingDependency(new HCatURI(newHCatDependency2), actionId);
    sleep(2000);
    Runnable recoveryRunnable = new RecoveryRunnable(0, 1, 1);
    recoveryRunnable.run();
    sleep(2000);
    // Recovery service should have discovered newHCatDependency2 and JMS Connection should exist
    // and newHCatDependency1 should be in PDMS waiting list
    assertTrue(jmsService.isListeningToTopic(hcatService.getJMSConnectionInfo(new URI(newHCatDependency2)), "hcat." + db + "." + table));
    checkCoordActionDependencies(actionId, newHCatDependency1);
    assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency2)));
    Collection<String> waitingActions = pdms.getWaitingActions(new HCatURI(newHCatDependency1));
    assertEquals(1, waitingActions.size());
    assertTrue(waitingActions.contains(actionId));
}

Example 3 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class TestHAPartitionDependencyManagerService method testPurgeMissingDependenciesForCache.

protected void testPurgeMissingDependenciesForCache(PartitionDependencyManagerService pdms) throws Exception {
    String actionId1 = "1234465451";
    String actionId2 = "1234465452";
    String actionId3 = "1234465453";
    // add partitions as missing
    HCatURI dep1 = new HCatURI("hcat://hcat-server1.domain.com:5080/mydb/mytbl1/dt=20120101;country=us");
    HCatURI dep2 = new HCatURI("hcat://hcat-server1.domain.com:5080/mydb/mytbl1/country=us;dt=20120101");
    HCatURI dep3 = new HCatURI("hcat://hcat-server2.domain.com:5080/mydb/mytbl2/dt=20120102;country=us");
    // actionId1-->(dep1,2), actionId2-->(dep2), actionId3-->(dep2,3)
    addMissingDependencyAndRegister(dep1, actionId1, pdms);
    addMissingDependencyAndRegister(dep2, actionId1, pdms);
    addMissingDependencyAndRegister(dep2, actionId2, pdms);
    addMissingDependencyAndRegister(dep2, actionId3, pdms);
    addMissingDependencyAndRegister(dep3, actionId3, pdms);
    List<String> waitingDep1 = (ArrayList<String>) pdms.getWaitingActions(dep1);
    assertEquals(waitingDep1.size(), 1);
    assertEquals(waitingDep1.get(0), actionId1);
    List<String> waitingDep2 = (ArrayList<String>) pdms.getWaitingActions(dep2);
    assertEquals(waitingDep2.size(), 3);
    for (String id : waitingDep2) {
        assertTrue(id.equals(actionId1) || id.equals(actionId2) || id.equals(actionId3));
    }
    List<String> waitingDep3 = (ArrayList<String>) pdms.getWaitingActions(dep3);
    assertEquals(waitingDep3.size(), 1);
    assertTrue(waitingDep3.get(0).equals(actionId3));
    // make only coordAction 1 to WAITING, the rest to RUNNING (only WAITING
    // remain dependency cache)
    ArrayList<JsonBean> insertList = new ArrayList<JsonBean>();
    CoordinatorActionBean coordAction1 = new CoordinatorActionBean();
    coordAction1.setId(actionId1);
    coordAction1.setStatus(Status.WAITING);
    insertList.add(coordAction1);
    CoordinatorActionBean coordAction2 = new CoordinatorActionBean();
    coordAction2.setId(actionId2);
    coordAction2.setStatus(Status.RUNNING);
    insertList.add(coordAction2);
    CoordinatorActionBean coordAction3 = new CoordinatorActionBean();
    coordAction3.setId(actionId3);
    coordAction3.setStatus(Status.RUNNING);
    insertList.add(coordAction3);
    BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(insertList, null, null);
    // run cache purge
    Services.get().getConf().setInt(PartitionDependencyManagerService.CACHE_PURGE_TTL, 0);
    pdms.runCachePurgeWorker();
    // only coord Action 1 still in dependency cache
    waitingDep1 = (ArrayList<String>) pdms.getWaitingActions(dep1);
    assertEquals(waitingDep1.size(), 1);
    assertTrue(waitingDep1.get(0).equals(actionId1));
    // only coord Action 1 still in dependency cache
    waitingDep2 = (ArrayList<String>) pdms.getWaitingActions(dep2);
    assertEquals(waitingDep2.size(), 1);
    assertTrue(waitingDep2.get(0).equals(actionId1));
    waitingDep3 = (ArrayList<String>) pdms.getWaitingActions(dep3);
    assertNull(waitingDep3);
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    // mytbl1 should be still in topic map
    assertTrue(hcatService.isRegisteredForNotification(dep1));
    // mytbl1 should be still in topic map
    assertTrue(hcatService.isRegisteredForNotification(dep2));
    // mytbl2 should NOT be in topic map
    assertFalse(hcatService.isRegisteredForNotification(dep3));
}

Example 4 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class TestHAPartitionDependencyManagerService method testCheckAfterActionDelete.

public void testCheckAfterActionDelete() throws Exception {
    Services.get().setService(ZKJobsConcurrencyService.class);
    Services.get().get(ConfigurationService.class).getConf().setInt(PartitionDependencyManagerService.CACHE_PURGE_TTL, 0);
    db = "default";
    table1 = "mytbl";
    table2 = "mytb2";
    part1 = "dt=20120101;country=us";
    part2 = "dt=20120102;country=us";
    part3 = "dt=20120103;country=us";
    String newHCatDependency1 = "hcat://" + server + "/" + db + "/" + table1 + "/" + part1;
    String newHCatDependency2 = "hcat://" + server + "/" + db + "/" + table1 + "/" + part2;
    String newHCatDependency3 = "hcat://" + server + "/" + db + "/" + table2 + "/" + part3;
    HCatURI dep1 = new HCatURI(newHCatDependency1);
    HCatURI dep2 = new HCatURI(newHCatDependency2);
    HCatURI dep3 = new HCatURI(newHCatDependency3);
    // create db, table and partitions
    populateTable();
    String actionId1 = addInitRecords(newHCatDependency1);
    String actionId2 = addInitRecords(newHCatDependency2);
    String actionId3 = addInitRecords(newHCatDependency3);
    PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
    pdms.init(Services.get());
    pdms.addMissingDependency(dep1, actionId1);
    pdms.addMissingDependency(dep2, actionId2);
    pdms.addMissingDependency(dep3, actionId3);
    pdms.runCachePurgeWorker();
    assertNotNull((Collection<String>) pdms.getWaitingActions(dep1));
    assertNotNull((Collection<String>) pdms.getWaitingActions(dep2));
    assertNotNull((Collection<String>) pdms.getWaitingActions(dep3));
    List<String> deleteList = new ArrayList<String>();
    deleteList.add(actionId1);
    JPAService jpaService = Services.get().get(JPAService.class);
    jpaService.execute(new CoordActionsDeleteJPAExecutor(deleteList));
    pdms.runCachePurgeWorker();
    assertNull((Collection<String>) pdms.getWaitingActions(dep1));
    assertNotNull((Collection<String>) pdms.getWaitingActions(dep2));
    assertNotNull((Collection<String>) pdms.getWaitingActions(dep3));
    deleteList.clear();
    deleteList.add(actionId2);
    jpaService.execute(new CoordActionsDeleteJPAExecutor(deleteList));
    pdms.runCachePurgeWorker();
    assertNull((Collection<String>) pdms.getWaitingActions(dep1));
    assertNull((Collection<String>) pdms.getWaitingActions(dep2));
    assertNotNull((Collection<String>) pdms.getWaitingActions(dep3));
}

Example 5 with HCatURI

use of org.apache.oozie.util.HCatURI in project oozie by apache.

the class TestPartitionDependencyManagerEhcache method testMaxElementsInMemory.

@Test
public void testMaxElementsInMemory() throws Exception {
    // maxElementsInMemory="500" overflowToDisk="false"
    setupServices("testmaxelementsinmemory");
    PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
    int numItems = 1000;
    for (int i = 0; i < numItems; i++) {
        HCatURI dep = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + i);
        pdms.addMissingDependency(dep, "" + i);
    }
    // First 500 should have been evicted. But it is LRU and the last 350 removed is between 250 and 750.
    for (int i = 0; i < 150; i++) {
        assertNull(pdms.getWaitingActions(new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + "" + i)));
    }
    int evicted = 0;
    for (int i = 150; i < 750; i++) {
        if (pdms.getWaitingActions(new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + "" + i)) == null) {
            evicted++;
        }
    }
    assertEquals(350, evicted);
    for (int i = 750; i < 1000; i++) {
        String actionID = "" + i;
        HCatURI dep = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + actionID);
        Collection<String> waitingActions = pdms.getWaitingActions(dep);
        assertNotNull(dep.toURIString() + " is missing in cache", waitingActions);
        assertTrue(dep.toURIString() + " is missing in cache", waitingActions.contains(actionID));
    }
}