Search in sources :

Example 1 with HCatAccessorService

use of org.apache.oozie.service.HCatAccessorService in project oozie by apache.

the class TestHCatMessageHandler method testCacheUpdateByMessage.

/**
 * Test that message is processed to update the dependency map and mark
 * partition as available
 */
@Test
public void testCacheUpdateByMessage() {
    try {
        String actionId1 = "1234465451";
        String actionId2 = "1234465452";
        String actionId3 = "1234465453";
        String actionId4 = "1234465454";
        // add partition as missing
        HCatURI dep1 = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/dt=20120101;country=us");
        HCatURI dep2 = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/country=us;dt=20120101");
        HCatURI dep3 = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/dt=20120102;country=us");
        HCatURI dep4 = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/dt=20120102;country=us;state=CA");
        PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
        pdms.addMissingDependency(dep1, actionId1);
        pdms.addMissingDependency(dep2, actionId2);
        pdms.addMissingDependency(dep3, actionId3);
        pdms.addMissingDependency(dep4, actionId4);
        assertTrue(pdms.getWaitingActions(dep1).contains(actionId1));
        assertTrue(pdms.getWaitingActions(dep2).contains(actionId2));
        assertTrue(pdms.getWaitingActions(dep3).contains(actionId3));
        assertTrue(pdms.getWaitingActions(dep4).contains(actionId4));
        // construct message
        List<Map<String, String>> partitionsList = new ArrayList<Map<String, String>>();
        partitionsList.add(getPartitionMap("dt=20120101;country=us;state=CA"));
        partitionsList.add(getPartitionMap("dt=20120101;country=us;state=NY"));
        JSONAddPartitionMessage jsonMsg = new JSONAddPartitionMessage("thrift://" + dep1.getServer(), "", dep1.getDb(), dep1.getTable(), partitionsList, System.currentTimeMillis());
        Message msg = session.createTextMessage(jsonMsg.toString());
        msg.setStringProperty(HCatConstants.HCAT_EVENT, HCatEventMessage.EventType.ADD_PARTITION.toString());
        // test message processing
        HCatMessageHandler hcatHandler = new HCatMessageHandler("hcat.server.com:5080");
        hcatHandler.process(msg);
        assertNull(pdms.getWaitingActions(dep1));
        assertNull(pdms.getWaitingActions(dep2));
        assertTrue(pdms.getWaitingActions(dep3).contains(actionId3));
        assertTrue(pdms.getWaitingActions(dep4).contains(actionId4));
        // test message processing through JMS notification listener
        partitionsList.clear();
        partitionsList.add(getPartitionMap("dt=20120102;country=us;state=CA"));
        partitionsList.add(getPartitionMap("dt=20120102;country=us;state=NY"));
        jsonMsg = new JSONAddPartitionMessage("thrift://" + dep1.getServer(), "", dep1.getDb(), dep1.getTable(), partitionsList, System.currentTimeMillis());
        msg = session.createTextMessage(jsonMsg.toString());
        msg.setStringProperty(HCatConstants.HCAT_EVENT, HCatEventMessage.EventType.ADD_PARTITION.toString());
        HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
        hcatService.registerForNotification(dep1, "hcat.topic1", hcatHandler);
        Topic topic = session.createTopic("hcat.topic1");
        MessageProducer producer = session.createProducer(topic);
        producer.send(msg);
        Thread.sleep(500);
        assertNull(pdms.getWaitingActions(dep3));
        assertNull(pdms.getWaitingActions(dep4));
        assertTrue(pdms.getAvailableDependencyURIs(actionId1).contains(dep1.getURI().toString()));
        assertTrue(pdms.getAvailableDependencyURIs(actionId2).contains(dep2.getURI().toString()));
        assertTrue(pdms.getAvailableDependencyURIs(actionId3).contains(dep3.getURI().toString()));
        assertTrue(pdms.getAvailableDependencyURIs(actionId4).contains(dep4.getURI().toString()));
    } catch (Exception e) {
        e.printStackTrace();
        fail("Exception: " + e.getMessage());
    }
}
Also used : HCatEventMessage(org.apache.hive.hcatalog.messaging.HCatEventMessage) JSONAddPartitionMessage(org.apache.hive.hcatalog.messaging.json.JSONAddPartitionMessage) Message(javax.jms.Message) HCatMessageHandler(org.apache.oozie.dependency.hcat.HCatMessageHandler) ArrayList(java.util.ArrayList) PartitionDependencyManagerService(org.apache.oozie.service.PartitionDependencyManagerService) JSONAddPartitionMessage(org.apache.hive.hcatalog.messaging.json.JSONAddPartitionMessage) HCatAccessorService(org.apache.oozie.service.HCatAccessorService) MessageProducer(javax.jms.MessageProducer) HCatURI(org.apache.oozie.util.HCatURI) Topic(javax.jms.Topic) Map(java.util.Map) Test(org.junit.Test)

Example 2 with HCatAccessorService

use of org.apache.oozie.service.HCatAccessorService in project oozie by apache.

the class TestCoordPushDependencyCheckXCommand method testUpdateCoordTableMultipleDepsV3.

@Test
public void testUpdateCoordTableMultipleDepsV3() throws Exception {
    // Test for two dependencies : one of them is already existing in the
    // hcat server. Other one is not.
    // Expected to see only first action in WAITING as we check for all dependencies.
    // Later make the other partition also available. action is expected to
    // be READY
    String db = "default";
    String table = "tablename";
    String newHCatDependency1 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=brazil";
    String newHCatDependency2 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=usa";
    String newHCatDependency = newHCatDependency1 + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency2;
    populateTable(db, table);
    String actionId = addInitRecords(newHCatDependency);
    checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
    // Checks for all missing dependencies
    new CoordPushDependencyCheckXCommand(actionId, true).call();
    checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.WAITING);
    PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    assertTrue(pdms.getWaitingActions(new HCatURI(newHCatDependency1)).contains(actionId));
    assertTrue(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
    assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency2)));
    // Make first dependency available
    addPartition(db, table, "dt=20120430;country=brazil");
    new CoordPushDependencyCheckXCommand(actionId).call();
    checkCoordAction(actionId, "", CoordinatorAction.Status.READY);
    assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency1)));
    assertFalse(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
}
Also used : HCatAccessorService(org.apache.oozie.service.HCatAccessorService) HCatURI(org.apache.oozie.util.HCatURI) PartitionDependencyManagerService(org.apache.oozie.service.PartitionDependencyManagerService) Test(org.junit.Test)

Example 3 with HCatAccessorService

use of org.apache.oozie.service.HCatAccessorService in project oozie by apache.

the class TestCoordPushDependencyCheckXCommand method testTimeOutWithException2.

@Test
public void testTimeOutWithException2() throws Exception {
    // Test timeout when table containing missing dependencies is dropped in between
    String db = "default";
    String table = "tablename";
    String newHCatDependency1 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=brazil";
    String newHCatDependency2 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=usa";
    String newHCatDependency = newHCatDependency1 + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency2;
    populateTable(db, table);
    String actionId = addInitRecords(newHCatDependency);
    checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
    new CoordPushDependencyCheckXCommand(actionId, true).call();
    checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.WAITING);
    PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    assertTrue(pdms.getWaitingActions(new HCatURI(newHCatDependency1)).contains(actionId));
    assertTrue(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
    // Timeout is 10 mins. Change action created time to before 12 min to make the action
    // timeout.
    long timeOutCreationTime = System.currentTimeMillis() - (12 * 60 * 1000);
    setCoordActionCreationTime(actionId, timeOutCreationTime);
    dropTable(db, table, true);
    try {
        new CoordPushDependencyCheckXCommand(actionId).call();
        fail();
    } catch (Exception e) {
        assertTrue(e.getMessage().contains("NoSuchObjectException"));
    }
    Thread.sleep(100);
    // Check for timeout status and unregistered missing dependencies
    checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.TIMEDOUT);
    assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency1)));
    assertFalse(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
}
Also used : HCatAccessorService(org.apache.oozie.service.HCatAccessorService) HCatURI(org.apache.oozie.util.HCatURI) PartitionDependencyManagerService(org.apache.oozie.service.PartitionDependencyManagerService) JPAExecutorException(org.apache.oozie.executor.jpa.JPAExecutorException) Test(org.junit.Test)

Example 4 with HCatAccessorService

use of org.apache.oozie.service.HCatAccessorService in project oozie by apache.

the class TestCoordPushDependencyCheckXCommand method testTimeOutWithException1.

@Test
public void testTimeOutWithException1() throws Exception {
    // Test timeout when missing dependencies are from a non existing table
    String newHCatDependency1 = "hcat://" + server + "/nodb/notable/dt=20120430;country=brazil";
    String newHCatDependency2 = "hcat://" + server + "/nodb/notable/dt=20120430;country=usa";
    String newHCatDependency = newHCatDependency1 + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency2;
    String actionId = addInitRecords(newHCatDependency);
    checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
    try {
        new CoordPushDependencyCheckXCommand(actionId, true).call();
        fail();
    } catch (Exception e) {
        assertTrue(e.getMessage().contains("NoSuchObjectException"));
    }
    checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
    PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency1)));
    assertFalse(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
    // Timeout is 10 mins. Change action created time to before 12 min to make the action
    // timeout.
    long timeOutCreationTime = System.currentTimeMillis() - (12 * 60 * 1000);
    setCoordActionCreationTime(actionId, timeOutCreationTime);
    try {
        new CoordPushDependencyCheckXCommand(actionId).call();
        fail();
    } catch (Exception e) {
        assertTrue(e.getMessage().contains("NoSuchObjectException"));
    }
    Thread.sleep(100);
    // Check for timeout status and unregistered missing dependencies
    checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.TIMEDOUT);
}
Also used : HCatAccessorService(org.apache.oozie.service.HCatAccessorService) HCatURI(org.apache.oozie.util.HCatURI) PartitionDependencyManagerService(org.apache.oozie.service.PartitionDependencyManagerService) JPAExecutorException(org.apache.oozie.executor.jpa.JPAExecutorException) Test(org.junit.Test)

Example 5 with HCatAccessorService

use of org.apache.oozie.service.HCatAccessorService in project oozie by apache.

the class HCatURIHandler method getHiveConf.

private HiveConf getHiveConf(URI uri, Configuration conf) throws HCatAccessorException {
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    if (hcatService.getHCatConf() != null) {
        conf = hcatService.getHCatConf();
    }
    HiveConf hiveConf = new HiveConf(conf, this.getClass());
    String serverURI = getMetastoreConnectURI(uri);
    if (!serverURI.equals("")) {
        hiveConf.set("hive.metastore.local", "false");
    }
    hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, serverURI);
    return hiveConf;
}
Also used : HCatAccessorService(org.apache.oozie.service.HCatAccessorService) HiveConf(org.apache.hadoop.hive.conf.HiveConf)

Aggregations

HCatAccessorService (org.apache.oozie.service.HCatAccessorService)14 HCatURI (org.apache.oozie.util.HCatURI)8 PartitionDependencyManagerService (org.apache.oozie.service.PartitionDependencyManagerService)7 Test (org.junit.Test)6 Map (java.util.Map)4 Collection (java.util.Collection)3 HashMap (java.util.HashMap)3 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)3 ConcurrentMap (java.util.concurrent.ConcurrentMap)3 ArrayList (java.util.ArrayList)2 HCatMessageHandler (org.apache.oozie.dependency.hcat.HCatMessageHandler)2 JPAExecutorException (org.apache.oozie.executor.jpa.JPAExecutorException)2 URISyntaxException (java.net.URISyntaxException)1 HashSet (java.util.HashSet)1 Message (javax.jms.Message)1 MessageProducer (javax.jms.MessageProducer)1 Topic (javax.jms.Topic)1 Configuration (org.apache.hadoop.conf.Configuration)1 HiveConf (org.apache.hadoop.hive.conf.HiveConf)1 HCatClient (org.apache.hive.hcatalog.api.HCatClient)1