Examples with HCatMessageHandler org.apache.oozie.dependency.hcat.HCatMessageHandler used on opensource projects

Example 1 with HCatMessageHandler

use of org.apache.oozie.dependency.hcat.HCatMessageHandler in project oozie by apache.

the class TestHCatMessageHandler method testDropEventTypeMessage.

/**
 * Test the other type of messages - DROP_PARTITION and DROP_TABLE are
 * handled with the correct log messages
 */
public void testDropEventTypeMessage() {
    try {
        // Set the log4j appender for getting the statements logged by
        // HCatMessageHandler
        Logger logger = Logger.getLogger(HCatMessageHandler.class);
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        Layout layout = new SimpleLayout();
        Appender appender = new WriterAppender(layout, out);
        logger.addAppender(appender);
        Message msg = session.createTextMessage("{" + "\"server\" : \"thrift://localhost:1234\"," + "\"db\" : \"default\"," + "\"table\" : \"newTable\"," + "\"timestamp\" : \"123456\"," + "\"partitions\" : [{ \"dt\" : \"2012_01_01\", \"grid\" : \"AB\" }]" + "}");
        msg.setStringProperty(HCatConstants.HCAT_EVENT, HCatEventMessage.EventType.DROP_PARTITION.toString());
        HCatMessageHandler hcatHandler = new HCatMessageHandler("localhost");
        hcatHandler.process(msg);
        // check logs to see appropriate error message
        String logMsg = out.toString();
        assertTrue(logMsg.contains(HCatEventMessage.EventType.DROP_PARTITION.toString()));
        msg.setStringProperty(HCatConstants.HCAT_EVENT, HCatEventMessage.EventType.DROP_TABLE.toString());
        hcatHandler.process(msg);
        // check logs to see appropriate error message
        logMsg = out.toString();
        assertTrue(logMsg.contains(HCatEventMessage.EventType.DROP_TABLE.toString()));
    } catch (Exception e) {
        fail("Exception caused " + e.getMessage());
    }
}

Example 2 with HCatMessageHandler

use of org.apache.oozie.dependency.hcat.HCatMessageHandler in project oozie by apache.

the class TestHCatMessageHandler method testCacheUpdateByMessage.

/**
 * Test that message is processed to update the dependency map and mark
 * partition as available
 */
@Test
public void testCacheUpdateByMessage() {
    try {
        String actionId1 = "1234465451";
        String actionId2 = "1234465452";
        String actionId3 = "1234465453";
        String actionId4 = "1234465454";
        // add partition as missing
        HCatURI dep1 = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/dt=20120101;country=us");
        HCatURI dep2 = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/country=us;dt=20120101");
        HCatURI dep3 = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/dt=20120102;country=us");
        HCatURI dep4 = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/dt=20120102;country=us;state=CA");
        PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
        pdms.addMissingDependency(dep1, actionId1);
        pdms.addMissingDependency(dep2, actionId2);
        pdms.addMissingDependency(dep3, actionId3);
        pdms.addMissingDependency(dep4, actionId4);
        assertTrue(pdms.getWaitingActions(dep1).contains(actionId1));
        assertTrue(pdms.getWaitingActions(dep2).contains(actionId2));
        assertTrue(pdms.getWaitingActions(dep3).contains(actionId3));
        assertTrue(pdms.getWaitingActions(dep4).contains(actionId4));
        // construct message
        List<Map<String, String>> partitionsList = new ArrayList<Map<String, String>>();
        partitionsList.add(getPartitionMap("dt=20120101;country=us;state=CA"));
        partitionsList.add(getPartitionMap("dt=20120101;country=us;state=NY"));
        JSONAddPartitionMessage jsonMsg = new JSONAddPartitionMessage("thrift://" + dep1.getServer(), "", dep1.getDb(), dep1.getTable(), partitionsList, System.currentTimeMillis());
        Message msg = session.createTextMessage(jsonMsg.toString());
        msg.setStringProperty(HCatConstants.HCAT_EVENT, HCatEventMessage.EventType.ADD_PARTITION.toString());
        // test message processing
        HCatMessageHandler hcatHandler = new HCatMessageHandler("hcat.server.com:5080");
        hcatHandler.process(msg);
        assertNull(pdms.getWaitingActions(dep1));
        assertNull(pdms.getWaitingActions(dep2));
        assertTrue(pdms.getWaitingActions(dep3).contains(actionId3));
        assertTrue(pdms.getWaitingActions(dep4).contains(actionId4));
        // test message processing through JMS notification listener
        partitionsList.clear();
        partitionsList.add(getPartitionMap("dt=20120102;country=us;state=CA"));
        partitionsList.add(getPartitionMap("dt=20120102;country=us;state=NY"));
        jsonMsg = new JSONAddPartitionMessage("thrift://" + dep1.getServer(), "", dep1.getDb(), dep1.getTable(), partitionsList, System.currentTimeMillis());
        msg = session.createTextMessage(jsonMsg.toString());
        msg.setStringProperty(HCatConstants.HCAT_EVENT, HCatEventMessage.EventType.ADD_PARTITION.toString());
        HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
        hcatService.registerForNotification(dep1, "hcat.topic1", hcatHandler);
        Topic topic = session.createTopic("hcat.topic1");
        MessageProducer producer = session.createProducer(topic);
        producer.send(msg);
        Thread.sleep(500);
        assertNull(pdms.getWaitingActions(dep3));
        assertNull(pdms.getWaitingActions(dep4));
        assertTrue(pdms.getAvailableDependencyURIs(actionId1).contains(dep1.getURI().toString()));
        assertTrue(pdms.getAvailableDependencyURIs(actionId2).contains(dep2.getURI().toString()));
        assertTrue(pdms.getAvailableDependencyURIs(actionId3).contains(dep3.getURI().toString()));
        assertTrue(pdms.getAvailableDependencyURIs(actionId4).contains(dep4.getURI().toString()));
    } catch (Exception e) {
        e.printStackTrace();
        fail("Exception: " + e.getMessage());
    }
}

Example 3 with HCatMessageHandler

use of org.apache.oozie.dependency.hcat.HCatMessageHandler in project oozie by apache.

the class TestJMSAccessorService method testConnectionRetry.

@Test
public void testConnectionRetry() throws Exception {
    services.destroy();
    services = super.setupServicesForHCatalog();
    int randomPort = 30000 + random.nextInt(10000);
    String brokerURl = "tcp://localhost:" + randomPort;
    Configuration servicesConf = services.getConf();
    servicesConf.set(JMSAccessorService.CONF_RETRY_INITIAL_DELAY, "1");
    servicesConf.set(JMSAccessorService.CONF_RETRY_MAX_ATTEMPTS, "3");
    servicesConf.set(HCatAccessorService.JMS_CONNECTIONS_PROPERTIES, "default=java.naming.factory.initial#" + ActiveMQConnFactory + ";" + "java.naming.provider.url#" + brokerURl + ";" + "connectionFactoryNames#" + "ConnectionFactory");
    services.init();
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    JMSAccessorService jmsService = Services.get().get(JMSAccessorService.class);
    String publisherAuthority = "hcat.server.com:5080";
    String topic = "topic.topic1";
    JMSConnectionInfo connInfo = hcatService.getJMSConnectionInfo(new URI("hcat://hcat.server.com:8020"));
    jmsService.registerForNotification(connInfo, topic, new HCatMessageHandler(publisherAuthority));
    assertFalse(jmsService.isListeningToTopic(connInfo, topic));
    assertTrue(jmsService.isConnectionInRetryList(connInfo));
    assertTrue(jmsService.isTopicInRetryList(connInfo, topic));
    // Start the broker and check if listening to topic now
    BrokerService broker = new BrokerService();
    broker.addConnector(brokerURl);
    broker.setDataDirectory(getTestCaseDir());
    broker.start();
    Thread.sleep(1000);
    assertTrue(jmsService.isListeningToTopic(connInfo, topic));
    assertFalse(jmsService.isConnectionInRetryList(connInfo));
    assertFalse(jmsService.isTopicInRetryList(connInfo, topic));
    broker.stop();
    jmsService.destroy();
}

Example 4 with HCatMessageHandler

use of org.apache.oozie.dependency.hcat.HCatMessageHandler in project oozie by apache.

the class EhcacheHCatDependencyCacheExtended method addMissingDependencyAndRegister.

protected void addMissingDependencyAndRegister(HCatURI hcatURI, String actionId, PartitionDependencyManagerService pdms) {
    pdms.addMissingDependency(hcatURI, actionId);
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    if (!hcatService.isRegisteredForNotification(hcatURI)) {
        hcatService.registerForNotification(hcatURI, hcatURI.getDb() + "." + hcatURI.getTable(), new HCatMessageHandler(hcatURI.getServer()));
    }
}

Example 5 with HCatMessageHandler

use of org.apache.oozie.dependency.hcat.HCatMessageHandler in project oozie by apache.

the class HCatURIHandler method registerForNotification.

@Override
public void registerForNotification(URI uri, Configuration conf, String user, String actionID) throws URIHandlerException {
    HCatURI hcatURI;
    try {
        hcatURI = new HCatURI(uri);
    } catch (URISyntaxException e) {
        throw new URIHandlerException(ErrorCode.E0906, uri, e);
    }
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    if (!hcatService.isRegisteredForNotification(hcatURI)) {
        HCatClient client = getHCatClient(uri, conf);
        try {
            String topic = client.getMessageBusTopicName(hcatURI.getDb(), hcatURI.getTable());
            if (topic == null) {
                return;
            }
            hcatService.registerForNotification(hcatURI, topic, new HCatMessageHandler(uri.getAuthority()));
        } catch (HCatException e) {
            throw new HCatAccessorException(ErrorCode.E1501, e);
        } finally {
            closeQuietly(client, null, true);
        }
    }
    PartitionDependencyManagerService pdmService = Services.get().get(PartitionDependencyManagerService.class);
    pdmService.addMissingDependency(hcatURI, actionID);
}