use of org.apache.oozie.service.HCatAccessorService in project oozie by apache.
the class TestHCatMessageHandler method testCacheUpdateByMessage.
/**
* Test that message is processed to update the dependency map and mark
* partition as available
*/
@Test
public void testCacheUpdateByMessage() {
try {
String actionId1 = "1234465451";
String actionId2 = "1234465452";
String actionId3 = "1234465453";
String actionId4 = "1234465454";
// add partition as missing
HCatURI dep1 = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/dt=20120101;country=us");
HCatURI dep2 = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/country=us;dt=20120101");
HCatURI dep3 = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/dt=20120102;country=us");
HCatURI dep4 = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/dt=20120102;country=us;state=CA");
PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
pdms.addMissingDependency(dep1, actionId1);
pdms.addMissingDependency(dep2, actionId2);
pdms.addMissingDependency(dep3, actionId3);
pdms.addMissingDependency(dep4, actionId4);
assertTrue(pdms.getWaitingActions(dep1).contains(actionId1));
assertTrue(pdms.getWaitingActions(dep2).contains(actionId2));
assertTrue(pdms.getWaitingActions(dep3).contains(actionId3));
assertTrue(pdms.getWaitingActions(dep4).contains(actionId4));
// construct message
List<Map<String, String>> partitionsList = new ArrayList<Map<String, String>>();
partitionsList.add(getPartitionMap("dt=20120101;country=us;state=CA"));
partitionsList.add(getPartitionMap("dt=20120101;country=us;state=NY"));
JSONAddPartitionMessage jsonMsg = new JSONAddPartitionMessage("thrift://" + dep1.getServer(), "", dep1.getDb(), dep1.getTable(), partitionsList, System.currentTimeMillis());
Message msg = session.createTextMessage(jsonMsg.toString());
msg.setStringProperty(HCatConstants.HCAT_EVENT, HCatEventMessage.EventType.ADD_PARTITION.toString());
// test message processing
HCatMessageHandler hcatHandler = new HCatMessageHandler("hcat.server.com:5080");
hcatHandler.process(msg);
assertNull(pdms.getWaitingActions(dep1));
assertNull(pdms.getWaitingActions(dep2));
assertTrue(pdms.getWaitingActions(dep3).contains(actionId3));
assertTrue(pdms.getWaitingActions(dep4).contains(actionId4));
// test message processing through JMS notification listener
partitionsList.clear();
partitionsList.add(getPartitionMap("dt=20120102;country=us;state=CA"));
partitionsList.add(getPartitionMap("dt=20120102;country=us;state=NY"));
jsonMsg = new JSONAddPartitionMessage("thrift://" + dep1.getServer(), "", dep1.getDb(), dep1.getTable(), partitionsList, System.currentTimeMillis());
msg = session.createTextMessage(jsonMsg.toString());
msg.setStringProperty(HCatConstants.HCAT_EVENT, HCatEventMessage.EventType.ADD_PARTITION.toString());
HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
hcatService.registerForNotification(dep1, "hcat.topic1", hcatHandler);
Topic topic = session.createTopic("hcat.topic1");
MessageProducer producer = session.createProducer(topic);
producer.send(msg);
Thread.sleep(500);
assertNull(pdms.getWaitingActions(dep3));
assertNull(pdms.getWaitingActions(dep4));
assertTrue(pdms.getAvailableDependencyURIs(actionId1).contains(dep1.getURI().toString()));
assertTrue(pdms.getAvailableDependencyURIs(actionId2).contains(dep2.getURI().toString()));
assertTrue(pdms.getAvailableDependencyURIs(actionId3).contains(dep3.getURI().toString()));
assertTrue(pdms.getAvailableDependencyURIs(actionId4).contains(dep4.getURI().toString()));
} catch (Exception e) {
e.printStackTrace();
fail("Exception: " + e.getMessage());
}
}
use of org.apache.oozie.service.HCatAccessorService in project oozie by apache.
the class TestCoordPushDependencyCheckXCommand method testUpdateCoordTableMultipleDepsV3.
@Test
public void testUpdateCoordTableMultipleDepsV3() throws Exception {
// Test for two dependencies : one of them is already existing in the
// hcat server. Other one is not.
// Expected to see only first action in WAITING as we check for all dependencies.
// Later make the other partition also available. action is expected to
// be READY
String db = "default";
String table = "tablename";
String newHCatDependency1 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=brazil";
String newHCatDependency2 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=usa";
String newHCatDependency = newHCatDependency1 + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency2;
populateTable(db, table);
String actionId = addInitRecords(newHCatDependency);
checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
// Checks for all missing dependencies
new CoordPushDependencyCheckXCommand(actionId, true).call();
checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.WAITING);
PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
assertTrue(pdms.getWaitingActions(new HCatURI(newHCatDependency1)).contains(actionId));
assertTrue(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency2)));
// Make first dependency available
addPartition(db, table, "dt=20120430;country=brazil");
new CoordPushDependencyCheckXCommand(actionId).call();
checkCoordAction(actionId, "", CoordinatorAction.Status.READY);
assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency1)));
assertFalse(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
}
use of org.apache.oozie.service.HCatAccessorService in project oozie by apache.
the class TestCoordPushDependencyCheckXCommand method testTimeOutWithException2.
@Test
public void testTimeOutWithException2() throws Exception {
// Test timeout when table containing missing dependencies is dropped in between
String db = "default";
String table = "tablename";
String newHCatDependency1 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=brazil";
String newHCatDependency2 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=usa";
String newHCatDependency = newHCatDependency1 + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency2;
populateTable(db, table);
String actionId = addInitRecords(newHCatDependency);
checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
new CoordPushDependencyCheckXCommand(actionId, true).call();
checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.WAITING);
PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
assertTrue(pdms.getWaitingActions(new HCatURI(newHCatDependency1)).contains(actionId));
assertTrue(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
// Timeout is 10 mins. Change action created time to before 12 min to make the action
// timeout.
long timeOutCreationTime = System.currentTimeMillis() - (12 * 60 * 1000);
setCoordActionCreationTime(actionId, timeOutCreationTime);
dropTable(db, table, true);
try {
new CoordPushDependencyCheckXCommand(actionId).call();
fail();
} catch (Exception e) {
assertTrue(e.getMessage().contains("NoSuchObjectException"));
}
Thread.sleep(100);
// Check for timeout status and unregistered missing dependencies
checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.TIMEDOUT);
assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency1)));
assertFalse(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
}
use of org.apache.oozie.service.HCatAccessorService in project oozie by apache.
the class TestCoordPushDependencyCheckXCommand method testTimeOutWithException1.
@Test
public void testTimeOutWithException1() throws Exception {
// Test timeout when missing dependencies are from a non existing table
String newHCatDependency1 = "hcat://" + server + "/nodb/notable/dt=20120430;country=brazil";
String newHCatDependency2 = "hcat://" + server + "/nodb/notable/dt=20120430;country=usa";
String newHCatDependency = newHCatDependency1 + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency2;
String actionId = addInitRecords(newHCatDependency);
checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
try {
new CoordPushDependencyCheckXCommand(actionId, true).call();
fail();
} catch (Exception e) {
assertTrue(e.getMessage().contains("NoSuchObjectException"));
}
checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency1)));
assertFalse(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
// Timeout is 10 mins. Change action created time to before 12 min to make the action
// timeout.
long timeOutCreationTime = System.currentTimeMillis() - (12 * 60 * 1000);
setCoordActionCreationTime(actionId, timeOutCreationTime);
try {
new CoordPushDependencyCheckXCommand(actionId).call();
fail();
} catch (Exception e) {
assertTrue(e.getMessage().contains("NoSuchObjectException"));
}
Thread.sleep(100);
// Check for timeout status and unregistered missing dependencies
checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.TIMEDOUT);
}
use of org.apache.oozie.service.HCatAccessorService in project oozie by apache.
the class HCatURIHandler method getHiveConf.
private HiveConf getHiveConf(URI uri, Configuration conf) throws HCatAccessorException {
HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
if (hcatService.getHCatConf() != null) {
conf = hcatService.getHCatConf();
}
HiveConf hiveConf = new HiveConf(conf, this.getClass());
String serverURI = getMetastoreConnectURI(uri);
if (!serverURI.equals("")) {
hiveConf.set("hive.metastore.local", "false");
}
hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, serverURI);
return hiveConf;
}
Aggregations