use of org.apache.oozie.service.URIHandlerService in project oozie by apache.
the class CoordPushDependencyCheckXCommand method unregisterMissingDependencies.
public static void unregisterMissingDependencies(List<String> missingDeps, String actionId) {
final XLog LOG = XLog.getLog(CoordPushDependencyCheckXCommand.class);
URIHandlerService uriService = Services.get().get(URIHandlerService.class);
for (String missingDep : missingDeps) {
try {
URI missingURI = new URI(missingDep);
URIHandler handler = uriService.getURIHandler(missingURI);
if (handler.unregisterFromNotification(missingURI, actionId)) {
LOG.debug("Successfully unregistered uri [{0}] from notifications", missingURI);
} else {
LOG.warn("Unable to unregister uri [{0}] from notifications", missingURI);
}
} catch (Exception e) {
LOG.warn("Exception while unregistering uri [{0}] from notifications", missingDep, e);
}
}
}
use of org.apache.oozie.service.URIHandlerService in project oozie by apache.
the class TestURIHandlerService method testGetAuthorityWithScheme.
@Test
public void testGetAuthorityWithScheme() throws Exception {
URIHandlerService uriService = new URIHandlerService();
URI uri = uriService.getAuthorityWithScheme("hdfs://nn1:8020/dataset/${YEAR}/${MONTH}");
assertEquals("hdfs://nn1:8020", uri.toString());
uri = uriService.getAuthorityWithScheme("hdfs://nn1:8020");
assertEquals("hdfs://nn1:8020", uri.toString());
uri = uriService.getAuthorityWithScheme("hdfs://nn1:8020/");
assertEquals("hdfs://nn1:8020", uri.toString());
uri = uriService.getAuthorityWithScheme("hdfs://///tmp/file");
assertEquals("hdfs:///", uri.toString());
uri = uriService.getAuthorityWithScheme("hdfs:///tmp/file");
assertEquals("hdfs:///", uri.toString());
uri = uriService.getAuthorityWithScheme("/tmp/file");
assertEquals("/", uri.toString());
}
use of org.apache.oozie.service.URIHandlerService in project oozie by apache.
the class CoordELFunctions method coord_latestRange_sync.
private static String coord_latestRange_sync(int startOffset, int endOffset) throws Exception {
final XLog LOG = XLog.getLog(CoordELFunctions.class);
final Thread currentThread = Thread.currentThread();
ELEvaluator eval = ELEvaluator.getCurrent();
String retVal = "";
// in minutes
int datasetFrequency = (int) getDSFrequency();
TimeUnit dsTimeUnit = getDSTimeUnit();
int[] instCount = new int[1];
boolean useCurrentTime = Services.get().getConf().getBoolean(LATEST_EL_USE_CURRENT_TIME, false);
Calendar nominalInstanceCal;
if (useCurrentTime) {
nominalInstanceCal = getCurrentInstance(new Date(), instCount);
} else {
nominalInstanceCal = getCurrentInstance(getActualTime(), instCount);
}
StringBuilder resolvedInstances = new StringBuilder();
StringBuilder resolvedURIPaths = new StringBuilder();
if (nominalInstanceCal != null) {
Calendar initInstance = getInitialInstanceCal();
SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
if (ds == null) {
throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
}
String uriTemplate = ds.getUriTemplate();
Configuration conf = (Configuration) eval.getVariable(CONFIGURATION);
if (conf == null) {
throw new RuntimeException("Associated Configuration should be defined with key " + CONFIGURATION);
}
int available = 0;
boolean resolved = false;
String user = ParamChecker.notEmpty((String) eval.getVariable(OozieClient.USER_NAME), OozieClient.USER_NAME);
String doneFlag = ds.getDoneFlag();
URIHandlerService uriService = Services.get().get(URIHandlerService.class);
URIHandler uriHandler = null;
Context uriContext = null;
try {
while (nominalInstanceCal.compareTo(initInstance) >= 0 && !currentThread.isInterrupted()) {
ELEvaluator uriEval = getUriEvaluator(nominalInstanceCal);
String uriPath = uriEval.evaluate(uriTemplate, String.class);
if (uriHandler == null) {
URI uri = new URI(uriPath);
uriHandler = uriService.getURIHandler(uri);
uriContext = uriHandler.getContext(uri, conf, user, true);
}
String uriWithDoneFlag = uriHandler.getURIWithDoneFlag(uriPath, doneFlag);
if (uriHandler.exists(new URI(uriWithDoneFlag), uriContext)) {
XLog.getLog(CoordELFunctions.class).debug("Found latest(" + available + "): " + uriWithDoneFlag);
if (available == startOffset) {
LOG.debug("Matched latest(" + available + "): " + uriWithDoneFlag);
resolved = true;
resolvedInstances.append(DateUtils.formatDateOozieTZ(nominalInstanceCal));
resolvedURIPaths.append(uriPath);
retVal = resolvedInstances.toString();
eval.setVariable(CoordELConstants.RESOLVED_PATH, resolvedURIPaths.toString());
break;
} else if (available <= endOffset) {
LOG.debug("Matched latest(" + available + "): " + uriWithDoneFlag);
resolvedInstances.append(DateUtils.formatDateOozieTZ(nominalInstanceCal)).append(INSTANCE_SEPARATOR);
resolvedURIPaths.append(uriPath).append(INSTANCE_SEPARATOR);
}
available--;
}
// nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), -datasetFrequency);
nominalInstanceCal = (Calendar) initInstance.clone();
instCount[0]--;
nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency);
// DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
}
if (!StringUtils.isEmpty(resolvedURIPaths.toString()) && eval.getVariable(CoordELConstants.RESOLVED_PATH) == null) {
eval.setVariable(CoordELConstants.RESOLVED_PATH, resolvedURIPaths.toString());
}
} finally {
if (uriContext != null) {
uriContext.destroy();
}
}
if (!resolved) {
// return unchanged latest function with variable 'is_resolved'
// to 'false'
eval.setVariable(CoordELConstants.IS_RESOLVED, Boolean.FALSE);
if (startOffset == endOffset) {
retVal = "${coord:latest(" + startOffset + ")}";
} else {
retVal = "${coord:latestRange(" + startOffset + "," + endOffset + ")}";
}
} else {
eval.setVariable(CoordELConstants.IS_RESOLVED, Boolean.TRUE);
}
} else {
// No feasible nominal time
eval.setVariable(CoordELConstants.IS_RESOLVED, Boolean.FALSE);
}
return retVal;
}
use of org.apache.oozie.service.URIHandlerService in project oozie by apache.
the class DependencyChecker method checkForAvailability.
/**
* Get the currently missing and available dependencies after checking the list of known missing
* dependencies against the source.
*
* @param missingDependencies known missing dependencies
* @param actionConf Configuration for the action
* @param stopOnFirstMissing Does not continue check for the rest of list if there is a missing
* dependency
* @return ActionDependency which has the list of missing and available dependencies
* @throws CommandException in case of error
*/
public static ActionDependency checkForAvailability(List<String> missingDependencies, Configuration actionConf, boolean stopOnFirstMissing) throws CommandException {
// OOZIE-1251. Don't initialize as static variable.
final XLog LOG = XLog.getLog(DependencyChecker.class);
String user = ParamChecker.notEmpty(actionConf.get(OozieClient.USER_NAME), OozieClient.USER_NAME);
List<String> missingDeps = new ArrayList<String>();
List<String> availableDeps = new ArrayList<String>();
URIHandlerService uriService = Services.get().get(URIHandlerService.class);
boolean continueChecking = true;
try {
for (int index = 0; index < missingDependencies.size(); index++) {
if (continueChecking) {
String dependency = missingDependencies.get(index);
URI uri = new URI(dependency);
URIHandler uriHandler = uriService.getURIHandler(uri);
LOG.debug("Checking for the availability of dependency [{0}] ", dependency);
if (uriHandler.exists(uri, actionConf, user)) {
LOG.debug("Dependency [{0}] is available", dependency);
availableDeps.add(dependency);
} else {
LOG.debug("Dependency [{0}] is missing", dependency);
missingDeps.add(dependency);
if (stopOnFirstMissing) {
continueChecking = false;
}
}
} else {
missingDeps.add(missingDependencies.get(index));
}
}
} catch (URISyntaxException e) {
throw new CommandException(ErrorCode.E0906, e.getMessage(), e);
} catch (URIHandlerException e) {
throw new CommandException(e);
}
return new ActionDependency(missingDeps, availableDeps);
}
use of org.apache.oozie.service.URIHandlerService in project oozie by apache.
the class HCatELFunctions method hcat_exists.
/* Workflow Parameterization EL functions */
/**
* Return true if partitions exists or false if not.
*
* @param uri hcatalog partition uri.
* @return <code>true</code> if the uri exists, <code>false</code> if it does not.
* @throws Exception
*/
public static boolean hcat_exists(String uri) throws Exception {
URI hcatURI = new URI(uri);
URIHandlerService uriService = Services.get().get(URIHandlerService.class);
URIHandler handler = uriService.getURIHandler(hcatURI);
WorkflowJob workflow = DagELFunctions.getWorkflow();
String user = workflow.getUser();
return handler.exists(hcatURI, EMPTY_CONF, user);
}
Aggregations