use of org.apache.oozie.dependency.URIHandler in project oozie by apache.
the class CoordELFunctions method coord_latestRange_sync.
private static String coord_latestRange_sync(int startOffset, int endOffset) throws Exception {
final XLog LOG = XLog.getLog(CoordELFunctions.class);
final Thread currentThread = Thread.currentThread();
ELEvaluator eval = ELEvaluator.getCurrent();
String retVal = "";
// in minutes
int datasetFrequency = (int) getDSFrequency();
TimeUnit dsTimeUnit = getDSTimeUnit();
int[] instCount = new int[1];
boolean useCurrentTime = Services.get().getConf().getBoolean(LATEST_EL_USE_CURRENT_TIME, false);
Calendar nominalInstanceCal;
if (useCurrentTime) {
nominalInstanceCal = getCurrentInstance(new Date(), instCount);
} else {
nominalInstanceCal = getCurrentInstance(getActualTime(), instCount);
}
StringBuilder resolvedInstances = new StringBuilder();
StringBuilder resolvedURIPaths = new StringBuilder();
if (nominalInstanceCal != null) {
Calendar initInstance = getInitialInstanceCal();
SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
if (ds == null) {
throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
}
String uriTemplate = ds.getUriTemplate();
Configuration conf = (Configuration) eval.getVariable(CONFIGURATION);
if (conf == null) {
throw new RuntimeException("Associated Configuration should be defined with key " + CONFIGURATION);
}
int available = 0;
boolean resolved = false;
String user = ParamChecker.notEmpty((String) eval.getVariable(OozieClient.USER_NAME), OozieClient.USER_NAME);
String doneFlag = ds.getDoneFlag();
URIHandlerService uriService = Services.get().get(URIHandlerService.class);
URIHandler uriHandler = null;
Context uriContext = null;
try {
while (nominalInstanceCal.compareTo(initInstance) >= 0 && !currentThread.isInterrupted()) {
ELEvaluator uriEval = getUriEvaluator(nominalInstanceCal);
String uriPath = uriEval.evaluate(uriTemplate, String.class);
if (uriHandler == null) {
URI uri = new URI(uriPath);
uriHandler = uriService.getURIHandler(uri);
uriContext = uriHandler.getContext(uri, conf, user, true);
}
String uriWithDoneFlag = uriHandler.getURIWithDoneFlag(uriPath, doneFlag);
if (uriHandler.exists(new URI(uriWithDoneFlag), uriContext)) {
XLog.getLog(CoordELFunctions.class).debug("Found latest(" + available + "): " + uriWithDoneFlag);
if (available == startOffset) {
LOG.debug("Matched latest(" + available + "): " + uriWithDoneFlag);
resolved = true;
resolvedInstances.append(DateUtils.formatDateOozieTZ(nominalInstanceCal));
resolvedURIPaths.append(uriPath);
retVal = resolvedInstances.toString();
eval.setVariable(CoordELConstants.RESOLVED_PATH, resolvedURIPaths.toString());
break;
} else if (available <= endOffset) {
LOG.debug("Matched latest(" + available + "): " + uriWithDoneFlag);
resolvedInstances.append(DateUtils.formatDateOozieTZ(nominalInstanceCal)).append(INSTANCE_SEPARATOR);
resolvedURIPaths.append(uriPath).append(INSTANCE_SEPARATOR);
}
available--;
}
// nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), -datasetFrequency);
nominalInstanceCal = (Calendar) initInstance.clone();
instCount[0]--;
nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency);
// DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
}
if (!StringUtils.isEmpty(resolvedURIPaths.toString()) && eval.getVariable(CoordELConstants.RESOLVED_PATH) == null) {
eval.setVariable(CoordELConstants.RESOLVED_PATH, resolvedURIPaths.toString());
}
} finally {
if (uriContext != null) {
uriContext.destroy();
}
}
if (!resolved) {
// return unchanged latest function with variable 'is_resolved'
// to 'false'
eval.setVariable(CoordELConstants.IS_RESOLVED, Boolean.FALSE);
if (startOffset == endOffset) {
retVal = "${coord:latest(" + startOffset + ")}";
} else {
retVal = "${coord:latestRange(" + startOffset + "," + endOffset + ")}";
}
} else {
eval.setVariable(CoordELConstants.IS_RESOLVED, Boolean.TRUE);
}
} else {
// No feasible nominal time
eval.setVariable(CoordELConstants.IS_RESOLVED, Boolean.FALSE);
}
return retVal;
}
use of org.apache.oozie.dependency.URIHandler in project oozie by apache.
the class HCatELFunctions method hcat_exists.
/* Workflow Parameterization EL functions */
/**
* Return true if partitions exists or false if not.
*
* @param uri hcatalog partition uri.
* @return <code>true</code> if the uri exists, <code>false</code> if it does not.
* @throws Exception
*/
public static boolean hcat_exists(String uri) throws Exception {
URI hcatURI = new URI(uri);
URIHandlerService uriService = Services.get().get(URIHandlerService.class);
URIHandler handler = uriService.getURIHandler(hcatURI);
WorkflowJob workflow = DagELFunctions.getWorkflow();
String user = workflow.getUser();
return handler.exists(hcatURI, EMPTY_CONF, user);
}
use of org.apache.oozie.dependency.URIHandler in project oozie by apache.
the class CoordOldInputDependency method getDependency.
@SuppressWarnings("unchecked")
private Map<String, ActionDependency> getDependency(CoordinatorActionBean coordAction) throws JDOMException, URIHandlerException {
Map<String, ActionDependency> dependenciesMap = new HashMap<String, ActionDependency>();
URIHandlerService uriService = Services.get().get(URIHandlerService.class);
Element eAction = XmlUtils.parseXml(coordAction.getActionXml());
Element inputList = eAction.getChild("input-events", eAction.getNamespace());
List<Element> eDataEvents = inputList.getChildren("data-in", eAction.getNamespace());
for (Element event : eDataEvents) {
Element uri = event.getChild("uris", event.getNamespace());
ActionDependency dependency = new ActionDependency();
if (uri != null) {
Element doneFlagElement = event.getChild("dataset", event.getNamespace()).getChild("done-flag", event.getNamespace());
String[] dataSets = uri.getText().split(CoordELFunctions.INSTANCE_SEPARATOR);
String doneFlag = CoordUtils.getDoneFlag(doneFlagElement);
for (String dataSet : dataSets) {
URIHandler uriHandler;
uriHandler = uriService.getURIHandler(dataSet);
dependency.getMissingDependencies().add(uriHandler.getURIWithDoneFlag(dataSet, doneFlag));
}
}
if (event.getChildTextTrim(CoordCommandUtils.UNRESOLVED_INSTANCES_TAG, event.getNamespace()) != null) {
ActionDependency unResolvedDependency = getUnResolvedDependency(coordAction, event);
dependency.getMissingDependencies().addAll(unResolvedDependency.getMissingDependencies());
dependency.setUriTemplate(unResolvedDependency.getUriTemplate());
}
dependenciesMap.put(event.getAttributeValue("name"), dependency);
}
return dependenciesMap;
}
use of org.apache.oozie.dependency.URIHandler in project oozie by apache.
the class CoordInputLogicEvaluatorPhaseThree method getPathWithoutDoneFlag.
private String getPathWithoutDoneFlag(String sPath, String dataSet) throws URIHandlerException {
if (dataSet == null) {
return sPath;
}
URIHandlerService service = Services.get().get(URIHandlerService.class);
URIHandler handler = service.getURIHandler(sPath);
return handler.getURIWithoutDoneFlag(sPath, eval.getVariable(".datain." + dataSet + ".doneFlag").toString());
}
use of org.apache.oozie.dependency.URIHandler in project oozie by apache.
the class TestLauncherHCatURIHandler method testDeleteTable.
public void testDeleteTable() throws Exception {
try {
createTestTable();
createPartitionForTestDelete(true, true);
URI hcatURI = getHCatURI(db, table, "year=2012;month=12;dt=02;country=us");
URIHandler uriHandler = uriService.getURIHandler(hcatURI);
assertTrue(uriHandler.exists(hcatURI, conf, getTestUser()));
hcatURI = getHCatURI(db, table);
assertTrue(uriHandler.exists(hcatURI, conf, getTestUser()));
LauncherURIHandlerFactory uriHandlerFactory = new LauncherURIHandlerFactory(uriService.getLauncherConfig());
LauncherURIHandler handler = uriHandlerFactory.getURIHandler(hcatURI);
handler.delete(hcatURI, conf);
assertFalse(uriHandler.exists(hcatURI, conf, getTestUser()));
} finally {
dropTestTable(true);
}
}
Aggregations