Search in sources :

Example 1 with SparkListenerJobStart

use of org.apache.spark.scheduler.SparkListenerJobStart in project zeppelin by apache.

the class SparkInterpreter method setupListeners.

static JobProgressListener setupListeners(SparkContext context) {
    JobProgressListener pl = new JobProgressListener(context.getConf()) {

        @Override
        public synchronized void onJobStart(SparkListenerJobStart jobStart) {
            super.onJobStart(jobStart);
            int jobId = jobStart.jobId();
            String jobGroupId = jobStart.properties().getProperty("spark.jobGroup.id");
            String jobUrl = getJobUrl(jobId);
            String noteId = Utils.getNoteId(jobGroupId);
            String paragraphId = Utils.getParagraphId(jobGroupId);
            if (jobUrl != null && noteId != null && paragraphId != null) {
                RemoteEventClientWrapper eventClient = ZeppelinContext.getEventClient();
                Map<String, String> infos = new java.util.HashMap<>();
                infos.put("jobUrl", jobUrl);
                infos.put("label", "SPARK JOB");
                infos.put("tooltip", "View in Spark web UI");
                if (eventClient != null) {
                    eventClient.onParaInfosReceived(noteId, paragraphId, infos);
                }
            }
        }

        private String getJobUrl(int jobId) {
            String jobUrl = null;
            if (sparkUrl != null) {
                jobUrl = sparkUrl + "/jobs/job?id=" + jobId;
            }
            return jobUrl;
        }
    };
    try {
        Object listenerBus = context.getClass().getMethod("listenerBus").invoke(context);
        Method[] methods = listenerBus.getClass().getMethods();
        Method addListenerMethod = null;
        for (Method m : methods) {
            if (!m.getName().equals("addListener")) {
                continue;
            }
            Class<?>[] parameterTypes = m.getParameterTypes();
            if (parameterTypes.length != 1) {
                continue;
            }
            if (!parameterTypes[0].isAssignableFrom(JobProgressListener.class)) {
                continue;
            }
            addListenerMethod = m;
            break;
        }
        if (addListenerMethod != null) {
            addListenerMethod.invoke(listenerBus, pl);
        } else {
            return null;
        }
    } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
        logger.error(e.toString(), e);
        return null;
    }
    return pl;
}
Also used : JobProgressListener(org.apache.spark.ui.jobs.JobProgressListener) SparkListenerJobStart(org.apache.spark.scheduler.SparkListenerJobStart) HashMap(scala.collection.mutable.HashMap) Method(java.lang.reflect.Method) RemoteEventClientWrapper(org.apache.zeppelin.interpreter.remote.RemoteEventClientWrapper) InvocationTargetException(java.lang.reflect.InvocationTargetException)

Aggregations

InvocationTargetException (java.lang.reflect.InvocationTargetException)1 Method (java.lang.reflect.Method)1 SparkListenerJobStart (org.apache.spark.scheduler.SparkListenerJobStart)1 JobProgressListener (org.apache.spark.ui.jobs.JobProgressListener)1 RemoteEventClientWrapper (org.apache.zeppelin.interpreter.remote.RemoteEventClientWrapper)1 HashMap (scala.collection.mutable.HashMap)1