Examples with SparkListenerJobStart org.apache.spark.scheduler.SparkListenerJobStart used on opensource projects

Example 1 with SparkListenerJobStart

use of org.apache.spark.scheduler.SparkListenerJobStart in project zeppelin by apache.

the class SparkInterpreter method setupListeners.

static JobProgressListener setupListeners(SparkContext context) {
    JobProgressListener pl = new JobProgressListener(context.getConf()) {

        @Override
        public synchronized void onJobStart(SparkListenerJobStart jobStart) {
            super.onJobStart(jobStart);
            int jobId = jobStart.jobId();
            String jobGroupId = jobStart.properties().getProperty("spark.jobGroup.id");
            String jobUrl = getJobUrl(jobId);
            String noteId = Utils.getNoteId(jobGroupId);
            String paragraphId = Utils.getParagraphId(jobGroupId);
            if (jobUrl != null && noteId != null && paragraphId != null) {
                RemoteEventClientWrapper eventClient = ZeppelinContext.getEventClient();
                Map<String, String> infos = new java.util.HashMap<>();
                infos.put("jobUrl", jobUrl);
                infos.put("label", "SPARK JOB");
                infos.put("tooltip", "View in Spark web UI");
                if (eventClient != null) {
                    eventClient.onParaInfosReceived(noteId, paragraphId, infos);
                }
            }
        }

        private String getJobUrl(int jobId) {
            String jobUrl = null;
            if (sparkUrl != null) {
                jobUrl = sparkUrl + "/jobs/job?id=" + jobId;
            }
            return jobUrl;
        }
    };
    try {
        Object listenerBus = context.getClass().getMethod("listenerBus").invoke(context);
        Method[] methods = listenerBus.getClass().getMethods();
        Method addListenerMethod = null;
        for (Method m : methods) {
            if (!m.getName().equals("addListener")) {
                continue;
            }
            Class<?>[] parameterTypes = m.getParameterTypes();
            if (parameterTypes.length != 1) {
                continue;
            }
            if (!parameterTypes[0].isAssignableFrom(JobProgressListener.class)) {
                continue;
            }
            addListenerMethod = m;
            break;
        }
        if (addListenerMethod != null) {
            addListenerMethod.invoke(listenerBus, pl);
        } else {
            return null;
        }
    } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
        logger.error(e.toString(), e);
        return null;
    }
    return pl;
}