Examples with DAGNode com.twitter.ambrose.model.DAGNode used on opensource projects

Example 11 with DAGNode

use of com.twitter.ambrose.model.DAGNode in project ambrose by twitter.

the class AmbroseHiveStatPublisher method send.

private void send(String jobIDStr, Map<String, Double> counterValues) {
    EmbeddedAmbroseHiveProgressReporter reporter = getEmbeddedProgressReporter();
    Configuration conf = SessionState.get().getConf();
    String queryId = AmbroseHiveUtil.getHiveQueryId(conf);
    Map<String, DAGNode<Job>> nodeIdToDAGNode = reporter.getNodeIdToDAGNode();
    DAGNode<Job> dagNode = nodeIdToDAGNode.get(nodeId);
    if (dagNode == null) {
        LOG.warn("jobStartedNotification - unrecorgnized operator name found for " + "jobId " + jobIDStr);
        return;
    }
    HiveJob job = (HiveJob) dagNode.getJob();
    // a job has been started
    if (job.getId() == null) {
        // job identifier on GUI
        job.setId(AmbroseHiveUtil.asDisplayId(queryId, jobIDStr, nodeId));
        reporter.addJobIdToNodeId(jobIDStr, nodeId);
        reporter.pushEvent(queryId, new Event.JobStartedEvent(dagNode));
    }
    try {
        boolean isUpdated = updateJobState();
        if (isUpdated && !reporter.getCompletedJobIds().contains(jobIDStr)) {
            Event<DAGNode<? extends Job>> event = null;
            job.setMapReduceJobState(jobProgress);
            if (jobProgress.isComplete()) {
                event = new Event.JobFinishedEvent(dagNode);
                // update reduce progress to 1 if we have no reducers
                int reducers = jobProgress.getTotalReducers();
                if (reducers == 0) {
                    jobProgress.setReduceProgress(1.0f);
                }
                // update job state
                job.setConfiguration(((HiveConf) conf).getAllProperties());
                job.setCounterGroupMap(AmbroseHiveUtil.counterGroupInfoMap(counterValues));
                reporter.addCompletedJobIds(jobIDStr);
                reporter.addJob(job);
            } else {
                event = new Event.JobProgressEvent(dagNode);
            }
            reporter.addJobIdToProgress(jobIDStr, getJobProgress());
            pushWorkflowProgress(queryId, reporter);
            reporter.pushEvent(queryId, event);
        }
    } catch (IOException e) {
        LOG.error("Error getting job info!", e);
    }
}

Example 12 with DAGNode

use of com.twitter.ambrose.model.DAGNode in project ambrose by twitter.

the class PigJobTest method doTestRoundTrip.

private void doTestRoundTrip(DAGNode<PigJob> expected) throws IOException {
    String asJson = expected.toJson();
    DAGNode asDAGNodeAgain = DAGNode.fromJson(asJson);
    assertEquals(expected.getName(), asDAGNodeAgain.getName());
    assertNotNull(asDAGNodeAgain.getJob());
    // assert that it's an instance of PigJob
    assertNotNull(asDAGNodeAgain.getJob() instanceof PigJob);
    assertJobEquals(expected.getJob(), (PigJob) asDAGNodeAgain.getJob());
}