Examples with Event com.twitter.ambrose.model.Event used on opensource projects

Example 6 with Event

use of com.twitter.ambrose.model.Event in project ambrose by twitter.

the class TestHRavenStatsReadService method main.

/**
   * Main method for testing reading from hraven
   */
public static void main(String[] args) throws IOException {
    //cluster!userName!appId!runId!timestamp!flowId
    String workflowId = args[0];
    HRavenStatsReadService service = new HRavenStatsReadService();
    Map<String, DAGNode> dagMap = service.getDagNodeNameMap(workflowId);
    if (dagMap == null) {
        print("No dagNodeNameMap found for " + workflowId);
    } else {
        print(String.format("Found %d dapMap entries", dagMap.size()));
        for (Map.Entry<String, DAGNode> entry : dagMap.entrySet()) {
            DAGNode node = entry.getValue();
            String jobId = node.getJob() != null ? node.getJob().getId() : null;
            print(String.format("%s: nodeName=%s jobId=%s successors=%s", entry.getKey(), node.getName(), jobId, node.getSuccessorNames()));
        }
    }
    List<Event> events = service.getEventsSinceId(workflowId, -1);
    print(String.format("Found %d events", events.size()));
    for (Event event : events) {
        print(String.format("%d %d %s %s", event.getId(), event.getTimestamp(), event.getType(), event.getPayload()));
    }
}

Example 7 with Event

use of com.twitter.ambrose.model.Event in project ambrose by twitter.

the class AmbroseHiveStatPublisher method send.

private void send(String jobIDStr, Map<String, Double> counterValues) {
    EmbeddedAmbroseHiveProgressReporter reporter = getEmbeddedProgressReporter();
    Configuration conf = SessionState.get().getConf();
    String queryId = AmbroseHiveUtil.getHiveQueryId(conf);
    Map<String, DAGNode<Job>> nodeIdToDAGNode = reporter.getNodeIdToDAGNode();
    DAGNode<Job> dagNode = nodeIdToDAGNode.get(nodeId);
    if (dagNode == null) {
        LOG.warn("jobStartedNotification - unrecorgnized operator name found for " + "jobId " + jobIDStr);
        return;
    }
    HiveJob job = (HiveJob) dagNode.getJob();
    // a job has been started
    if (job.getId() == null) {
        // job identifier on GUI
        job.setId(AmbroseHiveUtil.asDisplayId(queryId, jobIDStr, nodeId));
        reporter.addJobIdToNodeId(jobIDStr, nodeId);
        reporter.pushEvent(queryId, new Event.JobStartedEvent(dagNode));
    }
    try {
        boolean isUpdated = updateJobState();
        if (isUpdated && !reporter.getCompletedJobIds().contains(jobIDStr)) {
            Event<DAGNode<? extends Job>> event = null;
            job.setMapReduceJobState(jobProgress);
            if (jobProgress.isComplete()) {
                event = new Event.JobFinishedEvent(dagNode);
                // update reduce progress to 1 if we have no reducers
                int reducers = jobProgress.getTotalReducers();
                if (reducers == 0) {
                    jobProgress.setReduceProgress(1.0f);
                }
                // update job state
                job.setConfiguration(((HiveConf) conf).getAllProperties());
                job.setCounterGroupMap(AmbroseHiveUtil.counterGroupInfoMap(counterValues));
                reporter.addCompletedJobIds(jobIDStr);
                reporter.addJob(job);
            } else {
                event = new Event.JobProgressEvent(dagNode);
            }
            reporter.addJobIdToProgress(jobIDStr, getJobProgress());
            pushWorkflowProgress(queryId, reporter);
            reporter.pushEvent(queryId, event);
        }
    } catch (IOException e) {
        LOG.error("Error getting job info!", e);
    }
}

Example 8 with Event

use of com.twitter.ambrose.model.Event in project ambrose by twitter.

the class EmbeddedAmbroseHiveProgressReporter method initInternal.

@SuppressWarnings("unchecked")
private void initInternal() {
    try {
        Field eventMapField = AmbroseHiveUtil.getInternalField(InMemoryStatsService.class, "eventMap");
        _eventMap = (SortedMap<Integer, Event<?>>) eventMapField.get(service);
    } catch (Exception e) {
        LOG.fatal("Can't access to eventMap/dagNodeNameMap fields at " + InMemoryStatsService.class.getName() + "!");
        throw new RuntimeException("Incompatible Hive API found!", e);
    }
}

Example 9 with Event

use of com.twitter.ambrose.model.Event in project ambrose by twitter.

the class AmbrosePigProgressNotificationListener method jobStartedNotification.

/**
   * Called with a job is started. This is the first time that we are notified of a new jobId for a
   * launched job. Hence this method binds the jobId to the DAGNode and pushes a status event.
   * @param scriptId scriptId of the running script
   * @param assignedJobId the jobId assigned to the job started.
   */
@Override
public void jobStartedNotification(String scriptId, String assignedJobId) {
    log.info("jobStartedNotification - scriptId " + scriptId + "jobId " + assignedJobId);
    // up it's scope and bind the jobId to the DAGNode with the same scope.
    for (JobStats jobStats : pigConfig.getJobGraph()) {
        if (assignedJobId.equals(jobStats.getJobId())) {
            log.info("jobStartedNotification - scope " + jobStats.getName() + " is jobId " + assignedJobId);
            DAGNode<PigJob> node = this.dagNodeNameMap.get(jobStats.getName());
            if (node == null) {
                log.warn("jobStartedNotification - unrecognized operator name found (" + jobStats.getName() + ") for jobId " + assignedJobId);
                return;
            }
            PigJob job = node.getJob();
            job.setId(assignedJobId);
            mapReduceHelper.addMapReduceJobState(job, pigConfig.getJobClient());
            dagNodeJobIdMap.put(job.getId(), node);
            AmbroseUtils.pushEvent(statsWriteService, scriptId, new Event.JobStartedEvent(node));
        }
    }
}

Example 10 with Event

use of com.twitter.ambrose.model.Event in project ambrose by twitter.

the class HRavenStatsReadService method getEventsSinceId.

@SuppressWarnings("rawtypes")
@Override
public List<Event> getEventsSinceId(String workflowId, int eventId, int maxEvents) throws IOException {
    Preconditions.checkArgument(maxEvents > 0);
    WorkflowId id = WorkflowId.parseString(workflowId);
    FlowEventKey flowEventKey = new FlowEventKey(toFlowKey(id), eventId);
    List<FlowEvent> flowEventList = flowEventService.getFlowEventsSince(flowEventKey);
    // TODO push this limit into the FlowEventService
    int numElems = 0;
    List<Event> workflowEvents = Lists.newArrayListWithCapacity(maxEvents);
    for (FlowEvent flowEvent : flowEventList) {
        if (numElems >= maxEvents) {
            break;
        }
        String eventDataJson = flowEvent.getEventDataJSON();
        try {
            Event event = Event.fromJson(eventDataJson);
            numElems++;
            workflowEvents.add(event);
        } catch (JsonMappingException e) {
            LOG.error("Could not deserialize json: " + eventDataJson, e);
        }
    }
    return workflowEvents;
}