Examples with SummaryEvent org.apache.tez.dag.history.SummaryEvent used on opensource projects

Example 1 with SummaryEvent

use of org.apache.tez.dag.history.SummaryEvent in project tez by apache.

the class TestHistoryEventsProtoConversion method testSummaryProtoConversion.

private HistoryEvent testSummaryProtoConversion(HistoryEvent historyEvent) throws IOException, TezException {
    SummaryEvent event = (SummaryEvent) historyEvent;
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    HistoryEvent deserializedEvent = null;
    event.toSummaryProtoStream(os);
    os.flush();
    os.close();
    LOG.info("Serialized event to byte array" + ", eventType=" + historyEvent.getEventType() + ", bufLen=" + os.toByteArray().length);
    SummaryEventProto summaryEventProto = SummaryEventProto.parseDelimitedFrom(new ByteArrayInputStream(os.toByteArray()));
    deserializedEvent = ReflectionUtils.createClazzInstance(event.getClass().getName());
    ((SummaryEvent) deserializedEvent).fromSummaryProtoStream(summaryEventProto);
    return deserializedEvent;
}

Example 2 with SummaryEvent

use of org.apache.tez.dag.history.SummaryEvent in project tez by apache.

the class RecoveryService method handle.

public void handle(DAGHistoryEvent event) throws IOException {
    if (stopped.get()) {
        LOG.warn("Igoring event as service stopped, eventType" + event.getHistoryEvent().getEventType());
        return;
    }
    HistoryEventType eventType = event.getHistoryEvent().getEventType();
    if (recoveryFatalErrorOccurred.get()) {
        return;
    }
    if (!started.get()) {
        LOG.warn("Adding event of type " + eventType + " to queue as service not started");
        addToEventQueue(event);
        return;
    }
    TezDAGID dagId = event.getDagID();
    if (eventType.equals(HistoryEventType.DAG_SUBMITTED)) {
        DAGSubmittedEvent dagSubmittedEvent = (DAGSubmittedEvent) event.getHistoryEvent();
        String dagName = dagSubmittedEvent.getDAGName();
        if (dagName != null && dagName.startsWith(TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX)) {
            // Skip recording pre-warm DAG events
            skippedDAGs.add(dagId);
            return;
        }
    }
    if (dagId == null || skippedDAGs.contains(dagId)) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Skipping event for DAG" + ", eventType=" + eventType + ", dagId=" + (dagId == null ? "null" : dagId.toString()) + ", isSkippedDAG=" + (dagId == null ? "null" : skippedDAGs.contains(dagId)));
        }
        return;
    }
    if (event.getHistoryEvent() instanceof SummaryEvent) {
        synchronized (lock) {
            if (stopped.get()) {
                LOG.warn("Igoring event as service stopped, eventType" + event.getHistoryEvent().getEventType());
                return;
            }
            try {
                SummaryEvent summaryEvent = (SummaryEvent) event.getHistoryEvent();
                handleSummaryEvent(dagId, eventType, summaryEvent);
                if (summaryEvent.writeToRecoveryImmediately()) {
                    handleRecoveryEvent(event);
                    // outputStream may already be closed and removed
                    if (outputStreamMap.containsKey(event.getDagID())) {
                        doFlush(outputStreamMap.get(event.getDagID()), appContext.getClock().getTime());
                    }
                } else {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Queueing Non-immediate Summary/Recovery event of type" + eventType.name());
                    }
                    addToEventQueue(event);
                }
                if (eventType.equals(HistoryEventType.DAG_FINISHED)) {
                    LOG.info("DAG completed" + ", dagId=" + event.getDagID() + ", queueSize=" + eventQueue.size());
                    completedDAGs.add(dagId);
                    if (outputStreamMap.containsKey(dagId)) {
                        try {
                            outputStreamMap.get(dagId).close();
                            outputStreamMap.remove(dagId);
                        } catch (IOException ioe) {
                            LOG.warn("Error when trying to flush/close recovery file for" + " dag, dagId=" + event.getDagID());
                        }
                    }
                }
            } catch (IOException ioe) {
                LOG.error("Error handling summary event" + ", eventType=" + event.getHistoryEvent().getEventType(), ioe);
                createFatalErrorFlagDir();
                if (eventType.equals(HistoryEventType.DAG_SUBMITTED)) {
                    // Throw error to tell client that dag submission failed
                    throw ioe;
                }
            }
        }
    } else {
        // All other events just get queued
        if (LOG.isDebugEnabled()) {
            LOG.debug("Queueing Non-Summary Recovery event of type " + eventType.name());
        }
        addToEventQueue(event);
    }
}