Examples with VertexConfigurationDoneEvent org.apache.tez.dag.history.events.VertexConfigurationDoneEvent used on opensource projects

Example 1 with VertexConfigurationDoneEvent

use of org.apache.tez.dag.history.events.VertexConfigurationDoneEvent in project tez by apache.

the class VertexImpl method logVertexConfigurationDoneEvent.

void logVertexConfigurationDoneEvent() {
    if (recoveryData == null || !recoveryData.shouldSkipInit()) {
        Map<String, EdgeProperty> sourceEdgeProperties = new HashMap<String, EdgeProperty>();
        for (Map.Entry<Vertex, Edge> entry : this.sourceVertices.entrySet()) {
            sourceEdgeProperties.put(entry.getKey().getName(), entry.getValue().getEdgeProperty());
        }
        VertexConfigurationDoneEvent reconfigureDoneEvent = new VertexConfigurationDoneEvent(vertexId, clock.getTime(), numTasks, taskLocationHints == null ? null : VertexLocationHint.create(Lists.newArrayList(taskLocationHints)), sourceEdgeProperties, rootInputSpecs, setParallelismCalledFlag);
        this.appContext.getHistoryHandler().handle(new DAGHistoryEvent(getDAGId(), reconfigureDoneEvent));
    }
}

Example 2 with VertexConfigurationDoneEvent

use of org.apache.tez.dag.history.events.VertexConfigurationDoneEvent in project tez by apache.

the class VertexImpl method assignVertexManager.

private void assignVertexManager() throws TezException {
    // VertexReconfigureDoneEvent will be logged
    if (recoveryData != null && recoveryData.shouldSkipInit()) {
        // Replace the original VertexManager with NoOpVertexManager if the reconfiguration is done in the last AM attempt
        VertexConfigurationDoneEvent reconfigureDoneEvent = recoveryData.getVertexConfigurationDoneEvent();
        if (LOG.isInfoEnabled()) {
            LOG.info("VertexManager reconfiguration is done in the last AM Attempt" + ", use NoOpVertexManager to replace it, vertexId=" + logIdentifier);
            LOG.info("VertexReconfigureDoneEvent=" + reconfigureDoneEvent);
        }
        NonSyncByteArrayOutputStream out = new NonSyncByteArrayOutputStream();
        try {
            reconfigureDoneEvent.toProtoStream(out);
        } catch (IOException e) {
            throw new TezUncheckedException("Unable to deserilize VertexReconfigureDoneEvent");
        }
        this.vertexManager = new VertexManager(VertexManagerPluginDescriptor.create(NoOpVertexManager.class.getName()).setUserPayload(UserPayload.create(ByteBuffer.wrap(out.toByteArray()))), dagUgi, this, appContext, stateChangeNotifier);
        return;
    }
    boolean hasBipartite = false;
    boolean hasOneToOne = false;
    boolean hasCustom = false;
    if (sourceVertices != null) {
        for (Edge edge : sourceVertices.values()) {
            switch(edge.getEdgeProperty().getDataMovementType()) {
                case SCATTER_GATHER:
                    hasBipartite = true;
                    break;
                case ONE_TO_ONE:
                    hasOneToOne = true;
                    break;
                case BROADCAST:
                    break;
                case CUSTOM:
                    hasCustom = true;
                    break;
                default:
                    throw new TezUncheckedException("Unknown data movement type: " + edge.getEdgeProperty().getDataMovementType());
            }
        }
    }
    boolean hasUserVertexManager = vertexPlan.hasVertexManagerPlugin();
    if (hasUserVertexManager) {
        VertexManagerPluginDescriptor pluginDesc = DagTypeConverters.convertVertexManagerPluginDescriptorFromDAGPlan(vertexPlan.getVertexManagerPlugin());
        LOG.info("Setting user vertex manager plugin: " + pluginDesc.getClassName() + " on vertex: " + getLogIdentifier());
        vertexManager = new VertexManager(pluginDesc, dagUgi, this, appContext, stateChangeNotifier);
    } else {
        // Else we use the default ImmediateStartVertexManager
        if (inputsWithInitializers != null) {
            LOG.info("Setting vertexManager to RootInputVertexManager for " + logIdentifier);
            vertexManager = new VertexManager(RootInputVertexManager.createConfigBuilder(vertexConf).build(), dagUgi, this, appContext, stateChangeNotifier);
        } else if (hasOneToOne && !hasCustom) {
            LOG.info("Setting vertexManager to InputReadyVertexManager for " + logIdentifier);
            vertexManager = new VertexManager(VertexManagerPluginDescriptor.create(InputReadyVertexManager.class.getName()), dagUgi, this, appContext, stateChangeNotifier);
        } else if (hasBipartite && !hasCustom) {
            LOG.info("Setting vertexManager to ShuffleVertexManager for " + logIdentifier);
            // shuffle vertex manager needs a conf payload
            vertexManager = new VertexManager(ShuffleVertexManager.createConfigBuilder(vertexConf).build(), dagUgi, this, appContext, stateChangeNotifier);
        } else {
            // schedule all tasks upon vertex start. Default behavior.
            LOG.info("Setting vertexManager to ImmediateStartVertexManager for " + logIdentifier);
            vertexManager = new VertexManager(VertexManagerPluginDescriptor.create(ImmediateStartVertexManager.class.getName()), dagUgi, this, appContext, stateChangeNotifier);
        }
    }
}

Example 3 with VertexConfigurationDoneEvent

use of org.apache.tez.dag.history.events.VertexConfigurationDoneEvent in project tez by apache.

the class TestDAGRecovery method testVertexRecoverFromStart.

/**
 * RecoveryEvents:
 *  DAG:  DAGInitedEvent -> DAGStartedEvent
 *  V1:   VertexReconfigrationDoneEvent -> VertexInitializedEvent -> VertexStartedEvent
 *
 * V1 skip initialization.
 */
@Test(timeout = 5000)
public void testVertexRecoverFromStart() {
    initMockDAGRecoveryDataForVertex();
    List<TezEvent> inputGeneratedTezEvents = new ArrayList<TezEvent>();
    VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, "vertex1", 0L, v1InitedTime, v1NumTask, "", null, inputGeneratedTezEvents, null);
    VertexConfigurationDoneEvent v1ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v1Id, 0L, v1NumTask, null, null, null, true);
    VertexStartedEvent v1StartedEvent = new VertexStartedEvent(v1Id, 0L, v1StartedTime);
    VertexRecoveryData vertexRecoveryData = new VertexRecoveryData(v1InitedEvent, v1ReconfigureDoneEvent, v1StartedEvent, null, new HashMap<TezTaskID, TaskRecoveryData>(), false);
    doReturn(vertexRecoveryData).when(dagRecoveryData).getVertexRecoveryData(v1Id);
    DAGEventRecoverEvent recoveryEvent = new DAGEventRecoverEvent(dagId, dagRecoveryData);
    dag.handle(recoveryEvent);
    dispatcher.await();
    VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
    VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
    VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
    assertEquals(DAGState.RUNNING, dag.getState());
    // v1 skip initialization
    assertEquals(VertexState.RUNNING, v1.getState());
    assertEquals(v1InitedTime, v1.initedTime);
    assertEquals(v1StartedTime, v1.startedTime);
    assertEquals(v1NumTask, v1.getTotalTasks());
    assertEquals(VertexState.RUNNING, v2.getState());
    assertEquals(VertexState.RUNNING, v3.getState());
}

Example 4 with VertexConfigurationDoneEvent

use of org.apache.tez.dag.history.events.VertexConfigurationDoneEvent in project tez by apache.

the class TestDAGRecovery method initMockDAGRecoveryDataForTask.

// ///////////////////////////// Task ////////////////////////////////////////////////////////////
private void initMockDAGRecoveryDataForTask() {
    List<TezEvent> inputGeneratedTezEvents = new ArrayList<TezEvent>();
    VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, "vertex1", 0L, v1InitedTime, v1NumTask, "", null, inputGeneratedTezEvents, null);
    Map<String, InputSpecUpdate> rootInputSpecs = new HashMap<String, InputSpecUpdate>();
    VertexConfigurationDoneEvent v1ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v1Id, 0L, v1NumTask, null, null, rootInputSpecs, true);
    VertexStartedEvent v1StartedEvent = new VertexStartedEvent(v1Id, 0L, v1StartedTime);
    VertexRecoveryData v1RecoveryData = new VertexRecoveryData(v1InitedEvent, v1ReconfigureDoneEvent, v1StartedEvent, null, new HashMap<TezTaskID, TaskRecoveryData>(), false);
    DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, "user", "dagName", null);
    DAGStartedEvent dagStartedEvent = new DAGStartedEvent(dagId, dagStartedTime, "user", "dagName");
    doReturn(v1RecoveryData).when(dagRecoveryData).getVertexRecoveryData(v1Id);
    doReturn(dagInitedEvent).when(dagRecoveryData).getDAGInitializedEvent();
    doReturn(dagStartedEvent).when(dagRecoveryData).getDAGStartedEvent();
}

Example 5 with VertexConfigurationDoneEvent

use of org.apache.tez.dag.history.events.VertexConfigurationDoneEvent in project tez by apache.

the class TestHistoryEventTimelineConversion method testConvertVertexReconfigreDoneEvent.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testConvertVertexReconfigreDoneEvent() {
    TezVertexID vId = tezVertexID;
    Map<String, EdgeProperty> edgeMgrs = new HashMap<String, EdgeProperty>();
    edgeMgrs.put("a", EdgeProperty.create(EdgeManagerPluginDescriptor.create("a.class").setHistoryText("text"), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")));
    VertexConfigurationDoneEvent event = new VertexConfigurationDoneEvent(vId, 0L, 1, null, edgeMgrs, null, true);
    List<TimelineEntity> entities = HistoryEventTimelineConversion.convertToTimelineEntities(event);
    Assert.assertEquals(1, entities.size());
    TimelineEntity timelineEntity = entities.get(0);
    Assert.assertEquals(ATSConstants.TEZ_VERTEX_ID, timelineEntity.getEntityType());
    Assert.assertEquals(vId.toString(), timelineEntity.getEntityId());
    Assert.assertEquals(1, timelineEntity.getEvents().size());
    final Map<String, Set<Object>> primaryFilters = timelineEntity.getPrimaryFilters();
    Assert.assertEquals(2, primaryFilters.size());
    Assert.assertTrue(primaryFilters.get(ATSConstants.APPLICATION_ID).contains(applicationId.toString()));
    Assert.assertTrue(primaryFilters.get(EntityTypes.TEZ_DAG_ID.name()).contains(tezDAGID.toString()));
    TimelineEvent evt = timelineEntity.getEvents().get(0);
    Assert.assertEquals(HistoryEventType.VERTEX_CONFIGURE_DONE.name(), evt.getEventType());
    Assert.assertEquals(1, evt.getEventInfo().get(ATSConstants.NUM_TASKS));
    Assert.assertNotNull(evt.getEventInfo().get(ATSConstants.UPDATED_EDGE_MANAGERS));
    Map<String, Object> updatedEdgeMgrs = (Map<String, Object>) evt.getEventInfo().get(ATSConstants.UPDATED_EDGE_MANAGERS);
    Assert.assertEquals(1, updatedEdgeMgrs.size());
    Assert.assertTrue(updatedEdgeMgrs.containsKey("a"));
    Map<String, Object> updatedEdgeMgr = (Map<String, Object>) updatedEdgeMgrs.get("a");
    Assert.assertEquals(DataMovementType.CUSTOM.name(), updatedEdgeMgr.get(DAGUtils.DATA_MOVEMENT_TYPE_KEY));
    Assert.assertEquals("In", updatedEdgeMgr.get(DAGUtils.EDGE_DESTINATION_CLASS_KEY));
    Assert.assertEquals("a.class", updatedEdgeMgr.get(DAGUtils.EDGE_MANAGER_CLASS_KEY));
    Assert.assertEquals(1, timelineEntity.getOtherInfo().get(ATSConstants.NUM_TASKS));
}