Examples with VertexEvent org.apache.tez.dag.app.dag.event.VertexEvent used on opensource projects

Example 1 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class VertexImpl method setParallelismWrapper.

private void setParallelismWrapper(int parallelism, VertexLocationHint vertexLocationHint, Map<String, EdgeProperty> sourceEdgeProperties, Map<String, InputSpecUpdate> rootInputSpecUpdates, boolean fromVertexManager) throws AMUserCodeException {
    Preconditions.checkArgument(parallelism >= 0, "Parallelism must be >=0. Value: " + parallelism + " for vertex: " + logIdentifier);
    writeLock.lock();
    this.setParallelismCalledFlag = true;
    try {
        // disallow changing things after a vertex has started
        if (!tasksNotYetScheduled) {
            String msg = "setParallelism cannot be called after scheduling tasks. Vertex: " + getLogIdentifier();
            LOG.info(msg);
            throw new TezUncheckedException(msg);
        }
        if (fromVertexManager && canInitVertex()) {
            // vertex is fully defined. setParallelism has been called. VertexManager should have
            // informed us about this. Otherwise we would have notified listeners that we are fully
            // defined before we are actually fully defined
            Preconditions.checkState(vertexToBeReconfiguredByManager, "Vertex is fully configured but still" + " the reconfiguration API has been called. VertexManager must notify the framework using " + " context.vertexReconfigurationPlanned() before re-configuring the vertex." + " vertexId=" + logIdentifier);
        }
        // Input initializer/Vertex Manager/1-1 split expected to set parallelism.
        if (numTasks == -1) {
            if (getState() != VertexState.INITIALIZING) {
                throw new TezUncheckedException("Vertex state is not Initializing. Value: " + getState() + " for vertex: " + logIdentifier);
            }
            if (sourceEdgeProperties != null) {
                for (Map.Entry<String, EdgeProperty> entry : sourceEdgeProperties.entrySet()) {
                    LOG.info("Replacing edge manager for source:" + entry.getKey() + " destination: " + getLogIdentifier());
                    Vertex sourceVertex = appContext.getCurrentDAG().getVertex(entry.getKey());
                    Edge edge = sourceVertices.get(sourceVertex);
                    try {
                        edge.setEdgeProperty(entry.getValue());
                    } catch (Exception e) {
                        throw new TezUncheckedException("Fail to update EdgeProperty for Edge," + "sourceVertex:" + edge.getSourceVertexName() + "destinationVertex:" + edge.getDestinationVertexName(), e);
                    }
                }
            }
            if (rootInputSpecUpdates != null) {
                LOG.info("Got updated RootInputsSpecs: " + rootInputSpecUpdates.toString());
                // Sanity check for correct number of updates.
                for (Entry<String, InputSpecUpdate> rootInputSpecUpdateEntry : rootInputSpecUpdates.entrySet()) {
                    Preconditions.checkState(rootInputSpecUpdateEntry.getValue().isForAllWorkUnits() || (rootInputSpecUpdateEntry.getValue().getAllNumPhysicalInputs() != null && rootInputSpecUpdateEntry.getValue().getAllNumPhysicalInputs().size() == parallelism), "Not enough input spec updates for root input named " + rootInputSpecUpdateEntry.getKey());
                }
                this.rootInputSpecs.putAll(rootInputSpecUpdates);
            }
            int oldNumTasks = numTasks;
            this.numTasks = parallelism;
            stateChangeNotifier.stateChanged(vertexId, new VertexStateUpdateParallelismUpdated(vertexName, numTasks, oldNumTasks));
            this.createTasks();
            setVertexLocationHint(vertexLocationHint);
            LOG.info("Vertex " + getLogIdentifier() + " parallelism set to " + parallelism);
            if (canInitVertex()) {
                getEventHandler().handle(new VertexEvent(getVertexId(), VertexEventType.V_READY_TO_INIT));
            }
        } else {
            // This is an artificial restriction since there's no way of knowing whether a VertexManager
            // will attempt to update root input specs. When parallelism has not been initialized, the
            // Vertex will not be in started state so it's safe to update the specifications.
            // TODO TEZ-937 - add e mechanism to query vertex managers, or for VMs to indicate readines
            // for a vertex to start.
            Preconditions.checkState(rootInputSpecUpdates == null, "Root Input specs can only be updated when the vertex is configured with -1 tasks");
            int oldNumTasks = numTasks;
            // start buffering incoming events so that we can re-route existing events
            for (Edge edge : sourceVertices.values()) {
                edge.startEventBuffering();
            }
            if (parallelism == numTasks) {
                LOG.info("setParallelism same as current value: " + parallelism + " for vertex: " + logIdentifier);
                Preconditions.checkArgument(sourceEdgeProperties != null, "Source edge managers or RootInputSpecs must be set when not changing parallelism");
            } else {
                LOG.info("Resetting vertex location hints due to change in parallelism for vertex: " + logIdentifier);
                vertexLocationHint = null;
                if (parallelism > numTasks) {
                    addTasks((parallelism));
                } else if (parallelism < numTasks) {
                    removeTasks(parallelism);
                }
            }
            Preconditions.checkState(this.numTasks == parallelism, getLogIdentifier());
            // set new vertex location hints
            setVertexLocationHint(vertexLocationHint);
            LOG.info("Vertex " + getLogIdentifier() + " parallelism set to " + parallelism + " from " + oldNumTasks);
            // notify listeners
            stateChangeNotifier.stateChanged(vertexId, new VertexStateUpdateParallelismUpdated(vertexName, numTasks, oldNumTasks));
            assert tasks.size() == numTasks;
            // set new edge managers
            if (sourceEdgeProperties != null) {
                for (Map.Entry<String, EdgeProperty> entry : sourceEdgeProperties.entrySet()) {
                    LOG.info("Replacing edge manager for source:" + entry.getKey() + " destination: " + getLogIdentifier());
                    Vertex sourceVertex = appContext.getCurrentDAG().getVertex(entry.getKey());
                    Edge edge = sourceVertices.get(sourceVertex);
                    try {
                        edge.setEdgeProperty(entry.getValue());
                    } catch (Exception e) {
                        throw new TezUncheckedException(e);
                    }
                }
            }
            // stop buffering events
            for (Edge edge : sourceVertices.values()) {
                edge.stopEventBuffering();
            }
        }
    } finally {
        writeLock.unlock();
    }
}

Example 2 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class VertexImpl method handle.

@Override
public /**
 * The only entry point to change the Vertex.
 */
void handle(VertexEvent event) {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Processing VertexEvent " + event.getVertexId() + " of type " + event.getType() + " while in state " + getInternalState() + ". Event: " + event);
    }
    try {
        writeLock.lock();
        VertexState oldState = getInternalState();
        try {
            getStateMachine().doTransition(event.getType(), event);
        } catch (InvalidStateTransitonException e) {
            String message = "Invalid event " + event.getType() + " on vertex " + this.vertexName + " with vertexId " + this.vertexId + " at current state " + oldState;
            LOG.error("Can't handle " + message, e);
            addDiagnostic(message);
            eventHandler.handle(new VertexEvent(this.vertexId, VertexEventType.V_INTERNAL_ERROR));
        } catch (RuntimeException e) {
            String message = "Uncaught Exception when handling event " + event.getType() + " on vertex " + this.vertexName + " with vertexId " + this.vertexId + " at current state " + oldState;
            LOG.error(message, e);
            addDiagnostic(message);
            if (!internalErrorTriggered.getAndSet(true)) {
                eventHandler.handle(new VertexEvent(this.vertexId, VertexEventType.V_INTERNAL_ERROR));
            }
        }
        if (oldState != getInternalState()) {
            LOG.info(logIdentifier + " transitioned from " + oldState + " to " + getInternalState() + " due to event " + event.getType());
        }
    } finally {
        writeLock.unlock();
    }
}

Example 3 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class TestVertexImpl method testVertexRootInputSpecUpdateAll.

@Test(timeout = 5000)
public void testVertexRootInputSpecUpdateAll() throws Exception {
    useCustomInitializer = true;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithInputInitializer("TestInputInitializer");
    setupPostDagCreation();
    int expectedNumTasks = RootInputSpecUpdaterVertexManager.NUM_TASKS;
    VertexImplWithControlledInitializerManager v3 = (VertexImplWithControlledInitializerManager) vertices.get("vertex3");
    dispatcher.getEventHandler().handle(new VertexEvent(v3.getVertexId(), VertexEventType.V_INIT));
    dispatcher.await();
    Assert.assertEquals(VertexState.INITIALIZING, v3.getState());
    RootInputInitializerManagerControlled initializerManager1 = v3.getRootInputInitializerManager();
    initializerManager1.completeInputInitialization();
    dispatcher.await();
    Assert.assertEquals(VertexState.INITED, v3.getState());
    Assert.assertEquals(expectedNumTasks, v3.getTotalTasks());
    Assert.assertEquals(RootInputSpecUpdaterVertexManager.class.getName(), v3.getVertexManager().getPlugin().getClass().getName());
    Assert.assertEquals(true, initializerManager1.hasShutDown);
    for (int i = 0; i < expectedNumTasks; i++) {
        List<InputSpec> inputSpecs = v3.getInputSpecList(i);
        Assert.assertEquals(1, inputSpecs.size());
        Assert.assertEquals(4, inputSpecs.get(0).getPhysicalEdgeCount());
    }
}

Example 4 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class TestVertexImpl method testVertexWithInputDistributor.

@Test(timeout = 5000)
public void testVertexWithInputDistributor() throws Exception {
    useCustomInitializer = true;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithInputDistributor("TestInputInitializer");
    setupPostDagCreation();
    VertexImplWithControlledInitializerManager v1 = (VertexImplWithControlledInitializerManager) vertices.get("vertex1");
    VertexImplWithControlledInitializerManager v2 = (VertexImplWithControlledInitializerManager) vertices.get("vertex2");
    dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_INIT));
    dispatcher.await();
    Assert.assertEquals(VertexState.INITIALIZING, v1.getState());
    Assert.assertEquals(VertexState.INITIALIZING, v2.getState());
    RootInputInitializerManagerControlled initializerManager1 = v1.getRootInputInitializerManager();
    byte[] payload = new byte[0];
    initializerManager1.completeInputDistribution(payload);
    dispatcher.await();
    // edge is still null so its initializing
    Assert.assertEquals(VertexState.INITIALIZING, v1.getState());
    Assert.assertEquals(true, initializerManager1.hasShutDown);
    Assert.assertEquals(2, v1.getTotalTasks());
    Assert.assertArrayEquals(payload, v1.getInputSpecList(0).get(0).getInputDescriptor().getUserPayload().deepCopyAsArray());
    EdgeManagerPluginDescriptor mockEdgeManagerDescriptor = EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName());
    Edge e = v2.sourceVertices.get(v1);
    Assert.assertNull(e.getEdgeManager());
    e.setCustomEdgeManager(mockEdgeManagerDescriptor);
    dispatcher.await();
    Assert.assertEquals(VertexState.INITED, v1.getState());
    Assert.assertEquals(VertexState.INITED, v2.getState());
}

Example 5 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class TestVertexImpl method testExceptionFromVM_Initialize.

@Test(timeout = 5000)
public void testExceptionFromVM_Initialize() throws TezException {
    useCustomInitializer = true;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithVMException("TestInputInitializer", VMExceptionLocation.Initialize);
    setupPostDagCreation();
    VertexImplWithControlledInitializerManager v1 = (VertexImplWithControlledInitializerManager) vertices.get("vertex1");
    dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_INIT));
    dispatcher.await();
    Assert.assertEquals(VertexState.FAILED, v1.getState());
    String diagnostics = StringUtils.join(v1.getDiagnostics(), ",");
    Assert.assertTrue(diagnostics.contains(VMExceptionLocation.Initialize.name()));
    Assert.assertEquals(VertexTerminationCause.AM_USERCODE_FAILURE, v1.getTerminationCause());
}