Examples with VertexEvent org.apache.tez.dag.app.dag.event.VertexEvent used on opensource projects

Example 31 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class TestVertexImpl method testVertexWithMultipleInitializers2.

@Test(timeout = 5000)
public void testVertexWithMultipleInitializers2() throws Exception {
    useCustomInitializer = true;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithMultipleInitializers("TestInputInitializer");
    setupPostDagCreation();
    VertexImplWithControlledInitializerManager v1 = (VertexImplWithControlledInitializerManager) vertices.get("vertex1");
    dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_INIT));
    dispatcher.await();
    Assert.assertEquals(VertexState.INITIALIZING, v1.getState());
    RootInputInitializerManagerControlled initializerManager1 = v1.getRootInputInitializerManager();
    List<TaskLocationHint> v1Hints = createTaskLocationHints(5);
    // Complete initializer which does not set parallelism
    initializerManager1.completeInputInitialization(1);
    dispatcher.await();
    Assert.assertEquals(VertexState.INITIALIZING, v1.getState());
    Assert.assertEquals(1, v1.numInitializerCompletionsHandled);
    // Complete second initializer which sets parallelism
    initializerManager1.completeInputInitialization(0, 5, v1Hints);
    dispatcher.await();
    Assert.assertEquals(VertexState.INITED, v1.getState());
    Assert.assertEquals(2, v1.numInitializerCompletionsHandled);
}

Example 32 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class TestVertexImpl method testVertexRootInputSpecUpdatePerTask.

@Test(timeout = 5000)
public void testVertexRootInputSpecUpdatePerTask() throws Exception {
    useCustomInitializer = true;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithInputInitializer("TestInputInitializer");
    setupPostDagCreation();
    int expectedNumTasks = RootInputSpecUpdaterVertexManager.NUM_TASKS;
    VertexImplWithControlledInitializerManager v4 = (VertexImplWithControlledInitializerManager) vertices.get("vertex4");
    dispatcher.getEventHandler().handle(new VertexEvent(v4.getVertexId(), VertexEventType.V_INIT));
    dispatcher.await();
    Assert.assertEquals(VertexState.INITIALIZING, v4.getState());
    RootInputInitializerManagerControlled initializerManager1 = v4.getRootInputInitializerManager();
    initializerManager1.completeInputInitialization();
    dispatcher.await();
    Assert.assertEquals(VertexState.INITED, v4.getState());
    Assert.assertEquals(expectedNumTasks, v4.getTotalTasks());
    Assert.assertEquals(RootInputSpecUpdaterVertexManager.class.getName(), v4.getVertexManager().getPlugin().getClass().getName());
    Assert.assertEquals(true, initializerManager1.hasShutDown);
    for (int i = 0; i < expectedNumTasks; i++) {
        List<InputSpec> inputSpecs = v4.getInputSpecList(i);
        Assert.assertEquals(1, inputSpecs.size());
        Assert.assertEquals(i + 1, inputSpecs.get(0).getPhysicalEdgeCount());
    }
}

Example 33 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class TestVertexImpl method testNonExistOutputCommitter.

@Test(timeout = 5000)
public void testNonExistOutputCommitter() throws TezException {
    setupPreDagCreation();
    dagPlan = createDAGPlanWithNonExistOutputCommitter();
    setupPostDagCreation();
    VertexImpl v1 = vertices.get("vertex1");
    v1.handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_INIT));
    Assert.assertEquals(VertexState.FAILED, v1.getState());
    Assert.assertEquals(VertexTerminationCause.INIT_FAILURE, v1.getTerminationCause());
    Assert.assertTrue(StringUtils.join(v1.getDiagnostics(), "").contains("java.lang.ClassNotFoundException: non-exist-output-committer"));
}

Example 34 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class TestVertexImpl method testExceptionFromVM_OnRootVertexInitialized.

@Test(timeout = 5000)
public void testExceptionFromVM_OnRootVertexInitialized() throws Exception {
    useCustomInitializer = true;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithVMException("TestInputInitializer", VMExceptionLocation.OnRootVertexInitialized);
    setupPostDagCreation();
    VertexImplWithControlledInitializerManager v1 = (VertexImplWithControlledInitializerManager) vertices.get("vertex1");
    dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_INIT));
    dispatcher.await();
    RootInputInitializerManagerControlled initializerManager1 = v1.getRootInputInitializerManager();
    initializerManager1.completeInputInitialization();
    dispatcher.await();
    Assert.assertEquals(VertexManagerWithException.class, v1.vertexManager.getPlugin().getClass());
    Assert.assertEquals(VertexState.FAILED, v1.getState());
    Assert.assertTrue(initializerManager1.hasShutDown);
    String diagnostics = StringUtils.join(v1.getDiagnostics(), ",");
    Assert.assertTrue(diagnostics.contains(VMExceptionLocation.OnRootVertexInitialized.name()));
    Assert.assertEquals(VertexTerminationCause.AM_USERCODE_FAILURE, v1.getTerminationCause());
}

Example 35 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class TestVertexImpl method testVertexConfiguredDoneByVMBeforeEdgeDefined.

@Test(timeout = 5000)
public void testVertexConfiguredDoneByVMBeforeEdgeDefined() throws Exception {
    // Race when a source vertex manages to start before the target vertex has
    // been initialized
    setupPreDagCreation();
    dagPlan = createSamplerDAGPlan(true);
    setupPostDagCreation();
    VertexImpl vA = vertices.get("A");
    VertexImpl vB = vertices.get("B");
    VertexImpl vC = vertices.get("C");
    TestUpdateListener listener = new TestUpdateListener();
    updateTracker.registerForVertexUpdates(vB.getName(), EnumSet.of(org.apache.tez.dag.api.event.VertexState.CONFIGURED), listener);
    // fudge the vm so we can do custom stuff
    vB.vertexManager = new VertexManager(VertexManagerPluginDescriptor.create(VertexManagerPluginForTest.class.getName()), UserGroupInformation.getCurrentUser(), vB, appContext, mock(StateChangeNotifier.class));
    vB.vertexReconfigurationPlanned();
    dispatcher.getEventHandler().handle(new VertexEvent(vA.getVertexId(), VertexEventType.V_INIT));
    dispatcher.getEventHandler().handle(new VertexEvent(vA.getVertexId(), VertexEventType.V_START));
    dispatcher.await();
    Assert.assertEquals(VertexState.INITIALIZING, vA.getState());
    Assert.assertEquals(VertexState.INITIALIZING, vB.getState());
    Assert.assertEquals(VertexState.INITIALIZING, vC.getState());
    // setting the edge manager should vA to start
    EdgeManagerPluginDescriptor mockEdgeManagerDescriptor = EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName());
    Edge e = vC.sourceVertices.get(vA);
    Assert.assertNull(e.getEdgeManager());
    e.setCustomEdgeManager(mockEdgeManagerDescriptor);
    dispatcher.await();
    Assert.assertEquals(VertexState.RUNNING, vA.getState());
    Assert.assertEquals(VertexState.INITIALIZING, vB.getState());
    Assert.assertEquals(VertexState.INITIALIZING, vC.getState());
    // vB is not configured yet. Edge to C is not configured. So it should not send configured event
    // even thought VM says its doneConfiguring vertex
    vB.doneReconfiguringVertex();
    Assert.assertEquals(0, listener.events.size());
    // complete configuration and verify getting configured signal from vB
    EdgeProperty edgeProp = EdgeProperty.create(mockEdgeManagerDescriptor, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"));
    Map<String, EdgeProperty> edges = Maps.newHashMap();
    edges.put("B", edgeProp);
    vC.reconfigureVertex(2, vertexLocationHint, edges);
    dispatcher.await();
    Assert.assertEquals(1, listener.events.size());
    Assert.assertEquals(vB.getName(), listener.events.get(0).getVertexName());
    Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.CONFIGURED, listener.events.get(0).getVertexState());
    updateTracker.unregisterForVertexUpdates(vB.getName(), listener);
    Assert.assertEquals(VertexState.RUNNING, vA.getState());
    Assert.assertEquals(VertexState.RUNNING, vB.getState());
    Assert.assertEquals(VertexState.RUNNING, vC.getState());
    Assert.assertNotNull(vA.getTask(0));
    Assert.assertNotNull(vB.getTask(0));
    Assert.assertNotNull(vC.getTask(0));
}