Examples with DataMovementEvent org.apache.tez.runtime.api.events.DataMovementEvent used on opensource projects

Example 6 with DataMovementEvent

use of org.apache.tez.runtime.api.events.DataMovementEvent in project tez by apache.

the class TestDAGImpl method testEdgeManager_RouteDataMovementEventToDestinationWithLegacyRouting.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testEdgeManager_RouteDataMovementEventToDestinationWithLegacyRouting() {
    // Remove after legacy routing is removed
    setupDAGWithCustomEdge(ExceptionLocation.RouteDataMovementEventToDestination, true);
    dispatcher.getEventHandler().handle(new DAGEvent(dagWithCustomEdge.getID(), DAGEventType.DAG_INIT));
    dispatcher.getEventHandler().handle(new DAGEventStartDag(dagWithCustomEdge.getID(), null));
    dispatcher.await();
    Assert.assertEquals(DAGState.RUNNING, dagWithCustomEdge.getState());
    VertexImpl v1 = (VertexImpl) dagWithCustomEdge.getVertex("vertex1");
    VertexImpl v2 = (VertexImpl) dagWithCustomEdge.getVertex("vertex2");
    dispatcher.await();
    Task t1 = v2.getTask(0);
    TaskAttemptImpl ta1 = (TaskAttemptImpl) t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskId(), 0));
    DataMovementEvent daEvent = DataMovementEvent.create(ByteBuffer.wrap(new byte[0]));
    TezEvent tezEvent = new TezEvent(daEvent, new EventMetaData(EventProducerConsumerType.INPUT, "vertex1", "vertex2", ta1.getID()));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v2.getVertexId(), Lists.newArrayList(tezEvent)));
    dispatcher.await();
    Assert.assertEquals(VertexState.FAILED, v2.getState());
    Assert.assertEquals(VertexState.KILLED, v1.getState());
    String diag = StringUtils.join(v2.getDiagnostics(), ",");
    Assert.assertTrue(diag.contains(ExceptionLocation.RouteDataMovementEventToDestination.name()));
}

Example 7 with DataMovementEvent

use of org.apache.tez.runtime.api.events.DataMovementEvent in project tez by apache.

the class TestCompositeDataMovementEvent method testGetEvents.

@Test(timeout = 5000)
public void testGetEvents() {
    int numOutputs = 0;
    int startIndex = 1;
    CompositeDataMovementEvent cdme2 = CompositeDataMovementEvent.create(startIndex, numOutputs, userPayload);
    for (DataMovementEvent dme : cdme2.getEvents()) {
        numOutputs++;
    }
    Assert.assertEquals(numOutputs, cdme2.getCount());
}

Example 8 with DataMovementEvent

use of org.apache.tez.runtime.api.events.DataMovementEvent in project tez by apache.

the class TestReduceProcessor method testReduceProcessor.

@Test(timeout = 5000)
public void testReduceProcessor() throws Exception {
    final String dagName = "mrdag0";
    String mapVertexName = MultiStageMRConfigUtil.getInitialMapVertexName();
    String reduceVertexName = MultiStageMRConfigUtil.getFinalReduceVertexName();
    JobConf jobConf = new JobConf(defaultConf);
    setUpJobConf(jobConf);
    MRHelpers.translateMRConfToTez(jobConf);
    jobConf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, 0);
    jobConf.set(MRFrameworkConfigs.TASK_LOCAL_RESOURCE_DIR, new Path(workDir, "localized-resources").toUri().toString());
    jobConf.setBoolean(MRJobConfig.MR_TEZ_SPLITS_VIA_EVENTS, false);
    Path mapInput = new Path(workDir, "map0");
    MapUtils.generateInputSplit(localFs, workDir, jobConf, mapInput, 10);
    InputSpec mapInputSpec = new InputSpec("NullSrcVertex", InputDescriptor.create(MRInputLegacy.class.getName()).setUserPayload(UserPayload.create(ByteBuffer.wrap(MRRuntimeProtos.MRInputUserPayloadProto.newBuilder().setConfigurationBytes(TezUtils.createByteStringFromConf(jobConf)).build().toByteArray()))), 1);
    OutputSpec mapOutputSpec = new OutputSpec("NullDestVertex", OutputDescriptor.create(OrderedPartitionedKVOutput.class.getName()).setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1);
    // Run a map
    TestUmbilical testUmbilical = new TestUmbilical();
    TezSharedExecutor sharedExecutor = new TezSharedExecutor(jobConf);
    LogicalIOProcessorRuntimeTask mapTask = MapUtils.createLogicalTask(localFs, workDir, jobConf, 0, mapInput, testUmbilical, dagName, mapVertexName, Collections.singletonList(mapInputSpec), Collections.singletonList(mapOutputSpec), sharedExecutor);
    mapTask.initialize();
    mapTask.run();
    mapTask.close();
    // One VME, One DME
    Assert.assertEquals(2, testUmbilical.getEvents().size());
    Assert.assertEquals(EventType.VERTEX_MANAGER_EVENT, testUmbilical.getEvents().get(0).getEventType());
    Assert.assertEquals(EventType.COMPOSITE_DATA_MOVEMENT_EVENT, testUmbilical.getEvents().get(1).getEventType());
    CompositeDataMovementEvent cdmEvent = (CompositeDataMovementEvent) testUmbilical.getEvents().get(1).getEvent();
    Assert.assertEquals(1, cdmEvent.getCount());
    DataMovementEvent dme = cdmEvent.getEvents().iterator().next();
    dme.setTargetIndex(0);
    LOG.info("Starting reduce...");
    JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(dagName));
    JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager();
    Token<JobTokenIdentifier> shuffleToken = new Token<JobTokenIdentifier>(identifier, jobTokenSecretManager);
    shuffleToken.setService(identifier.getJobId());
    jobConf.setOutputFormat(SequenceFileOutputFormat.class);
    jobConf.set(MRFrameworkConfigs.TASK_LOCAL_RESOURCE_DIR, new Path(workDir, "localized-resources").toUri().toString());
    jobConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
    FileOutputFormat.setOutputPath(jobConf, new Path(workDir, "output"));
    ProcessorDescriptor reduceProcessorDesc = ProcessorDescriptor.create(ReduceProcessor.class.getName()).setUserPayload(TezUtils.createUserPayloadFromConf(jobConf));
    InputSpec reduceInputSpec = new InputSpec(mapVertexName, InputDescriptor.create(OrderedGroupedInputLegacy.class.getName()).setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1);
    OutputSpec reduceOutputSpec = new OutputSpec("NullDestinationVertex", OutputDescriptor.create(MROutputLegacy.class.getName()).setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1);
    // Now run a reduce
    TaskSpec taskSpec = new TaskSpec(TezTestUtils.getMockTaskAttemptId(0, 1, 0, 0), dagName, reduceVertexName, -1, reduceProcessorDesc, Collections.singletonList(reduceInputSpec), Collections.singletonList(reduceOutputSpec), null, null);
    Map<String, ByteBuffer> serviceConsumerMetadata = new HashMap<String, ByteBuffer>();
    String auxiliaryService = jobConf.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID, TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT);
    serviceConsumerMetadata.put(auxiliaryService, ShuffleUtils.convertJobTokenToBytes(shuffleToken));
    Map<String, String> serviceProviderEnvMap = new HashMap<String, String>();
    ByteBuffer shufflePortBb = ByteBuffer.allocate(4).putInt(0, 8000);
    AuxiliaryServiceHelper.setServiceDataIntoEnv(auxiliaryService, shufflePortBb, serviceProviderEnvMap);
    LogicalIOProcessorRuntimeTask task = new LogicalIOProcessorRuntimeTask(taskSpec, 0, jobConf, new String[] { workDir.toString() }, new TestUmbilical(), serviceConsumerMetadata, serviceProviderEnvMap, HashMultimap.<String, String>create(), null, "", new ExecutionContextImpl("localhost"), Runtime.getRuntime().maxMemory(), true, new DefaultHadoopShim(), sharedExecutor);
    List<Event> destEvents = new LinkedList<Event>();
    destEvents.add(dme);
    task.initialize();
    OrderedGroupedInputLegacy sortedOut = (OrderedGroupedInputLegacy) task.getInputs().values().iterator().next();
    sortedOut.handleEvents(destEvents);
    task.run();
    task.close();
    sharedExecutor.shutdownNow();
    // MRTask mrTask = (MRTask)t.getProcessor();
    // TODO NEWTEZ Verify the partitioner has not been created
    // Likely not applicable anymore.
    // Assert.assertNull(mrTask.getPartitioner());
    // Only a task commit happens, hence the data is still in the temporary directory.
    Path reduceOutputDir = new Path(new Path(workDir, "output"), "_temporary/0/" + IDConverter.toMRTaskIdForOutput(TezTestUtils.getMockTaskId(0, 1, 0)));
    Path reduceOutputFile = new Path(reduceOutputDir, "part-v001-o000-00000");
    SequenceFile.Reader reader = new SequenceFile.Reader(localFs, reduceOutputFile, jobConf);
    LongWritable key = new LongWritable();
    Text value = new Text();
    long prev = Long.MIN_VALUE;
    while (reader.next(key, value)) {
        if (prev != Long.MIN_VALUE) {
            Assert.assertTrue(prev < key.get());
            prev = key.get();
        }
    }
    reader.close();
}

Example 9 with DataMovementEvent

use of org.apache.tez.runtime.api.events.DataMovementEvent in project tez by apache.

the class TestMockDAGAppMaster method testBasicEvents.

@Test(timeout = 5000)
public void testBasicEvents() throws Exception {
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
    tezClient.start();
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    mockApp.eventsDelegate = new TestEventsDelegate();
    DAG dag = DAG.create("testBasicEvents");
    Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 2);
    Vertex vB = Vertex.create("B", ProcessorDescriptor.create("Proc.class"), 2);
    Vertex vC = Vertex.create("C", ProcessorDescriptor.create("Proc.class"), 2);
    Vertex vD = Vertex.create("D", ProcessorDescriptor.create("Proc.class"), 2);
    dag.addVertex(vA).addVertex(vB).addVertex(vC).addVertex(vD).addEdge(Edge.create(vA, vB, EdgeProperty.create(DataMovementType.BROADCAST, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")))).addEdge(Edge.create(vA, vC, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")))).addEdge(Edge.create(vA, vD, EdgeProperty.create(DataMovementType.ONE_TO_ONE, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"))));
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    mockLauncher.startScheduling(true);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    VertexImpl vImpl = (VertexImpl) dagImpl.getVertex(vB.getName());
    TaskImpl tImpl = (TaskImpl) vImpl.getTask(1);
    TezTaskAttemptID taId = TezTaskAttemptID.getInstance(tImpl.getTaskId(), 0);
    List<TezEvent> tEvents = vImpl.getTaskAttemptTezEvents(taId, 0, 0, 1000).getEvents();
    // 2 from vA
    Assert.assertEquals(2, tEvents.size());
    Assert.assertEquals(vA.getName(), tEvents.get(0).getDestinationInfo().getEdgeVertexName());
    Assert.assertEquals(0, ((DataMovementEvent) tEvents.get(0).getEvent()).getSourceIndex());
    Assert.assertEquals(vA.getName(), tEvents.get(1).getDestinationInfo().getEdgeVertexName());
    Assert.assertEquals(0, ((DataMovementEvent) tEvents.get(1).getEvent()).getSourceIndex());
    int targetIndex1 = ((DataMovementEvent) tEvents.get(0).getEvent()).getTargetIndex();
    int targetIndex2 = ((DataMovementEvent) tEvents.get(1).getEvent()).getTargetIndex();
    // order of vA task completion can change order of events
    Assert.assertTrue("t1: " + targetIndex1 + " t2: " + targetIndex2, (targetIndex1 == 0 && targetIndex2 == 1) || (targetIndex1 == 1 && targetIndex2 == 0));
    vImpl = (VertexImpl) dagImpl.getVertex(vC.getName());
    tImpl = (TaskImpl) vImpl.getTask(1);
    taId = TezTaskAttemptID.getInstance(tImpl.getTaskId(), 0);
    tEvents = vImpl.getTaskAttemptTezEvents(taId, 0, 0, 1000).getEvents();
    // 2 from vA
    Assert.assertEquals(2, tEvents.size());
    Assert.assertEquals(vA.getName(), tEvents.get(0).getDestinationInfo().getEdgeVertexName());
    Assert.assertEquals(1, ((CompositeRoutedDataMovementEvent) tEvents.get(0).getEvent()).getSourceIndex());
    Assert.assertEquals(vA.getName(), tEvents.get(1).getDestinationInfo().getEdgeVertexName());
    Assert.assertEquals(1, ((CompositeRoutedDataMovementEvent) tEvents.get(1).getEvent()).getSourceIndex());
    targetIndex1 = ((CompositeRoutedDataMovementEvent) tEvents.get(0).getEvent()).getTargetIndex();
    targetIndex2 = ((CompositeRoutedDataMovementEvent) tEvents.get(1).getEvent()).getTargetIndex();
    // order of vA task completion can change order of events
    Assert.assertTrue("t1: " + targetIndex1 + " t2: " + targetIndex2, (targetIndex1 == 0 && targetIndex2 == 1) || (targetIndex1 == 1 && targetIndex2 == 0));
    vImpl = (VertexImpl) dagImpl.getVertex(vD.getName());
    tImpl = (TaskImpl) vImpl.getTask(1);
    taId = TezTaskAttemptID.getInstance(tImpl.getTaskId(), 0);
    tEvents = vImpl.getTaskAttemptTezEvents(taId, 0, 0, 1000).getEvents();
    // 1 from vA
    Assert.assertEquals(1, tEvents.size());
    Assert.assertEquals(vA.getName(), tEvents.get(0).getDestinationInfo().getEdgeVertexName());
    Assert.assertEquals(0, ((DataMovementEvent) tEvents.get(0).getEvent()).getTargetIndex());
    Assert.assertEquals(0, ((DataMovementEvent) tEvents.get(0).getEvent()).getSourceIndex());
    tezClient.stop();
}

Example 10 with DataMovementEvent

use of org.apache.tez.runtime.api.events.DataMovementEvent in project tez by apache.

the class TestVertexImpl method testRouteEvent_RecoveredEvent.

@Test(timeout = 5000)
public void testRouteEvent_RecoveredEvent() throws IOException {
    doReturn(historyEventHandler).when(appContext).getHistoryHandler();
    doReturn(true).when(appContext).isRecoveryEnabled();
    initAllVertices(VertexState.INITED);
    VertexImpl v1 = (VertexImpl) vertices.get("vertex1");
    VertexImpl v2 = (VertexImpl) vertices.get("vertex2");
    VertexImpl v3 = (VertexImpl) vertices.get("vertex3");
    startVertex(v1);
    startVertex(v2);
    TezTaskID taskId = TezTaskID.getInstance(v1.getVertexId(), 0);
    v1.handle(new VertexEventTaskCompleted(taskId, TaskState.SUCCEEDED));
    DataMovementEvent dmEvent = DataMovementEvent.create(0, ByteBuffer.wrap(new byte[0]));
    TezTaskAttemptID taId = TezTaskAttemptID.getInstance(taskId, 0);
    TezEvent tezEvent1 = new TezEvent(dmEvent, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", "vertex3", taId));
    v1.handle(new VertexEventRouteEvent(v1.getVertexId(), Lists.newArrayList(tezEvent1)));
    dispatcher.await();
    assertTrue(v3.pendingTaskEvents.size() != 0);
    // ArgumentCaptor<DAGHistoryEvent> argCaptor = ArgumentCaptor.forClass(DAGHistoryEvent.class);
    // verify(historyEventHandler, atLeast(1)).handle(argCaptor.capture());
    // verifyHistoryEvents(argCaptor.getAllValues(), HistoryEventType.VERTEX_DATA_MOVEMENT_EVENTS_GENERATED, 1);
    v3.scheduleTasks(Lists.newArrayList(ScheduleTaskRequest.create(0, null)));
    dispatcher.await();
    assertTrue(v3.pendingTaskEvents.size() == 0);
// recovery events is not only handled one time
// argCaptor = ArgumentCaptor.forClass(DAGHistoryEvent.class);
// verify(historyEventHandler, atLeast(1)).handle(argCaptor.capture());
// verifyHistoryEvents(argCaptor.getAllValues(), HistoryEventType.VERTEX_DATA_MOVEMENT_EVENTS_GENERATED, 1);
}