Examples with EdgeProperty org.apache.tez.dag.api.EdgeProperty used on opensource projects

Example 16 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class TestEdge method testEdgeManagerPluginCtxGetVertexGroupName.

@Test(timeout = 5000)
public void testEdgeManagerPluginCtxGetVertexGroupName() throws TezException {
    EdgeManagerPluginDescriptor edgeManagerDescriptor = EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName());
    EdgeProperty edgeProp = EdgeProperty.create(edgeManagerDescriptor, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"));
    Edge edge = new Edge(edgeProp, null, null);
    Vertex srcV = mock(Vertex.class), destV = mock(Vertex.class);
    String srcName = "srcV", destName = "destV";
    when(srcV.getName()).thenReturn(srcName);
    when(destV.getName()).thenReturn(destName);
    edge.setSourceVertex(srcV);
    edge.setDestinationVertex(destV);
    assertNull(edge.edgeManager.getContext().getVertexGroupName());
    String group = "group";
    when(destV.getGroupInputSpecList()).thenReturn(Arrays.asList(new GroupInputSpec(group, Arrays.asList("v1", "v3"), null)));
    assertNull(edge.edgeManager.getContext().getVertexGroupName());
    when(destV.getGroupInputSpecList()).thenReturn(Arrays.asList(new GroupInputSpec(group, Arrays.asList(srcName, "v3"), null)));
    assertEquals(group, edge.edgeManager.getContext().getVertexGroupName());
}

Example 17 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class TestRootInputVertexManager method testTezDrainCompletionsOnVertexStart.

@Test
public void testTezDrainCompletionsOnVertexStart() throws IOException {
    Configuration conf = new Configuration();
    RootInputVertexManager manager = null;
    HashMap<String, EdgeProperty> mockInputVertices = new HashMap<String, EdgeProperty>();
    String mockSrcVertexId1 = "Vertex1";
    EdgeProperty eProp1 = EdgeProperty.create(EdgeProperty.DataMovementType.BROADCAST, EdgeProperty.DataSourceType.PERSISTED, EdgeProperty.SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in"));
    VertexManagerPluginContext mockContext = mock(VertexManagerPluginContext.class);
    when(mockContext.getVertexStatistics(any(String.class))).thenReturn(mock(VertexStatistics.class));
    when(mockContext.getInputVertexEdgeProperties()).thenReturn(mockInputVertices);
    when(mockContext.getVertexNumTasks(mockSrcVertexId1)).thenReturn(3);
    mockInputVertices.put(mockSrcVertexId1, eProp1);
    // check initialization
    manager = createRootInputVertexManager(conf, mockContext, 0.1f, 0.1f);
    Assert.assertEquals(0, manager.numSourceTasksCompleted);
    manager.onVertexStarted(Collections.singletonList(createTaskAttemptIdentifier(mockSrcVertexId1, 0)));
    Assert.assertEquals(1, manager.numSourceTasksCompleted);
}

Example 18 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class TestRootInputVertexManager method testRootInputVertexManagerSlowStart.

@Test(timeout = 5000)
public void testRootInputVertexManagerSlowStart() {
    Configuration conf = new Configuration();
    RootInputVertexManager manager = null;
    HashMap<String, EdgeProperty> mockInputVertices = new HashMap<String, EdgeProperty>();
    String mockSrcVertexId1 = "Vertex1";
    EdgeProperty eProp1 = EdgeProperty.create(EdgeProperty.DataMovementType.BROADCAST, EdgeProperty.DataSourceType.PERSISTED, EdgeProperty.SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in"));
    String mockSrcVertexId2 = "Vertex2";
    EdgeProperty eProp2 = EdgeProperty.create(EdgeProperty.DataMovementType.BROADCAST, EdgeProperty.DataSourceType.PERSISTED, EdgeProperty.SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in"));
    String mockManagedVertexId = "Vertex3";
    VertexManagerPluginContext mockContext = mock(VertexManagerPluginContext.class);
    when(mockContext.getVertexStatistics(any(String.class))).thenReturn(mock(VertexStatistics.class));
    when(mockContext.getInputVertexEdgeProperties()).thenReturn(mockInputVertices);
    when(mockContext.getVertexName()).thenReturn(mockManagedVertexId);
    when(mockContext.getVertexNumTasks(mockManagedVertexId)).thenReturn(3);
    mockInputVertices.put(mockSrcVertexId1, eProp1);
    mockInputVertices.put(mockSrcVertexId2, eProp2);
    // check initialization
    manager = createRootInputVertexManager(conf, mockContext, 0.1f, 0.1f);
    final List<Integer> scheduledTasks = Lists.newLinkedList();
    doAnswer(new ScheduledTasksAnswer(scheduledTasks)).when(mockContext).scheduleTasks(anyList());
    // source vertices have 0 tasks. immediate start of all managed tasks
    when(mockContext.getVertexNumTasks(mockSrcVertexId1)).thenReturn(0);
    when(mockContext.getVertexNumTasks(mockSrcVertexId2)).thenReturn(0);
    manager.onVertexStarted(emptyCompletions);
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2, VertexState.CONFIGURED));
    Assert.assertEquals(manager.pendingTasks.size(), 0);
    // all tasks scheduled
    Assert.assertEquals(scheduledTasks.size(), 3);
    when(mockContext.getVertexNumTasks(mockSrcVertexId1)).thenReturn(2);
    when(mockContext.getVertexNumTasks(mockSrcVertexId2)).thenReturn(2);
    try {
        // source vertex have some tasks. min < 0.
        manager = createRootInputVertexManager(conf, mockContext, -0.1f, 0.0f);
        // should not come here
        Assert.assertTrue(false);
    } catch (IllegalArgumentException e) {
        Assert.assertTrue(e.getMessage().contains("Invalid values for slowStartMinFraction"));
    }
    try {
        // source vertex have some tasks. max > 1.
        manager = createRootInputVertexManager(conf, mockContext, 0.0f, 95.0f);
        // should not come here
        Assert.assertTrue(false);
    } catch (IllegalArgumentException e) {
        Assert.assertTrue(e.getMessage().contains("Invalid values for slowStartMinFraction"));
    }
    try {
        // source vertex have some tasks. min > max
        manager = createRootInputVertexManager(conf, mockContext, 0.5f, 0.3f);
        // should not come here
        Assert.assertTrue(false);
    } catch (IllegalArgumentException e) {
        Assert.assertTrue(e.getMessage().contains("Invalid values for slowStartMinFraction"));
    }
    // source vertex have some tasks. min > default and max undefined
    int numTasks = 20;
    when(mockContext.getVertexNumTasks(mockSrcVertexId1)).thenReturn(numTasks);
    when(mockContext.getVertexNumTasks(mockSrcVertexId2)).thenReturn(numTasks);
    scheduledTasks.clear();
    manager = createRootInputVertexManager(conf, mockContext, 0.975f, null);
    manager.onVertexStarted(emptyCompletions);
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2, VertexState.CONFIGURED));
    Assert.assertEquals(3, manager.pendingTasks.size());
    Assert.assertEquals(numTasks * 2, manager.totalNumSourceTasks);
    Assert.assertEquals(0, manager.numSourceTasksCompleted);
    float completedTasksThreshold = 0.975f * numTasks;
    // Finish all tasks before exceeding the threshold
    for (String mockSrcVertex : new String[] { mockSrcVertexId1, mockSrcVertexId2 }) {
        for (int i = 0; i < mockContext.getVertexNumTasks(mockSrcVertex); ++i) {
            // complete 0th tasks outside the loop
            manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertex, i + 1));
            if ((i + 2) >= completedTasksThreshold) {
                // stop before completing more than min/max source tasks
                break;
            }
        }
    }
    // Since we haven't exceeded the threshold, all tasks are still pending
    Assert.assertEquals(manager.totalTasksToSchedule, manager.pendingTasks.size());
    // no tasks scheduled
    Assert.assertEquals(0, scheduledTasks.size());
    // Cross the threshold min/max threshold to schedule all tasks
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId1, 0));
    Assert.assertEquals(3, manager.pendingTasks.size());
    Assert.assertEquals(0, scheduledTasks.size());
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId2, 0));
    Assert.assertEquals(0, manager.pendingTasks.size());
    // all tasks scheduled
    Assert.assertEquals(manager.totalTasksToSchedule, scheduledTasks.size());
    // reset vertices for next test
    when(mockContext.getVertexNumTasks(mockSrcVertexId1)).thenReturn(2);
    when(mockContext.getVertexNumTasks(mockSrcVertexId2)).thenReturn(2);
    // source vertex have some tasks. min, max, 0
    manager = createRootInputVertexManager(conf, mockContext, 0.0f, 0.0f);
    manager.onVertexStarted(emptyCompletions);
    Assert.assertEquals(manager.totalTasksToSchedule, 3);
    Assert.assertEquals(manager.numSourceTasksCompleted, 0);
    // all source vertices need to be configured
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2, VertexState.CONFIGURED));
    Assert.assertEquals(manager.totalNumSourceTasks, 4);
    Assert.assertEquals(manager.pendingTasks.size(), 0);
    // all tasks scheduled
    Assert.assertEquals(scheduledTasks.size(), 3);
    // min, max > 0 and min, max
    manager = createRootInputVertexManager(conf, mockContext, 0.25f, 0.25f);
    manager.onVertexStarted(emptyCompletions);
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2, VertexState.CONFIGURED));
    // no tasks scheduled
    Assert.assertEquals(manager.pendingTasks.size(), 3);
    Assert.assertEquals(manager.totalNumSourceTasks, 4);
    // task completion from non-bipartite stage does nothing
    // no tasks scheduled
    Assert.assertEquals(manager.pendingTasks.size(), 3);
    Assert.assertEquals(manager.totalNumSourceTasks, 4);
    Assert.assertEquals(manager.numSourceTasksCompleted, 0);
    // task completion on only 1 SG edge does nothing
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId2, 0));
    // no tasks scheduled
    Assert.assertEquals(manager.pendingTasks.size(), 3);
    Assert.assertEquals(manager.totalNumSourceTasks, 4);
    Assert.assertEquals(manager.numSourceTasksCompleted, 1);
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId1, 0));
    Assert.assertEquals(manager.pendingTasks.size(), 0);
    // all tasks scheduled
    Assert.assertEquals(scheduledTasks.size(), 3);
    Assert.assertEquals(manager.numSourceTasksCompleted, 2);
    // min, max > 0 and min, max, absolute max 1.0
    manager = createRootInputVertexManager(conf, mockContext, 1.0f, 1.0f);
    manager.onVertexStarted(emptyCompletions);
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2, VertexState.CONFIGURED));
    // no tasks scheduled
    Assert.assertEquals(manager.pendingTasks.size(), 3);
    Assert.assertEquals(manager.totalNumSourceTasks, 4);
    // task completion from non-bipartite stage does nothing
    // no tasks scheduled
    Assert.assertEquals(manager.pendingTasks.size(), 3);
    Assert.assertEquals(manager.totalNumSourceTasks, 4);
    Assert.assertEquals(manager.numSourceTasksCompleted, 0);
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId1, 0));
    Assert.assertEquals(manager.pendingTasks.size(), 3);
    Assert.assertEquals(manager.numSourceTasksCompleted, 1);
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId1, 1));
    Assert.assertEquals(manager.pendingTasks.size(), 3);
    Assert.assertEquals(manager.numSourceTasksCompleted, 2);
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId2, 0));
    Assert.assertEquals(manager.pendingTasks.size(), 3);
    Assert.assertEquals(manager.numSourceTasksCompleted, 3);
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId2, 1));
    Assert.assertEquals(manager.pendingTasks.size(), 0);
    // all tasks scheduled
    Assert.assertEquals(scheduledTasks.size(), 3);
    Assert.assertEquals(manager.numSourceTasksCompleted, 4);
    // min, max > 0 and min, max
    manager = createRootInputVertexManager(conf, mockContext, 1.0f, 1.0f);
    manager.onVertexStarted(emptyCompletions);
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2, VertexState.CONFIGURED));
    // no tasks scheduled
    Assert.assertEquals(manager.pendingTasks.size(), 3);
    Assert.assertEquals(manager.totalNumSourceTasks, 4);
    // task completion from non-bipartite stage does nothing
    // no tasks scheduled
    Assert.assertEquals(manager.pendingTasks.size(), 3);
    Assert.assertEquals(manager.totalNumSourceTasks, 4);
    Assert.assertEquals(manager.numSourceTasksCompleted, 0);
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId1, 0));
    Assert.assertEquals(manager.pendingTasks.size(), 3);
    Assert.assertEquals(manager.numSourceTasksCompleted, 1);
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId1, 1));
    Assert.assertEquals(manager.pendingTasks.size(), 3);
    Assert.assertEquals(manager.numSourceTasksCompleted, 2);
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId2, 0));
    Assert.assertEquals(manager.pendingTasks.size(), 3);
    Assert.assertEquals(manager.numSourceTasksCompleted, 3);
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId2, 1));
    Assert.assertEquals(manager.pendingTasks.size(), 0);
    // all tasks scheduled
    Assert.assertEquals(scheduledTasks.size(), 3);
    Assert.assertEquals(manager.numSourceTasksCompleted, 4);
    // reset vertices for next test
    when(mockContext.getVertexNumTasks(mockSrcVertexId1)).thenReturn(4);
    when(mockContext.getVertexNumTasks(mockSrcVertexId2)).thenReturn(4);
    // min, max > and min < max
    manager = createRootInputVertexManager(conf, mockContext, 0.25f, 0.75f);
    manager.onVertexStarted(emptyCompletions);
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2, VertexState.CONFIGURED));
    // no tasks scheduled
    Assert.assertEquals(manager.pendingTasks.size(), 3);
    Assert.assertEquals(manager.totalNumSourceTasks, 8);
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId1, 0));
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId2, 1));
    Assert.assertEquals(manager.pendingTasks.size(), 3);
    Assert.assertEquals(manager.numSourceTasksCompleted, 2);
    // completion of same task again should not get counted
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId2, 1));
    Assert.assertEquals(manager.pendingTasks.size(), 3);
    Assert.assertEquals(manager.numSourceTasksCompleted, 2);
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId1, 1));
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId2, 0));
    Assert.assertEquals(manager.pendingTasks.size(), 1);
    // 2 task scheduled
    Assert.assertEquals(scheduledTasks.size(), 2);
    Assert.assertEquals(manager.numSourceTasksCompleted, 4);
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId1, 2));
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId2, 2));
    Assert.assertEquals(manager.pendingTasks.size(), 0);
    // 1 tasks scheduled
    Assert.assertEquals(scheduledTasks.size(), 1);
    Assert.assertEquals(manager.numSourceTasksCompleted, 6);
    scheduledTasks.clear();
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId1, // we are done. no action
    3));
    Assert.assertEquals(manager.pendingTasks.size(), 0);
    // no task scheduled
    Assert.assertEquals(scheduledTasks.size(), 0);
    Assert.assertEquals(manager.numSourceTasksCompleted, 7);
    // min, max > and min < max
    manager = createRootInputVertexManager(conf, mockContext, 0.25f, 1.0f);
    manager.onVertexStarted(emptyCompletions);
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2, VertexState.CONFIGURED));
    // no tasks scheduled
    Assert.assertEquals(manager.pendingTasks.size(), 3);
    Assert.assertEquals(manager.totalNumSourceTasks, 8);
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId1, 0));
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId2, 1));
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId1, 1));
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId2, 0));
    Assert.assertEquals(manager.pendingTasks.size(), 2);
    // 1 task scheduled
    Assert.assertEquals(scheduledTasks.size(), 1);
    Assert.assertEquals(manager.numSourceTasksCompleted, 4);
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId1, 2));
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId2, 2));
    Assert.assertEquals(manager.pendingTasks.size(), 1);
    // 1 task scheduled
    Assert.assertEquals(scheduledTasks.size(), 1);
    Assert.assertEquals(manager.numSourceTasksCompleted, 6);
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId1, 3));
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId2, 3));
    Assert.assertEquals(manager.pendingTasks.size(), 0);
    // no task scheduled
    Assert.assertEquals(scheduledTasks.size(), 1);
    Assert.assertEquals(manager.numSourceTasksCompleted, 8);
    // if there is single task to schedule, it should be schedule when src
    // completed fraction is more than min slow start fraction
    scheduledTasks.clear();
    when(mockContext.getVertexNumTasks(mockManagedVertexId)).thenReturn(1);
    manager = createRootInputVertexManager(conf, mockContext, 0.25f, 0.75f);
    manager.onVertexStarted(emptyCompletions);
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2, VertexState.CONFIGURED));
    // no tasks scheduled
    Assert.assertEquals(manager.pendingTasks.size(), 1);
    Assert.assertEquals(manager.totalNumSourceTasks, 8);
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId1, 0));
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId2, 1));
    Assert.assertEquals(manager.pendingTasks.size(), 1);
    // no task scheduled
    Assert.assertEquals(scheduledTasks.size(), 0);
    Assert.assertEquals(manager.numSourceTasksCompleted, 2);
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId1, 1));
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId2, 0));
    Assert.assertEquals(manager.pendingTasks.size(), 0);
    // 1 task scheduled
    Assert.assertEquals(scheduledTasks.size(), 1);
    Assert.assertEquals(manager.numSourceTasksCompleted, 4);
    scheduledTasks.clear();
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId1, 2));
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId2, 2));
    Assert.assertEquals(manager.pendingTasks.size(), 0);
    // no task scheduled
    Assert.assertEquals(scheduledTasks.size(), 0);
    Assert.assertEquals(manager.numSourceTasksCompleted, 6);
    scheduledTasks.clear();
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId1, // we are done. no action
    3));
    Assert.assertEquals(manager.pendingTasks.size(), 0);
    // no task scheduled
    Assert.assertEquals(scheduledTasks.size(), 0);
    Assert.assertEquals(manager.numSourceTasksCompleted, 7);
}

Example 19 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class TestHistoryEventTimelineConversion method testConvertVertexReconfigreDoneEvent.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testConvertVertexReconfigreDoneEvent() {
    TezVertexID vId = tezVertexID;
    Map<String, EdgeProperty> edgeMgrs = new HashMap<String, EdgeProperty>();
    edgeMgrs.put("a", EdgeProperty.create(EdgeManagerPluginDescriptor.create("a.class").setHistoryText("text"), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")));
    VertexConfigurationDoneEvent event = new VertexConfigurationDoneEvent(vId, 0L, 1, null, edgeMgrs, null, true);
    List<TimelineEntity> entities = HistoryEventTimelineConversion.convertToTimelineEntities(event);
    Assert.assertEquals(1, entities.size());
    TimelineEntity timelineEntity = entities.get(0);
    Assert.assertEquals(ATSConstants.TEZ_VERTEX_ID, timelineEntity.getEntityType());
    Assert.assertEquals(vId.toString(), timelineEntity.getEntityId());
    Assert.assertEquals(1, timelineEntity.getEvents().size());
    final Map<String, Set<Object>> primaryFilters = timelineEntity.getPrimaryFilters();
    Assert.assertEquals(2, primaryFilters.size());
    Assert.assertTrue(primaryFilters.get(ATSConstants.APPLICATION_ID).contains(applicationId.toString()));
    Assert.assertTrue(primaryFilters.get(EntityTypes.TEZ_DAG_ID.name()).contains(tezDAGID.toString()));
    TimelineEvent evt = timelineEntity.getEvents().get(0);
    Assert.assertEquals(HistoryEventType.VERTEX_CONFIGURE_DONE.name(), evt.getEventType());
    Assert.assertEquals(1, evt.getEventInfo().get(ATSConstants.NUM_TASKS));
    Assert.assertNotNull(evt.getEventInfo().get(ATSConstants.UPDATED_EDGE_MANAGERS));
    Map<String, Object> updatedEdgeMgrs = (Map<String, Object>) evt.getEventInfo().get(ATSConstants.UPDATED_EDGE_MANAGERS);
    Assert.assertEquals(1, updatedEdgeMgrs.size());
    Assert.assertTrue(updatedEdgeMgrs.containsKey("a"));
    Map<String, Object> updatedEdgeMgr = (Map<String, Object>) updatedEdgeMgrs.get("a");
    Assert.assertEquals(DataMovementType.CUSTOM.name(), updatedEdgeMgr.get(DAGUtils.DATA_MOVEMENT_TYPE_KEY));
    Assert.assertEquals("In", updatedEdgeMgr.get(DAGUtils.EDGE_DESTINATION_CLASS_KEY));
    Assert.assertEquals("a.class", updatedEdgeMgr.get(DAGUtils.EDGE_MANAGER_CLASS_KEY));
    Assert.assertEquals(1, timelineEntity.getOtherInfo().get(ATSConstants.NUM_TASKS));
}

Example 20 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class HistoryEventTimelineConversion method convertVertexReconfigureDoneEvent.

private static TimelineEntity convertVertexReconfigureDoneEvent(VertexConfigurationDoneEvent event) {
    TimelineEntity atsEntity = new TimelineEntity();
    atsEntity.setEntityId(event.getVertexID().toString());
    atsEntity.setEntityType(EntityTypes.TEZ_VERTEX_ID.name());
    atsEntity.addPrimaryFilter(ATSConstants.APPLICATION_ID, event.getVertexID().getDAGId().getApplicationId().toString());
    atsEntity.addPrimaryFilter(EntityTypes.TEZ_DAG_ID.name(), event.getVertexID().getDAGId().toString());
    TimelineEvent updateEvt = new TimelineEvent();
    updateEvt.setEventType(HistoryEventType.VERTEX_CONFIGURE_DONE.name());
    updateEvt.setTimestamp(event.getReconfigureDoneTime());
    Map<String, Object> eventInfo = new HashMap<String, Object>();
    if (event.getSourceEdgeProperties() != null && !event.getSourceEdgeProperties().isEmpty()) {
        Map<String, Object> updatedEdgeManagers = new HashMap<String, Object>();
        for (Entry<String, EdgeProperty> entry : event.getSourceEdgeProperties().entrySet()) {
            updatedEdgeManagers.put(entry.getKey(), DAGUtils.convertEdgeProperty(entry.getValue()));
        }
        eventInfo.put(ATSConstants.UPDATED_EDGE_MANAGERS, updatedEdgeManagers);
    }
    eventInfo.put(ATSConstants.NUM_TASKS, event.getNumTasks());
    updateEvt.setEventInfo(eventInfo);
    atsEntity.addEvent(updateEvt);
    atsEntity.addOtherInfo(ATSConstants.NUM_TASKS, event.getNumTasks());
    return atsEntity;
}