Examples with EdgeProperty org.apache.tez.dag.api.EdgeProperty used on opensource projects

Example 6 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class TestFairCartesianProductVertexManager method testParallelismTwoSkewedSource.

@Test(timeout = 5000)
public void testParallelismTwoSkewedSource() throws Exception {
    setupDAGVertexOnly(100, 10000, 10, 10);
    vertexManager.onVertexStateUpdated(new VertexStateUpdate("v0", VertexState.CONFIGURED));
    vertexManager.onVertexStateUpdated(new VertexStateUpdate("v1", VertexState.CONFIGURED));
    vertexManager.onVertexManagerEventReceived(getVMEvent(15000, "v0", 0));
    for (int i = 0; i < 30; i++) {
        vertexManager.onVertexManagerEventReceived(getVMEvent(1, "v1", i));
    }
    verify(ctx, times(1)).reconfigureVertex(eq(99), any(VertexLocationHint.class), edgePropertiesCaptor.capture());
    Map<String, EdgeProperty> edgeProperties = edgePropertiesCaptor.getValue();
    verifyEdgeProperties(edgeProperties.get("v0"), new String[] { "v0", "v1" }, new int[] { 99, 1 }, 100);
}

Example 7 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class TestFairCartesianProductVertexManager method testZeroSrcTask.

@Test(timeout = 5000)
public void testZeroSrcTask() throws Exception {
    ctx = mock(VertexManagerPluginContext.class);
    vertexManager = new FairCartesianProductVertexManager(ctx);
    when(ctx.getVertexNumTasks("v0")).thenReturn(2);
    when(ctx.getVertexNumTasks("v1")).thenReturn(0);
    CartesianProductConfigProto.Builder builder = CartesianProductConfigProto.newBuilder();
    builder.setIsPartitioned(false).addSources("v0").addSources("v1").addNumChunks(2).addNumChunks(3).setMaxParallelism(6);
    CartesianProductConfigProto config = builder.build();
    Map<String, EdgeProperty> edgePropertyMap = new HashMap<>();
    edgePropertyMap.put("v0", EdgeProperty.create(EdgeManagerPluginDescriptor.create(CartesianProductEdgeManager.class.getName()), null, null, null, null));
    edgePropertyMap.put("v1", EdgeProperty.create(EdgeManagerPluginDescriptor.create(CartesianProductEdgeManager.class.getName()), null, null, null, null));
    when(ctx.getInputVertexEdgeProperties()).thenReturn(edgePropertyMap);
    vertexManager.initialize(config);
    vertexManager.onVertexStateUpdated(new VertexStateUpdate("v0", VertexState.CONFIGURED));
    vertexManager.onVertexStateUpdated(new VertexStateUpdate("v1", VertexState.CONFIGURED));
    vertexManager.onVertexStarted(new ArrayList<TaskAttemptIdentifier>());
    vertexManager.onSourceTaskCompleted(getTaId("v0", 0));
    vertexManager.onSourceTaskCompleted(getTaId("v0", 1));
}

Example 8 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class TestFairCartesianProductVertexManager method testDAGVertexGroup.

@Test(timeout = 5000)
public void testDAGVertexGroup() throws Exception {
    setupDAGVertexGroup(100, 1, 1);
    for (int i = 0; i < 3; i++) {
        vertexManager.onVertexStateUpdated(new VertexStateUpdate("v" + i, VertexState.CONFIGURED));
    }
    vertexManager.onVertexManagerEventReceived(getVMEvent(100, "v0", 0));
    vertexManager.onVertexManagerEventReceived(getVMEvent(10, "v1", 0));
    vertexManager.onVertexManagerEventReceived(getVMEvent(5, "v2", 0));
    vertexManager.onVertexManagerEventReceived(getVMEvent(5, "v2", 1));
    verify(ctx, times(1)).reconfigureVertex(eq(100), any(VertexLocationHint.class), edgePropertiesCaptor.capture());
    Map<String, EdgeProperty> edgeProperties = edgePropertiesCaptor.getValue();
    for (int i = 0; i < 3; i++) {
        verifyEdgeProperties(edgeProperties.get("v" + i), new String[] { "v0", "g0" }, new int[] { 20, 5 }, 100);
    }
    vertexManager.onVertexStarted(null);
    vertexManager.onSourceTaskCompleted(getTaId("v0", 0));
    vertexManager.onSourceTaskCompleted(getTaId("v1", 0));
    verifyScheduleRequest(0);
    vertexManager.onSourceTaskCompleted(getTaId("v2", 0));
    verifyScheduleRequest(1, 0, 5, 10, 15, 20, 25, 30, 35, 40, 45);
    vertexManager.onSourceTaskCompleted(getTaId("v1", 1));
    verifyScheduleRequest(1);
    vertexManager.onSourceTaskCompleted(getTaId("v2", 1));
    verifyScheduleRequest(2, 1, 6, 11, 16, 21, 26, 31, 36, 41, 46);
}

Example 9 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class TestUnorderedPartitionedKVEdgeConfig method testHistoryText.

@Test(timeout = 2000)
public void testHistoryText() {
    UnorderedPartitionedKVEdgeConfig.Builder builder = UnorderedPartitionedKVEdgeConfig.newBuilder("KEY", "VALUE", "PARTITIONER");
    Configuration fromConf = new Configuration(false);
    fromConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, true);
    builder.setFromConfiguration(fromConf);
    UnorderedPartitionedKVEdgeConfig kvEdgeConfig = builder.build();
    checkHistoryText(kvEdgeConfig.getInputHistoryText());
    checkHistoryText(kvEdgeConfig.getOutputHistoryText());
    EdgeProperty defaultEdgeProperty = builder.build().createDefaultEdgeProperty();
    checkHistoryText(defaultEdgeProperty.getEdgeDestination().getHistoryText());
    checkHistoryText(defaultEdgeProperty.getEdgeSource().getHistoryText());
    EdgeManagerPluginDescriptor descriptor = mock(EdgeManagerPluginDescriptor.class);
    EdgeProperty edgeProperty = builder.build().createDefaultCustomEdgeProperty(descriptor);
    checkHistoryText(edgeProperty.getEdgeDestination().getHistoryText());
    checkHistoryText(edgeProperty.getEdgeSource().getHistoryText());
}

Example 10 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class TestInputReadyVertexManager method testDelayedConfigureOneToOne.

@Test(timeout = 5000)
public void testDelayedConfigureOneToOne() throws Exception {
    HashMap<String, EdgeProperty> mockInputVertices = new HashMap<String, EdgeProperty>();
    String mockSrcVertexId1 = "Vertex1";
    EdgeProperty eProp1 = EdgeProperty.create(EdgeProperty.DataMovementType.ONE_TO_ONE, EdgeProperty.DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in"));
    String mockManagedVertexId = "Vertex";
    VertexManagerPluginContext mockContext = mock(VertexManagerPluginContext.class);
    when(mockContext.getInputVertexEdgeProperties()).thenReturn(mockInputVertices);
    when(mockContext.getVertexName()).thenReturn(mockManagedVertexId);
    when(mockContext.getVertexNumTasks(mockManagedVertexId)).thenReturn(3);
    when(mockContext.getVertexNumTasks(mockSrcVertexId1)).thenReturn(3);
    mockInputVertices.put(mockSrcVertexId1, eProp1);
    InputReadyVertexManager manager = new InputReadyVertexManager(mockContext);
    manager.initialize();
    verify(mockContext, times(1)).vertexReconfigurationPlanned();
    verify(mockContext, times(0)).scheduleTasks(requestCaptor.capture());
    // ok to have source task complete before anything else
    manager.onSourceTaskCompleted(TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId1, 1));
    // first own vertex started
    manager.onVertexStarted(Collections.singletonList(TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId1, 0)));
    // no scheduling as we are not configured yet
    verify(mockContext, times(0)).scheduleTasks(requestCaptor.capture());
    // then source vertex configured. now we start
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
    verify(mockContext, times(1)).doneReconfiguringVertex();
    verify(mockContext, times(2)).scheduleTasks(requestCaptor.capture());
    manager.onSourceTaskCompleted(TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId1, 2));
    verify(mockContext, times(3)).scheduleTasks(requestCaptor.capture());
    Assert.assertEquals(1, requestCaptor.getValue().size());
    Assert.assertEquals(2, requestCaptor.getValue().get(0).getTaskIndex());
    Assert.assertEquals(mockSrcVertexId1, requestCaptor.getValue().get(0).getTaskLocationHint().getAffinitizedTask().getVertexName());
    Assert.assertEquals(2, requestCaptor.getValue().get(0).getTaskLocationHint().getAffinitizedTask().getTaskIndex());
}