Examples with EdgeProperty org.apache.tez.dag.api.EdgeProperty used on opensource projects

Example 56 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class TestOrderedPartitionedKVEdgeConfig method testHistoryText.

@Test(timeout = 2000)
public void testHistoryText() {
    OrderedPartitionedKVEdgeConfig.Builder builder = OrderedPartitionedKVEdgeConfig.newBuilder("KEY", "VALUE", "PARTITIONER");
    Configuration fromConf = new Configuration(false);
    fromConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, true);
    builder.setFromConfiguration(fromConf);
    OrderedPartitionedKVEdgeConfig kvEdgeConfig = builder.build();
    checkHistoryText(kvEdgeConfig.getInputHistoryText());
    checkHistoryText(kvEdgeConfig.getOutputHistoryText());
    EdgeProperty defaultEdgeProperty = builder.build().createDefaultEdgeProperty();
    checkHistoryText(defaultEdgeProperty.getEdgeDestination().getHistoryText());
    checkHistoryText(defaultEdgeProperty.getEdgeSource().getHistoryText());
    EdgeManagerPluginDescriptor descriptor = mock(EdgeManagerPluginDescriptor.class);
    EdgeProperty edgeProperty = builder.build().createDefaultCustomEdgeProperty(descriptor);
    checkHistoryText(edgeProperty.getEdgeDestination().getHistoryText());
    checkHistoryText(edgeProperty.getEdgeSource().getHistoryText());
}

Example 57 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class TestUnorderedKVEdgeConfig method testHistoryText.

@Test(timeout = 2000)
public void testHistoryText() {
    UnorderedKVEdgeConfig.Builder builder = UnorderedKVEdgeConfig.newBuilder("KEY", "VALUE");
    Configuration fromConf = new Configuration(false);
    fromConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, true);
    builder.setFromConfiguration(fromConf);
    UnorderedKVEdgeConfig kvEdgeConfig = builder.build();
    checkHistoryText(kvEdgeConfig.getInputHistoryText());
    checkHistoryText(kvEdgeConfig.getOutputHistoryText());
    EdgeProperty defaultEdgeProperty = builder.build().createDefaultBroadcastEdgeProperty();
    checkHistoryText(defaultEdgeProperty.getEdgeDestination().getHistoryText());
    checkHistoryText(defaultEdgeProperty.getEdgeSource().getHistoryText());
    defaultEdgeProperty = builder.build().createDefaultOneToOneEdgeProperty();
    checkHistoryText(defaultEdgeProperty.getEdgeDestination().getHistoryText());
    checkHistoryText(defaultEdgeProperty.getEdgeSource().getHistoryText());
    EdgeManagerPluginDescriptor descriptor = mock(EdgeManagerPluginDescriptor.class);
    EdgeProperty edgeProperty = builder.build().createDefaultCustomEdgeProperty(descriptor);
    checkHistoryText(edgeProperty.getEdgeDestination().getHistoryText());
    checkHistoryText(edgeProperty.getEdgeSource().getHistoryText());
}

Example 58 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class TestFairCartesianProductVertexManager method testDAGVertexOnlyGroupByMinOpsPerWorker.

@Test(timeout = 5000)
public void testDAGVertexOnlyGroupByMinOpsPerWorker() throws Exception {
    setupDAGVertexOnly(100, 10000, 10, 10);
    vertexManager.onVertexStateUpdated(new VertexStateUpdate("v0", VertexState.CONFIGURED));
    vertexManager.onVertexStateUpdated(new VertexStateUpdate("v1", VertexState.CONFIGURED));
    for (int i = 0; i < 20; i++) {
        vertexManager.onVertexManagerEventReceived(getVMEvent(20, "v0", i));
    }
    for (int i = 0; i < 30; i++) {
        vertexManager.onVertexManagerEventReceived(getVMEvent(10, "v1", i));
    }
    verify(ctx, times(1)).reconfigureVertex(eq(12), any(VertexLocationHint.class), edgePropertiesCaptor.capture());
    Map<String, EdgeProperty> edgeProperties = edgePropertiesCaptor.getValue();
    verifyEdgeProperties(edgeProperties.get("v0"), new String[] { "v0", "v1" }, new int[] { 4, 3 }, 100);
    verifyEdgeProperties(edgeProperties.get("v1"), new String[] { "v0", "v1" }, new int[] { 4, 3 }, 100);
    vertexManager.onVertexStarted(null);
    verifyScheduleRequest(0);
    for (int i = 0; i < 5; i++) {
        vertexManager.onSourceTaskCompleted(getTaId("v0", i));
    }
    for (int i = 0; i < 10; i++) {
        vertexManager.onSourceTaskCompleted(getTaId("v1", 10 + i));
    }
    verifyScheduleRequest(1, 1);
}

Example 59 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class TestFairCartesianProductVertexManager method testDAGVertexOnlyGroupByMaxParallelism.

@Test(timeout = 5000)
public void testDAGVertexOnlyGroupByMaxParallelism() throws Exception {
    setupDAGVertexOnly(30, 1, 30, 1);
    vertexManager.onVertexStateUpdated(new VertexStateUpdate("v0", VertexState.CONFIGURED));
    vertexManager.onVertexStateUpdated(new VertexStateUpdate("v1", VertexState.CONFIGURED));
    vertexManager.onVertexManagerEventReceived(getVMEvent(250, "v0", 0));
    verify(ctx, never()).reconfigureVertex(anyInt(), any(VertexLocationHint.class), anyMapOf(String.class, EdgeProperty.class));
    vertexManager.onVertexManagerEventReceived(getVMEvent(200, "v1", 0));
    verify(ctx, times(1)).reconfigureVertex(eq(30), any(VertexLocationHint.class), edgePropertiesCaptor.capture());
    Map<String, EdgeProperty> edgeProperties = edgePropertiesCaptor.getValue();
    verifyEdgeProperties(edgeProperties.get("v0"), new String[] { "v0", "v1" }, new int[] { 5, 6 }, 30);
    verifyVertexGroupInfo(edgeProperties.get("v0"), 0);
    verifyEdgeProperties(edgeProperties.get("v1"), new String[] { "v0", "v1" }, new int[] { 5, 6 }, 30);
    verifyVertexGroupInfo(edgeProperties.get("v1"), 0);
    vertexManager.onVertexStarted(null);
    verifyScheduleRequest(0);
    vertexManager.onSourceTaskCompleted(getTaId("v0", 0));
    vertexManager.onSourceTaskCompleted(getTaId("v1", 0));
    verifyScheduleRequest(1, 0, 6, 1, 7);
    vertexManager.onSourceTaskCompleted(getTaId("v0", 1));
    verifyScheduleRequest(2, 12, 13, 18, 19, 24, 25);
}

Example 60 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class TestFairCartesianProductVertexManager method testParallelismThreeSkewedSource.

@Test(timeout = 5000)
public void testParallelismThreeSkewedSource() throws Exception {
    when(ctx.getInputVertexEdgeProperties()).thenReturn(getEdgePropertyMap(3));
    setSrcParallelism(ctx, 10, 2, 3, 4);
    CartesianProductConfigProto.Builder builder = CartesianProductConfigProto.newBuilder();
    builder.setIsPartitioned(false).addSources("v0").addSources("v1").addSources("v2").setMaxParallelism(100).setMinOpsPerWorker(10000).setNumPartitionsForFairCase(10);
    vertexManager.initialize(builder.build());
    vertexManager.onVertexStateUpdated(new VertexStateUpdate("v0", VertexState.CONFIGURED));
    vertexManager.onVertexStateUpdated(new VertexStateUpdate("v1", VertexState.CONFIGURED));
    vertexManager.onVertexStateUpdated(new VertexStateUpdate("v2", VertexState.CONFIGURED));
    vertexManager.onVertexManagerEventReceived(getVMEvent(60000, "v0", 0));
    vertexManager.onVertexManagerEventReceived(getVMEvent(4000, "v1", 0));
    for (int i = 0; i < 40; i++) {
        vertexManager.onVertexManagerEventReceived(getVMEvent(3, "v2", i));
    }
    verify(ctx, times(1)).reconfigureVertex(eq(93), any(VertexLocationHint.class), edgePropertiesCaptor.capture());
    Map<String, EdgeProperty> edgeProperties = edgePropertiesCaptor.getValue();
    verifyEdgeProperties(edgeProperties.get("v0"), new String[] { "v0", "v1", "v2" }, new int[] { 31, 3, 1 }, 100);
}