Examples with CartesianProductConfigProto org.apache.tez.runtime.library.cartesianproduct.CartesianProductUserPayload.CartesianProductConfigProto used on opensource projects

Example 1 with CartesianProductConfigProto

use of org.apache.tez.runtime.library.cartesianproduct.CartesianProductUserPayload.CartesianProductConfigProto in project tez by apache.

the class TestFairCartesianProductVertexManager method verifyEdgeProperties.

private void verifyEdgeProperties(EdgeProperty edgeProperty, String[] sources, int[] numChunksPerSrc, int maxParallelism) throws InvalidProtocolBufferException {
    CartesianProductConfigProto config = CartesianProductConfigProto.parseFrom(ByteString.copyFrom(edgeProperty.getEdgeManagerDescriptor().getUserPayload().getPayload()));
    assertArrayEquals(sources, config.getSourcesList().toArray());
    assertArrayEquals(numChunksPerSrc, Ints.toArray(config.getNumChunksList()));
    assertEquals(maxParallelism, config.getMaxParallelism());
}

Example 2 with CartesianProductConfigProto

use of org.apache.tez.runtime.library.cartesianproduct.CartesianProductUserPayload.CartesianProductConfigProto in project tez by apache.

the class TestFairCartesianProductVertexManager method verifyVertexGroupInfo.

private void verifyVertexGroupInfo(EdgeProperty edgeProperty, int positionInGroup, int... numTaskPerVertexInGroup) throws InvalidProtocolBufferException {
    CartesianProductConfigProto config = CartesianProductConfigProto.parseFrom(ByteString.copyFrom(edgeProperty.getEdgeManagerDescriptor().getUserPayload().getPayload()));
    assertEquals(positionInGroup, config.getPositionInGroup());
    int i = 0;
    for (int numTask : numTaskPerVertexInGroup) {
        assertEquals(numTask, config.getNumTaskPerVertexInGroup(i));
        i++;
    }
}

Example 3 with CartesianProductConfigProto

use of org.apache.tez.runtime.library.cartesianproduct.CartesianProductUserPayload.CartesianProductConfigProto in project tez by apache.

the class TestFairCartesianProductVertexManager method testZeroSrcTask.

@Test(timeout = 5000)
public void testZeroSrcTask() throws Exception {
    ctx = mock(VertexManagerPluginContext.class);
    vertexManager = new FairCartesianProductVertexManager(ctx);
    when(ctx.getVertexNumTasks("v0")).thenReturn(2);
    when(ctx.getVertexNumTasks("v1")).thenReturn(0);
    CartesianProductConfigProto.Builder builder = CartesianProductConfigProto.newBuilder();
    builder.setIsPartitioned(false).addSources("v0").addSources("v1").addNumChunks(2).addNumChunks(3).setMaxParallelism(6);
    CartesianProductConfigProto config = builder.build();
    Map<String, EdgeProperty> edgePropertyMap = new HashMap<>();
    edgePropertyMap.put("v0", EdgeProperty.create(EdgeManagerPluginDescriptor.create(CartesianProductEdgeManager.class.getName()), null, null, null, null));
    edgePropertyMap.put("v1", EdgeProperty.create(EdgeManagerPluginDescriptor.create(CartesianProductEdgeManager.class.getName()), null, null, null, null));
    when(ctx.getInputVertexEdgeProperties()).thenReturn(edgePropertyMap);
    vertexManager.initialize(config);
    vertexManager.onVertexStateUpdated(new VertexStateUpdate("v0", VertexState.CONFIGURED));
    vertexManager.onVertexStateUpdated(new VertexStateUpdate("v1", VertexState.CONFIGURED));
    vertexManager.onVertexStarted(new ArrayList<TaskAttemptIdentifier>());
    vertexManager.onSourceTaskCompleted(getTaId("v0", 0));
    vertexManager.onSourceTaskCompleted(getTaId("v0", 1));
}

Example 4 with CartesianProductConfigProto

use of org.apache.tez.runtime.library.cartesianproduct.CartesianProductUserPayload.CartesianProductConfigProto in project tez by apache.

the class TestCartesianProductConfig method testFairCartesianProductConfig.

@Test(timeout = 5000)
public void testFairCartesianProductConfig() {
    List<String> sourceVertices = new ArrayList<>();
    sourceVertices.add("v0");
    sourceVertices.add("v1");
    CartesianProductConfig config = new CartesianProductConfig(sourceVertices);
    // conf not set
    CartesianProductConfigProto proto = config.toProto(conf);
    assertEquals(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MAX_PARALLELISM_DEFAULT, proto.getMaxParallelism());
    assertEquals(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MIN_OPS_PER_WORKER_DEFAULT, proto.getMinOpsPerWorker());
    assertEquals(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_ENABLE_GROUPING_DEFAULT, proto.getEnableGrouping());
    assertFalse(proto.hasNumPartitionsForFairCase());
    assertFalse(proto.hasGroupingFraction());
    // conf set
    conf.setInt(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MAX_PARALLELISM, 1000);
    conf.setLong(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MIN_OPS_PER_WORKER, 1000000);
    conf.setBoolean(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_ENABLE_GROUPING, false);
    conf.setFloat(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_GROUPING_FRACTION, 0.75f);
    conf.setInt(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_NUM_PARTITIONS, 25);
    proto = config.toProto(conf);
    assertEquals(1000, proto.getMaxParallelism());
    assertEquals(1000000, proto.getMinOpsPerWorker());
    assertFalse(proto.getEnableGrouping());
    assertEquals(0.75f, proto.getGroupingFraction(), 0.01);
    assertEquals(25, proto.getNumPartitionsForFairCase());
}

Example 5 with CartesianProductConfigProto

use of org.apache.tez.runtime.library.cartesianproduct.CartesianProductUserPayload.CartesianProductConfigProto in project tez by apache.

the class CartesianProductConfig method toProto.

protected CartesianProductConfigProto toProto(TezConfiguration conf) {
    CartesianProductConfigProto.Builder builder = CartesianProductConfigProto.newBuilder();
    builder.setIsPartitioned(this.isPartitioned).addAllSources(Arrays.asList(sources));
    if (isPartitioned) {
        builder.addAllNumPartitions(Ints.asList(numPartitions));
        if (filterDescriptor != null) {
            builder.setFilterClassName(filterDescriptor.getClassName());
            UserPayload filterUesrPayload = filterDescriptor.getUserPayload();
            if (filterUesrPayload != null) {
                builder.setFilterUserPayload(ByteString.copyFrom(filterUesrPayload.getPayload()));
            }
        }
    }
    if (conf != null) {
        builder.setMinFraction(conf.getFloat(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MIN_FRACTION, CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MIN_FRACTION_DEFAULT));
        builder.setMaxFraction(conf.getFloat(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MAX_FRACTION, CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MAX_FRACTION_DEFAULT));
        builder.setMaxParallelism(conf.getInt(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MAX_PARALLELISM, CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MAX_PARALLELISM_DEFAULT));
        builder.setMinOpsPerWorker(conf.getLong(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MIN_OPS_PER_WORKER, CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MIN_OPS_PER_WORKER_DEFAULT));
        builder.setEnableGrouping(conf.getBoolean(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_ENABLE_GROUPING, CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_ENABLE_GROUPING_DEFAULT));
        if (conf.get(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_GROUPING_FRACTION) != null) {
            builder.setGroupingFraction(Float.parseFloat(conf.get(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_GROUPING_FRACTION)));
            Preconditions.checkArgument(0 < builder.getGroupingFraction() && builder.getGroupingFraction() <= 1, "grouping fraction should be larger than 0 and less" + " or equal to 1, current value: " + builder.getGroupingFraction());
        }
        if (conf.get(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_NUM_PARTITIONS) != null) {
            builder.setNumPartitionsForFairCase(Integer.parseInt(conf.get(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_NUM_PARTITIONS)));
            Preconditions.checkArgument(builder.getNumPartitionsForFairCase() > 0, "Number of partitions for fair cartesian product should be positive integer");
        }
    }
    Preconditions.checkArgument(builder.getMinFraction() <= builder.getMaxFraction(), "min fraction(" + builder.getMinFraction() + ") should be less than max fraction(" + builder.getMaxFraction() + ") in cartesian product slow start");
    Preconditions.checkArgument(builder.getMaxParallelism() > 0, "max parallelism must be positive, currently is " + builder.getMaxParallelism());
    Preconditions.checkArgument(builder.getMinOpsPerWorker() > 0, "Min ops per worker must be positive, currently is " + builder.getMinOpsPerWorker());
    return builder.build();
}