Examples with TezNamedEntityDescriptorProto org.apache.tez.dag.api.records.DAGProtos.TezNamedEntityDescriptorProto used on opensource projects

Example 1 with TezNamedEntityDescriptorProto

use of org.apache.tez.dag.api.records.DAGProtos.TezNamedEntityDescriptorProto in project tez by apache.

the class DAGAppMaster method parsePlugin.

@VisibleForTesting
public static void parsePlugin(List<NamedEntityDescriptor> resultList, BiMap<String, Integer> pluginMap, List<TezNamedEntityDescriptorProto> namedEntityDescriptorProtos, boolean tezYarnEnabled, boolean uberEnabled, UserPayload defaultPayload) {
    if (tezYarnEnabled) {
        // Default classnames will be populated by individual components
        NamedEntityDescriptor r = new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null).setUserPayload(defaultPayload);
        addDescriptor(resultList, pluginMap, r);
    }
    if (uberEnabled) {
        // Default classnames will be populated by individual components
        NamedEntityDescriptor r = new NamedEntityDescriptor(TezConstants.getTezUberServicePluginName(), null).setUserPayload(defaultPayload);
        addDescriptor(resultList, pluginMap, r);
    }
    if (namedEntityDescriptorProtos != null) {
        for (TezNamedEntityDescriptorProto namedEntityDescriptorProto : namedEntityDescriptorProtos) {
            NamedEntityDescriptor namedEntityDescriptor = DagTypeConverters.convertNamedDescriptorFromProto(namedEntityDescriptorProto);
            addDescriptor(resultList, pluginMap, namedEntityDescriptor);
        }
    }
}

Example 2 with TezNamedEntityDescriptorProto

use of org.apache.tez.dag.api.records.DAGProtos.TezNamedEntityDescriptorProto in project tez by apache.

the class TestDAGAppMaster method testPluginParsing.

@Test(timeout = 5000)
public void testPluginParsing() throws IOException {
    BiMap<String, Integer> pluginMap = HashBiMap.create();
    Configuration conf = new Configuration(false);
    conf.set("testkey", "testval");
    UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf);
    List<TezNamedEntityDescriptorProto> entityDescriptors = new LinkedList<>();
    List<NamedEntityDescriptor> entities;
    // Test empty descriptor list, yarn enabled
    pluginMap.clear();
    entities = new LinkedList<>();
    DAGAppMaster.parsePlugin(entities, pluginMap, null, true, false, defaultPayload);
    assertEquals(1, pluginMap.size());
    assertEquals(1, entities.size());
    assertTrue(pluginMap.containsKey(TezConstants.getTezYarnServicePluginName()));
    assertTrue(0 == pluginMap.get(TezConstants.getTezYarnServicePluginName()));
    assertEquals("testval", TezUtils.createConfFromUserPayload(entities.get(0).getUserPayload()).get("testkey"));
    // Test empty descriptor list, uber enabled
    pluginMap.clear();
    entities = new LinkedList<>();
    DAGAppMaster.parsePlugin(entities, pluginMap, null, false, true, defaultPayload);
    assertEquals(1, pluginMap.size());
    assertEquals(1, entities.size());
    assertTrue(pluginMap.containsKey(TezConstants.getTezUberServicePluginName()));
    assertTrue(0 == pluginMap.get(TezConstants.getTezUberServicePluginName()));
    assertEquals("testval", TezUtils.createConfFromUserPayload(entities.get(0).getUserPayload()).get("testkey"));
    // Test empty descriptor list, yarn enabled, uber enabled
    pluginMap.clear();
    entities = new LinkedList<>();
    DAGAppMaster.parsePlugin(entities, pluginMap, null, true, true, defaultPayload);
    assertEquals(2, pluginMap.size());
    assertEquals(2, entities.size());
    assertTrue(pluginMap.containsKey(TezConstants.getTezYarnServicePluginName()));
    assertTrue(0 == pluginMap.get(TezConstants.getTezYarnServicePluginName()));
    assertTrue(pluginMap.containsKey(TezConstants.getTezUberServicePluginName()));
    assertTrue(1 == pluginMap.get(TezConstants.getTezUberServicePluginName()));
    String pluginName = "d1";
    ByteBuffer bb = ByteBuffer.allocate(4);
    bb.putInt(0, 3);
    TezNamedEntityDescriptorProto d1 = TezNamedEntityDescriptorProto.newBuilder().setName(pluginName).setEntityDescriptor(DAGProtos.TezEntityDescriptorProto.newBuilder().setClassName("d1Class").setTezUserPayload(TezUserPayloadProto.newBuilder().setUserPayload(ByteString.copyFrom(bb)))).build();
    entityDescriptors.add(d1);
    // Test descriptor, no yarn, no uber
    pluginMap.clear();
    entities = new LinkedList<>();
    DAGAppMaster.parsePlugin(entities, pluginMap, entityDescriptors, false, false, defaultPayload);
    assertEquals(1, pluginMap.size());
    assertEquals(1, entities.size());
    assertTrue(pluginMap.containsKey(pluginName));
    assertTrue(0 == pluginMap.get(pluginName));
    // Test descriptor, yarn and uber
    pluginMap.clear();
    entities = new LinkedList<>();
    DAGAppMaster.parsePlugin(entities, pluginMap, entityDescriptors, true, true, defaultPayload);
    assertEquals(3, pluginMap.size());
    assertEquals(3, entities.size());
    assertTrue(pluginMap.containsKey(TezConstants.getTezYarnServicePluginName()));
    assertTrue(0 == pluginMap.get(TezConstants.getTezYarnServicePluginName()));
    assertTrue(pluginMap.containsKey(TezConstants.getTezUberServicePluginName()));
    assertTrue(1 == pluginMap.get(TezConstants.getTezUberServicePluginName()));
    assertTrue(pluginMap.containsKey(pluginName));
    assertTrue(2 == pluginMap.get(pluginName));
    entityDescriptors.clear();
}

Example 3 with TezNamedEntityDescriptorProto

use of org.apache.tez.dag.api.records.DAGProtos.TezNamedEntityDescriptorProto in project tez by apache.

the class DagTypeConverters method convertNamedEntityCollectionToProto.

public static List<TezNamedEntityDescriptorProto> convertNamedEntityCollectionToProto(NamedEntityDescriptor[] namedEntityDescriptors) {
    List<TezNamedEntityDescriptorProto> list = Lists.newArrayListWithCapacity(namedEntityDescriptors.length);
    for (NamedEntityDescriptor namedEntity : namedEntityDescriptors) {
        TezNamedEntityDescriptorProto namedEntityProto = convertNamedEntityToProto(namedEntity);
        list.add(namedEntityProto);
    }
    return list;
}

Example 4 with TezNamedEntityDescriptorProto

use of org.apache.tez.dag.api.records.DAGProtos.TezNamedEntityDescriptorProto in project tez by apache.

the class DagTypeConverters method convertNamedEntityToProto.

public static TezNamedEntityDescriptorProto convertNamedEntityToProto(NamedEntityDescriptor namedEntityDescriptor) {
    TezNamedEntityDescriptorProto.Builder builder = TezNamedEntityDescriptorProto.newBuilder();
    builder.setName(namedEntityDescriptor.getEntityName());
    DAGProtos.TezEntityDescriptorProto entityProto = DagTypeConverters.convertToDAGPlan(namedEntityDescriptor);
    builder.setEntityDescriptor(entityProto);
    return builder.build();
}