Examples with ShuffleVertexManager org.apache.tez.dag.library.vertexmanager.ShuffleVertexManager used on opensource projects

Example 1 with ShuffleVertexManager

use of org.apache.tez.dag.library.vertexmanager.ShuffleVertexManager in project tez by apache.

the class VertexImpl method assignVertexManager.

private void assignVertexManager() throws TezException {
    // VertexReconfigureDoneEvent will be logged
    if (recoveryData != null && recoveryData.shouldSkipInit()) {
        // Replace the original VertexManager with NoOpVertexManager if the reconfiguration is done in the last AM attempt
        VertexConfigurationDoneEvent reconfigureDoneEvent = recoveryData.getVertexConfigurationDoneEvent();
        if (LOG.isInfoEnabled()) {
            LOG.info("VertexManager reconfiguration is done in the last AM Attempt" + ", use NoOpVertexManager to replace it, vertexId=" + logIdentifier);
            LOG.info("VertexReconfigureDoneEvent=" + reconfigureDoneEvent);
        }
        NonSyncByteArrayOutputStream out = new NonSyncByteArrayOutputStream();
        try {
            reconfigureDoneEvent.toProtoStream(out);
        } catch (IOException e) {
            throw new TezUncheckedException("Unable to deserilize VertexReconfigureDoneEvent");
        }
        this.vertexManager = new VertexManager(VertexManagerPluginDescriptor.create(NoOpVertexManager.class.getName()).setUserPayload(UserPayload.create(ByteBuffer.wrap(out.toByteArray()))), dagUgi, this, appContext, stateChangeNotifier);
        return;
    }
    boolean hasBipartite = false;
    boolean hasOneToOne = false;
    boolean hasCustom = false;
    if (sourceVertices != null) {
        for (Edge edge : sourceVertices.values()) {
            switch(edge.getEdgeProperty().getDataMovementType()) {
                case SCATTER_GATHER:
                    hasBipartite = true;
                    break;
                case ONE_TO_ONE:
                    hasOneToOne = true;
                    break;
                case BROADCAST:
                    break;
                case CUSTOM:
                    hasCustom = true;
                    break;
                default:
                    throw new TezUncheckedException("Unknown data movement type: " + edge.getEdgeProperty().getDataMovementType());
            }
        }
    }
    boolean hasUserVertexManager = vertexPlan.hasVertexManagerPlugin();
    if (hasUserVertexManager) {
        VertexManagerPluginDescriptor pluginDesc = DagTypeConverters.convertVertexManagerPluginDescriptorFromDAGPlan(vertexPlan.getVertexManagerPlugin());
        LOG.info("Setting user vertex manager plugin: " + pluginDesc.getClassName() + " on vertex: " + getLogIdentifier());
        vertexManager = new VertexManager(pluginDesc, dagUgi, this, appContext, stateChangeNotifier);
    } else {
        // Else we use the default ImmediateStartVertexManager
        if (inputsWithInitializers != null) {
            LOG.info("Setting vertexManager to RootInputVertexManager for " + logIdentifier);
            vertexManager = new VertexManager(RootInputVertexManager.createConfigBuilder(vertexConf).build(), dagUgi, this, appContext, stateChangeNotifier);
        } else if (hasOneToOne && !hasCustom) {
            LOG.info("Setting vertexManager to InputReadyVertexManager for " + logIdentifier);
            vertexManager = new VertexManager(VertexManagerPluginDescriptor.create(InputReadyVertexManager.class.getName()), dagUgi, this, appContext, stateChangeNotifier);
        } else if (hasBipartite && !hasCustom) {
            LOG.info("Setting vertexManager to ShuffleVertexManager for " + logIdentifier);
            // shuffle vertex manager needs a conf payload
            vertexManager = new VertexManager(ShuffleVertexManager.createConfigBuilder(vertexConf).build(), dagUgi, this, appContext, stateChangeNotifier);
        } else {
            // schedule all tasks upon vertex start. Default behavior.
            LOG.info("Setting vertexManager to ImmediateStartVertexManager for " + logIdentifier);
            vertexManager = new VertexManager(VertexManagerPluginDescriptor.create(ImmediateStartVertexManager.class.getName()), dagUgi, this, appContext, stateChangeNotifier);
        }
    }
}