Examples with EdgeProperty org.apache.tez.dag.api.EdgeProperty used on opensource projects

Example 21 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class InputReadyVertexManager method initialize.

@Override
public void initialize() {
    // this will prevent vertex from starting until we notify we are done
    getContext().vertexReconfigurationPlanned();
    Map<String, EdgeProperty> edges = getContext().getInputVertexEdgeProperties();
    // wait for sources and self to start
    numConfiguredSources = 0;
    configured = new AtomicBoolean(false);
    started = new AtomicBoolean(false);
    for (String entry : edges.keySet()) {
        getContext().registerForVertexStateUpdates(entry, EnumSet.of(VertexState.CONFIGURED));
    }
}

Example 22 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class InputReadyVertexManager method configure.

private void configure() {
    Preconditions.checkState(!configured.get(), "Vertex: " + getContext().getVertexName());
    int numManagedTasks = getContext().getVertexNumTasks(getContext().getVertexName());
    LOG.info("Managing " + numManagedTasks + " tasks for vertex: " + getContext().getVertexName());
    // find out about all input edge types. If there is a custom edge then
    // TODO Until TEZ-1013 we cannot handle custom input formats
    Map<String, EdgeProperty> edges = getContext().getInputVertexEdgeProperties();
    int oneToOneSrcTaskCount = 0;
    numOneToOneEdges = 0;
    for (Map.Entry<String, EdgeProperty> entry : edges.entrySet()) {
        EdgeProperty edgeProp = entry.getValue();
        String srcVertex = entry.getKey();
        int numSrcTasks = getContext().getVertexNumTasks(srcVertex);
        switch(edgeProp.getDataMovementType()) {
            case CUSTOM:
                throw new TezUncheckedException("Cannot handle custom edge");
            case ONE_TO_ONE:
                numOneToOneEdges++;
                if (oneToOneSrcTaskCount == 0) {
                    oneToOneSrcTaskCount = numSrcTasks;
                } else if (oneToOneSrcTaskCount != numSrcTasks) {
                    throw new TezUncheckedException("All 1-1 source vertices must have identical concurrency");
                }
                break;
            case SCATTER_GATHER:
            case BROADCAST:
                break;
            default:
                throw new TezUncheckedException("Unknown edge type: " + edgeProp.getDataMovementType());
        }
        srcVertexInfo.put(srcVertex, new SourceVertexInfo(numSrcTasks, edgeProp));
    }
    if (numOneToOneEdges > 0) {
        Preconditions.checkState(oneToOneSrcTaskCount >= 0, "Vertex: " + getContext().getVertexName());
        if (oneToOneSrcTaskCount != numManagedTasks) {
            numManagedTasks = oneToOneSrcTaskCount;
            // must change parallelism to make them the same
            LOG.info("Update parallelism of vertex: " + getContext().getVertexName() + " to " + oneToOneSrcTaskCount + " to match source 1-1 vertices.");
            getContext().reconfigureVertex(oneToOneSrcTaskCount, null, null);
        }
        oneToOneSrcTasksDoneCount = new int[oneToOneSrcTaskCount];
        oneToOneLocationHints = new TaskLocationHint[oneToOneSrcTaskCount];
    }
    Preconditions.checkState(numManagedTasks >= 0, "Vertex: " + getContext().getVertexName());
    taskIsStarted = new boolean[numManagedTasks];
    // allow scheduling
    configured.set(true);
    getContext().doneReconfiguringVertex();
    trySchedulingPendingCompletions();
}

Example 23 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class ShuffleVertexManagerBase method onVertexStarted.

@Override
public synchronized void onVertexStarted(List<TaskAttemptIdentifier> completions) {
    // examine edges after vertex started because until then these may not have been defined
    Map<String, EdgeProperty> inputs = getContext().getInputVertexEdgeProperties();
    for (Map.Entry<String, EdgeProperty> entry : inputs.entrySet()) {
        srcVertexInfo.put(entry.getKey(), createSourceVertexInfo(entry.getValue(), getContext().getVertexNumTasks(getContext().getVertexName())));
        // TODO what if derived class has already called this
        // register for status update from all source vertices
        getContext().registerForVertexStateUpdates(entry.getKey(), EnumSet.of(VertexState.CONFIGURED));
        if (entry.getValue().getDataMovementType() == DataMovementType.SCATTER_GATHER) {
            bipartiteSources++;
        }
    }
    onVertexStartedCheck();
    for (VertexStateUpdate stateUpdate : pendingStateUpdates) {
        handleVertexStateUpdate(stateUpdate);
    }
    pendingStateUpdates.clear();
    // track the tasks in this vertex
    updatePendingTasks();
    for (VertexManagerEvent vmEvent : pendingVMEvents) {
        handleVertexManagerEvent(vmEvent);
    }
    pendingVMEvents.clear();
    LOG.info("OnVertexStarted vertex: {} with {} source tasks and {} pending" + " tasks", getContext().getVertexName(), totalNumBipartiteSourceTasks, totalTasksToSchedule);
    if (completions != null) {
        for (TaskAttemptIdentifier attempt : completions) {
            onSourceTaskCompleted(attempt);
        }
    }
    onVertexStartedDone.set(true);
    // for the special case when source has 0 tasks or min fraction == 0
    processPendingTasks(null);
}

Example 24 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class ShuffleVertexManagerBase method reconfigVertex.

private void reconfigVertex(final int finalTaskParallelism) {
    Map<String, EdgeProperty> edgeProperties = new HashMap<String, EdgeProperty>(bipartiteSources);
    Iterable<Map.Entry<String, SourceVertexInfo>> bipartiteItr = getBipartiteInfo();
    for (Map.Entry<String, SourceVertexInfo> entry : bipartiteItr) {
        String vertex = entry.getKey();
        EdgeProperty oldEdgeProp = entry.getValue().edgeProperty;
        EdgeProperty newEdgeProp = EdgeProperty.create(entry.getValue().newDescriptor, oldEdgeProp.getDataSourceType(), oldEdgeProp.getSchedulingType(), oldEdgeProp.getEdgeSource(), oldEdgeProp.getEdgeDestination());
        edgeProperties.put(vertex, newEdgeProp);
    }
    getContext().reconfigureVertex(finalTaskParallelism, null, edgeProperties);
}

Example 25 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class HistoryEventJsonConversion method convertVertexReconfigureDoneEvent.

private static JSONObject convertVertexReconfigureDoneEvent(VertexConfigurationDoneEvent event) throws JSONException {
    JSONObject jsonObject = new JSONObject();
    jsonObject.put(ATSConstants.ENTITY, event.getVertexID().toString());
    jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name());
    // Events
    JSONArray events = new JSONArray();
    JSONObject updateEvent = new JSONObject();
    updateEvent.put(ATSConstants.TIMESTAMP, event.getReconfigureDoneTime());
    updateEvent.put(ATSConstants.EVENT_TYPE, HistoryEventType.VERTEX_CONFIGURE_DONE.name());
    JSONObject eventInfo = new JSONObject();
    eventInfo.put(ATSConstants.NUM_TASKS, event.getNumTasks());
    if (event.getSourceEdgeProperties() != null && !event.getSourceEdgeProperties().isEmpty()) {
        JSONObject updatedEdgeManagers = new JSONObject();
        for (Entry<String, EdgeProperty> entry : event.getSourceEdgeProperties().entrySet()) {
            updatedEdgeManagers.put(entry.getKey(), new JSONObject(DAGUtils.convertEdgeProperty(entry.getValue())));
        }
        eventInfo.put(ATSConstants.UPDATED_EDGE_MANAGERS, updatedEdgeManagers);
    }
    updateEvent.put(ATSConstants.EVENT_INFO, eventInfo);
    events.put(updateEvent);
    jsonObject.put(ATSConstants.EVENTS, events);
    // Other info
    JSONObject otherInfo = new JSONObject();
    jsonObject.put(ATSConstants.OTHER_INFO, otherInfo);
    // TODO add more on all other updated information
    return jsonObject;
}