Examples with EdgeProperty org.apache.tez.dag.api.EdgeProperty used on opensource projects

Example 41 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class FairCartesianProductVertexManager method initialize.

@Override
public void initialize(CartesianProductConfigProto config) throws Exception {
    this.config = config;
    maxParallelism = config.hasMaxParallelism() ? config.getMaxParallelism() : CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MAX_PARALLELISM_DEFAULT;
    enableGrouping = config.hasEnableGrouping() ? config.getEnableGrouping() : CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_ENABLE_GROUPING_DEFAULT;
    minOpsPerWorker = config.hasMinOpsPerWorker() ? config.getMinOpsPerWorker() : CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MIN_OPS_PER_WORKER_DEFAULT;
    sourceList = config.getSourcesList();
    if (config.hasNumPartitionsForFairCase()) {
        numPartitions = config.getNumPartitionsForFairCase();
    } else {
        numPartitions = (int) Math.pow(maxParallelism, 1.0 / sourceList.size());
    }
    for (Map.Entry<String, EdgeProperty> e : getContext().getInputVertexEdgeProperties().entrySet()) {
        if (e.getValue().getDataMovementType() == CUSTOM && e.getValue().getEdgeManagerDescriptor().getClassName().equals(CartesianProductEdgeManager.class.getName())) {
            srcVerticesByName.put(e.getKey(), new SrcVertex());
            srcVerticesByName.get(e.getKey()).name = e.getKey();
            getContext().registerForVertexStateUpdates(e.getKey(), EnumSet.of(VertexState.CONFIGURED));
            numCPSrcNotInConfigureState++;
        } else {
            getContext().registerForVertexStateUpdates(e.getKey(), EnumSet.of(VertexState.RUNNING));
            numBroadcastSrcNotInRunningState++;
        }
    }
    Map<String, List<String>> srcGroups = getContext().getInputVertexGroups();
    for (int i = 0; i < sourceList.size(); i++) {
        String srcName = sourceList.get(i);
        Source source = new Source();
        source.position = i;
        if (srcGroups.containsKey(srcName)) {
            source.name = srcName;
            for (String srcVName : srcGroups.get(srcName)) {
                source.srcVertices.add(srcVerticesByName.get(srcVName));
                srcVerticesByName.get(srcVName).source = source;
            }
        } else {
            source.name = srcName;
            source.srcVertices.add(srcVerticesByName.get(srcName));
            srcVerticesByName.get(srcName).source = source;
        }
        sourcesByName.put(srcName, source);
    }
    minNumRecordForEstimation = (long) Math.pow(minOpsPerWorker * maxParallelism, 1.0 / sourceList.size());
    numChunksPerSrc = new int[sourcesByName.size()];
    getContext().vertexReconfigurationPlanned();
}

Example 42 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class UnorderedKVEdgeConfig method createDefaultOneToOneEdgeProperty.

/**
 * This is a convenience method for the typical usage of this edge, and creates an instance of
 * {@link org.apache.tez.dag.api.EdgeProperty} which is likely to be used. </p>
 * If custom edge properties are required, the methods to get the relevant payloads should be
 * used. </p>
 * * In this case - DataMovementType.ONE_TO_ONE, EdgeProperty.DataSourceType.PERSISTED,
 * EdgeProperty.SchedulingType.SEQUENTIAL
 *
 * @return an {@link org.apache.tez.dag.api.EdgeProperty} instance
 */
public EdgeProperty createDefaultOneToOneEdgeProperty() {
    EdgeProperty edgeProperty = EdgeProperty.create(EdgeProperty.DataMovementType.ONE_TO_ONE, EdgeProperty.DataSourceType.PERSISTED, EdgeProperty.SchedulingType.SEQUENTIAL, OutputDescriptor.create(getOutputClassName()).setUserPayload(getOutputPayload()), InputDescriptor.create(getInputClassName()).setUserPayload(getInputPayload()));
    Utils.setEdgePropertyHistoryText(this, edgeProperty);
    return edgeProperty;
}

Example 43 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class UnorderedKVEdgeConfig method createDefaultCustomEdgeProperty.

/**
 * This is a convenience method for creating an Edge descriptor based on the specified
 * EdgeManagerDescriptor.
 *
 * @param edgeManagerDescriptor the custom edge specification
 * @return an {@link org.apache.tez.dag.api.EdgeProperty} instance
 */
public EdgeProperty createDefaultCustomEdgeProperty(EdgeManagerPluginDescriptor edgeManagerDescriptor) {
    Preconditions.checkNotNull(edgeManagerDescriptor, "EdgeManagerDescriptor cannot be null");
    EdgeProperty edgeProperty = EdgeProperty.create(edgeManagerDescriptor, EdgeProperty.DataSourceType.PERSISTED, EdgeProperty.SchedulingType.SEQUENTIAL, OutputDescriptor.create(getOutputClassName()).setUserPayload(getOutputPayload()), InputDescriptor.create(getInputClassName()).setUserPayload(getInputPayload()));
    Utils.setEdgePropertyHistoryText(this, edgeProperty);
    return edgeProperty;
}

Example 44 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class UnorderedKVEdgeConfig method createDefaultBroadcastEdgeProperty.

/**
 * This is a convenience method for the typical usage of this edge, and creates an instance of
 * {@link org.apache.tez.dag.api.EdgeProperty} which is likely to be used. </p>
 * If custom edge properties are required, the methods to get the relevant payloads should be
 * used. </p>
 * * In this case - DataMovementType.BROADCAST, EdgeProperty.DataSourceType.PERSISTED,
 * EdgeProperty.SchedulingType.SEQUENTIAL
 *
 * @return an {@link org.apache.tez.dag.api.EdgeProperty} instance
 */
public EdgeProperty createDefaultBroadcastEdgeProperty() {
    EdgeProperty edgeProperty = EdgeProperty.create(EdgeProperty.DataMovementType.BROADCAST, EdgeProperty.DataSourceType.PERSISTED, EdgeProperty.SchedulingType.SEQUENTIAL, OutputDescriptor.create(getOutputClassName()).setUserPayload(getOutputPayload()), InputDescriptor.create(getInputClassName()).setUserPayload(getInputPayload()));
    Utils.setEdgePropertyHistoryText(this, edgeProperty);
    return edgeProperty;
}

Example 45 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class VertexImpl method setParallelism.

@Override
public void setParallelism(int parallelism, VertexLocationHint vertexLocationHint, Map<String, EdgeManagerPluginDescriptor> sourceEdgeManagers, Map<String, InputSpecUpdate> rootInputSpecUpdates, boolean fromVertexManager) throws AMUserCodeException {
    // temporarily support conversion of edge manager to edge property
    Map<String, EdgeProperty> sourceEdgeProperties = Maps.newHashMap();
    readLock.lock();
    try {
        if (sourceEdgeManagers != null && !sourceEdgeManagers.isEmpty()) {
            for (Edge e : sourceVertices.values()) {
                EdgeManagerPluginDescriptor newEdge = sourceEdgeManagers.get(e.getSourceVertexName());
                EdgeProperty oldEdge = e.getEdgeProperty();
                if (newEdge != null) {
                    sourceEdgeProperties.put(e.getSourceVertexName(), EdgeProperty.create(newEdge, oldEdge.getDataSourceType(), oldEdge.getSchedulingType(), oldEdge.getEdgeSource(), oldEdge.getEdgeDestination()));
                }
            }
        }
    } finally {
        readLock.unlock();
    }
    setParallelismWrapper(parallelism, vertexLocationHint, sourceEdgeProperties, rootInputSpecUpdates, fromVertexManager);
}