Examples with InputAttemptIdentifier org.apache.tez.runtime.library.common.InputAttemptIdentifier used on opensource projects

Example 31 with InputAttemptIdentifier

use of org.apache.tez.runtime.library.common.InputAttemptIdentifier in project tez by apache.

the class ShuffleScheduler method getMapsForHost.

public synchronized List<InputAttemptIdentifier> getMapsForHost(MapHost host) {
    List<InputAttemptIdentifier> origList = host.getAndClearKnownMaps();
    ListMultimap<Integer, InputAttemptIdentifier> dedupedList = LinkedListMultimap.create();
    Iterator<InputAttemptIdentifier> listItr = origList.iterator();
    while (listItr.hasNext()) {
        // we may want to try all versions of the input but with current retry
        // behavior older ones are likely to be lost and should be ignored.
        // This may be removed after TEZ-914
        InputAttemptIdentifier id = listItr.next();
        if (inputShouldBeConsumed(id)) {
            Integer inputNumber = Integer.valueOf(id.getInputIdentifier());
            List<InputAttemptIdentifier> oldIdList = dedupedList.get(inputNumber);
            if (oldIdList == null || oldIdList.isEmpty()) {
                dedupedList.put(inputNumber, id);
                continue;
            }
            // In case of pipelined shuffle, we can have multiple spills. In such cases, we can have
            // more than one item in the oldIdList.
            boolean addIdentifierToList = false;
            Iterator<InputAttemptIdentifier> oldIdIterator = oldIdList.iterator();
            while (oldIdIterator.hasNext()) {
                InputAttemptIdentifier oldId = oldIdIterator.next();
                // no need to add if spill ids are same
                if (id.canRetrieveInputInChunks()) {
                    if (oldId.getSpillEventId() == id.getSpillEventId()) {
                        // TODO: need to handle deterministic spills later.
                        addIdentifierToList = false;
                        continue;
                    } else if (oldId.getAttemptNumber() == id.getAttemptNumber()) {
                        // but with different spill id.
                        addIdentifierToList = true;
                        break;
                    }
                }
                // if its from different attempt, take the latest attempt
                if (oldId.getAttemptNumber() < id.getAttemptNumber()) {
                    // remove existing identifier
                    oldIdIterator.remove();
                    LOG.warn("Old Src for InputIndex: " + inputNumber + " with attemptNumber: " + oldId.getAttemptNumber() + " was not determined to be invalid. Ignoring it for now in favour of " + id.getAttemptNumber());
                    addIdentifierToList = true;
                    break;
                }
            }
            if (addIdentifierToList) {
                dedupedList.put(inputNumber, id);
            }
        } else {
            LOG.info("Ignoring finished or obsolete source: " + id);
        }
    }
    // Compute the final list, limited by NUM_FETCHERS_AT_ONCE
    List<InputAttemptIdentifier> result = new ArrayList<InputAttemptIdentifier>();
    int includedMaps = 0;
    int totalSize = dedupedList.size();
    for (Integer inputIndex : dedupedList.keySet()) {
        List<InputAttemptIdentifier> attemptIdentifiers = dedupedList.get(inputIndex);
        for (InputAttemptIdentifier inputAttemptIdentifier : attemptIdentifiers) {
            if (includedMaps++ >= maxTaskOutputAtOnce) {
                host.addKnownMap(inputAttemptIdentifier);
            } else {
                if (inputAttemptIdentifier.canRetrieveInputInChunks()) {
                    ShuffleEventInfo shuffleEventInfo = pipelinedShuffleInfoEventsMap.get(inputAttemptIdentifier.getInputIdentifier());
                    if (shuffleEventInfo != null) {
                        shuffleEventInfo.scheduledForDownload = true;
                    }
                }
                result.add(inputAttemptIdentifier);
            }
        }
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("assigned " + includedMaps + " of " + totalSize + " to " + host + " to " + Thread.currentThread().getName());
    }
    return result;
}

Example 32 with InputAttemptIdentifier

use of org.apache.tez.runtime.library.common.InputAttemptIdentifier in project tez by apache.

the class ShuffleUtils method constructInputURL.

public static URL constructInputURL(String baseURI, Collection<InputAttemptIdentifier> inputs, boolean keepAlive) throws MalformedURLException {
    StringBuilder url = new StringBuilder(baseURI);
    boolean first = true;
    for (InputAttemptIdentifier input : inputs) {
        if (first) {
            first = false;
            url.append(input.getPathComponent());
        } else {
            url.append(",").append(input.getPathComponent());
        }
    }
    // Refer MAPREDUCE-5787 to enable/disable keep-alive in the cluster.
    if (keepAlive) {
        url.append("&keepAlive=true");
    }
    return new URL(url.toString());
}

Example 33 with InputAttemptIdentifier

use of org.apache.tez.runtime.library.common.InputAttemptIdentifier in project tez by apache.

the class ShuffleInputEventHandlerImpl method constructInputAttemptIdentifier.

/**
 * Helper method to create InputAttemptIdentifier
 *
 * @param targetIndex
 * @param targetIndexCount
 * @param version
 * @param shufflePayload
 * @param isShared
 * @return CompositeInputAttemptIdentifier
 */
private CompositeInputAttemptIdentifier constructInputAttemptIdentifier(int targetIndex, int targetIndexCount, int version, DataMovementEventPayloadProto shufflePayload, boolean isShared) {
    String pathComponent = (shufflePayload.hasPathComponent()) ? shufflePayload.getPathComponent() : null;
    CompositeInputAttemptIdentifier srcAttemptIdentifier = null;
    if (shufflePayload.hasSpillId()) {
        int spillEventId = shufflePayload.getSpillId();
        boolean lastEvent = shufflePayload.getLastEvent();
        InputAttemptIdentifier.SPILL_INFO spillInfo = (lastEvent) ? InputAttemptIdentifier.SPILL_INFO.FINAL_UPDATE : InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE;
        srcAttemptIdentifier = new CompositeInputAttemptIdentifier(targetIndex, version, pathComponent, isShared, spillInfo, spillEventId, targetIndexCount);
    } else {
        srcAttemptIdentifier = new CompositeInputAttemptIdentifier(targetIndex, version, pathComponent, isShared, targetIndexCount);
    }
    return srcAttemptIdentifier;
}

Example 34 with InputAttemptIdentifier

use of org.apache.tez.runtime.library.common.InputAttemptIdentifier in project tez by apache.

the class FetcherOrderedGrouped method setupLocalDiskFetch.

@VisibleForTesting
protected void setupLocalDiskFetch(MapHost host) throws InterruptedException {
    // Get completed maps on 'host'
    List<InputAttemptIdentifier> srcAttempts = scheduler.getMapsForHost(host);
    // especially at the tail of large jobs
    if (srcAttempts.size() == 0) {
        return;
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Fetcher " + id + " going to fetch (local disk) from " + host + " for: " + srcAttempts + ", partition range: " + minPartition + "-" + maxPartition);
    }
    // List of maps to be fetched yet
    populateRemainingMap(srcAttempts);
    try {
        final Iterator<InputAttemptIdentifier> iter = remaining.values().iterator();
        while (iter.hasNext()) {
            // Avoid fetching more if already stopped
            if (stopped) {
                return;
            }
            InputAttemptIdentifier srcAttemptId = iter.next();
            MapOutput mapOutput = null;
            boolean hasFailures = false;
            // Fetch partition count number of map outputs (handles auto-reduce case)
            for (int curPartition = minPartition; curPartition <= maxPartition; curPartition++) {
                try {
                    long startTime = System.currentTimeMillis();
                    // Partition id is the base partition id plus the relative offset
                    int reduceId = host.getPartitionId() + curPartition - minPartition;
                    srcAttemptId = scheduler.getIdentifierForFetchedOutput(srcAttemptId.getPathComponent(), reduceId);
                    Path filename = getShuffleInputFileName(srcAttemptId.getPathComponent(), null);
                    TezIndexRecord indexRecord = getIndexRecord(srcAttemptId.getPathComponent(), reduceId);
                    if (!indexRecord.hasData()) {
                        continue;
                    }
                    mapOutput = getMapOutputForDirectDiskFetch(srcAttemptId, filename, indexRecord);
                    long endTime = System.currentTimeMillis();
                    scheduler.copySucceeded(srcAttemptId, host, indexRecord.getPartLength(), indexRecord.getRawLength(), (endTime - startTime), mapOutput, true);
                } catch (IOException | InternalError e) {
                    if (mapOutput != null) {
                        mapOutput.abort();
                    }
                    if (!stopped) {
                        hasFailures = true;
                        ioErrs.increment(1);
                        scheduler.copyFailed(srcAttemptId, host, true, false, true);
                        LOG.warn("Failed to read local disk output of " + srcAttemptId + " from " + host.getHostIdentifier(), e);
                    } else {
                        if (LOG.isDebugEnabled()) {
                            LOG.debug("Ignoring fetch error during local disk copy since fetcher has already been stopped");
                        }
                        return;
                    }
                }
            }
            if (!hasFailures) {
                iter.remove();
            }
        }
    } finally {
        putBackRemainingMapOutputs(host);
    }
}

Example 35 with InputAttemptIdentifier

use of org.apache.tez.runtime.library.common.InputAttemptIdentifier in project tez by apache.

the class FetcherOrderedGrouped method putBackRemainingMapOutputs.

@VisibleForTesting
protected void putBackRemainingMapOutputs(MapHost host) {
    // Cycle through remaining MapOutputs
    boolean isFirst = true;
    InputAttemptIdentifier first = null;
    for (InputAttemptIdentifier left : remaining.values()) {
        if (isFirst) {
            first = left;
            isFirst = false;
            continue;
        }
        scheduler.putBackKnownMapOutput(host, left);
    }
    if (first != null) {
        // Empty remaining list.
        scheduler.putBackKnownMapOutput(host, first);
    }
}