Examples with InputAttemptIdentifier org.apache.tez.runtime.library.common.InputAttemptIdentifier used on opensource projects

Example 21 with InputAttemptIdentifier

use of org.apache.tez.runtime.library.common.InputAttemptIdentifier in project tez by apache.

the class TestIFile method testConcatenatedZlibPadding.

// test concatenated zlib input - as in multiple map outputs during shuffle
// This specific input is valid but the decompressor can leave lingering
// bytes between segments. If the lingering bytes aren't handled correctly,
// the stream will get out-of-sync.
@Test(timeout = 5000)
public void testConcatenatedZlibPadding() throws IOException, URISyntaxException {
    byte[] bytes;
    long compTotal = 0;
    // Known raw and compressed lengths of input
    long[] raws = { 2392, 102314, 42576, 31432, 25090 };
    long[] compressed = { 723, 25396, 10926, 8203, 6665 };
    CompressionCodecFactory codecFactory = new CompressionCodecFactory(new Configuration());
    codec = codecFactory.getCodecByClassName("org.apache.hadoop.io.compress.DefaultCodec");
    URL url = getClass().getClassLoader().getResource("TestIFile_concatenated_compressed.bin");
    assertNotEquals("IFileinput file must exist", null, url);
    Path p = new Path(url.toURI());
    FSDataInputStream inStream = localFs.open(p);
    for (int i = 0; i < 5; i++) {
        bytes = new byte[(int) raws[i]];
        assertEquals("Compressed stream out-of-sync", inStream.getPos(), compTotal);
        IFile.Reader.readToMemory(bytes, inStream, (int) compressed[i], codec, false, -1);
        compTotal += compressed[i];
        // Now read the data
        InMemoryReader inMemReader = new InMemoryReader(null, new InputAttemptIdentifier(0, 0), bytes, 0, bytes.length);
        DataInputBuffer keyIn = new DataInputBuffer();
        DataInputBuffer valIn = new DataInputBuffer();
        Deserializer<Text> keyDeserializer;
        Deserializer<IntWritable> valDeserializer;
        SerializationFactory serializationFactory = new SerializationFactory(defaultConf);
        keyDeserializer = serializationFactory.getDeserializer(Text.class);
        valDeserializer = serializationFactory.getDeserializer(IntWritable.class);
        keyDeserializer.open(keyIn);
        valDeserializer.open(valIn);
        while (inMemReader.nextRawKey(keyIn)) {
            inMemReader.nextRawValue(valIn);
        }
    }
    inStream.close();
}

Example 22 with InputAttemptIdentifier

use of org.apache.tez.runtime.library.common.InputAttemptIdentifier in project tez by apache.

the class ShuffleManager method constructFetcherForHost.

@VisibleForTesting
Fetcher constructFetcherForHost(InputHost inputHost, Configuration conf) {
    Path lockDisk = null;
    if (sharedFetchEnabled) {
        // pick a single lock disk from the edge name's hashcode + host hashcode
        final int h = Math.abs(Objects.hashCode(this.srcNameTrimmed, inputHost.getHost()));
        lockDisk = new Path(this.localDisks[h % this.localDisks.length], "locks");
    }
    FetcherBuilder fetcherBuilder = new FetcherBuilder(ShuffleManager.this, httpConnectionParams, inputManager, inputContext.getApplicationId(), inputContext.getDagIdentifier(), jobTokenSecretMgr, srcNameTrimmed, conf, localFs, localDirAllocator, lockDisk, localDiskFetchEnabled, sharedFetchEnabled, localhostName, shufflePort, asyncHttp, verifyDiskChecksum, compositeFetch);
    if (codec != null) {
        fetcherBuilder.setCompressionParameters(codec);
    }
    fetcherBuilder.setIFileParams(ifileReadAhead, ifileReadAheadLength);
    // Remove obsolete inputs from the list being given to the fetcher. Also
    // remove from the obsolete list.
    PartitionToInputs pendingInputsOfOnePartitionRange = inputHost.clearAndGetOnePartitionRange();
    int includedMaps = 0;
    for (Iterator<InputAttemptIdentifier> inputIter = pendingInputsOfOnePartitionRange.getInputs().iterator(); inputIter.hasNext(); ) {
        InputAttemptIdentifier input = inputIter.next();
        // For pipelined shuffle.
        if (!validateInputAttemptForPipelinedShuffle(input)) {
            continue;
        }
        // Avoid adding attempts which have already completed.
        boolean alreadyCompleted;
        if (input instanceof CompositeInputAttemptIdentifier) {
            CompositeInputAttemptIdentifier compositeInput = (CompositeInputAttemptIdentifier) input;
            int nextClearBit = completedInputSet.nextClearBit(compositeInput.getInputIdentifier());
            int maxClearBit = compositeInput.getInputIdentifier() + compositeInput.getInputIdentifierCount();
            alreadyCompleted = nextClearBit > maxClearBit;
        } else {
            alreadyCompleted = completedInputSet.get(input.getInputIdentifier());
        }
        // Avoid adding attempts which have already completed or have been marked as OBSOLETE
        if (alreadyCompleted || obsoletedInputs.contains(input)) {
            inputIter.remove();
            continue;
        }
        // Check if max threshold is met
        if (includedMaps >= maxTaskOutputAtOnce) {
            inputIter.remove();
            // add to inputHost
            inputHost.addKnownInput(pendingInputsOfOnePartitionRange.getPartition(), pendingInputsOfOnePartitionRange.getPartitionCount(), input);
        } else {
            includedMaps++;
        }
    }
    if (inputHost.getNumPendingPartitions() > 0) {
        // add it to queue
        pendingHosts.add(inputHost);
    }
    for (InputAttemptIdentifier input : pendingInputsOfOnePartitionRange.getInputs()) {
        ShuffleEventInfo eventInfo = shuffleInfoEventsMap.get(input.getInputIdentifier());
        if (eventInfo != null) {
            eventInfo.scheduledForDownload = true;
        }
    }
    fetcherBuilder.assignWork(inputHost.getHost(), inputHost.getPort(), pendingInputsOfOnePartitionRange.getPartition(), pendingInputsOfOnePartitionRange.getPartitionCount(), pendingInputsOfOnePartitionRange.getInputs());
    if (LOG.isDebugEnabled()) {
        LOG.debug("Created Fetcher for host: " + inputHost.getHost() + ", info: " + inputHost.getAdditionalInfo() + ", with inputs: " + pendingInputsOfOnePartitionRange);
    }
    return fetcherBuilder.build();
}

Example 23 with InputAttemptIdentifier

use of org.apache.tez.runtime.library.common.InputAttemptIdentifier in project tez by apache.

the class ShuffleInputEventHandlerImpl method processInputFailedEvent.

private void processInputFailedEvent(InputFailedEvent ife) {
    InputAttemptIdentifier srcAttemptIdentifier = new InputAttemptIdentifier(ife.getTargetIndex(), ife.getVersion());
    shuffleManager.obsoleteKnownInput(srcAttemptIdentifier);
}

Example 24 with InputAttemptIdentifier

use of org.apache.tez.runtime.library.common.InputAttemptIdentifier in project tez by apache.

the class ShuffleInputEventHandlerImpl method processCompositeRoutedDataMovementEvent.

private void processCompositeRoutedDataMovementEvent(CompositeRoutedDataMovementEvent crdme, DataMovementEventPayloadProto shufflePayload, BitSet emptyPartitionsBitSet) throws IOException {
    int partitionId = crdme.getSourceIndex();
    if (LOG.isDebugEnabled()) {
        LOG.debug("DME srcIdx: " + partitionId + ", targetIndex: " + crdme.getTargetIndex() + ", count:" + crdme.getCount() + ", attemptNum: " + crdme.getVersion() + ", payload: " + ShuffleUtils.stringify(shufflePayload));
    }
    if (shufflePayload.hasEmptyPartitions()) {
        CompositeInputAttemptIdentifier compositeInputAttemptIdentifier = constructInputAttemptIdentifier(crdme.getTargetIndex(), crdme.getCount(), crdme.getVersion(), shufflePayload, false);
        boolean allPartitionsEmpty = true;
        for (int i = 0; i < crdme.getCount(); i++) {
            int srcPartitionId = partitionId + i;
            allPartitionsEmpty &= emptyPartitionsBitSet.get(srcPartitionId);
            if (emptyPartitionsBitSet.get(srcPartitionId)) {
                InputAttemptIdentifier srcAttemptIdentifier = compositeInputAttemptIdentifier.expand(i);
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Source partition: " + srcPartitionId + " did not generate any data. SrcAttempt: [" + srcAttemptIdentifier + "]. Not fetching.");
                }
                numDmeEventsNoData.getAndIncrement();
                shuffleManager.addCompletedInputWithNoData(srcAttemptIdentifier);
            }
        }
        if (allPartitionsEmpty) {
            return;
        }
    }
    CompositeInputAttemptIdentifier srcAttemptIdentifier = constructInputAttemptIdentifier(crdme.getTargetIndex(), crdme.getCount(), crdme.getVersion(), shufflePayload, (useSharedInputs && partitionId == 0));
    shuffleManager.addKnownInput(shufflePayload.getHost(), shufflePayload.getPort(), srcAttemptIdentifier, partitionId);
}

Example 25 with InputAttemptIdentifier

use of org.apache.tez.runtime.library.common.InputAttemptIdentifier in project tez by apache.

the class FetcherOrderedGrouped method copyMapOutput.

protected InputAttemptIdentifier[] copyMapOutput(MapHost host, DataInputStream input, InputAttemptIdentifier inputAttemptIdentifier) throws FetcherReadTimeoutException {
    MapOutput mapOutput = null;
    InputAttemptIdentifier srcAttemptId = null;
    long decompressedLength = 0;
    long compressedLength = 0;
    try {
        long startTime = System.currentTimeMillis();
        int partitionCount = 1;
        if (this.compositeFetch) {
            // Multiple partitions are fetched
            partitionCount = WritableUtils.readVInt(input);
        }
        ArrayList<MapOutputStat> mapOutputStats = new ArrayList<>(partitionCount);
        for (int mapOutputIndex = 0; mapOutputIndex < partitionCount; mapOutputIndex++) {
            MapOutputStat mapOutputStat = null;
            try {
                // Read the shuffle header
                ShuffleHeader header = new ShuffleHeader();
                // TODO Review: Multiple header reads in case of status WAIT ?
                header.readFields(input);
                if (!header.mapId.startsWith(InputAttemptIdentifier.PATH_PREFIX)) {
                    if (!stopped) {
                        badIdErrs.increment(1);
                        LOG.warn("Invalid map id: " + header.mapId + ", expected to start with " + InputAttemptIdentifier.PATH_PREFIX + ", partition: " + header.forReduce);
                        return new InputAttemptIdentifier[] { getNextRemainingAttempt() };
                    } else {
                        if (LOG.isDebugEnabled()) {
                            LOG.debug("Already shutdown. Ignoring invalid map id error");
                        }
                        return EMPTY_ATTEMPT_ID_ARRAY;
                    }
                }
                if (header.getCompressedLength() == 0) {
                    // Empty partitions are already accounted for
                    continue;
                }
                mapOutputStat = new MapOutputStat(scheduler.getIdentifierForFetchedOutput(header.mapId, header.forReduce), header.uncompressedLength, header.compressedLength, header.forReduce);
                mapOutputStats.add(mapOutputStat);
            } catch (IllegalArgumentException e) {
                if (!stopped) {
                    badIdErrs.increment(1);
                    LOG.warn("Invalid map id ", e);
                    // the remaining because we dont know where to start reading from. YARN-1773
                    return new InputAttemptIdentifier[] { getNextRemainingAttempt() };
                } else {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Already shutdown. Ignoring invalid map id error. Exception: " + e.getClass().getName() + ", Message: " + e.getMessage());
                    }
                    return EMPTY_ATTEMPT_ID_ARRAY;
                }
            }
            // Do some basic sanity verification
            if (!verifySanity(mapOutputStat.compressedLength, mapOutputStat.decompressedLength, mapOutputStat.forReduce, remaining, mapOutputStat.srcAttemptId)) {
                if (!stopped) {
                    srcAttemptId = mapOutputStat.srcAttemptId;
                    if (srcAttemptId == null) {
                        srcAttemptId = getNextRemainingAttempt();
                        LOG.warn("Was expecting " + srcAttemptId + " but got null");
                    }
                    assert (srcAttemptId != null);
                    return new InputAttemptIdentifier[] { srcAttemptId };
                } else {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Already stopped. Ignoring verification failure.");
                    }
                    return EMPTY_ATTEMPT_ID_ARRAY;
                }
            }
            if (LOG.isDebugEnabled()) {
                LOG.debug("header: " + mapOutputStat.srcAttemptId + ", len: " + mapOutputStat.compressedLength + ", decomp len: " + mapOutputStat.decompressedLength);
            }
        }
        for (MapOutputStat mapOutputStat : mapOutputStats) {
            // Get the location for the map output - either in-memory or on-disk
            srcAttemptId = mapOutputStat.srcAttemptId;
            decompressedLength = mapOutputStat.decompressedLength;
            compressedLength = mapOutputStat.compressedLength;
            try {
                mapOutput = allocator.reserve(srcAttemptId, decompressedLength, compressedLength, id);
            } catch (IOException e) {
                if (!stopped) {
                    // Kill the reduce attempt
                    ioErrs.increment(1);
                    scheduler.reportLocalError(e);
                } else {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Already stopped. Ignoring error from merger.reserve");
                    }
                }
                return EMPTY_ATTEMPT_ID_ARRAY;
            }
            // Check if we can shuffle *now* ...
            if (mapOutput.getType() == Type.WAIT) {
                LOG.info("fetcher#" + id + " - MergerManager returned Status.WAIT ...");
                // Not an error but wait to process data.
                return EMPTY_ATTEMPT_ID_ARRAY;
            }
            // Go!
            if (LOG.isDebugEnabled()) {
                LOG.debug("fetcher#" + id + " about to shuffle output of map " + mapOutput.getAttemptIdentifier() + " decomp: " + decompressedLength + " len: " + compressedLength + " to " + mapOutput.getType());
            }
            if (mapOutput.getType() == Type.MEMORY) {
                ShuffleUtils.shuffleToMemory(mapOutput.getMemory(), input, (int) decompressedLength, (int) compressedLength, codec, ifileReadAhead, ifileReadAheadLength, LOG, mapOutput.getAttemptIdentifier());
            } else if (mapOutput.getType() == Type.DISK) {
                ShuffleUtils.shuffleToDisk(mapOutput.getDisk(), host.getHostIdentifier(), input, compressedLength, decompressedLength, LOG, mapOutput.getAttemptIdentifier(), ifileReadAhead, ifileReadAheadLength, verifyDiskChecksum);
            } else {
                throw new IOException("Unknown mapOutput type while fetching shuffle data:" + mapOutput.getType());
            }
            // Inform the shuffle scheduler
            long endTime = System.currentTimeMillis();
            // Reset retryStartTime as map task make progress if retried before.
            retryStartTime = 0;
            scheduler.copySucceeded(srcAttemptId, host, compressedLength, decompressedLength, endTime - startTime, mapOutput, false);
        }
        remaining.remove(inputAttemptIdentifier.toString());
    } catch (IOException | InternalError ioe) {
        if (stopped) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Not reporting fetch failure for exception during data copy: [" + ioe.getClass().getName() + ", " + ioe.getMessage() + "]");
            }
            cleanupCurrentConnection(true);
            if (mapOutput != null) {
                // Release resources
                mapOutput.abort();
            }
            // Don't need to put back - since that's handled by the invoker
            return EMPTY_ATTEMPT_ID_ARRAY;
        }
        if (shouldRetry(host, ioe)) {
            // release mem/file handles
            if (mapOutput != null) {
                mapOutput.abort();
            }
            throw new FetcherReadTimeoutException(ioe);
        }
        ioErrs.increment(1);
        if (srcAttemptId == null || mapOutput == null) {
            LOG.info("fetcher#" + id + " failed to read map header" + srcAttemptId + " decomp: " + decompressedLength + ", " + compressedLength, ioe);
            if (srcAttemptId == null) {
                return remaining.values().toArray(new InputAttemptIdentifier[remaining.values().size()]);
            } else {
                return new InputAttemptIdentifier[] { srcAttemptId };
            }
        }
        LOG.warn("Failed to shuffle output of " + srcAttemptId + " from " + host.getHostIdentifier(), ioe);
        // Inform the shuffle-scheduler
        mapOutput.abort();
        return new InputAttemptIdentifier[] { srcAttemptId };
    }
    return null;
}