Examples with IOInterruptedException org.apache.tez.runtime.library.api.IOInterruptedException used on opensource projects

Example 1 with IOInterruptedException

use of org.apache.tez.runtime.library.api.IOInterruptedException in project tez by apache.

the class TestOrderedGroupedKVInput method testInterruptWhileAwaitingInput.

@Test(timeout = 5000)
public void testInterruptWhileAwaitingInput() throws IOException, TezException {
    InputContext inputContext = createMockInputContext();
    OrderedGroupedKVInput kvInput = new OrderedGroupedKVInputForTest(inputContext, 10);
    kvInput.initialize();
    kvInput.start();
    try {
        kvInput.getReader();
        Assert.fail("getReader should not return since underlying inputs are not ready");
    } catch (IOException e) {
        Assert.assertTrue(e instanceof IOInterruptedException);
    }
}

Example 2 with IOInterruptedException

use of org.apache.tez.runtime.library.api.IOInterruptedException in project tez by apache.

the class MultiMROutput method getWriter.

@Override
public KeyValueWriterWithBasePath getWriter() throws IOException {
    return new KeyValueWriterWithBasePath() {

        @SuppressWarnings("unchecked")
        @Override
        public void write(Object key, Object value) throws IOException {
            throw new UnsupportedOperationException("Write without basePath isn't supported.");
        }

        @SuppressWarnings("unchecked")
        @Override
        public void write(Object key, Object value, String basePath) throws IOException {
            if (basePath == null) {
                throw new UnsupportedOperationException("Write without basePath isn't supported.");
            }
            if (basePath.length() > 0 && basePath.charAt(0) == '/') {
                // written outside the output committer's task work path.
                throw new UnsupportedOperationException("Write with absolute basePath isn't supported.");
            }
            if (useNewApi) {
                try {
                    getNewRecordWriter(newApiTaskAttemptContext, basePath).write(key, value);
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                    throw new IOInterruptedException("Interrupted while writing next key-value", e);
                }
            } else {
                getOldRecordWriter(basePath).write(key, value);
            }
            outputRecordCounter.increment(1);
            getContext().notifyProgress();
        }
    };
}

Example 3 with IOInterruptedException

use of org.apache.tez.runtime.library.api.IOInterruptedException in project tez by apache.

the class PipelinedSorter method spill.

public boolean spill(boolean ignoreEmptySpills) throws IOException {
    FSDataOutputStream out = null;
    try {
        try {
            boolean ret = merger.ready();
            // then return directly without spilling
            if (!ret && ignoreEmptySpills) {
                return false;
            }
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            LOG.info(outputContext.getDestinationVertexName() + ": Interrupted while waiting for mergers to complete");
            throw new IOInterruptedException(outputContext.getDestinationVertexName() + ": Interrupted while waiting for mergers to complete", e);
        }
        // create spill file
        final long size = capacity + +(partitions * APPROX_HEADER_LENGTH);
        final TezSpillRecord spillRec = new TezSpillRecord(partitions);
        final Path filename = mapOutputFile.getSpillFileForWrite(numSpills, size);
        spillFilePaths.put(numSpills, filename);
        out = rfs.create(filename, true, 4096);
        if (!SPILL_FILE_PERMS.equals(SPILL_FILE_PERMS.applyUMask(FsPermission.getUMask(conf)))) {
            rfs.setPermission(filename, SPILL_FILE_PERMS);
        }
        LOG.info(outputContext.getDestinationVertexName() + ": Spilling to " + filename.toString());
        for (int i = 0; i < partitions; ++i) {
            if (isThreadInterrupted()) {
                return false;
            }
            outputContext.notifyProgress();
            TezRawKeyValueIterator kvIter = merger.filter(i);
            // write merged output to disk
            long segmentStart = out.getPos();
            Writer writer = null;
            boolean hasNext = kvIter.hasNext();
            if (hasNext || !sendEmptyPartitionDetails) {
                writer = new Writer(conf, out, keyClass, valClass, codec, spilledRecordsCounter, null, merger.needsRLE());
            }
            if (combiner == null) {
                while (kvIter.next()) {
                    writer.append(kvIter.getKey(), kvIter.getValue());
                }
            } else {
                if (hasNext) {
                    runCombineProcessor(kvIter, writer);
                }
            }
            long rawLength = 0;
            long partLength = 0;
            // close
            if (writer != null) {
                writer.close();
                rawLength = writer.getRawLength();
                partLength = writer.getCompressedLength();
            }
            adjustSpillCounters(rawLength, partLength);
            // record offsets
            final TezIndexRecord rec = new TezIndexRecord(segmentStart, rawLength, partLength);
            spillRec.putIndex(rec, i);
            if (!isFinalMergeEnabled() && reportPartitionStats()) {
                partitionStats[i] += partLength;
            }
        }
        Path indexFilename = mapOutputFile.getSpillIndexFileForWrite(numSpills, partitions * MAP_OUTPUT_INDEX_RECORD_LENGTH);
        spillFileIndexPaths.put(numSpills, indexFilename);
        spillRec.writeToFile(indexFilename, conf);
        // TODO: honor cache limits
        indexCacheList.add(spillRec);
        ++numSpills;
        if (!isFinalMergeEnabled()) {
            fileOutputByteCounter.increment(rfs.getFileStatus(filename).getLen());
            // No final merge. Set the number of files offered via shuffle-handler
            numShuffleChunks.setValue(numSpills);
        }
        return true;
    } finally {
        if (out != null) {
            out.close();
        }
    }
}

Example 4 with IOInterruptedException

use of org.apache.tez.runtime.library.api.IOInterruptedException in project tez by apache.

the class ExternalSorter method runCombineProcessor.

protected void runCombineProcessor(TezRawKeyValueIterator kvIter, Writer writer) throws IOException {
    try {
        outputContext.notifyProgress();
        combiner.combine(kvIter, writer);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new IOInterruptedException("Combiner interrupted", e);
    }
}

Example 5 with IOInterruptedException

use of org.apache.tez.runtime.library.api.IOInterruptedException in project tez by apache.

the class UnorderedKVReader method moveToNextInput.

/**
 * Moves to the next available input. This method may block if the input is not ready yet.
 * Also takes care of closing the previous input.
 *
 * @return true if the next input exists, false otherwise
 * @throws IOException
 */
private boolean moveToNextInput() throws IOException {
    if (currentReader != null) {
        // Close the current reader.
        totalBytesRead.getAndAdd(currentReader.bytesRead);
        currentReader.close();
        /**
         * clear reader explicitly. Otherwise this could point to stale reference when next() is
         * called and end up throwing EOF exception from IFIle. Ref: TEZ-2348
         */
        currentReader = null;
        currentFetchedInput.free();
    }
    try {
        currentFetchedInput = shuffleManager.getNextInput();
    } catch (InterruptedException e) {
        LOG.warn("Interrupted while waiting for next available input", e);
        Thread.currentThread().interrupt();
        throw new IOInterruptedException(e);
    }
    if (currentFetchedInput == null) {
        hasCompletedProcessing();
        // No more inputs
        return false;
    } else {
        currentReader = openIFileReader(currentFetchedInput);
        totalFileBytes.getAndAdd(currentReader.getLength());
        return true;
    }
}