Examples with SocketOutputStream org.apache.hadoop.net.SocketOutputStream used on opensource projects

Example 1 with SocketOutputStream

use of org.apache.hadoop.net.SocketOutputStream in project hadoop by apache.

the class BlockSender method sendPacket.

/**
   * Sends a packet with up to maxChunks chunks of data.
   * 
   * @param pkt buffer used for writing packet data
   * @param maxChunks maximum number of chunks to send
   * @param out stream to send data to
   * @param transferTo use transferTo to send data
   * @param throttler used for throttling data transfer bandwidth
   */
private int sendPacket(ByteBuffer pkt, int maxChunks, OutputStream out, boolean transferTo, DataTransferThrottler throttler) throws IOException {
    int dataLen = (int) Math.min(endOffset - offset, (chunkSize * (long) maxChunks));
    // Number of chunks be sent in the packet
    int numChunks = numberOfChunks(dataLen);
    int checksumDataLen = numChunks * checksumSize;
    int packetLen = dataLen + checksumDataLen + 4;
    boolean lastDataPacket = offset + dataLen == endOffset && dataLen > 0;
    // The packet buffer is organized as follows:
    // _______HHHHCCCCD?D?D?D?
    //        ^   ^
    //        |   \ checksumOff
    //        \ headerOff
    // _ padding, since the header is variable-length
    // H = header and length prefixes
    // C = checksums
    // D? = data, if transferTo is false.
    int headerLen = writePacketHeader(pkt, dataLen, packetLen);
    // Per above, the header doesn't start at the beginning of the
    // buffer
    int headerOff = pkt.position() - headerLen;
    int checksumOff = pkt.position();
    byte[] buf = pkt.array();
    if (checksumSize > 0 && ris.getChecksumIn() != null) {
        readChecksum(buf, checksumOff, checksumDataLen);
        // write in progress that we need to use to get last checksum
        if (lastDataPacket && lastChunkChecksum != null) {
            int start = checksumOff + checksumDataLen - checksumSize;
            byte[] updatedChecksum = lastChunkChecksum.getChecksum();
            if (updatedChecksum != null) {
                System.arraycopy(updatedChecksum, 0, buf, start, checksumSize);
            }
        }
    }
    int dataOff = checksumOff + checksumDataLen;
    if (!transferTo) {
        // normal transfer
        ris.readDataFully(buf, dataOff, dataLen);
        if (verifyChecksum) {
            verifyChecksum(buf, dataOff, dataLen, numChunks, checksumOff);
        }
    }
    try {
        if (transferTo) {
            SocketOutputStream sockOut = (SocketOutputStream) out;
            // First write header and checksums
            sockOut.write(buf, headerOff, dataOff - headerOff);
            // no need to flush since we know out is not a buffered stream
            FileChannel fileCh = ((FileInputStream) ris.getDataIn()).getChannel();
            LongWritable waitTime = new LongWritable();
            LongWritable transferTime = new LongWritable();
            fileIoProvider.transferToSocketFully(ris.getVolumeRef().getVolume(), sockOut, fileCh, blockInPosition, dataLen, waitTime, transferTime);
            datanode.metrics.addSendDataPacketBlockedOnNetworkNanos(waitTime.get());
            datanode.metrics.addSendDataPacketTransferNanos(transferTime.get());
            blockInPosition += dataLen;
        } else {
            // normal transfer
            out.write(buf, headerOff, dataOff + dataLen - headerOff);
        }
    } catch (IOException e) {
        if (e instanceof SocketTimeoutException) {
        /*
         * writing to client timed out.  This happens if the client reads
         * part of a block and then decides not to read the rest (but leaves
         * the socket open).
         * 
         * Reporting of this case is done in DataXceiver#run
         */
        } else {
            /* Exception while writing to the client. Connection closure from
         * the other end is mostly the case and we do not care much about
         * it. But other things can go wrong, especially in transferTo(),
         * which we do not want to ignore.
         *
         * The message parsing below should not be considered as a good
         * coding example. NEVER do it to drive a program logic. NEVER.
         * It was done here because the NIO throws an IOException for EPIPE.
         */
            String ioem = e.getMessage();
            if (!ioem.startsWith("Broken pipe") && !ioem.startsWith("Connection reset")) {
                LOG.error("BlockSender.sendChunks() exception: ", e);
                datanode.getBlockScanner().markSuspectBlock(ris.getVolumeRef().getVolume().getStorageID(), block);
            }
        }
        throw ioeToSocketException(e);
    }
    if (throttler != null) {
        // rebalancing so throttle
        throttler.throttle(packetLen);
    }
    return dataLen;
}

Example 2 with SocketOutputStream

use of org.apache.hadoop.net.SocketOutputStream in project hadoop by apache.

the class BlockSender method doSendBlock.

private long doSendBlock(DataOutputStream out, OutputStream baseStream, DataTransferThrottler throttler) throws IOException {
    if (out == null) {
        throw new IOException("out stream is null");
    }
    initialOffset = offset;
    long totalRead = 0;
    OutputStream streamForSendChunks = out;
    lastCacheDropOffset = initialOffset;
    if (isLongRead() && ris.getDataInFd() != null) {
        // Advise that this file descriptor will be accessed sequentially.
        ris.dropCacheBehindReads(block.getBlockName(), 0, 0, POSIX_FADV_SEQUENTIAL);
    }
    // Trigger readahead of beginning of file if configured.
    manageOsCache();
    final long startTime = ClientTraceLog.isDebugEnabled() ? System.nanoTime() : 0;
    try {
        int maxChunksPerPacket;
        int pktBufSize = PacketHeader.PKT_MAX_HEADER_LEN;
        boolean transferTo = transferToAllowed && !verifyChecksum && baseStream instanceof SocketOutputStream && ris.getDataIn() instanceof FileInputStream;
        if (transferTo) {
            FileChannel fileChannel = ((FileInputStream) ris.getDataIn()).getChannel();
            blockInPosition = fileChannel.position();
            streamForSendChunks = baseStream;
            maxChunksPerPacket = numberOfChunks(TRANSFERTO_BUFFER_SIZE);
            // Smaller packet size to only hold checksum when doing transferTo
            pktBufSize += checksumSize * maxChunksPerPacket;
        } else {
            maxChunksPerPacket = Math.max(1, numberOfChunks(IO_FILE_BUFFER_SIZE));
            // Packet size includes both checksum and data
            pktBufSize += (chunkSize + checksumSize) * maxChunksPerPacket;
        }
        ByteBuffer pktBuf = ByteBuffer.allocate(pktBufSize);
        while (endOffset > offset && !Thread.currentThread().isInterrupted()) {
            manageOsCache();
            long len = sendPacket(pktBuf, maxChunksPerPacket, streamForSendChunks, transferTo, throttler);
            offset += len;
            totalRead += len + (numberOfChunks(len) * checksumSize);
            seqno++;
        }
        // If this thread was interrupted, then it did not send the full block.
        if (!Thread.currentThread().isInterrupted()) {
            try {
                // send an empty packet to mark the end of the block
                sendPacket(pktBuf, maxChunksPerPacket, streamForSendChunks, transferTo, throttler);
                out.flush();
            } catch (IOException e) {
                //socket error
                throw ioeToSocketException(e);
            }
            sentEntireByteRange = true;
        }
    } finally {
        if ((clientTraceFmt != null) && ClientTraceLog.isDebugEnabled()) {
            final long endTime = System.nanoTime();
            ClientTraceLog.debug(String.format(clientTraceFmt, totalRead, initialOffset, endTime - startTime));
        }
        close();
    }
    return totalRead;
}