Examples with DataChecksum org.apache.hadoop.util.DataChecksum used on opensource projects

Example 16 with DataChecksum

use of org.apache.hadoop.util.DataChecksum in project hadoop by apache.

the class BlockMetadataHeader method preadHeader.

/**
   * Read the header without changing the position of the FileChannel.
   * This is used by the client for short-circuit reads.
   *
   * @param fc The FileChannel to read.
   * @return the Metadata Header.
   * @throws IOException on error.
   */
public static BlockMetadataHeader preadHeader(FileChannel fc) throws IOException {
    final byte[] arr = new byte[getHeaderSize()];
    ByteBuffer buf = ByteBuffer.wrap(arr);
    while (buf.hasRemaining()) {
        if (fc.read(buf, 0) <= 0) {
            throw new EOFException("unexpected EOF while reading " + "metadata file header");
        }
    }
    short version = (short) ((arr[0] << 8) | (arr[1] & 0xff));
    DataChecksum dataChecksum = DataChecksum.newDataChecksum(arr, 2);
    return new BlockMetadataHeader(version, dataChecksum);
}

Example 17 with DataChecksum

use of org.apache.hadoop.util.DataChecksum in project hadoop by apache.

the class DataXceiver method replaceBlock.

@Override
public void replaceBlock(final ExtendedBlock block, final StorageType storageType, final Token<BlockTokenIdentifier> blockToken, final String delHint, final DatanodeInfo proxySource) throws IOException {
    updateCurrentThreadName("Replacing block " + block + " from " + delHint);
    DataOutputStream replyOut = new DataOutputStream(getOutputStream());
    checkAccess(replyOut, true, block, blockToken, Op.REPLACE_BLOCK, BlockTokenIdentifier.AccessMode.REPLACE);
    if (!dataXceiverServer.balanceThrottler.acquire()) {
        // not able to start
        String msg = "Not able to receive block " + block.getBlockId() + " from " + peer.getRemoteAddressString() + " because threads " + "quota is exceeded.";
        LOG.warn(msg);
        sendResponse(ERROR, msg);
        return;
    }
    Socket proxySock = null;
    DataOutputStream proxyOut = null;
    Status opStatus = SUCCESS;
    String errMsg = null;
    DataInputStream proxyReply = null;
    boolean IoeDuringCopyBlockOperation = false;
    try {
        // Move the block to different storage in the same datanode
        if (proxySource.equals(datanode.getDatanodeId())) {
            ReplicaInfo oldReplica = datanode.data.moveBlockAcrossStorage(block, storageType);
            if (oldReplica != null) {
                LOG.info("Moved " + block + " from StorageType " + oldReplica.getVolume().getStorageType() + " to " + storageType);
            }
        } else {
            block.setNumBytes(dataXceiverServer.estimateBlockSize);
            // get the output stream to the proxy
            final String dnAddr = proxySource.getXferAddr(connectToDnViaHostname);
            if (LOG.isDebugEnabled()) {
                LOG.debug("Connecting to datanode " + dnAddr);
            }
            InetSocketAddress proxyAddr = NetUtils.createSocketAddr(dnAddr);
            proxySock = datanode.newSocket();
            NetUtils.connect(proxySock, proxyAddr, dnConf.socketTimeout);
            proxySock.setTcpNoDelay(dnConf.getDataTransferServerTcpNoDelay());
            proxySock.setSoTimeout(dnConf.socketTimeout);
            proxySock.setKeepAlive(true);
            OutputStream unbufProxyOut = NetUtils.getOutputStream(proxySock, dnConf.socketWriteTimeout);
            InputStream unbufProxyIn = NetUtils.getInputStream(proxySock);
            DataEncryptionKeyFactory keyFactory = datanode.getDataEncryptionKeyFactoryForBlock(block);
            IOStreamPair saslStreams = datanode.saslClient.socketSend(proxySock, unbufProxyOut, unbufProxyIn, keyFactory, blockToken, proxySource);
            unbufProxyOut = saslStreams.out;
            unbufProxyIn = saslStreams.in;
            proxyOut = new DataOutputStream(new BufferedOutputStream(unbufProxyOut, smallBufferSize));
            proxyReply = new DataInputStream(new BufferedInputStream(unbufProxyIn, ioFileBufferSize));
            /* send request to the proxy */
            IoeDuringCopyBlockOperation = true;
            new Sender(proxyOut).copyBlock(block, blockToken);
            IoeDuringCopyBlockOperation = false;
            // receive the response from the proxy
            BlockOpResponseProto copyResponse = BlockOpResponseProto.parseFrom(PBHelperClient.vintPrefixed(proxyReply));
            String logInfo = "copy block " + block + " from " + proxySock.getRemoteSocketAddress();
            DataTransferProtoUtil.checkBlockOpStatus(copyResponse, logInfo, true);
            // get checksum info about the block we're copying
            ReadOpChecksumInfoProto checksumInfo = copyResponse.getReadOpChecksumInfo();
            DataChecksum remoteChecksum = DataTransferProtoUtil.fromProto(checksumInfo.getChecksum());
            // open a block receiver and check if the block does not exist
            setCurrentBlockReceiver(getBlockReceiver(block, storageType, proxyReply, proxySock.getRemoteSocketAddress().toString(), proxySock.getLocalSocketAddress().toString(), null, 0, 0, 0, "", null, datanode, remoteChecksum, CachingStrategy.newDropBehind(), false, false));
            // receive a block
            blockReceiver.receiveBlock(null, null, replyOut, null, dataXceiverServer.balanceThrottler, null, true);
            // notify name node
            final Replica r = blockReceiver.getReplica();
            datanode.notifyNamenodeReceivedBlock(block, delHint, r.getStorageUuid(), r.isOnTransientStorage());
            LOG.info("Moved " + block + " from " + peer.getRemoteAddressString() + ", delHint=" + delHint);
        }
    } catch (IOException ioe) {
        opStatus = ERROR;
        if (ioe instanceof BlockPinningException) {
            opStatus = Status.ERROR_BLOCK_PINNED;
        }
        errMsg = "opReplaceBlock " + block + " received exception " + ioe;
        LOG.info(errMsg);
        if (!IoeDuringCopyBlockOperation) {
            // Don't double count IO errors
            incrDatanodeNetworkErrors();
        }
        throw ioe;
    } finally {
        // receive the last byte that indicates the proxy released its thread resource
        if (opStatus == SUCCESS && proxyReply != null) {
            try {
                proxyReply.readChar();
            } catch (IOException ignored) {
            }
        }
        // now release the thread resource
        dataXceiverServer.balanceThrottler.release();
        // send response back
        try {
            sendResponse(opStatus, errMsg);
        } catch (IOException ioe) {
            LOG.warn("Error writing reply back to " + peer.getRemoteAddressString());
            incrDatanodeNetworkErrors();
        }
        IOUtils.closeStream(proxyOut);
        IOUtils.closeStream(blockReceiver);
        IOUtils.closeStream(proxyReply);
        IOUtils.closeStream(replyOut);
    }
    //update metrics
    datanode.metrics.addReplaceBlockOp(elapsed());
}

Example 18 with DataChecksum

use of org.apache.hadoop.util.DataChecksum in project hadoop by apache.

the class LocalReplica method truncateBlock.

public static void truncateBlock(FsVolumeSpi volume, File blockFile, File metaFile, long oldlen, long newlen, FileIoProvider fileIoProvider) throws IOException {
    LOG.info("truncateBlock: blockFile=" + blockFile + ", metaFile=" + metaFile + ", oldlen=" + oldlen + ", newlen=" + newlen);
    if (newlen == oldlen) {
        return;
    }
    if (newlen > oldlen) {
        throw new IOException("Cannot truncate block to from oldlen (=" + oldlen + ") to newlen (=" + newlen + ")");
    }
    // fis is closed by BlockMetadataHeader.readHeader.
    final FileInputStream fis = fileIoProvider.getFileInputStream(volume, metaFile);
    DataChecksum dcs = BlockMetadataHeader.readHeader(fis).getChecksum();
    int checksumsize = dcs.getChecksumSize();
    int bpc = dcs.getBytesPerChecksum();
    long n = (newlen - 1) / bpc + 1;
    long newmetalen = BlockMetadataHeader.getHeaderSize() + n * checksumsize;
    long lastchunkoffset = (n - 1) * bpc;
    int lastchunksize = (int) (newlen - lastchunkoffset);
    byte[] b = new byte[Math.max(lastchunksize, checksumsize)];
    try (RandomAccessFile blockRAF = fileIoProvider.getRandomAccessFile(volume, blockFile, "rw")) {
        //truncate blockFile
        blockRAF.setLength(newlen);
        //read last chunk
        blockRAF.seek(lastchunkoffset);
        blockRAF.readFully(b, 0, lastchunksize);
    }
    //compute checksum
    dcs.update(b, 0, lastchunksize);
    dcs.writeValue(b, 0, false);
    //update metaFile
    try (RandomAccessFile metaRAF = fileIoProvider.getRandomAccessFile(volume, metaFile, "rw")) {
        metaRAF.setLength(newmetalen);
        metaRAF.seek(newmetalen - checksumsize);
        metaRAF.write(b, 0, checksumsize);
    }
}

Example 19 with DataChecksum

use of org.apache.hadoop.util.DataChecksum in project hadoop by apache.

the class FsDatasetImpl method computeChecksum.

/**
   * Compute and store the checksum for a block file that does not already have
   * its checksum computed.
   *
   * @param srcReplica source {@link ReplicaInfo}, containing only the checksum
   *     header, not a calculated checksum
   * @param dstMeta destination meta file, into which this method will write a
   *     full computed checksum
   * @param smallBufferSize buffer size to use
   * @param conf the {@link Configuration}
   * @throws IOException
   */
static void computeChecksum(ReplicaInfo srcReplica, File dstMeta, int smallBufferSize, final Configuration conf) throws IOException {
    final File srcMeta = new File(srcReplica.getMetadataURI());
    DataChecksum checksum;
    try (FileInputStream fis = srcReplica.getFileIoProvider().getFileInputStream(srcReplica.getVolume(), srcMeta)) {
        checksum = BlockMetadataHeader.readDataChecksum(fis, DFSUtilClient.getIoFileBufferSize(conf), srcMeta);
    }
    final byte[] data = new byte[1 << 16];
    final byte[] crcs = new byte[checksum.getChecksumSize(data.length)];
    DataOutputStream metaOut = null;
    try {
        File parentFile = dstMeta.getParentFile();
        if (parentFile != null) {
            if (!parentFile.mkdirs() && !parentFile.isDirectory()) {
                throw new IOException("Destination '" + parentFile + "' directory cannot be created");
            }
        }
        metaOut = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(dstMeta), smallBufferSize));
        BlockMetadataHeader.writeHeader(metaOut, checksum);
        int offset = 0;
        try (InputStream dataIn = srcReplica.getDataInputStream(0)) {
            for (int n; (n = dataIn.read(data, offset, data.length - offset)) != -1; ) {
                if (n > 0) {
                    n += offset;
                    offset = n % checksum.getBytesPerChecksum();
                    final int length = n - offset;
                    if (length > 0) {
                        checksum.calculateChunkedSums(data, 0, length, crcs, 0);
                        metaOut.write(crcs, 0, checksum.getChecksumSize(length));
                        System.arraycopy(data, length, data, 0, offset);
                    }
                }
            }
        }
        // calculate and write the last crc
        checksum.calculateChunkedSums(data, 0, offset, crcs, 0);
        metaOut.write(crcs, 0, 4);
    } finally {
        IOUtils.cleanup(null, metaOut);
    }
}

Example 20 with DataChecksum

use of org.apache.hadoop.util.DataChecksum in project hadoop by apache.

the class MappableBlock method verifyChecksum.

/**
   * Verifies the block's checksum. This is an I/O intensive operation.
   */
private static void verifyChecksum(long length, FileInputStream metaIn, FileChannel blockChannel, String blockFileName) throws IOException, ChecksumException {
    // Verify the checksum from the block's meta file
    // Get the DataChecksum from the meta file header
    BlockMetadataHeader header = BlockMetadataHeader.readHeader(new DataInputStream(new BufferedInputStream(metaIn, BlockMetadataHeader.getHeaderSize())));
    FileChannel metaChannel = null;
    try {
        metaChannel = metaIn.getChannel();
        if (metaChannel == null) {
            throw new IOException("Block InputStream meta file has no FileChannel.");
        }
        DataChecksum checksum = header.getChecksum();
        final int bytesPerChecksum = checksum.getBytesPerChecksum();
        final int checksumSize = checksum.getChecksumSize();
        final int numChunks = (8 * 1024 * 1024) / bytesPerChecksum;
        ByteBuffer blockBuf = ByteBuffer.allocate(numChunks * bytesPerChecksum);
        ByteBuffer checksumBuf = ByteBuffer.allocate(numChunks * checksumSize);
        // Verify the checksum
        int bytesVerified = 0;
        while (bytesVerified < length) {
            Preconditions.checkState(bytesVerified % bytesPerChecksum == 0, "Unexpected partial chunk before EOF");
            assert bytesVerified % bytesPerChecksum == 0;
            int bytesRead = fillBuffer(blockChannel, blockBuf);
            if (bytesRead == -1) {
                throw new IOException("checksum verification failed: premature EOF");
            }
            blockBuf.flip();
            // Number of read chunks, including partial chunk at end
            int chunks = (bytesRead + bytesPerChecksum - 1) / bytesPerChecksum;
            checksumBuf.limit(chunks * checksumSize);
            fillBuffer(metaChannel, checksumBuf);
            checksumBuf.flip();
            checksum.verifyChunkedSums(blockBuf, checksumBuf, blockFileName, bytesVerified);
            // Success
            bytesVerified += bytesRead;
            blockBuf.clear();
            checksumBuf.clear();
        }
    } finally {
        IOUtils.closeQuietly(metaChannel);
    }
}