Search in sources :

Example 11 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class ByteRangeInputStream method openInputStream.

@VisibleForTesting
protected InputStreamAndFileLength openInputStream(long startOffset) throws IOException {
    if (startOffset < 0) {
        throw new EOFException("Negative Position");
    }
    // Use the original url if no resolved url exists, eg. if
    // it's the first time a request is made.
    final boolean resolved = resolvedURL.getURL() != null;
    final URLOpener opener = resolved ? resolvedURL : originalURL;
    final HttpURLConnection connection = opener.connect(startOffset, resolved);
    resolvedURL.setURL(getResolvedUrl(connection));
    InputStream in = connection.getInputStream();
    final Long length;
    final Map<String, List<String>> headers = connection.getHeaderFields();
    if (isChunkedTransferEncoding(headers)) {
        // file length is not known
        length = null;
    } else {
        // for non-chunked transfer-encoding, get content-length
        final String cl = connection.getHeaderField(HttpHeaders.CONTENT_LENGTH);
        if (cl == null) {
            throw new IOException(HttpHeaders.CONTENT_LENGTH + " is missing: " + headers);
        }
        final long streamlength = Long.parseLong(cl);
        length = startOffset + streamlength;
        // Java has a bug with >2GB request streams.  It won't bounds check
        // the reads so the transfer blocks until the server times out
        in = new BoundedInputStream(in, streamlength);
    }
    return new InputStreamAndFileLength(length, in);
}
Also used : BoundedInputStream(org.apache.commons.io.input.BoundedInputStream) FSInputStream(org.apache.hadoop.fs.FSInputStream) InputStream(java.io.InputStream) IOException(java.io.IOException) HttpURLConnection(java.net.HttpURLConnection) BoundedInputStream(org.apache.commons.io.input.BoundedInputStream) EOFException(java.io.EOFException) List(java.util.List) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 12 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class BlockTokenIdentifier method writeProtobuf.

@VisibleForTesting
void writeProtobuf(DataOutput out) throws IOException {
    BlockTokenSecretProto secret = PBHelperClient.convert(this);
    out.write(secret.toByteArray());
}
Also used : BlockTokenSecretProto(org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockTokenSecretProto) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 13 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class BlockTokenIdentifier method readFieldsProtobuf.

@VisibleForTesting
void readFieldsProtobuf(DataInput in) throws IOException {
    BlockTokenSecretProto blockTokenSecretProto = BlockTokenSecretProto.parseFrom((DataInputStream) in);
    expiryDate = blockTokenSecretProto.getExpiryDate();
    keyId = blockTokenSecretProto.getKeyId();
    if (blockTokenSecretProto.hasUserId()) {
        userId = blockTokenSecretProto.getUserId();
    } else {
        userId = null;
    }
    if (blockTokenSecretProto.hasBlockPoolId()) {
        blockPoolId = blockTokenSecretProto.getBlockPoolId();
    } else {
        blockPoolId = null;
    }
    blockId = blockTokenSecretProto.getBlockId();
    for (int i = 0; i < blockTokenSecretProto.getModesCount(); i++) {
        AccessModeProto accessModeProto = blockTokenSecretProto.getModes(i);
        modes.add(PBHelperClient.convert(accessModeProto));
    }
    useProto = true;
}
Also used : AccessModeProto(org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.AccessModeProto) BlockTokenSecretProto(org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockTokenSecretProto) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 14 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class RpcProgramNfs3 method write.

@VisibleForTesting
WRITE3Response write(XDR xdr, Channel channel, int xid, SecurityHandler securityHandler, SocketAddress remoteAddress) {
    WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3_OK);
    DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
    if (dfsClient == null) {
        response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
        return response;
    }
    WRITE3Request request;
    try {
        request = WRITE3Request.deserialize(xdr);
    } catch (IOException e) {
        LOG.error("Invalid WRITE request");
        return new WRITE3Response(Nfs3Status.NFS3ERR_INVAL);
    }
    long offset = request.getOffset();
    int count = request.getCount();
    WriteStableHow stableHow = request.getStableHow();
    byte[] data = request.getData().array();
    if (data.length < count) {
        LOG.error("Invalid argument, data size is less than count in request");
        return new WRITE3Response(Nfs3Status.NFS3ERR_INVAL);
    }
    FileHandle handle = request.getHandle();
    if (LOG.isDebugEnabled()) {
        LOG.debug("NFS WRITE fileId: " + handle.getFileId() + " offset: " + offset + " length: " + count + " stableHow: " + stableHow.getValue() + " xid: " + xid + " client: " + remoteAddress);
    }
    Nfs3FileAttributes preOpAttr = null;
    try {
        preOpAttr = writeManager.getFileAttr(dfsClient, handle, iug);
        if (preOpAttr == null) {
            LOG.error("Can't get path for fileId: " + handle.getFileId());
            return new WRITE3Response(Nfs3Status.NFS3ERR_STALE);
        }
        if (!checkAccessPrivilege(remoteAddress, AccessPrivilege.READ_WRITE)) {
            return new WRITE3Response(Nfs3Status.NFS3ERR_ACCES, new WccData(Nfs3Utils.getWccAttr(preOpAttr), preOpAttr), 0, stableHow, Nfs3Constant.WRITE_COMMIT_VERF);
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug("requested offset=" + offset + " and current filesize=" + preOpAttr.getSize());
        }
        writeManager.handleWrite(dfsClient, request, channel, xid, preOpAttr);
    } catch (IOException e) {
        LOG.info("Error writing to fileId " + handle.getFileId() + " at offset " + offset + " and length " + data.length, e);
        // Try to return WccData
        Nfs3FileAttributes postOpAttr = null;
        try {
            postOpAttr = writeManager.getFileAttr(dfsClient, handle, iug);
        } catch (IOException e1) {
            LOG.info("Can't get postOpAttr for fileId: " + handle.getFileId(), e1);
        }
        WccAttr attr = preOpAttr == null ? null : Nfs3Utils.getWccAttr(preOpAttr);
        WccData fileWcc = new WccData(attr, postOpAttr);
        int status = mapErrorStatus(e);
        return new WRITE3Response(status, fileWcc, 0, request.getStableHow(), Nfs3Constant.WRITE_COMMIT_VERF);
    }
    return null;
}
Also used : DFSClient(org.apache.hadoop.hdfs.DFSClient) WccData(org.apache.hadoop.nfs.nfs3.response.WccData) WriteStableHow(org.apache.hadoop.nfs.nfs3.Nfs3Constant.WriteStableHow) FileHandle(org.apache.hadoop.nfs.nfs3.FileHandle) Nfs3FileAttributes(org.apache.hadoop.nfs.nfs3.Nfs3FileAttributes) WRITE3Request(org.apache.hadoop.nfs.nfs3.request.WRITE3Request) WccAttr(org.apache.hadoop.nfs.nfs3.response.WccAttr) IOException(java.io.IOException) WRITE3Response(org.apache.hadoop.nfs.nfs3.response.WRITE3Response) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 15 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class RpcProgramNfs3 method lookup.

@VisibleForTesting
LOOKUP3Response lookup(XDR xdr, SecurityHandler securityHandler, SocketAddress remoteAddress) {
    LOOKUP3Response response = new LOOKUP3Response(Nfs3Status.NFS3_OK);
    if (!checkAccessPrivilege(remoteAddress, AccessPrivilege.READ_ONLY)) {
        response.setStatus(Nfs3Status.NFS3ERR_ACCES);
        return response;
    }
    DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
    if (dfsClient == null) {
        response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
        return response;
    }
    LOOKUP3Request request;
    try {
        request = LOOKUP3Request.deserialize(xdr);
    } catch (IOException e) {
        LOG.error("Invalid LOOKUP request");
        return new LOOKUP3Response(Nfs3Status.NFS3ERR_INVAL);
    }
    FileHandle dirHandle = request.getHandle();
    String fileName = request.getName();
    if (LOG.isDebugEnabled()) {
        LOG.debug("NFS LOOKUP dir fileId: " + dirHandle.getFileId() + " name: " + fileName + " client: " + remoteAddress);
    }
    try {
        String dirFileIdPath = Nfs3Utils.getFileIdPath(dirHandle);
        Nfs3FileAttributes postOpObjAttr = writeManager.getFileAttr(dfsClient, dirHandle, fileName);
        if (postOpObjAttr == null) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("NFS LOOKUP fileId: " + dirHandle.getFileId() + " name: " + fileName + " does not exist");
            }
            Nfs3FileAttributes postOpDirAttr = Nfs3Utils.getFileAttr(dfsClient, dirFileIdPath, iug);
            return new LOOKUP3Response(Nfs3Status.NFS3ERR_NOENT, null, null, postOpDirAttr);
        }
        Nfs3FileAttributes postOpDirAttr = Nfs3Utils.getFileAttr(dfsClient, dirFileIdPath, iug);
        if (postOpDirAttr == null) {
            LOG.info("Can't get path for dir fileId: " + dirHandle.getFileId());
            return new LOOKUP3Response(Nfs3Status.NFS3ERR_STALE);
        }
        FileHandle fileHandle = new FileHandle(postOpObjAttr.getFileId());
        return new LOOKUP3Response(Nfs3Status.NFS3_OK, fileHandle, postOpObjAttr, postOpDirAttr);
    } catch (IOException e) {
        LOG.warn("Exception ", e);
        int status = mapErrorStatus(e);
        return new LOOKUP3Response(status);
    }
}
Also used : DFSClient(org.apache.hadoop.hdfs.DFSClient) FileHandle(org.apache.hadoop.nfs.nfs3.FileHandle) LOOKUP3Request(org.apache.hadoop.nfs.nfs3.request.LOOKUP3Request) Nfs3FileAttributes(org.apache.hadoop.nfs.nfs3.Nfs3FileAttributes) IOException(java.io.IOException) LOOKUP3Response(org.apache.hadoop.nfs.nfs3.response.LOOKUP3Response) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Aggregations

VisibleForTesting (com.google.common.annotations.VisibleForTesting)760 IOException (java.io.IOException)128 ArrayList (java.util.ArrayList)52 Path (java.nio.file.Path)46 Map (java.util.Map)46 File (java.io.File)40 HashMap (java.util.HashMap)34 Path (org.apache.hadoop.fs.Path)30 ImmutableList (com.google.common.collect.ImmutableList)28 Matcher (java.util.regex.Matcher)26 List (java.util.List)24 SourcePath (com.facebook.buck.rules.SourcePath)20 ImmutableMap (com.google.common.collect.ImmutableMap)20 HashSet (java.util.HashSet)19 FileStatus (org.apache.hadoop.fs.FileStatus)19 FileHandle (org.apache.hadoop.nfs.nfs3.FileHandle)19 DFSClient (org.apache.hadoop.hdfs.DFSClient)18 Nfs3FileAttributes (org.apache.hadoop.nfs.nfs3.Nfs3FileAttributes)18 ImmutableSet (com.google.common.collect.ImmutableSet)16 CigarElement (htsjdk.samtools.CigarElement)13