Examples with FileSystem alluxio.client.file.FileSystem used on opensource projects

Example 81 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class S3RestServiceHandler method createObjectOrUploadPart.

/**
 * @summary uploads an object or part of an object in multipart upload
 * @param authorization header parameter authorization
 * @param contentMD5 the optional Base64 encoded 128-bit MD5 digest of the object
 * @param copySource the source path to copy the new file from
 * @param decodedLength the length of the content when in aws-chunked encoding
 * @param contentLength the total length of the request body
 * @param bucket the bucket name
 * @param object the object name
 * @param partNumber the identification of the part of the object in multipart upload,
 *                   otherwise null
 * @param uploadId the upload ID of the multipart upload, otherwise null
 * @param is the request body
 * @return the response object
 */
@PUT
@Path(OBJECT_PARAM)
@Consumes(MediaType.WILDCARD)
public Response createObjectOrUploadPart(@HeaderParam("Authorization") String authorization, @HeaderParam("Content-MD5") final String contentMD5, @HeaderParam("x-amz-copy-source") String copySource, @HeaderParam("x-amz-decoded-content-length") String decodedLength, @HeaderParam("Content-Length") String contentLength, @PathParam("bucket") final String bucket, @PathParam("object") final String object, @QueryParam("partNumber") final Integer partNumber, @QueryParam("uploadId") final Long uploadId, final InputStream is) {
    return S3RestUtils.call(bucket, () -> {
        Preconditions.checkNotNull(bucket, "required 'bucket' parameter is missing");
        Preconditions.checkNotNull(object, "required 'object' parameter is missing");
        Preconditions.checkArgument((partNumber == null && uploadId == null) || (partNumber != null && uploadId != null), "'partNumber' and 'uploadId' parameter should appear together or be " + "missing together.");
        String bucketPath = S3RestUtils.parsePath(AlluxioURI.SEPARATOR + bucket);
        final FileSystem fs = getFileSystem(authorization);
        S3RestUtils.checkPathIsAlluxioDirectory(fs, bucketPath);
        String objectPath = bucketPath + AlluxioURI.SEPARATOR + object;
        CreateDirectoryPOptions dirOptions = CreateDirectoryPOptions.newBuilder().setRecursive(true).setAllowExists(true).build();
        if (objectPath.endsWith(AlluxioURI.SEPARATOR)) {
            // Need to create a folder
            try {
                fs.createDirectory(new AlluxioURI(objectPath), dirOptions);
            } catch (FileAlreadyExistsException e) {
                // ok if directory already exists the user wanted to create it anyway
                LOG.warn("attempting to create dir which already exists");
            } catch (IOException | AlluxioException e) {
                throw S3RestUtils.toObjectS3Exception(e, objectPath);
            }
            return Response.ok().build();
        }
        if (partNumber != null) {
            // This object is part of a multipart upload, should be uploaded into the temporary
            // directory first.
            String tmpDir = S3RestUtils.getMultipartTemporaryDirForObject(bucketPath, object);
            S3RestUtils.checkUploadId(fs, new AlluxioURI(tmpDir), uploadId);
            objectPath = tmpDir + AlluxioURI.SEPARATOR + partNumber;
        }
        AlluxioURI objectURI = new AlluxioURI(objectPath);
        // remove exist object
        deleteExistObject(fs, objectURI);
        CreateFilePOptions filePOptions = CreateFilePOptions.newBuilder().setRecursive(true).setWriteType(S3RestUtils.getS3WriteType()).build();
        // not copying from an existing file
        if (copySource == null) {
            try {
                MessageDigest md5 = MessageDigest.getInstance("MD5");
                // The request body can be in the aws-chunked encoding format, or not encoded at all
                // determine if it's encoded, and then which parts of the stream to read depending on
                // the encoding type.
                boolean isChunkedEncoding = decodedLength != null;
                int toRead;
                InputStream readStream = is;
                if (isChunkedEncoding) {
                    toRead = Integer.parseInt(decodedLength);
                    readStream = new ChunkedEncodingInputStream(is);
                } else {
                    toRead = Integer.parseInt(contentLength);
                }
                FileOutStream os = fs.createFile(objectURI, filePOptions);
                try (DigestOutputStream digestOutputStream = new DigestOutputStream(os, md5)) {
                    long read = ByteStreams.copy(ByteStreams.limit(readStream, toRead), digestOutputStream);
                    if (read < toRead) {
                        throw new IOException(String.format("Failed to read all required bytes from the stream. Read %d/%d", read, toRead));
                    }
                }
                byte[] digest = md5.digest();
                String base64Digest = BaseEncoding.base64().encode(digest);
                if (contentMD5 != null && !contentMD5.equals(base64Digest)) {
                    // The object may be corrupted, delete the written object and return an error.
                    try {
                        fs.delete(objectURI, DeletePOptions.newBuilder().setRecursive(true).build());
                    } catch (Exception e2) {
                    // intend to continue and return BAD_DIGEST S3Exception.
                    }
                    throw new S3Exception(objectURI.getPath(), S3ErrorCode.BAD_DIGEST);
                }
                String entityTag = Hex.encodeHexString(digest);
                return Response.ok().tag(entityTag).build();
            } catch (Exception e) {
                throw S3RestUtils.toObjectS3Exception(e, objectPath);
            }
        } else {
            try (FileInStream in = fs.openFile(new AlluxioURI(!copySource.startsWith(AlluxioURI.SEPARATOR) ? AlluxioURI.SEPARATOR + copySource : copySource));
                FileOutStream out = fs.createFile(objectURI)) {
                MessageDigest md5 = MessageDigest.getInstance("MD5");
                try (DigestOutputStream digestOut = new DigestOutputStream(out, md5)) {
                    IOUtils.copyLarge(in, digestOut, new byte[8 * Constants.MB]);
                    byte[] digest = md5.digest();
                    String entityTag = Hex.encodeHexString(digest);
                    return new CopyObjectResult(entityTag, System.currentTimeMillis());
                } catch (IOException e) {
                    try {
                        out.cancel();
                    } catch (Throwable t2) {
                        e.addSuppressed(t2);
                    }
                    throw e;
                }
            } catch (Exception e) {
                throw S3RestUtils.toObjectS3Exception(e, objectPath);
            }
        }
    });
}

Example 82 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class S3RestServiceHandler method getObjectOrListParts.

/**
 * @summary downloads an object or list parts of the object in multipart upload
 * @param authorization header parameter authorization
 * @param bucket the bucket name
 * @param object the object name
 * @param uploadId the ID of the multipart upload, if not null, listing parts of the object
 * @param range the http range header
 * @return the response object
 */
@GET
@Path(OBJECT_PARAM)
@Produces({ MediaType.APPLICATION_XML, MediaType.APPLICATION_OCTET_STREAM })
public Response getObjectOrListParts(@HeaderParam("Authorization") String authorization, @HeaderParam("Range") final String range, @PathParam("bucket") final String bucket, @PathParam("object") final String object, @QueryParam("uploadId") final Long uploadId) {
    Preconditions.checkNotNull(bucket, "required 'bucket' parameter is missing");
    Preconditions.checkNotNull(object, "required 'object' parameter is missing");
    final FileSystem fs = getFileSystem(authorization);
    if (uploadId != null) {
        return listParts(fs, bucket, object, uploadId);
    } else {
        return getObject(fs, bucket, object, range);
    }
}

Example 83 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class S3RestServiceHandler method initiateOrCompleteMultipartUpload.

/**
 * @summary initiates or completes a multipart upload based on query parameters
 * @param authorization header parameter authorization
 * @param bucket the bucket name
 * @param object the object name
 * @param uploads the query parameter specifying that this request is to initiate a multipart
 *                upload instead of uploading an object through HTTP multipart forms
 * @param uploadId the ID of the multipart upload to be completed
 * @return the response object
 */
@POST
@Path(OBJECT_PARAM)
// endpoints where the only difference is the query parameter.
@Consumes({ MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_XML })
public Response initiateOrCompleteMultipartUpload(@HeaderParam("Authorization") String authorization, @PathParam("bucket") final String bucket, @PathParam("object") final String object, @QueryParam("uploads") final String uploads, @QueryParam("uploadId") final Long uploadId) {
    Preconditions.checkArgument(uploads != null || uploadId != null, "parameter 'uploads' or 'uploadId' should exist");
    final FileSystem fileSystem = getFileSystem(authorization);
    if (uploads != null) {
        return initiateMultipartUpload(fileSystem, bucket, object);
    } else {
        return completeMultipartUpload(fileSystem, bucket, object, uploadId);
    }
}

Example 84 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class S3RestServiceHandler method deleteBucket.

/**
 * @summary deletes a bucket
 * @param authorization header parameter authorization
 * @param bucket the bucket name
 * @return the response object
 */
@DELETE
@Path(BUCKET_PARAM)
public Response deleteBucket(@HeaderParam("Authorization") String authorization, @PathParam("bucket") final String bucket) {
    return S3RestUtils.call(bucket, () -> {
        Preconditions.checkNotNull(bucket, "required 'bucket' parameter is missing");
        String bucketPath = S3RestUtils.parsePath(AlluxioURI.SEPARATOR + bucket);
        final FileSystem fs = getFileSystem(authorization);
        S3RestUtils.checkPathIsAlluxioDirectory(fs, bucketPath);
        // Delete the bucket.
        DeletePOptions options = DeletePOptions.newBuilder().setAlluxioOnly(ServerConfiguration.get(PropertyKey.PROXY_S3_DELETE_TYPE).equals(Constants.S3_DELETE_IN_ALLUXIO_ONLY)).build();
        try {
            fs.delete(new AlluxioURI(bucketPath), options);
        } catch (Exception e) {
            throw S3RestUtils.toBucketS3Exception(e, bucketPath);
        }
        return Response.Status.NO_CONTENT;
    });
}

Example 85 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class FileSystemMasterClientIntegrationTest method masterUnavailable.

@Test(timeout = 300000)
@LocalAlluxioClusterResource.Config(confParams = { PropertyKey.Name.USER_RPC_RETRY_MAX_DURATION, "10s" })
public void masterUnavailable() throws Exception {
    mLocalAlluxioClusterResource.get().getLocalAlluxioMaster().stop();
    Thread thread = new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                Thread.sleep(3000);
                mLocalAlluxioClusterResource.get().getLocalAlluxioMaster().start();
            } catch (InterruptedException e) {
                throw Throwables.propagate(e);
            }
        }
    });
    thread.start();
    FileSystem fileSystem = mLocalAlluxioClusterResource.get().getClient();
    fileSystem.listStatus(new AlluxioURI("/"));
    thread.join();
}