Examples with CreateDirectoryPOptions alluxio.grpc.CreateDirectoryPOptions used on opensource projects

Example 6 with CreateDirectoryPOptions

use of alluxio.grpc.CreateDirectoryPOptions in project alluxio by Alluxio.

the class UfsJournalIntegrationTest method delete.

/**
 * Tests file and directory creation and deletion.
 */
@Test
public void delete() throws Exception {
    CreateDirectoryPOptions recMkdir = CreateDirectoryPOptions.newBuilder().setRecursive(true).build();
    DeletePOptions recDelete = DeletePOptions.newBuilder().setRecursive(true).build();
    for (int i = 0; i < 10; i++) {
        String dirPath = "/i" + i;
        mFileSystem.createDirectory(new AlluxioURI(dirPath), recMkdir);
        for (int j = 0; j < 10; j++) {
            CreateFilePOptions option = CreateFilePOptions.newBuilder().setBlockSizeBytes((i + j + 1) * 64).build();
            String filePath = dirPath + "/j" + j;
            mFileSystem.createFile(new AlluxioURI(filePath), option).close();
            if (j >= 5) {
                mFileSystem.delete(new AlluxioURI(filePath), recDelete);
            }
        }
        if (i >= 5) {
            mFileSystem.delete(new AlluxioURI(dirPath), recDelete);
        }
    }
    mLocalAlluxioCluster.stopFS();
    deleteTestUtil();
    deleteFsMasterJournalLogs();
    deleteTestUtil();
}

Example 7 with CreateDirectoryPOptions

use of alluxio.grpc.CreateDirectoryPOptions in project alluxio by Alluxio.

the class AbstractFileSystem method mkdirs.

/**
 * Attempts to create a folder with the specified path. Parent directories will be created.
 *
 * @param path path to create
 * @param permission permissions to grant the created folder
 * @return true if the indicated folder is created successfully or already exists
 */
@Override
public boolean mkdirs(Path path, FsPermission permission) throws IOException {
    LOG.debug("mkdirs({}, {})", path, permission);
    if (mStatistics != null) {
        mStatistics.incrementWriteOps(1);
    }
    AlluxioURI uri = getAlluxioPath(path);
    CreateDirectoryPOptions options = CreateDirectoryPOptions.newBuilder().setRecursive(true).setAllowExists(true).setMode(new Mode(permission.toShort()).toProto()).build();
    try {
        mFileSystem.createDirectory(uri, options);
        return true;
    } catch (AlluxioException e) {
        throw new IOException(e);
    }
}

Example 8 with CreateDirectoryPOptions

use of alluxio.grpc.CreateDirectoryPOptions in project alluxio by Alluxio.

the class S3RestServiceHandler method createBucket.

/**
 * @summary creates a bucket
 * @param authorization header parameter authorization
 * @param bucket the bucket name
 * @return the response object
 */
@PUT
@Path(BUCKET_PARAM)
public Response createBucket(@HeaderParam("Authorization") String authorization, @PathParam("bucket") final String bucket) {
    return S3RestUtils.call(bucket, () -> {
        Preconditions.checkNotNull(bucket, "required 'bucket' parameter is missing");
        String bucketPath = S3RestUtils.parsePath(AlluxioURI.SEPARATOR + bucket);
        final FileSystem fs = getFileSystem(authorization);
        // Create the bucket.
        CreateDirectoryPOptions options = CreateDirectoryPOptions.newBuilder().setWriteType(S3RestUtils.getS3WriteType()).build();
        try {
            fs.createDirectory(new AlluxioURI(bucketPath), options);
        } catch (Exception e) {
            throw S3RestUtils.toBucketS3Exception(e, bucketPath);
        }
        return Response.Status.OK;
    });
}

Example 9 with CreateDirectoryPOptions

use of alluxio.grpc.CreateDirectoryPOptions in project alluxio by Alluxio.

the class S3RestServiceHandler method createObjectOrUploadPart.

/**
 * @summary uploads an object or part of an object in multipart upload
 * @param authorization header parameter authorization
 * @param contentMD5 the optional Base64 encoded 128-bit MD5 digest of the object
 * @param copySource the source path to copy the new file from
 * @param decodedLength the length of the content when in aws-chunked encoding
 * @param contentLength the total length of the request body
 * @param bucket the bucket name
 * @param object the object name
 * @param partNumber the identification of the part of the object in multipart upload,
 *                   otherwise null
 * @param uploadId the upload ID of the multipart upload, otherwise null
 * @param is the request body
 * @return the response object
 */
@PUT
@Path(OBJECT_PARAM)
@Consumes(MediaType.WILDCARD)
public Response createObjectOrUploadPart(@HeaderParam("Authorization") String authorization, @HeaderParam("Content-MD5") final String contentMD5, @HeaderParam("x-amz-copy-source") String copySource, @HeaderParam("x-amz-decoded-content-length") String decodedLength, @HeaderParam("Content-Length") String contentLength, @PathParam("bucket") final String bucket, @PathParam("object") final String object, @QueryParam("partNumber") final Integer partNumber, @QueryParam("uploadId") final Long uploadId, final InputStream is) {
    return S3RestUtils.call(bucket, () -> {
        Preconditions.checkNotNull(bucket, "required 'bucket' parameter is missing");
        Preconditions.checkNotNull(object, "required 'object' parameter is missing");
        Preconditions.checkArgument((partNumber == null && uploadId == null) || (partNumber != null && uploadId != null), "'partNumber' and 'uploadId' parameter should appear together or be " + "missing together.");
        String bucketPath = S3RestUtils.parsePath(AlluxioURI.SEPARATOR + bucket);
        final FileSystem fs = getFileSystem(authorization);
        S3RestUtils.checkPathIsAlluxioDirectory(fs, bucketPath);
        String objectPath = bucketPath + AlluxioURI.SEPARATOR + object;
        CreateDirectoryPOptions dirOptions = CreateDirectoryPOptions.newBuilder().setRecursive(true).setAllowExists(true).build();
        if (objectPath.endsWith(AlluxioURI.SEPARATOR)) {
            // Need to create a folder
            try {
                fs.createDirectory(new AlluxioURI(objectPath), dirOptions);
            } catch (FileAlreadyExistsException e) {
                // ok if directory already exists the user wanted to create it anyway
                LOG.warn("attempting to create dir which already exists");
            } catch (IOException | AlluxioException e) {
                throw S3RestUtils.toObjectS3Exception(e, objectPath);
            }
            return Response.ok().build();
        }
        if (partNumber != null) {
            // This object is part of a multipart upload, should be uploaded into the temporary
            // directory first.
            String tmpDir = S3RestUtils.getMultipartTemporaryDirForObject(bucketPath, object);
            S3RestUtils.checkUploadId(fs, new AlluxioURI(tmpDir), uploadId);
            objectPath = tmpDir + AlluxioURI.SEPARATOR + partNumber;
        }
        AlluxioURI objectURI = new AlluxioURI(objectPath);
        // remove exist object
        deleteExistObject(fs, objectURI);
        CreateFilePOptions filePOptions = CreateFilePOptions.newBuilder().setRecursive(true).setWriteType(S3RestUtils.getS3WriteType()).build();
        // not copying from an existing file
        if (copySource == null) {
            try {
                MessageDigest md5 = MessageDigest.getInstance("MD5");
                // The request body can be in the aws-chunked encoding format, or not encoded at all
                // determine if it's encoded, and then which parts of the stream to read depending on
                // the encoding type.
                boolean isChunkedEncoding = decodedLength != null;
                int toRead;
                InputStream readStream = is;
                if (isChunkedEncoding) {
                    toRead = Integer.parseInt(decodedLength);
                    readStream = new ChunkedEncodingInputStream(is);
                } else {
                    toRead = Integer.parseInt(contentLength);
                }
                FileOutStream os = fs.createFile(objectURI, filePOptions);
                try (DigestOutputStream digestOutputStream = new DigestOutputStream(os, md5)) {
                    long read = ByteStreams.copy(ByteStreams.limit(readStream, toRead), digestOutputStream);
                    if (read < toRead) {
                        throw new IOException(String.format("Failed to read all required bytes from the stream. Read %d/%d", read, toRead));
                    }
                }
                byte[] digest = md5.digest();
                String base64Digest = BaseEncoding.base64().encode(digest);
                if (contentMD5 != null && !contentMD5.equals(base64Digest)) {
                    // The object may be corrupted, delete the written object and return an error.
                    try {
                        fs.delete(objectURI, DeletePOptions.newBuilder().setRecursive(true).build());
                    } catch (Exception e2) {
                    // intend to continue and return BAD_DIGEST S3Exception.
                    }
                    throw new S3Exception(objectURI.getPath(), S3ErrorCode.BAD_DIGEST);
                }
                String entityTag = Hex.encodeHexString(digest);
                return Response.ok().tag(entityTag).build();
            } catch (Exception e) {
                throw S3RestUtils.toObjectS3Exception(e, objectPath);
            }
        } else {
            try (FileInStream in = fs.openFile(new AlluxioURI(!copySource.startsWith(AlluxioURI.SEPARATOR) ? AlluxioURI.SEPARATOR + copySource : copySource));
                FileOutStream out = fs.createFile(objectURI)) {
                MessageDigest md5 = MessageDigest.getInstance("MD5");
                try (DigestOutputStream digestOut = new DigestOutputStream(out, md5)) {
                    IOUtils.copyLarge(in, digestOut, new byte[8 * Constants.MB]);
                    byte[] digest = md5.digest();
                    String entityTag = Hex.encodeHexString(digest);
                    return new CopyObjectResult(entityTag, System.currentTimeMillis());
                } catch (IOException e) {
                    try {
                        out.cancel();
                    } catch (Throwable t2) {
                        e.addSuppressed(t2);
                    }
                    throw e;
                }
            } catch (Exception e) {
                throw S3RestUtils.toObjectS3Exception(e, objectPath);
            }
        }
    });
}

Example 10 with CreateDirectoryPOptions

use of alluxio.grpc.CreateDirectoryPOptions in project alluxio by Alluxio.

the class S3RestServiceHandler method initiateMultipartUpload.

private Response initiateMultipartUpload(final FileSystem fs, final String bucket, final String object) {
    return S3RestUtils.call(bucket, () -> {
        String bucketPath = S3RestUtils.parsePath(AlluxioURI.SEPARATOR + bucket);
        S3RestUtils.checkPathIsAlluxioDirectory(fs, bucketPath);
        String objectPath = bucketPath + AlluxioURI.SEPARATOR + object;
        AlluxioURI multipartTemporaryDir = new AlluxioURI(S3RestUtils.getMultipartTemporaryDirForObject(bucketPath, object));
        // remove exist object
        deleteExistObject(fs, new AlluxioURI(objectPath));
        // remove exist multipartTemporaryDir
        deleteExistObject(fs, multipartTemporaryDir, true);
        CreateDirectoryPOptions options = CreateDirectoryPOptions.newBuilder().setRecursive(true).setWriteType(S3RestUtils.getS3WriteType()).build();
        try {
            if (fs.exists(multipartTemporaryDir)) {
                if (MultipartUploadCleaner.apply(fs, bucket, object)) {
                    throw new S3Exception(multipartTemporaryDir.getPath(), S3ErrorCode.UPLOAD_ALREADY_EXISTS);
                }
            }
            fs.createDirectory(multipartTemporaryDir, options);
            // Use the file ID of multipartTemporaryDir as the upload ID.
            long uploadId = fs.getStatus(multipartTemporaryDir).getFileId();
            MultipartUploadCleaner.apply(fs, bucket, object, uploadId);
            return new InitiateMultipartUploadResult(bucket, object, Long.toString(uploadId));
        } catch (Exception e) {
            throw S3RestUtils.toObjectS3Exception(e, objectPath);
        }
    });
}