Examples with Copy com.amazonaws.services.s3.transfer.Copy used on opensource projects

Example 1 with Copy

use of com.amazonaws.services.s3.transfer.Copy in project hadoop by apache.

the class S3AFileSystem method copyFile.

/**
   * Copy a single object in the bucket via a COPY operation.
   * @param srcKey source object path
   * @param dstKey destination object path
   * @param size object size
   * @throws AmazonClientException on failures inside the AWS SDK
   * @throws InterruptedIOException the operation was interrupted
   * @throws IOException Other IO problems
   */
private void copyFile(String srcKey, String dstKey, long size) throws IOException, InterruptedIOException, AmazonClientException {
    LOG.debug("copyFile {} -> {} ", srcKey, dstKey);
    try {
        ObjectMetadata srcom = getObjectMetadata(srcKey);
        ObjectMetadata dstom = cloneObjectMetadata(srcom);
        setOptionalObjectMetadata(dstom);
        CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucket, srcKey, bucket, dstKey);
        setOptionalCopyObjectRequestParameters(copyObjectRequest);
        copyObjectRequest.setCannedAccessControlList(cannedACL);
        copyObjectRequest.setNewObjectMetadata(dstom);
        ProgressListener progressListener = new ProgressListener() {

            public void progressChanged(ProgressEvent progressEvent) {
                switch(progressEvent.getEventType()) {
                    case TRANSFER_PART_COMPLETED_EVENT:
                        incrementWriteOperations();
                        break;
                    default:
                        break;
                }
            }
        };
        Copy copy = transfers.copy(copyObjectRequest);
        copy.addProgressListener(progressListener);
        try {
            copy.waitForCopyResult();
            incrementWriteOperations();
            instrumentation.filesCopied(1, size);
        } catch (InterruptedException e) {
            throw new InterruptedIOException("Interrupted copying " + srcKey + " to " + dstKey + ", cancelling");
        }
    } catch (AmazonClientException e) {
        throw translateException("copyFile(" + srcKey + ", " + dstKey + ")", srcKey, e);
    }
}

Example 2 with Copy

use of com.amazonaws.services.s3.transfer.Copy in project hadoop by apache.

the class S3AFileSystem method innerRename.

/**
   * The inner rename operation. See {@link #rename(Path, Path)} for
   * the description of the operation.
   * This operation throws an exception on any failure which needs to be
   * reported and downgraded to a failure. That is: if a rename
   * @param src path to be renamed
   * @param dst new path after rename
   * @throws RenameFailedException if some criteria for a state changing
   * rename was not met. This means work didn't happen; it's not something
   * which is reported upstream to the FileSystem APIs, for which the semantics
   * of "false" are pretty vague.
   * @throws FileNotFoundException there's no source file.
   * @throws IOException on IO failure.
   * @throws AmazonClientException on failures inside the AWS SDK
   */
private boolean innerRename(Path src, Path dst) throws RenameFailedException, FileNotFoundException, IOException, AmazonClientException {
    LOG.debug("Rename path {} to {}", src, dst);
    incrementStatistic(INVOCATION_RENAME);
    String srcKey = pathToKey(src);
    String dstKey = pathToKey(dst);
    if (srcKey.isEmpty()) {
        throw new RenameFailedException(src, dst, "source is root directory");
    }
    if (dstKey.isEmpty()) {
        throw new RenameFailedException(src, dst, "dest is root directory");
    }
    // get the source file status; this raises a FNFE if there is no source
    // file.
    S3AFileStatus srcStatus = getFileStatus(src);
    if (srcKey.equals(dstKey)) {
        LOG.debug("rename: src and dest refer to the same file or directory: {}", dst);
        throw new RenameFailedException(src, dst, "source and dest refer to the same file or directory").withExitCode(srcStatus.isFile());
    }
    S3AFileStatus dstStatus = null;
    try {
        dstStatus = getFileStatus(dst);
        // whether or not it can be the destination of the rename.
        if (srcStatus.isDirectory()) {
            if (dstStatus.isFile()) {
                throw new RenameFailedException(src, dst, "source is a directory and dest is a file").withExitCode(srcStatus.isFile());
            } else if (!dstStatus.isEmptyDirectory()) {
                throw new RenameFailedException(src, dst, "Destination is a non-empty directory").withExitCode(false);
            }
        // at this point the destination is an empty directory
        } else {
            // empty or not
            if (dstStatus.isFile()) {
                throw new RenameFailedException(src, dst, "Cannot rename onto an existing file").withExitCode(false);
            }
        }
    } catch (FileNotFoundException e) {
        LOG.debug("rename: destination path {} not found", dst);
        // Parent must exist
        Path parent = dst.getParent();
        if (!pathToKey(parent).isEmpty()) {
            try {
                S3AFileStatus dstParentStatus = getFileStatus(dst.getParent());
                if (!dstParentStatus.isDirectory()) {
                    throw new RenameFailedException(src, dst, "destination parent is not a directory");
                }
            } catch (FileNotFoundException e2) {
                throw new RenameFailedException(src, dst, "destination has no parent ");
            }
        }
    }
    // Ok! Time to start
    if (srcStatus.isFile()) {
        LOG.debug("rename: renaming file {} to {}", src, dst);
        if (dstStatus != null && dstStatus.isDirectory()) {
            String newDstKey = dstKey;
            if (!newDstKey.endsWith("/")) {
                newDstKey = newDstKey + "/";
            }
            String filename = srcKey.substring(pathToKey(src.getParent()).length() + 1);
            newDstKey = newDstKey + filename;
            copyFile(srcKey, newDstKey, srcStatus.getLen());
        } else {
            copyFile(srcKey, dstKey, srcStatus.getLen());
        }
        innerDelete(srcStatus, false);
    } else {
        LOG.debug("rename: renaming directory {} to {}", src, dst);
        // This is a directory to directory copy
        if (!dstKey.endsWith("/")) {
            dstKey = dstKey + "/";
        }
        if (!srcKey.endsWith("/")) {
            srcKey = srcKey + "/";
        }
        //Verify dest is not a child of the source directory
        if (dstKey.startsWith(srcKey)) {
            throw new RenameFailedException(srcKey, dstKey, "cannot rename a directory to a subdirectory o fitself ");
        }
        List<DeleteObjectsRequest.KeyVersion> keysToDelete = new ArrayList<>();
        if (dstStatus != null && dstStatus.isEmptyDirectory()) {
            // delete unnecessary fake directory.
            keysToDelete.add(new DeleteObjectsRequest.KeyVersion(dstKey));
        }
        ListObjectsRequest request = new ListObjectsRequest();
        request.setBucketName(bucket);
        request.setPrefix(srcKey);
        request.setMaxKeys(maxKeys);
        ObjectListing objects = listObjects(request);
        while (true) {
            for (S3ObjectSummary summary : objects.getObjectSummaries()) {
                keysToDelete.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
                String newDstKey = dstKey + summary.getKey().substring(srcKey.length());
                copyFile(summary.getKey(), newDstKey, summary.getSize());
                if (keysToDelete.size() == MAX_ENTRIES_TO_DELETE) {
                    removeKeys(keysToDelete, true, false);
                }
            }
            if (objects.isTruncated()) {
                objects = continueListObjects(objects);
            } else {
                if (!keysToDelete.isEmpty()) {
                    removeKeys(keysToDelete, false, false);
                }
                break;
            }
        }
    }
    if (src.getParent() != dst.getParent()) {
        deleteUnnecessaryFakeDirectories(dst.getParent());
        createFakeDirectoryIfNecessary(src.getParent());
    }
    return true;
}

Example 3 with Copy

use of com.amazonaws.services.s3.transfer.Copy in project ice by Netflix.

the class MapDb method upload.

void upload() {
    AmazonS3Client s3Client = AwsUtils.getAmazonS3Client();
    File dir = new File(config.localDir);
    File[] files = dir.listFiles(new FilenameFilter() {

        public boolean accept(File file, String fileName) {
            return fileName.startsWith(dbName);
        }
    });
    for (File file : files) s3Client.putObject(config.workS3BucketName, config.workS3BucketPrefix + file.getName(), file);
    for (File file : files) s3Client.putObject(config.workS3BucketName, config.workS3BucketPrefix + "copy" + file.getName(), file);
}

Example 4 with Copy

use of com.amazonaws.services.s3.transfer.Copy in project alluxio by Alluxio.

the class S3AUnderFileSystem method createInstance.

/**
   * Constructs a new instance of {@link S3AUnderFileSystem}.
   *
   * @param uri the {@link AlluxioURI} for this UFS
   * @return the created {@link S3AUnderFileSystem} instance
   */
public static S3AUnderFileSystem createInstance(AlluxioURI uri) {
    String bucketName = uri.getHost();
    // Set the aws credential system properties based on Alluxio properties, if they are set
    if (Configuration.containsKey(PropertyKey.S3A_ACCESS_KEY)) {
        System.setProperty(SDKGlobalConfiguration.ACCESS_KEY_SYSTEM_PROPERTY, Configuration.get(PropertyKey.S3A_ACCESS_KEY));
    }
    if (Configuration.containsKey(PropertyKey.S3A_SECRET_KEY)) {
        System.setProperty(SDKGlobalConfiguration.SECRET_KEY_SYSTEM_PROPERTY, Configuration.get(PropertyKey.S3A_SECRET_KEY));
    }
    // Checks, in order, env variables, system properties, profile file, and instance profile
    AWSCredentialsProvider credentials = new AWSCredentialsProviderChain(new DefaultAWSCredentialsProviderChain());
    // Set the client configuration based on Alluxio configuration values
    ClientConfiguration clientConf = new ClientConfiguration();
    // Socket timeout
    clientConf.setSocketTimeout(Configuration.getInt(PropertyKey.UNDERFS_S3A_SOCKET_TIMEOUT_MS));
    // HTTP protocol
    if (Configuration.getBoolean(PropertyKey.UNDERFS_S3A_SECURE_HTTP_ENABLED)) {
        clientConf.setProtocol(Protocol.HTTPS);
    } else {
        clientConf.setProtocol(Protocol.HTTP);
    }
    // Proxy host
    if (Configuration.containsKey(PropertyKey.UNDERFS_S3_PROXY_HOST)) {
        clientConf.setProxyHost(Configuration.get(PropertyKey.UNDERFS_S3_PROXY_HOST));
    }
    // Proxy port
    if (Configuration.containsKey(PropertyKey.UNDERFS_S3_PROXY_PORT)) {
        clientConf.setProxyPort(Configuration.getInt(PropertyKey.UNDERFS_S3_PROXY_PORT));
    }
    int numAdminThreads = Configuration.getInt(PropertyKey.UNDERFS_S3_ADMIN_THREADS_MAX);
    int numTransferThreads = Configuration.getInt(PropertyKey.UNDERFS_S3_UPLOAD_THREADS_MAX);
    int numThreads = Configuration.getInt(PropertyKey.UNDERFS_S3_THREADS_MAX);
    if (numThreads < numAdminThreads + numTransferThreads) {
        LOG.warn("Configured s3 max threads: {} is less than # admin threads: {} plus transfer " + "threads {}. Using admin threads + transfer threads as max threads instead.");
        numThreads = numAdminThreads + numTransferThreads;
    }
    clientConf.setMaxConnections(numThreads);
    // Set client request timeout for all requests since multipart copy is used, and copy parts can
    // only be set with the client configuration.
    clientConf.setRequestTimeout(Configuration.getInt(PropertyKey.UNDERFS_S3A_REQUEST_TIMEOUT));
    AmazonS3Client amazonS3Client = new AmazonS3Client(credentials, clientConf);
    // Set a custom endpoint.
    if (Configuration.containsKey(PropertyKey.UNDERFS_S3_ENDPOINT)) {
        amazonS3Client.setEndpoint(Configuration.get(PropertyKey.UNDERFS_S3_ENDPOINT));
    }
    // Disable DNS style buckets, this enables path style requests.
    if (Configuration.getBoolean(PropertyKey.UNDERFS_S3_DISABLE_DNS_BUCKETS)) {
        S3ClientOptions clientOptions = S3ClientOptions.builder().setPathStyleAccess(true).build();
        amazonS3Client.setS3ClientOptions(clientOptions);
    }
    ExecutorService service = ExecutorServiceFactories.fixedThreadPoolExecutorServiceFactory("alluxio-s3-transfer-manager-worker", numTransferThreads).create();
    TransferManager transferManager = new TransferManager(amazonS3Client, service);
    TransferManagerConfiguration transferConf = new TransferManagerConfiguration();
    transferConf.setMultipartCopyThreshold(MULTIPART_COPY_THRESHOLD);
    transferManager.setConfiguration(transferConf);
    // Default to readable and writable by the user.
    short bucketMode = (short) 700;
    // There is no known account owner by default.
    String accountOwner = "";
    // if ACL enabled inherit bucket acl for all the objects.
    if (Configuration.getBoolean(PropertyKey.UNDERFS_S3A_INHERIT_ACL)) {
        String accountOwnerId = amazonS3Client.getS3AccountOwner().getId();
        // Gets the owner from user-defined static mapping from S3 canonical user
        // id to Alluxio user name.
        String owner = CommonUtils.getValueFromStaticMapping(Configuration.get(PropertyKey.UNDERFS_S3_OWNER_ID_TO_USERNAME_MAPPING), accountOwnerId);
        // If there is no user-defined mapping, use the display name.
        if (owner == null) {
            owner = amazonS3Client.getS3AccountOwner().getDisplayName();
        }
        accountOwner = owner == null ? accountOwnerId : owner;
        AccessControlList acl = amazonS3Client.getBucketAcl(bucketName);
        bucketMode = S3AUtils.translateBucketAcl(acl, accountOwnerId);
    }
    return new S3AUnderFileSystem(uri, amazonS3Client, bucketName, bucketMode, accountOwner, transferManager);
}

Example 5 with Copy

use of com.amazonaws.services.s3.transfer.Copy in project alluxio by Alluxio.

the class S3AUnderFileSystem method copyObject.

@Override
protected boolean copyObject(String src, String dst) {
    LOG.debug("Copying {} to {}", src, dst);
    // Retry copy for a few times, in case some AWS internal errors happened during copy.
    int retries = 3;
    for (int i = 0; i < retries; i++) {
        try {
            CopyObjectRequest request = new CopyObjectRequest(mBucketName, src, mBucketName, dst);
            if (Configuration.getBoolean(PropertyKey.UNDERFS_S3A_SERVER_SIDE_ENCRYPTION_ENABLED)) {
                ObjectMetadata meta = new ObjectMetadata();
                meta.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
                request.setNewObjectMetadata(meta);
            }
            mManager.copy(request).waitForCopyResult();
            return true;
        } catch (AmazonClientException | InterruptedException e) {
            LOG.error("Failed to copy file {} to {}", src, dst, e);
            if (i != retries - 1) {
                LOG.error("Retrying copying file {} to {}", src, dst);
            }
        }
    }
    LOG.error("Failed to copy file {} to {}, after {} retries", src, dst, retries);
    return false;
}