Search in sources :

Example 6 with DataStorageException

use of com.epam.pipeline.entity.datastorage.DataStorageException in project cloud-pipeline by epam.

the class NFSStorageProvider method createFolder.

@Override
public DataStorageFolder createFolder(NFSDataStorage dataStorage, String path) throws DataStorageException {
    File dataStorageDir = mount(dataStorage);
    File folder = new File(dataStorageDir, path);
    if (!folder.mkdirs()) {
        throw new DataStorageException(messageHelper.getMessage(MessageConstants.ERROR_DATASTORAGE_NFS_CREATE_FOLDER, dataStorage.getPath()));
    }
    try {
        setUmask(folder);
    } catch (IOException e) {
        throw new DataStorageException(messageHelper.getMessage(MessageConstants.ERROR_DATASTORAGE_CANNOT_CREATE_FILE, folder.getPath()), e);
    }
    return new DataStorageFolder(path, folder);
}
Also used : DataStorageException(com.epam.pipeline.entity.datastorage.DataStorageException) IOException(java.io.IOException) DataStorageFile(com.epam.pipeline.entity.datastorage.DataStorageFile) File(java.io.File) DataStorageFolder(com.epam.pipeline.entity.datastorage.DataStorageFolder)

Example 7 with DataStorageException

use of com.epam.pipeline.entity.datastorage.DataStorageException in project cloud-pipeline by epam.

the class NFSStorageProvider method deleteFile.

@Override
public void deleteFile(NFSDataStorage dataStorage, String path, String version, Boolean totally) throws DataStorageException {
    File dataStorageDir = mount(dataStorage);
    File file = new File(dataStorageDir, path);
    try {
        Files.delete(file.toPath());
    } catch (IOException e) {
        throw new DataStorageException(e);
    }
}
Also used : DataStorageException(com.epam.pipeline.entity.datastorage.DataStorageException) IOException(java.io.IOException) DataStorageFile(com.epam.pipeline.entity.datastorage.DataStorageFile) File(java.io.File)

Example 8 with DataStorageException

use of com.epam.pipeline.entity.datastorage.DataStorageException in project cloud-pipeline by epam.

the class NFSStorageProvider method deleteFolder.

@Override
public void deleteFolder(NFSDataStorage dataStorage, String path, Boolean totally) throws DataStorageException {
    File dataStorageDir = mount(dataStorage);
    File folder = new File(dataStorageDir, path);
    try {
        FileUtils.deleteDirectory(folder);
    } catch (IOException e) {
        throw new DataStorageException(e);
    }
}
Also used : DataStorageException(com.epam.pipeline.entity.datastorage.DataStorageException) IOException(java.io.IOException) DataStorageFile(com.epam.pipeline.entity.datastorage.DataStorageFile) File(java.io.File)

Example 9 with DataStorageException

use of com.epam.pipeline.entity.datastorage.DataStorageException in project cloud-pipeline by epam.

the class NFSStorageProvider method getItems.

@Override
public DataStorageListing getItems(NFSDataStorage dataStorage, String path, Boolean showVersion, Integer pageSize, String marker) {
    File dataStorageRoot = mount(dataStorage);
    File dir = path != null ? new File(dataStorageRoot, path) : dataStorageRoot;
    long offset = StringUtils.isNumeric(marker) ? Long.parseLong(marker) : 1;
    try (Stream<Path> dirStream = Files.walk(dir.toPath(), 1)) {
        List<AbstractDataStorageItem> dataStorageItems = dirStream.sorted().skip(// First element is a directory itself
        offset).limit(pageSize).map(p -> {
            File file = p.toFile();
            AbstractDataStorageItem item;
            if (file.isDirectory()) {
                item = new DataStorageFolder();
            } else {
                // set size if it's a file
                DataStorageFile dataStorageFile = new DataStorageFile();
                dataStorageFile.setSize(file.length());
                dataStorageFile.setChanged(S3Constants.getAwsDateFormat().format(new Date(file.lastModified())));
                item = dataStorageFile;
            }
            item.setName(file.getName());
            item.setPath(dataStorageRoot.toURI().relativize(file.toURI()).getPath());
            return item;
        }).collect(Collectors.toList());
        DataStorageListing listing = new DataStorageListing();
        listing.setResults(dataStorageItems);
        Long nextOffset = offset + pageSize;
        try (Stream<Path> nextStream = Files.walk(dir.toPath(), 1)) {
            if (nextStream.skip(nextOffset).findFirst().isPresent()) {
                listing.setNextPageMarker(nextOffset.toString());
            }
        }
        return listing;
    } catch (IOException e) {
        throw new DataStorageException(e);
    }
}
Also used : Path(java.nio.file.Path) Arrays(java.util.Arrays) Date(java.util.Date) S3Constants(com.epam.pipeline.manager.datastorage.providers.aws.s3.S3Constants) LoggerFactory(org.slf4j.LoggerFactory) SystemPreferences(com.epam.pipeline.manager.preference.SystemPreferences) Autowired(org.springframework.beans.factory.annotation.Autowired) StringUtils(org.apache.commons.lang3.StringUtils) Matcher(java.util.regex.Matcher) MessageHelper(com.epam.pipeline.common.MessageHelper) ByteArrayInputStream(java.io.ByteArrayInputStream) Map(java.util.Map) DataStorageItemContent(com.epam.pipeline.entity.datastorage.DataStorageItemContent) DataStorageListing(com.epam.pipeline.entity.datastorage.DataStorageListing) Path(java.nio.file.Path) PosixFilePermission(java.nio.file.attribute.PosixFilePermission) DataStorageDownloadFileUrl(com.epam.pipeline.entity.datastorage.DataStorageDownloadFileUrl) Set(java.util.Set) Collectors(java.util.stream.Collectors) FileNotFoundException(java.io.FileNotFoundException) DataStorageFile(com.epam.pipeline.entity.datastorage.DataStorageFile) IOUtils(org.apache.commons.io.IOUtils) List(java.util.List) Stream(java.util.stream.Stream) CmdExecutionException(com.epam.pipeline.exception.CmdExecutionException) DataStorageStreamingContent(com.epam.pipeline.entity.datastorage.DataStorageStreamingContent) CmdExecutor(com.epam.pipeline.manager.CmdExecutor) Pattern(java.util.regex.Pattern) FileContentUtils(com.epam.pipeline.utils.FileContentUtils) DataStorageDao(com.epam.pipeline.dao.datastorage.DataStorageDao) MessageConstants(com.epam.pipeline.common.MessageConstants) DataStorageFolder(com.epam.pipeline.entity.datastorage.DataStorageFolder) BufferedOutputStream(java.io.BufferedOutputStream) Value(org.springframework.beans.factory.annotation.Value) Service(org.springframework.stereotype.Service) DataStorageException(com.epam.pipeline.entity.datastorage.DataStorageException) PreferenceManager(com.epam.pipeline.manager.preference.PreferenceManager) Logger(org.slf4j.Logger) Files(java.nio.file.Files) SystemUtils(org.apache.commons.lang3.SystemUtils) FileOutputStream(java.io.FileOutputStream) AbstractDataStorage(com.epam.pipeline.entity.datastorage.AbstractDataStorage) FileUtils(org.apache.commons.io.FileUtils) IOException(java.io.IOException) DataStorageType(com.epam.pipeline.entity.datastorage.DataStorageType) FileInputStream(java.io.FileInputStream) NFSDataStorage(com.epam.pipeline.entity.datastorage.nfs.NFSDataStorage) File(java.io.File) AbstractDataStorageItem(com.epam.pipeline.entity.datastorage.AbstractDataStorageItem) Paths(java.nio.file.Paths) Collections(java.util.Collections) Assert(org.springframework.util.Assert) InputStream(java.io.InputStream) DataStorageFile(com.epam.pipeline.entity.datastorage.DataStorageFile) IOException(java.io.IOException) Date(java.util.Date) DataStorageFolder(com.epam.pipeline.entity.datastorage.DataStorageFolder) DataStorageListing(com.epam.pipeline.entity.datastorage.DataStorageListing) DataStorageException(com.epam.pipeline.entity.datastorage.DataStorageException) AbstractDataStorageItem(com.epam.pipeline.entity.datastorage.AbstractDataStorageItem) DataStorageFile(com.epam.pipeline.entity.datastorage.DataStorageFile) File(java.io.File)

Example 10 with DataStorageException

use of com.epam.pipeline.entity.datastorage.DataStorageException in project cloud-pipeline by epam.

the class S3Helper method moveFolder.

public DataStorageFolder moveFolder(String bucket, String rawOldPath, String rawNewPath) throws DataStorageException {
    if (StringUtils.isNullOrEmpty(rawOldPath) || StringUtils.isNullOrEmpty(rawNewPath)) {
        throw new DataStorageException(PATH_SHOULD_NOT_BE_EMPTY_MESSAGE);
    }
    final String oldPath = withTrailingDelimiter(rawOldPath);
    final String newPath = withTrailingDelimiter(rawNewPath);
    final String folderFullPath = newPath.substring(0, newPath.length() - 1);
    String[] parts = newPath.split(S3Constants.DELIMITER);
    final String folderName = parts[parts.length - 1];
    AmazonS3 client = getDefaultS3Client();
    if (!itemExists(client, bucket, oldPath, true)) {
        throw new DataStorageException(String.format("Folder '%s' does not exist", oldPath));
    }
    if (itemExists(client, bucket, newPath, true)) {
        throw new DataStorageException(String.format("Folder '%s' already exists", newPath));
    }
    ListObjectsRequest req = new ListObjectsRequest();
    req.setBucketName(bucket);
    ObjectListing listing = client.listObjects(req);
    boolean listingFinished = false;
    List<String> oldKeys = new ArrayList<>();
    while (!listingFinished) {
        for (S3ObjectSummary s3ObjectSummary : listing.getObjectSummaries()) {
            if (s3ObjectSummary.getSize() > COPYING_FILE_SIZE_LIMIT) {
                throw new DataStorageException(String.format("Moving folder '%s' was aborted because " + "some of its files '%s' size exceeds the limit of %s bytes", oldPath, s3ObjectSummary.getKey(), COPYING_FILE_SIZE_LIMIT));
            }
            String relativePath = s3ObjectSummary.getKey();
            if (relativePath.startsWith(oldPath)) {
                oldKeys.add(relativePath);
            }
        }
        if (listing.isTruncated()) {
            listing = client.listNextBatchOfObjects(listing);
        } else {
            listingFinished = true;
        }
    }
    final List<MoveObjectRequest> moveRequests = oldKeys.stream().map(oldKey -> new MoveObjectRequest(oldKey, newPath + oldKey.substring(oldPath.length()))).collect(Collectors.toList());
    moveS3Objects(client, bucket, moveRequests);
    DataStorageFolder folder = new DataStorageFolder();
    folder.setName(folderName);
    folder.setPath(folderFullPath);
    return folder;
}
Also used : BucketVersioningConfiguration(com.amazonaws.services.s3.model.BucketVersioningConfiguration) AmazonS3ClientBuilder(com.amazonaws.services.s3.AmazonS3ClientBuilder) CORSRule(com.amazonaws.services.s3.model.CORSRule) DefaultAWSCredentialsProviderChain(com.amazonaws.auth.DefaultAWSCredentialsProviderChain) BucketTaggingConfiguration(com.amazonaws.services.s3.model.BucketTaggingConfiguration) URL(java.net.URL) Date(java.util.Date) LoggerFactory(org.slf4j.LoggerFactory) StoragePolicy(com.epam.pipeline.entity.datastorage.StoragePolicy) ObjectListing(com.amazonaws.services.s3.model.ObjectListing) IOUtils(com.amazonaws.util.IOUtils) TagSet(com.amazonaws.services.s3.model.TagSet) ListObjectsV2Request(com.amazonaws.services.s3.model.ListObjectsV2Request) ByteArrayInputStream(java.io.ByteArrayInputStream) ObjectMetadata(com.amazonaws.services.s3.model.ObjectMetadata) BucketCrossOriginConfiguration(com.amazonaws.services.s3.model.BucketCrossOriginConfiguration) ListUtils(org.apache.commons.collections4.ListUtils) Map(java.util.Map) DataStorageItemContent(com.epam.pipeline.entity.datastorage.DataStorageItemContent) DataStorageListing(com.epam.pipeline.entity.datastorage.DataStorageListing) ParseException(java.text.ParseException) AWSCredentials(com.amazonaws.auth.AWSCredentials) HttpMethod(com.amazonaws.HttpMethod) DateFormat(java.text.DateFormat) GeneratePresignedUrlRequest(com.amazonaws.services.s3.model.GeneratePresignedUrlRequest) DataStorageItemType(com.epam.pipeline.entity.datastorage.DataStorageItemType) ServerSideEncryptionConfiguration(com.amazonaws.services.s3.model.ServerSideEncryptionConfiguration) TimeZone(java.util.TimeZone) DataStorageDownloadFileUrl(com.epam.pipeline.entity.datastorage.DataStorageDownloadFileUrl) DefaultAwsRegionProviderChain(com.amazonaws.regions.DefaultAwsRegionProviderChain) Set(java.util.Set) AwsRegion(com.epam.pipeline.entity.region.AwsRegion) Collectors(java.util.stream.Collectors) CreateBucketRequest(com.amazonaws.services.s3.model.CreateBucketRequest) SetObjectTaggingRequest(com.amazonaws.services.s3.model.SetObjectTaggingRequest) DataStorageFile(com.epam.pipeline.entity.datastorage.DataStorageFile) List(java.util.List) DataStorageStreamingContent(com.epam.pipeline.entity.datastorage.DataStorageStreamingContent) CollectionUtils(org.springframework.util.CollectionUtils) LifecycleFilter(com.amazonaws.services.s3.model.lifecycle.LifecycleFilter) SetBucketVersioningConfigurationRequest(com.amazonaws.services.s3.model.SetBucketVersioningConfigurationRequest) FileContentUtils(com.epam.pipeline.utils.FileContentUtils) LifecyclePrefixPredicate(com.amazonaws.services.s3.model.lifecycle.LifecyclePrefixPredicate) SimpleDateFormat(java.text.SimpleDateFormat) HashMap(java.util.HashMap) GetObjectRequest(com.amazonaws.services.s3.model.GetObjectRequest) SSEAlgorithm(com.amazonaws.services.s3.model.SSEAlgorithm) VersionListing(com.amazonaws.services.s3.model.VersionListing) DataStorageFolder(com.epam.pipeline.entity.datastorage.DataStorageFolder) ListObjectsV2Result(com.amazonaws.services.s3.model.ListObjectsV2Result) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) BucketLifecycleConfiguration(com.amazonaws.services.s3.model.BucketLifecycleConfiguration) S3Object(com.amazonaws.services.s3.model.S3Object) ServerSideEncryptionRule(com.amazonaws.services.s3.model.ServerSideEncryptionRule) AmazonS3(com.amazonaws.services.s3.AmazonS3) ObjectTagging(com.amazonaws.services.s3.model.ObjectTagging) S3ObjectSummary(com.amazonaws.services.s3.model.S3ObjectSummary) DataStorageException(com.epam.pipeline.entity.datastorage.DataStorageException) MapUtils(org.apache.commons.collections4.MapUtils) AmazonS3Exception(com.amazonaws.services.s3.model.AmazonS3Exception) SetBucketEncryptionRequest(com.amazonaws.services.s3.model.SetBucketEncryptionRequest) Logger(org.slf4j.Logger) ListObjectsRequest(com.amazonaws.services.s3.model.ListObjectsRequest) ServerSideEncryptionByDefault(com.amazonaws.services.s3.model.ServerSideEncryptionByDefault) AbstractDataStorage(com.epam.pipeline.entity.datastorage.AbstractDataStorage) IOException(java.io.IOException) Tag(com.amazonaws.services.s3.model.Tag) Bucket(com.amazonaws.services.s3.model.Bucket) GetObjectTaggingRequest(com.amazonaws.services.s3.model.GetObjectTaggingRequest) PutObjectRequest(com.amazonaws.services.s3.model.PutObjectRequest) AbstractDataStorageItem(com.epam.pipeline.entity.datastorage.AbstractDataStorageItem) StorageClass(com.amazonaws.services.s3.model.StorageClass) SdkClientException(com.amazonaws.SdkClientException) Headers(com.amazonaws.services.s3.Headers) ListVersionsRequest(com.amazonaws.services.s3.model.ListVersionsRequest) StringUtils(com.amazonaws.util.StringUtils) S3VersionSummary(com.amazonaws.services.s3.model.S3VersionSummary) GetObjectMetadataRequest(com.amazonaws.services.s3.model.GetObjectMetadataRequest) Collections(java.util.Collections) Assert(org.springframework.util.Assert) InputStream(java.io.InputStream) AmazonS3(com.amazonaws.services.s3.AmazonS3) ArrayList(java.util.ArrayList) ObjectListing(com.amazonaws.services.s3.model.ObjectListing) S3ObjectSummary(com.amazonaws.services.s3.model.S3ObjectSummary) DataStorageFolder(com.epam.pipeline.entity.datastorage.DataStorageFolder) ListObjectsRequest(com.amazonaws.services.s3.model.ListObjectsRequest) DataStorageException(com.epam.pipeline.entity.datastorage.DataStorageException)

Aggregations

DataStorageException (com.epam.pipeline.entity.datastorage.DataStorageException)28 IOException (java.io.IOException)14 DataStorageFile (com.epam.pipeline.entity.datastorage.DataStorageFile)12 File (java.io.File)11 AmazonS3 (com.amazonaws.services.s3.AmazonS3)8 AbstractDataStorage (com.epam.pipeline.entity.datastorage.AbstractDataStorage)5 DataStorageItemContent (com.epam.pipeline.entity.datastorage.DataStorageItemContent)5 AmazonS3Exception (com.amazonaws.services.s3.model.AmazonS3Exception)4 ListObjectsRequest (com.amazonaws.services.s3.model.ListObjectsRequest)4 ObjectListing (com.amazonaws.services.s3.model.ObjectListing)4 ObjectMetadata (com.amazonaws.services.s3.model.ObjectMetadata)4 DataStorageFolder (com.epam.pipeline.entity.datastorage.DataStorageFolder)4 DataStorageStreamingContent (com.epam.pipeline.entity.datastorage.DataStorageStreamingContent)4 ByteArrayInputStream (java.io.ByteArrayInputStream)4 SdkClientException (com.amazonaws.SdkClientException)3 S3ObjectSummary (com.amazonaws.services.s3.model.S3ObjectSummary)3 InputStream (java.io.InputStream)3 Date (java.util.Date)3 List (java.util.List)3 Test (org.junit.Test)3