Examples with DataStorageListing com.epam.pipeline.entity.datastorage.DataStorageListing used on opensource projects

Example 1 with DataStorageListing

use of com.epam.pipeline.entity.datastorage.DataStorageListing in project cloud-pipeline by epam.

the class NFSStorageProviderTest method testCreateFileFolderAndList.

@Test
public void testCreateFileFolderAndList() {
    NFSDataStorage dataStorage = new NFSDataStorage(0L, TEST_STORAGE_NAME, TEST_PATH + 2 + ":/test");
    nfsProvider.createStorage(dataStorage);
    String testFileName = "testFile.txt";
    nfsProvider.createFile(dataStorage, testFileName, "testContent".getBytes());
    File dataStorageRoot = new File(testMountPoint, TEST_PATH + 2 + "/test");
    File testFile = new File(dataStorageRoot, testFileName);
    Assert.assertTrue(testFile.exists());
    String testFolderName = "testFolder";
    nfsProvider.createFolder(dataStorage, testFolderName);
    File testFolder = new File(dataStorageRoot, testFolderName);
    Assert.assertTrue(testFolder.exists());
    DataStorageListing listing = nfsProvider.getItems(dataStorage, null, false, DEFAULT_PAGE_SIZE, null);
    Assert.assertFalse(listing.getResults().isEmpty());
    Optional<AbstractDataStorageItem> loadedFile = listing.getResults().stream().filter(i -> i.getType() == DataStorageItemType.File).findFirst();
    Assert.assertTrue(loadedFile.isPresent());
    Assert.assertEquals(testFileName, loadedFile.get().getName());
    Assert.assertEquals(testFileName, loadedFile.get().getPath());
    Assert.assertNotNull(((DataStorageFile) loadedFile.get()).getChanged());
    Optional<AbstractDataStorageItem> loadedFolder = listing.getResults().stream().filter(i -> i.getType() == DataStorageItemType.Folder).findFirst();
    Assert.assertTrue(loadedFolder.isPresent());
    Assert.assertEquals(testFolderName, loadedFolder.get().getName());
    Assert.assertEquals(testFolderName + "/", loadedFolder.get().getPath());
    Assert.assertNull(listing.getNextPageMarker());
    listing = nfsProvider.getItems(dataStorage, null, false, 1, null);
    Assert.assertEquals("2", listing.getNextPageMarker());
    listing = nfsProvider.getItems(dataStorage, null, false, 1, listing.getNextPageMarker());
    Assert.assertNull(listing.getNextPageMarker());
    Assert.assertFalse(listing.getResults().isEmpty());
}

Example 2 with DataStorageListing

use of com.epam.pipeline.entity.datastorage.DataStorageListing in project cloud-pipeline by epam.

the class NFSStorageProvider method getItems.

@Override
public DataStorageListing getItems(NFSDataStorage dataStorage, String path, Boolean showVersion, Integer pageSize, String marker) {
    File dataStorageRoot = mount(dataStorage);
    File dir = path != null ? new File(dataStorageRoot, path) : dataStorageRoot;
    long offset = StringUtils.isNumeric(marker) ? Long.parseLong(marker) : 1;
    try (Stream<Path> dirStream = Files.walk(dir.toPath(), 1)) {
        List<AbstractDataStorageItem> dataStorageItems = dirStream.sorted().skip(// First element is a directory itself
        offset).limit(pageSize).map(p -> {
            File file = p.toFile();
            AbstractDataStorageItem item;
            if (file.isDirectory()) {
                item = new DataStorageFolder();
            } else {
                // set size if it's a file
                DataStorageFile dataStorageFile = new DataStorageFile();
                dataStorageFile.setSize(file.length());
                dataStorageFile.setChanged(S3Constants.getAwsDateFormat().format(new Date(file.lastModified())));
                item = dataStorageFile;
            }
            item.setName(file.getName());
            item.setPath(dataStorageRoot.toURI().relativize(file.toURI()).getPath());
            return item;
        }).collect(Collectors.toList());
        DataStorageListing listing = new DataStorageListing();
        listing.setResults(dataStorageItems);
        Long nextOffset = offset + pageSize;
        try (Stream<Path> nextStream = Files.walk(dir.toPath(), 1)) {
            if (nextStream.skip(nextOffset).findFirst().isPresent()) {
                listing.setNextPageMarker(nextOffset.toString());
            }
        }
        return listing;
    } catch (IOException e) {
        throw new DataStorageException(e);
    }
}

Example 3 with DataStorageListing

use of com.epam.pipeline.entity.datastorage.DataStorageListing in project cloud-pipeline by epam.

the class S3Helper method listVersions.

private DataStorageListing listVersions(AmazonS3 client, String bucket, String requestPath, Integer pageSize, String marker) {
    ListVersionsRequest request = new ListVersionsRequest().withBucketName(bucket).withPrefix(requestPath).withDelimiter(S3Constants.DELIMITER);
    if (StringUtils.hasValue(marker)) {
        request.setKeyMarker(marker);
    }
    if (pageSize != null) {
        request.setMaxResults(pageSize);
    }
    VersionListing versionListing;
    List<AbstractDataStorageItem> items = new ArrayList<>();
    Map<String, DataStorageFile> itemKeys = new HashMap<>();
    String previous = null;
    do {
        versionListing = client.listVersions(request);
        for (String commonPrefix : versionListing.getCommonPrefixes()) {
            if (checkListingSize(pageSize, items, itemKeys)) {
                items.addAll(itemKeys.values());
                return new DataStorageListing(previous, items);
            }
            previous = getPreviousKey(previous, commonPrefix);
            AbstractDataStorageItem folder = parseFolder(requestPath, commonPrefix);
            items.add(folder);
        }
        for (S3VersionSummary versionSummary : versionListing.getVersionSummaries()) {
            if (!pathMatch(requestPath, versionSummary.getKey())) {
                continue;
            }
            DataStorageFile file = AbstractS3ObjectWrapper.getWrapper(versionSummary).convertToStorageFile(requestPath);
            if (file == null) {
                continue;
            }
            if (!itemKeys.containsKey(file.getName())) {
                if (checkListingSize(pageSize, items, itemKeys)) {
                    items.addAll(itemKeys.values());
                    return new DataStorageListing(previous, items);
                }
                previous = getPreviousKey(previous, versionSummary.getKey());
                Map<String, AbstractDataStorageItem> versions = new LinkedHashMap<>();
                versions.put(file.getVersion(), file.copy(file));
                file.setVersions(versions);
                itemKeys.put(file.getName(), file);
            } else {
                DataStorageFile item = itemKeys.get(file.getName());
                Map<String, AbstractDataStorageItem> versions = item.getVersions();
                versions.put(file.getVersion(), file.copy(file));
                if (isLaterVersion(file.getChanged(), item.getChanged())) {
                    file.setVersions(versions);
                    itemKeys.put(file.getName(), file);
                }
            }
        }
        request.setKeyMarker(versionListing.getNextKeyMarker());
        request.setVersionIdMarker(versionListing.getNextVersionIdMarker());
    } while (versionListing.isTruncated());
    items.addAll(itemKeys.values());
    String returnToken = versionListing.isTruncated() ? previous : null;
    return new DataStorageListing(returnToken, items);
}

Example 4 with DataStorageListing

use of com.epam.pipeline.entity.datastorage.DataStorageListing in project cloud-pipeline by epam.

the class S3Helper method listFiles.

private DataStorageListing listFiles(AmazonS3 client, String bucket, String requestPath, Integer pageSize, String marker) {
    ListObjectsV2Request req = new ListObjectsV2Request();
    req.setBucketName(bucket);
    req.setPrefix(requestPath);
    req.setDelimiter(S3Constants.DELIMITER);
    if (pageSize != null) {
        req.setMaxKeys(pageSize);
    }
    if (StringUtils.hasValue(marker)) {
        req.setStartAfter(marker);
    }
    ListObjectsV2Result listing;
    List<AbstractDataStorageItem> items = new ArrayList<>();
    String previous = null;
    do {
        listing = client.listObjectsV2(req);
        for (String name : listing.getCommonPrefixes()) {
            previous = getPreviousKey(previous, name);
            items.add(parseFolder(requestPath, name));
        }
        for (S3ObjectSummary s3ObjectSummary : listing.getObjectSummaries()) {
            DataStorageFile file = AbstractS3ObjectWrapper.getWrapper(s3ObjectSummary).convertToStorageFile(requestPath);
            if (file != null) {
                previous = getPreviousKey(previous, s3ObjectSummary.getKey());
                items.add(file);
            }
        }
        req.setContinuationToken(listing.getNextContinuationToken());
    } while (listing.isTruncated() && (pageSize == null || items.size() < pageSize));
    String returnToken = listing.isTruncated() ? previous : null;
    return new DataStorageListing(returnToken, items);
}

Example 5 with DataStorageListing

use of com.epam.pipeline.entity.datastorage.DataStorageListing in project cloud-pipeline by epam.

the class S3Helper method getItems.

public DataStorageListing getItems(String bucket, String path, Boolean showVersion, Integer pageSize, String marker) {
    String requestPath = path;
    if (requestPath == null) {
        requestPath = "";
    }
    AmazonS3 client = getDefaultS3Client();
    if (!StringUtils.isNullOrEmpty(requestPath)) {
        DataStorageItemType type = checkItemType(client, bucket, requestPath, showVersion);
        if (type == DataStorageItemType.Folder && !requestPath.endsWith(S3Constants.DELIMITER)) {
            requestPath += S3Constants.DELIMITER;
        }
    }
    DataStorageListing result = showVersion ? listVersions(client, bucket, requestPath, pageSize, marker) : listFiles(client, bucket, requestPath, pageSize, marker);
    result.getResults().sort(AbstractDataStorageItem.getStorageItemComparator());
    return result;
}