use of com.epam.pipeline.entity.datastorage.nfs.NFSDataStorage in project cloud-pipeline by epam.
the class NFSStorageProviderTest method testMoveDeleteFile.
@Test
public void testMoveDeleteFile() {
NFSDataStorage dataStorage = new NFSDataStorage(0L, "testStorage", TEST_PATH + 3 + ":/test");
nfsProvider.createStorage(dataStorage);
String testFileName = "testFile.txt";
String testFolderName = "testFolder";
String testFolder2Name = "testFolder2";
nfsProvider.createFile(dataStorage, testFileName, "testContent".getBytes());
nfsProvider.createFolder(dataStorage, testFolderName);
nfsProvider.createFolder(dataStorage, testFolder2Name);
File dataStorageRoot = new File(testMountPoint, TEST_PATH + 3 + "/test");
String newFilePath = testFolderName + "/" + testFileName;
DataStorageFile file = nfsProvider.moveFile(dataStorage, testFileName, newFilePath);
Assert.assertEquals(newFilePath, file.getPath());
File oldFileLocation = new File(dataStorageRoot, testFileName);
File newFileLocation = new File(dataStorageRoot, newFilePath);
Assert.assertTrue(newFileLocation.exists());
Assert.assertFalse(oldFileLocation.exists());
String newFolder2Path = testFolderName + "/" + testFolder2Name;
DataStorageFolder folder = nfsProvider.moveFolder(dataStorage, testFolder2Name, newFolder2Path);
Assert.assertEquals(newFolder2Path, folder.getPath());
File oldFolderLocation = new File(dataStorageRoot, testFolder2Name);
File newFolderLocation = new File(dataStorageRoot, newFolder2Path);
Assert.assertTrue(newFolderLocation.exists());
Assert.assertFalse(oldFolderLocation.exists());
nfsProvider.deleteFile(dataStorage, newFilePath, null, true);
Assert.assertFalse(newFileLocation.exists());
nfsProvider.deleteFolder(dataStorage, newFolder2Path, true);
Assert.assertFalse(newFolderLocation.exists());
}
use of com.epam.pipeline.entity.datastorage.nfs.NFSDataStorage in project cloud-pipeline by epam.
the class NFSStorageProviderTest method testCreateDeleteStorage.
@Test
@Transactional(propagation = Propagation.REQUIRES_NEW)
public void testCreateDeleteStorage() {
NFSDataStorage dataStorage = new NFSDataStorage(0L, TEST_STORAGE_NAME, TEST_PATH + 1 + ":root/" + STORAGE_NAME);
dataStorage.setOwner("test@user.com");
String path = nfsProvider.createStorage(dataStorage);
dataStorageDao.createDataStorage(dataStorage);
Assert.assertEquals(dataStorage.getPath(), path);
NFSDataStorage dataStorage2 = new NFSDataStorage(1L, TEST_STORAGE_NAME, TEST_PATH + 1 + ":root/" + STORAGE_NAME + 1);
dataStorage2.setOwner("test@user.com");
String path2 = nfsProvider.createStorage(dataStorage2);
dataStorageDao.createDataStorage(dataStorage2);
Assert.assertEquals(dataStorage2.getPath(), path2);
File mountRootDir = new File(testMountPoint, TEST_PATH + 1 + "/root");
Assert.assertTrue(mountRootDir.exists());
File dataStorageRoot = new File(mountRootDir.getPath() + "/" + STORAGE_NAME);
Assert.assertTrue(dataStorageRoot.exists());
nfsProvider.deleteStorage(dataStorage);
Assert.assertFalse(dataStorageRoot.exists());
Assert.assertTrue(mountRootDir.exists());
// emulate that database doesn't contain datastorages with mountRootDir
dataStorageDao.deleteDataStorage(dataStorage.getId());
nfsProvider.deleteStorage(dataStorage2);
Assert.assertFalse(mountRootDir.exists());
}
use of com.epam.pipeline.entity.datastorage.nfs.NFSDataStorage in project cloud-pipeline by epam.
the class NFSStorageProviderTest method testCreateFileFolderAndList.
@Test
public void testCreateFileFolderAndList() {
NFSDataStorage dataStorage = new NFSDataStorage(0L, TEST_STORAGE_NAME, TEST_PATH + 2 + ":/test");
nfsProvider.createStorage(dataStorage);
String testFileName = "testFile.txt";
nfsProvider.createFile(dataStorage, testFileName, "testContent".getBytes());
File dataStorageRoot = new File(testMountPoint, TEST_PATH + 2 + "/test");
File testFile = new File(dataStorageRoot, testFileName);
Assert.assertTrue(testFile.exists());
String testFolderName = "testFolder";
nfsProvider.createFolder(dataStorage, testFolderName);
File testFolder = new File(dataStorageRoot, testFolderName);
Assert.assertTrue(testFolder.exists());
DataStorageListing listing = nfsProvider.getItems(dataStorage, null, false, DEFAULT_PAGE_SIZE, null);
Assert.assertFalse(listing.getResults().isEmpty());
Optional<AbstractDataStorageItem> loadedFile = listing.getResults().stream().filter(i -> i.getType() == DataStorageItemType.File).findFirst();
Assert.assertTrue(loadedFile.isPresent());
Assert.assertEquals(testFileName, loadedFile.get().getName());
Assert.assertEquals(testFileName, loadedFile.get().getPath());
Assert.assertNotNull(((DataStorageFile) loadedFile.get()).getChanged());
Optional<AbstractDataStorageItem> loadedFolder = listing.getResults().stream().filter(i -> i.getType() == DataStorageItemType.Folder).findFirst();
Assert.assertTrue(loadedFolder.isPresent());
Assert.assertEquals(testFolderName, loadedFolder.get().getName());
Assert.assertEquals(testFolderName + "/", loadedFolder.get().getPath());
Assert.assertNull(listing.getNextPageMarker());
listing = nfsProvider.getItems(dataStorage, null, false, 1, null);
Assert.assertEquals("2", listing.getNextPageMarker());
listing = nfsProvider.getItems(dataStorage, null, false, 1, listing.getNextPageMarker());
Assert.assertNull(listing.getNextPageMarker());
Assert.assertFalse(listing.getResults().isEmpty());
}
use of com.epam.pipeline.entity.datastorage.nfs.NFSDataStorage in project cloud-pipeline by epam.
the class NFSStorageProvider method getItems.
@Override
public DataStorageListing getItems(NFSDataStorage dataStorage, String path, Boolean showVersion, Integer pageSize, String marker) {
File dataStorageRoot = mount(dataStorage);
File dir = path != null ? new File(dataStorageRoot, path) : dataStorageRoot;
long offset = StringUtils.isNumeric(marker) ? Long.parseLong(marker) : 1;
try (Stream<Path> dirStream = Files.walk(dir.toPath(), 1)) {
List<AbstractDataStorageItem> dataStorageItems = dirStream.sorted().skip(// First element is a directory itself
offset).limit(pageSize).map(p -> {
File file = p.toFile();
AbstractDataStorageItem item;
if (file.isDirectory()) {
item = new DataStorageFolder();
} else {
// set size if it's a file
DataStorageFile dataStorageFile = new DataStorageFile();
dataStorageFile.setSize(file.length());
dataStorageFile.setChanged(S3Constants.getAwsDateFormat().format(new Date(file.lastModified())));
item = dataStorageFile;
}
item.setName(file.getName());
item.setPath(dataStorageRoot.toURI().relativize(file.toURI()).getPath());
return item;
}).collect(Collectors.toList());
DataStorageListing listing = new DataStorageListing();
listing.setResults(dataStorageItems);
Long nextOffset = offset + pageSize;
try (Stream<Path> nextStream = Files.walk(dir.toPath(), 1)) {
if (nextStream.skip(nextOffset).findFirst().isPresent()) {
listing.setNextPageMarker(nextOffset.toString());
}
}
return listing;
} catch (IOException e) {
throw new DataStorageException(e);
}
}
use of com.epam.pipeline.entity.datastorage.nfs.NFSDataStorage in project cloud-pipeline by epam.
the class PipelineConfigurationManagerTest method testGetUnregisteredPipelineConfiguration.
@Test
@Transactional(propagation = Propagation.REQUIRES_NEW)
@WithMockUser(username = TEST_OWNER1)
public void testGetUnregisteredPipelineConfiguration() {
PipelineStart vo = getPipelineStartVO();
PipelineConfiguration config = pipelineConfigurationManager.getPipelineConfiguration(vo);
Assert.assertFalse(config.getBuckets().isEmpty());
Assert.assertFalse(config.getNfsMountOptions().isEmpty());
String[] buckets = config.getBuckets().split(";");
Assert.assertEquals(2, buckets.length);
for (String bucket : buckets) {
Assert.assertTrue(dataStorages.stream().anyMatch(ds -> bucket.equals(ds.getPathMask())));
}
String[] nfsOptions = config.getNfsMountOptions().split(";");
Assert.assertEquals(1, nfsOptions.length);
for (String option : nfsOptions) {
if (StringUtils.isNotBlank(option)) {
Assert.assertTrue(dataStorages.stream().filter(ds -> ds instanceof NFSDataStorage).anyMatch(ds -> {
NFSDataStorage nfsDs = (NFSDataStorage) ds;
return nfsDs.getMountOptions().equals(option) || option.equals(nfsDs.getMountOptions() + ",ro");
}));
}
}
String[] mountPoints = config.getMountPoints().split(";");
for (String mountPoint : mountPoints) {
if (StringUtils.isNotBlank(mountPoint)) {
Assert.assertTrue(dataStorages.stream().anyMatch(ds -> mountPoint.equals(ds.getMountPoint())));
}
}
// Assert.assertTrue(Arrays.stream(nfsOptions).anyMatch(o -> o.endsWith(",ro")));
}
Aggregations