Search in sources :

Example 1 with NFSDataStorage

use of com.epam.pipeline.entity.datastorage.nfs.NFSDataStorage in project cloud-pipeline by epam.

the class NFSStorageProviderTest method testMoveDeleteFile.

@Test
public void testMoveDeleteFile() {
    NFSDataStorage dataStorage = new NFSDataStorage(0L, "testStorage", TEST_PATH + 3 + ":/test");
    nfsProvider.createStorage(dataStorage);
    String testFileName = "testFile.txt";
    String testFolderName = "testFolder";
    String testFolder2Name = "testFolder2";
    nfsProvider.createFile(dataStorage, testFileName, "testContent".getBytes());
    nfsProvider.createFolder(dataStorage, testFolderName);
    nfsProvider.createFolder(dataStorage, testFolder2Name);
    File dataStorageRoot = new File(testMountPoint, TEST_PATH + 3 + "/test");
    String newFilePath = testFolderName + "/" + testFileName;
    DataStorageFile file = nfsProvider.moveFile(dataStorage, testFileName, newFilePath);
    Assert.assertEquals(newFilePath, file.getPath());
    File oldFileLocation = new File(dataStorageRoot, testFileName);
    File newFileLocation = new File(dataStorageRoot, newFilePath);
    Assert.assertTrue(newFileLocation.exists());
    Assert.assertFalse(oldFileLocation.exists());
    String newFolder2Path = testFolderName + "/" + testFolder2Name;
    DataStorageFolder folder = nfsProvider.moveFolder(dataStorage, testFolder2Name, newFolder2Path);
    Assert.assertEquals(newFolder2Path, folder.getPath());
    File oldFolderLocation = new File(dataStorageRoot, testFolder2Name);
    File newFolderLocation = new File(dataStorageRoot, newFolder2Path);
    Assert.assertTrue(newFolderLocation.exists());
    Assert.assertFalse(oldFolderLocation.exists());
    nfsProvider.deleteFile(dataStorage, newFilePath, null, true);
    Assert.assertFalse(newFileLocation.exists());
    nfsProvider.deleteFolder(dataStorage, newFolder2Path, true);
    Assert.assertFalse(newFolderLocation.exists());
}
Also used : DataStorageFile(com.epam.pipeline.entity.datastorage.DataStorageFile) NFSDataStorage(com.epam.pipeline.entity.datastorage.nfs.NFSDataStorage) Matchers.anyString(org.mockito.Matchers.anyString) File(java.io.File) DataStorageFile(com.epam.pipeline.entity.datastorage.DataStorageFile) DataStorageFolder(com.epam.pipeline.entity.datastorage.DataStorageFolder) AbstractSpringTest(com.epam.pipeline.AbstractSpringTest) Test(org.junit.Test)

Example 2 with NFSDataStorage

use of com.epam.pipeline.entity.datastorage.nfs.NFSDataStorage in project cloud-pipeline by epam.

the class NFSStorageProviderTest method testCreateDeleteStorage.

@Test
@Transactional(propagation = Propagation.REQUIRES_NEW)
public void testCreateDeleteStorage() {
    NFSDataStorage dataStorage = new NFSDataStorage(0L, TEST_STORAGE_NAME, TEST_PATH + 1 + ":root/" + STORAGE_NAME);
    dataStorage.setOwner("test@user.com");
    String path = nfsProvider.createStorage(dataStorage);
    dataStorageDao.createDataStorage(dataStorage);
    Assert.assertEquals(dataStorage.getPath(), path);
    NFSDataStorage dataStorage2 = new NFSDataStorage(1L, TEST_STORAGE_NAME, TEST_PATH + 1 + ":root/" + STORAGE_NAME + 1);
    dataStorage2.setOwner("test@user.com");
    String path2 = nfsProvider.createStorage(dataStorage2);
    dataStorageDao.createDataStorage(dataStorage2);
    Assert.assertEquals(dataStorage2.getPath(), path2);
    File mountRootDir = new File(testMountPoint, TEST_PATH + 1 + "/root");
    Assert.assertTrue(mountRootDir.exists());
    File dataStorageRoot = new File(mountRootDir.getPath() + "/" + STORAGE_NAME);
    Assert.assertTrue(dataStorageRoot.exists());
    nfsProvider.deleteStorage(dataStorage);
    Assert.assertFalse(dataStorageRoot.exists());
    Assert.assertTrue(mountRootDir.exists());
    // emulate that database doesn't contain datastorages with mountRootDir
    dataStorageDao.deleteDataStorage(dataStorage.getId());
    nfsProvider.deleteStorage(dataStorage2);
    Assert.assertFalse(mountRootDir.exists());
}
Also used : NFSDataStorage(com.epam.pipeline.entity.datastorage.nfs.NFSDataStorage) Matchers.anyString(org.mockito.Matchers.anyString) File(java.io.File) DataStorageFile(com.epam.pipeline.entity.datastorage.DataStorageFile) AbstractSpringTest(com.epam.pipeline.AbstractSpringTest) Test(org.junit.Test) Transactional(org.springframework.transaction.annotation.Transactional)

Example 3 with NFSDataStorage

use of com.epam.pipeline.entity.datastorage.nfs.NFSDataStorage in project cloud-pipeline by epam.

the class NFSStorageProviderTest method testCreateFileFolderAndList.

@Test
public void testCreateFileFolderAndList() {
    NFSDataStorage dataStorage = new NFSDataStorage(0L, TEST_STORAGE_NAME, TEST_PATH + 2 + ":/test");
    nfsProvider.createStorage(dataStorage);
    String testFileName = "testFile.txt";
    nfsProvider.createFile(dataStorage, testFileName, "testContent".getBytes());
    File dataStorageRoot = new File(testMountPoint, TEST_PATH + 2 + "/test");
    File testFile = new File(dataStorageRoot, testFileName);
    Assert.assertTrue(testFile.exists());
    String testFolderName = "testFolder";
    nfsProvider.createFolder(dataStorage, testFolderName);
    File testFolder = new File(dataStorageRoot, testFolderName);
    Assert.assertTrue(testFolder.exists());
    DataStorageListing listing = nfsProvider.getItems(dataStorage, null, false, DEFAULT_PAGE_SIZE, null);
    Assert.assertFalse(listing.getResults().isEmpty());
    Optional<AbstractDataStorageItem> loadedFile = listing.getResults().stream().filter(i -> i.getType() == DataStorageItemType.File).findFirst();
    Assert.assertTrue(loadedFile.isPresent());
    Assert.assertEquals(testFileName, loadedFile.get().getName());
    Assert.assertEquals(testFileName, loadedFile.get().getPath());
    Assert.assertNotNull(((DataStorageFile) loadedFile.get()).getChanged());
    Optional<AbstractDataStorageItem> loadedFolder = listing.getResults().stream().filter(i -> i.getType() == DataStorageItemType.Folder).findFirst();
    Assert.assertTrue(loadedFolder.isPresent());
    Assert.assertEquals(testFolderName, loadedFolder.get().getName());
    Assert.assertEquals(testFolderName + "/", loadedFolder.get().getPath());
    Assert.assertNull(listing.getNextPageMarker());
    listing = nfsProvider.getItems(dataStorage, null, false, 1, null);
    Assert.assertEquals("2", listing.getNextPageMarker());
    listing = nfsProvider.getItems(dataStorage, null, false, 1, listing.getNextPageMarker());
    Assert.assertNull(listing.getNextPageMarker());
    Assert.assertFalse(listing.getResults().isEmpty());
}
Also used : DataStorageListing(com.epam.pipeline.entity.datastorage.DataStorageListing) BeforeClass(org.junit.BeforeClass) DataStorageDao(com.epam.pipeline.dao.datastorage.DataStorageDao) Mock(org.mockito.Mock) Autowired(org.springframework.beans.factory.annotation.Autowired) AbstractSpringTest(com.epam.pipeline.AbstractSpringTest) Matchers.anyString(org.mockito.Matchers.anyString) DataStorageFolder(com.epam.pipeline.entity.datastorage.DataStorageFolder) MockitoAnnotations(org.mockito.MockitoAnnotations) Propagation(org.springframework.transaction.annotation.Propagation) DataStorageListing(com.epam.pipeline.entity.datastorage.DataStorageListing) Before(org.junit.Before) AfterClass(org.junit.AfterClass) DataStorageItemType(com.epam.pipeline.entity.datastorage.DataStorageItemType) FileUtils(org.apache.commons.io.FileUtils) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) File(java.io.File) NFSDataStorage(com.epam.pipeline.entity.datastorage.nfs.NFSDataStorage) AbstractDataStorageItem(com.epam.pipeline.entity.datastorage.AbstractDataStorageItem) DataStorageFile(com.epam.pipeline.entity.datastorage.DataStorageFile) Whitebox(org.mockito.internal.util.reflection.Whitebox) Optional(java.util.Optional) CmdExecutor(com.epam.pipeline.manager.CmdExecutor) Assert(org.junit.Assert) Transactional(org.springframework.transaction.annotation.Transactional) NFSDataStorage(com.epam.pipeline.entity.datastorage.nfs.NFSDataStorage) AbstractDataStorageItem(com.epam.pipeline.entity.datastorage.AbstractDataStorageItem) Matchers.anyString(org.mockito.Matchers.anyString) File(java.io.File) DataStorageFile(com.epam.pipeline.entity.datastorage.DataStorageFile) AbstractSpringTest(com.epam.pipeline.AbstractSpringTest) Test(org.junit.Test)

Example 4 with NFSDataStorage

use of com.epam.pipeline.entity.datastorage.nfs.NFSDataStorage in project cloud-pipeline by epam.

the class NFSStorageProvider method getItems.

@Override
public DataStorageListing getItems(NFSDataStorage dataStorage, String path, Boolean showVersion, Integer pageSize, String marker) {
    File dataStorageRoot = mount(dataStorage);
    File dir = path != null ? new File(dataStorageRoot, path) : dataStorageRoot;
    long offset = StringUtils.isNumeric(marker) ? Long.parseLong(marker) : 1;
    try (Stream<Path> dirStream = Files.walk(dir.toPath(), 1)) {
        List<AbstractDataStorageItem> dataStorageItems = dirStream.sorted().skip(// First element is a directory itself
        offset).limit(pageSize).map(p -> {
            File file = p.toFile();
            AbstractDataStorageItem item;
            if (file.isDirectory()) {
                item = new DataStorageFolder();
            } else {
                // set size if it's a file
                DataStorageFile dataStorageFile = new DataStorageFile();
                dataStorageFile.setSize(file.length());
                dataStorageFile.setChanged(S3Constants.getAwsDateFormat().format(new Date(file.lastModified())));
                item = dataStorageFile;
            }
            item.setName(file.getName());
            item.setPath(dataStorageRoot.toURI().relativize(file.toURI()).getPath());
            return item;
        }).collect(Collectors.toList());
        DataStorageListing listing = new DataStorageListing();
        listing.setResults(dataStorageItems);
        Long nextOffset = offset + pageSize;
        try (Stream<Path> nextStream = Files.walk(dir.toPath(), 1)) {
            if (nextStream.skip(nextOffset).findFirst().isPresent()) {
                listing.setNextPageMarker(nextOffset.toString());
            }
        }
        return listing;
    } catch (IOException e) {
        throw new DataStorageException(e);
    }
}
Also used : Path(java.nio.file.Path) Arrays(java.util.Arrays) Date(java.util.Date) S3Constants(com.epam.pipeline.manager.datastorage.providers.aws.s3.S3Constants) LoggerFactory(org.slf4j.LoggerFactory) SystemPreferences(com.epam.pipeline.manager.preference.SystemPreferences) Autowired(org.springframework.beans.factory.annotation.Autowired) StringUtils(org.apache.commons.lang3.StringUtils) Matcher(java.util.regex.Matcher) MessageHelper(com.epam.pipeline.common.MessageHelper) ByteArrayInputStream(java.io.ByteArrayInputStream) Map(java.util.Map) DataStorageItemContent(com.epam.pipeline.entity.datastorage.DataStorageItemContent) DataStorageListing(com.epam.pipeline.entity.datastorage.DataStorageListing) Path(java.nio.file.Path) PosixFilePermission(java.nio.file.attribute.PosixFilePermission) DataStorageDownloadFileUrl(com.epam.pipeline.entity.datastorage.DataStorageDownloadFileUrl) Set(java.util.Set) Collectors(java.util.stream.Collectors) FileNotFoundException(java.io.FileNotFoundException) DataStorageFile(com.epam.pipeline.entity.datastorage.DataStorageFile) IOUtils(org.apache.commons.io.IOUtils) List(java.util.List) Stream(java.util.stream.Stream) CmdExecutionException(com.epam.pipeline.exception.CmdExecutionException) DataStorageStreamingContent(com.epam.pipeline.entity.datastorage.DataStorageStreamingContent) CmdExecutor(com.epam.pipeline.manager.CmdExecutor) Pattern(java.util.regex.Pattern) FileContentUtils(com.epam.pipeline.utils.FileContentUtils) DataStorageDao(com.epam.pipeline.dao.datastorage.DataStorageDao) MessageConstants(com.epam.pipeline.common.MessageConstants) DataStorageFolder(com.epam.pipeline.entity.datastorage.DataStorageFolder) BufferedOutputStream(java.io.BufferedOutputStream) Value(org.springframework.beans.factory.annotation.Value) Service(org.springframework.stereotype.Service) DataStorageException(com.epam.pipeline.entity.datastorage.DataStorageException) PreferenceManager(com.epam.pipeline.manager.preference.PreferenceManager) Logger(org.slf4j.Logger) Files(java.nio.file.Files) SystemUtils(org.apache.commons.lang3.SystemUtils) FileOutputStream(java.io.FileOutputStream) AbstractDataStorage(com.epam.pipeline.entity.datastorage.AbstractDataStorage) FileUtils(org.apache.commons.io.FileUtils) IOException(java.io.IOException) DataStorageType(com.epam.pipeline.entity.datastorage.DataStorageType) FileInputStream(java.io.FileInputStream) NFSDataStorage(com.epam.pipeline.entity.datastorage.nfs.NFSDataStorage) File(java.io.File) AbstractDataStorageItem(com.epam.pipeline.entity.datastorage.AbstractDataStorageItem) Paths(java.nio.file.Paths) Collections(java.util.Collections) Assert(org.springframework.util.Assert) InputStream(java.io.InputStream) DataStorageFile(com.epam.pipeline.entity.datastorage.DataStorageFile) IOException(java.io.IOException) Date(java.util.Date) DataStorageFolder(com.epam.pipeline.entity.datastorage.DataStorageFolder) DataStorageListing(com.epam.pipeline.entity.datastorage.DataStorageListing) DataStorageException(com.epam.pipeline.entity.datastorage.DataStorageException) AbstractDataStorageItem(com.epam.pipeline.entity.datastorage.AbstractDataStorageItem) DataStorageFile(com.epam.pipeline.entity.datastorage.DataStorageFile) File(java.io.File)

Example 5 with NFSDataStorage

use of com.epam.pipeline.entity.datastorage.nfs.NFSDataStorage in project cloud-pipeline by epam.

the class PipelineConfigurationManagerTest method testGetUnregisteredPipelineConfiguration.

@Test
@Transactional(propagation = Propagation.REQUIRES_NEW)
@WithMockUser(username = TEST_OWNER1)
public void testGetUnregisteredPipelineConfiguration() {
    PipelineStart vo = getPipelineStartVO();
    PipelineConfiguration config = pipelineConfigurationManager.getPipelineConfiguration(vo);
    Assert.assertFalse(config.getBuckets().isEmpty());
    Assert.assertFalse(config.getNfsMountOptions().isEmpty());
    String[] buckets = config.getBuckets().split(";");
    Assert.assertEquals(2, buckets.length);
    for (String bucket : buckets) {
        Assert.assertTrue(dataStorages.stream().anyMatch(ds -> bucket.equals(ds.getPathMask())));
    }
    String[] nfsOptions = config.getNfsMountOptions().split(";");
    Assert.assertEquals(1, nfsOptions.length);
    for (String option : nfsOptions) {
        if (StringUtils.isNotBlank(option)) {
            Assert.assertTrue(dataStorages.stream().filter(ds -> ds instanceof NFSDataStorage).anyMatch(ds -> {
                NFSDataStorage nfsDs = (NFSDataStorage) ds;
                return nfsDs.getMountOptions().equals(option) || option.equals(nfsDs.getMountOptions() + ",ro");
            }));
        }
    }
    String[] mountPoints = config.getMountPoints().split(";");
    for (String mountPoint : mountPoints) {
        if (StringUtils.isNotBlank(mountPoint)) {
            Assert.assertTrue(dataStorages.stream().anyMatch(ds -> mountPoint.equals(ds.getMountPoint())));
        }
    }
// Assert.assertTrue(Arrays.stream(nfsOptions).anyMatch(o -> o.endsWith(",ro")));
}
Also used : DirtiesContext(org.springframework.test.annotation.DirtiesContext) TestApplicationWithAclSecurity(com.epam.pipeline.app.TestApplicationWithAclSecurity) ToolGroup(com.epam.pipeline.entity.pipeline.ToolGroup) DataStorageDao(com.epam.pipeline.dao.datastorage.DataStorageDao) AclTestDao(com.epam.pipeline.dao.util.AclTestDao) Autowired(org.springframework.beans.factory.annotation.Autowired) PipeConfValueVO(com.epam.pipeline.entity.configuration.PipeConfValueVO) StringUtils(org.apache.commons.lang3.StringUtils) ArrayList(java.util.ArrayList) Propagation(org.springframework.transaction.annotation.Propagation) AbstractManagerTest(com.epam.pipeline.manager.AbstractManagerTest) AclPermission(com.epam.pipeline.security.acl.AclPermission) PipelineConfiguration(com.epam.pipeline.entity.configuration.PipelineConfiguration) Before(org.junit.Before) ToolDao(com.epam.pipeline.dao.tool.ToolDao) AbstractDataStorage(com.epam.pipeline.entity.datastorage.AbstractDataStorage) Test(org.junit.Test) DockerRegistryDao(com.epam.pipeline.dao.docker.DockerRegistryDao) NFSDataStorage(com.epam.pipeline.entity.datastorage.nfs.NFSDataStorage) DockerRegistry(com.epam.pipeline.entity.pipeline.DockerRegistry) Tool(com.epam.pipeline.entity.pipeline.Tool) List(java.util.List) WithMockUser(org.springframework.security.test.context.support.WithMockUser) ContextConfiguration(org.springframework.test.context.ContextConfiguration) S3bucketDataStorage(com.epam.pipeline.entity.datastorage.aws.S3bucketDataStorage) PipelineStart(com.epam.pipeline.entity.pipeline.run.PipelineStart) Assert(org.junit.Assert) ToolGroupDao(com.epam.pipeline.dao.tool.ToolGroupDao) Collections(java.util.Collections) Transactional(org.springframework.transaction.annotation.Transactional) NFSDataStorage(com.epam.pipeline.entity.datastorage.nfs.NFSDataStorage) PipelineConfiguration(com.epam.pipeline.entity.configuration.PipelineConfiguration) PipelineStart(com.epam.pipeline.entity.pipeline.run.PipelineStart) WithMockUser(org.springframework.security.test.context.support.WithMockUser) AbstractManagerTest(com.epam.pipeline.manager.AbstractManagerTest) Test(org.junit.Test) Transactional(org.springframework.transaction.annotation.Transactional)

Aggregations

NFSDataStorage (com.epam.pipeline.entity.datastorage.nfs.NFSDataStorage)9 Test (org.junit.Test)6 AbstractSpringTest (com.epam.pipeline.AbstractSpringTest)5 DataStorageFile (com.epam.pipeline.entity.datastorage.DataStorageFile)5 File (java.io.File)4 Before (org.junit.Before)4 Matchers.anyString (org.mockito.Matchers.anyString)4 Transactional (org.springframework.transaction.annotation.Transactional)4 DataStorageDao (com.epam.pipeline.dao.datastorage.DataStorageDao)3 DataStorageFolder (com.epam.pipeline.entity.datastorage.DataStorageFolder)3 S3bucketDataStorage (com.epam.pipeline.entity.datastorage.aws.S3bucketDataStorage)3 Autowired (org.springframework.beans.factory.annotation.Autowired)3 AbstractDataStorage (com.epam.pipeline.entity.datastorage.AbstractDataStorage)2 AbstractDataStorageItem (com.epam.pipeline.entity.datastorage.AbstractDataStorageItem)2 DataStorageListing (com.epam.pipeline.entity.datastorage.DataStorageListing)2 DockerRegistry (com.epam.pipeline.entity.pipeline.DockerRegistry)2 Tool (com.epam.pipeline.entity.pipeline.Tool)2 ToolGroup (com.epam.pipeline.entity.pipeline.ToolGroup)2 CmdExecutor (com.epam.pipeline.manager.CmdExecutor)2 AclPermission (com.epam.pipeline.security.acl.AclPermission)2