Examples with FileUtil org.apache.samza.util.FileUtil used on opensource projects

Example 16 with FileUtil

use of org.apache.samza.util.FileUtil in project samza by apache.

the class TestTransactionalStateTaskRestoreManager method testSetupStoreDirs.

@Test
public void testSetupStoreDirs() {
    TaskModel mockTaskModel = mock(TaskModel.class);
    TaskName taskName = new TaskName("Partition 0");
    when(mockTaskModel.getTaskName()).thenReturn(taskName);
    Partition taskChangelogPartition = new Partition(0);
    when(mockTaskModel.getChangelogPartition()).thenReturn(taskChangelogPartition);
    String store1Name = "store1";
    StorageEngine store1Engine = mock(StorageEngine.class);
    StoreProperties mockStore1Properties = mock(StoreProperties.class);
    when(store1Engine.getStoreProperties()).thenReturn(mockStore1Properties);
    when(mockStore1Properties.isLoggedStore()).thenReturn(true);
    when(mockStore1Properties.isPersistedToDisk()).thenReturn(true);
    String store2Name = "store2";
    StorageEngine store2Engine = mock(StorageEngine.class);
    StoreProperties mockStore2Properties = mock(StoreProperties.class);
    when(store2Engine.getStoreProperties()).thenReturn(mockStore2Properties);
    // non-logged store
    when(mockStore2Properties.isLoggedStore()).thenReturn(false);
    when(mockStore2Properties.isPersistedToDisk()).thenReturn(true);
    Map<String, StorageEngine> mockStoreEngines = ImmutableMap.of(store1Name, store1Engine, store2Name, store2Engine);
    File mockStore1DirToRetain = mock(File.class);
    // there will be no dir to retain for non-logged persistent stores
    ImmutableMap<String, File> storeDirsToRetain = ImmutableMap.of(store1Name, mockStore1DirToRetain);
    ListMultimap<String, File> storeDirsToDelete = ArrayListMultimap.create();
    File mockStore1CurrentDir = mock(File.class);
    Path mockStore1CurrentDirPath = mock(Path.class);
    when(mockStore1CurrentDir.toPath()).thenReturn(mockStore1CurrentDirPath);
    File mockStore2CurrentDir = mock(File.class);
    Path mockStore2CurrentDirPath = mock(Path.class);
    when(mockStore2CurrentDir.toPath()).thenReturn(mockStore2CurrentDirPath);
    File mockStore1DirToDelete1 = mock(File.class);
    File mockStore1DirToDelete2 = mock(File.class);
    File mockStore2DirToDelete1 = mock(File.class);
    File mockStore2DirToDelete2 = mock(File.class);
    storeDirsToDelete.put(store1Name, mockStore1CurrentDir);
    storeDirsToDelete.put(store1Name, mockStore1DirToDelete1);
    storeDirsToDelete.put(store1Name, mockStore1DirToDelete2);
    storeDirsToDelete.put(store2Name, mockStore2CurrentDir);
    storeDirsToDelete.put(store2Name, mockStore2DirToDelete1);
    storeDirsToDelete.put(store2Name, mockStore2DirToDelete2);
    StoreActions storeActions = new StoreActions(storeDirsToRetain, storeDirsToDelete, ImmutableMap.of());
    StorageManagerUtil mockStorageManagerUtil = mock(StorageManagerUtil.class);
    FileUtil mockFileUtil = mock(FileUtil.class);
    File mockLoggedStoreBaseDir = mock(File.class);
    File mockNonLoggedStoreBaseDir = mock(File.class);
    when(mockStorageManagerUtil.getTaskStoreDir(eq(mockLoggedStoreBaseDir), eq(store1Name), any(), any())).thenReturn(mockStore1CurrentDir);
    when(mockStorageManagerUtil.getTaskStoreDir(eq(mockNonLoggedStoreBaseDir), eq(store2Name), any(), any())).thenReturn(mockStore2CurrentDir);
    when(mockFileUtil.exists(eq(mockStore1CurrentDirPath))).thenReturn(false);
    when(mockFileUtil.exists(eq(mockStore2CurrentDirPath))).thenReturn(// will not be true in reality since current dir is always cleared, but return true for testing
    true);
    TransactionalStateTaskRestoreManager.setupStoreDirs(mockTaskModel, mockStoreEngines, storeActions, mockStorageManagerUtil, mockFileUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir);
    // verify that store directories to delete are deleted
    verify(mockFileUtil, times(1)).rm(mockStore1CurrentDir);
    verify(mockFileUtil, times(1)).rm(mockStore1DirToDelete1);
    verify(mockFileUtil, times(1)).rm(mockStore1DirToDelete2);
    verify(mockFileUtil, times(1)).rm(mockStore2CurrentDir);
    verify(mockFileUtil, times(1)).rm(mockStore2DirToDelete1);
    verify(mockFileUtil, times(1)).rm(mockStore2DirToDelete2);
    // verify that store checkpoint directories to retain are moved to (empty) current dirs only for store 1
    // setupStoreDirs doesn't guarantee that the dir is empty by itself, but the dir will be part of dirs to delete.
    verify(mockStorageManagerUtil, times(1)).restoreCheckpointFiles(any(), any());
    verify(mockFileUtil, times(1)).exists(mockStore1CurrentDirPath);
    verify(mockFileUtil, times(1)).createDirectories(mockStore1CurrentDirPath);
    verify(mockFileUtil, times(1)).exists(mockStore2CurrentDirPath);
    // should not be called since exists == true
    verify(mockFileUtil, never()).createDirectories(mockStore2CurrentDirPath);
    verifyNoMoreInteractions(mockFileUtil);
}

Example 17 with FileUtil

use of org.apache.samza.util.FileUtil in project samza by apache.

the class BlobStoreTestUtil method createLocalDir.

public static Path createLocalDir(String files) throws IOException {
    Path tempDirPath = Files.createTempDirectory(TEMP_DIR_PREFIX);
    File tempDirFile = tempDirPath.toFile();
    String tempDirPathString = tempDirPath.toAbsolutePath().toString();
    if (files.length() == 2)
        return tempDirPath;
    String[] paths = files.substring(1, files.length() - 1).split(",");
    for (String path : paths) {
        path = path.trim();
        if (!path.contains("/")) {
            Path filePath = Files.createFile(Paths.get(tempDirPathString, path));
            // file contents == file name
            new FileUtil().writeToTextFile(filePath.toFile(), path, false);
            filePath.toFile().deleteOnExit();
        } else {
            String dirs = path.substring(0, path.lastIndexOf("/"));
            String file = path.substring(path.lastIndexOf("/") + 1);
            Path directories = Files.createDirectories(Paths.get(tempDirPathString, dirs));
            if (!StringUtils.isBlank(file)) {
                // can be blank for empty directories
                Path filePath = Paths.get(directories.toAbsolutePath().toString(), file);
                Files.createFile(filePath);
                // file contents == file name
                new FileUtil().writeToTextFile(filePath.toFile(), file, false);
                filePath.toFile().deleteOnExit();
            }
        }
    }
    deleteDirRecursivelyOnExit(tempDirFile);
    return tempDirPath;
}