Examples with UfsDirectoryStatus alluxio.underfs.UfsDirectoryStatus used on opensource projects

Example 1 with UfsDirectoryStatus

use of alluxio.underfs.UfsDirectoryStatus in project alluxio by Alluxio.

the class FileSystemMasterSyncMetadataTest method listStatusWithSyncMetadataAndEmptyS3Owner.

@Test
public void listStatusWithSyncMetadataAndEmptyS3Owner() throws Exception {
    AlluxioURI ufsMount = setupMockUfsS3Mount();
    short mode = ModeUtils.getUMask("0700").toShort();
    // Mock dir1 ufs path
    AlluxioURI dir1Path = ufsMount.join("dir1");
    UfsDirectoryStatus dir1Status = new UfsDirectoryStatus(dir1Path.getPath(), "", "", mode);
    Mockito.when(mUfs.getFingerprint(dir1Path.toString())).thenReturn(Fingerprint.create("s3", dir1Status).serialize());
    Mockito.when(mUfs.exists(dir1Path.toString())).thenReturn(true);
    Mockito.when(mUfs.isDirectory(dir1Path.toString())).thenReturn(true);
    Mockito.when(mUfs.isFile(dir1Path.toString())).thenReturn(false);
    Mockito.when(mUfs.getStatus(dir1Path.toString())).thenReturn(dir1Status);
    Mockito.when(mUfs.getDirectoryStatus(dir1Path.toString())).thenReturn(dir1Status);
    // Mock nested ufs path
    AlluxioURI nestedFilePath = ufsMount.join("dir1").join("file1");
    UfsFileStatus nestedFileStatus = new UfsFileStatus(nestedFilePath.getPath(), "dummy", 0, null, "", "", mode, 1024);
    Mockito.when(mUfs.getFingerprint(nestedFilePath.toString())).thenReturn(Fingerprint.create("s3", nestedFileStatus).serialize());
    Mockito.when(mUfs.getStatus(nestedFilePath.toString())).thenReturn(nestedFileStatus);
    Mockito.when(mUfs.isDirectory(nestedFilePath.toString())).thenReturn(false);
    Mockito.when(mUfs.isFile(nestedFilePath.toString())).thenReturn(true);
    Mockito.when(mUfs.getFileStatus(nestedFilePath.toString())).thenReturn(nestedFileStatus);
    Mockito.when(mUfs.exists(nestedFilePath.toString())).thenReturn(true);
    // Create directory in Alluxio only
    AlluxioURI dir1 = new AlluxioURI("/mnt/local/dir1");
    mFileSystemMaster.createDirectory(dir1, CreateDirectoryContext.defaults());
    // Mock creating the same directory and nested file in UFS out of band
    Mockito.when(mUfs.listStatus(eq(dir1Path.toString()))).thenReturn(new UfsStatus[] { new UfsFileStatus("file1", "dummy", 0, null, "", "", mode, 1024) });
    // List with sync.interval=0
    List<FileInfo> fileInfoList = mFileSystemMaster.listStatus(dir1, ListStatusContext.mergeFrom(ListStatusPOptions.newBuilder().setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setSyncIntervalMs(0).build())));
    assertEquals(1, fileInfoList.size());
    // Verify owner/group is not empty
    FileInfo mountLocalInfo = mFileSystemMaster.getFileInfo(new AlluxioURI("/mnt/local"), GetStatusContext.defaults());
    assertEquals(mountLocalInfo.getOwner(), mFileSystemMaster.getFileInfo(dir1, GetStatusContext.defaults()).getOwner());
    assertEquals(mountLocalInfo.getGroup(), mFileSystemMaster.getFileInfo(dir1, GetStatusContext.defaults()).getGroup());
    AlluxioURI file1 = new AlluxioURI("/mnt/local/dir1/file1");
    assertEquals(mountLocalInfo.getOwner(), mFileSystemMaster.getFileInfo(file1, GetStatusContext.defaults()).getOwner());
    assertEquals(mountLocalInfo.getGroup(), mFileSystemMaster.getFileInfo(file1, GetStatusContext.defaults()).getGroup());
}

Example 2 with UfsDirectoryStatus

use of alluxio.underfs.UfsDirectoryStatus in project alluxio by Alluxio.

the class HdfsUnderFileSystem method listStatus.

@Override
@Nullable
public UfsStatus[] listStatus(String path) throws IOException {
    FileStatus[] files = listStatusInternal(path);
    if (files == null) {
        return null;
    }
    UfsStatus[] rtn = new UfsStatus[files.length];
    int i = 0;
    for (FileStatus status : files) {
        // only return the relative path, to keep consistent with java.io.File.list()
        UfsStatus retStatus;
        if (!status.isDir()) {
            String contentHash = UnderFileSystemUtils.approximateContentHash(status.getLen(), status.getModificationTime());
            retStatus = new UfsFileStatus(status.getPath().getName(), contentHash, status.getLen(), status.getModificationTime(), status.getOwner(), status.getGroup(), status.getPermission().toShort(), status.getBlockSize());
        } else {
            retStatus = new UfsDirectoryStatus(status.getPath().getName(), status.getOwner(), status.getGroup(), status.getPermission().toShort(), status.getModificationTime());
        }
        rtn[i++] = retStatus;
    }
    return rtn;
}

Example 3 with UfsDirectoryStatus

use of alluxio.underfs.UfsDirectoryStatus in project alluxio by Alluxio.

the class HdfsUnderFileSystem method getDirectoryStatus.

@Override
public UfsDirectoryStatus getDirectoryStatus(String path) throws IOException {
    Path tPath = new Path(path);
    FileSystem hdfs = getFs();
    FileStatus fs = hdfs.getFileStatus(tPath);
    return new UfsDirectoryStatus(path, fs.getOwner(), fs.getGroup(), fs.getPermission().toShort(), fs.getModificationTime());
}

Example 4 with UfsDirectoryStatus

use of alluxio.underfs.UfsDirectoryStatus in project alluxio by Alluxio.

the class LocalUnderFileSystem method listStatus.

@Override
public UfsStatus[] listStatus(String path) throws IOException {
    path = stripPath(path);
    File file = new File(path);
    // By default, exists follows symlinks. If the symlink is invalid .exists() returns false
    File[] files = mSkipBrokenSymlinks ? file.listFiles(File::exists) : file.listFiles();
    if (files != null) {
        UfsStatus[] rtn = new UfsStatus[files.length];
        int i = 0;
        for (File f : files) {
            // TODO(adit): do we need extra call for attributes?
            PosixFileAttributes attr = Files.readAttributes(Paths.get(f.getPath()), PosixFileAttributes.class);
            short mode = FileUtils.translatePosixPermissionToMode(attr.permissions());
            UfsStatus retStatus;
            if (f.isDirectory()) {
                retStatus = new UfsDirectoryStatus(f.getName(), attr.owner().getName(), attr.group().getName(), mode, f.lastModified());
            } else {
                String contentHash = UnderFileSystemUtils.approximateContentHash(f.length(), f.lastModified());
                retStatus = new UfsFileStatus(f.getName(), contentHash, f.length(), f.lastModified(), attr.owner().getName(), attr.group().getName(), mode, mUfsConf.getBytes(PropertyKey.USER_BLOCK_SIZE_BYTES_DEFAULT));
            }
            rtn[i++] = retStatus;
        }
        return rtn;
    } else {
        return null;
    }
}

Example 5 with UfsDirectoryStatus

use of alluxio.underfs.UfsDirectoryStatus in project alluxio by Alluxio.

the class FileSystemMasterRestartIntegrationTest method unavailableUfsRecursiveCreate.

@Test
public void unavailableUfsRecursiveCreate() throws Exception {
    String ufsBase = "test://test/";
    UnderFileSystemFactory mockUfsFactory = Mockito.mock(UnderFileSystemFactory.class);
    Mockito.when(mockUfsFactory.supportsPath(ArgumentMatchers.anyString(), ArgumentMatchers.any())).thenReturn(Boolean.FALSE);
    Mockito.when(mockUfsFactory.supportsPath(ArgumentMatchers.eq(ufsBase), ArgumentMatchers.any())).thenReturn(Boolean.TRUE);
    UnderFileSystem mockUfs = Mockito.mock(UnderFileSystem.class);
    UfsDirectoryStatus ufsStatus = new UfsDirectoryStatus("test", "owner", "group", (short) 511);
    Mockito.when(mockUfsFactory.create(ArgumentMatchers.eq(ufsBase), ArgumentMatchers.any())).thenReturn(mockUfs);
    Mockito.when(mockUfs.isDirectory(ufsBase)).thenReturn(true);
    Mockito.when(mockUfs.resolveUri(new AlluxioURI(ufsBase), "")).thenReturn(new AlluxioURI(ufsBase));
    Mockito.when(mockUfs.resolveUri(new AlluxioURI(ufsBase), "/dir1")).thenReturn(new AlluxioURI(ufsBase + "/dir1"));
    Mockito.when(mockUfs.getExistingDirectoryStatus(ufsBase)).thenReturn(ufsStatus);
    Mockito.when(mockUfs.mkdirs(ArgumentMatchers.eq(ufsBase + "/dir1"), ArgumentMatchers.any())).thenThrow(new IOException("ufs unavailable"));
    Mockito.when(mockUfs.getStatus(ufsBase)).thenReturn(ufsStatus);
    UnderFileSystemFactoryRegistry.register(mockUfsFactory);
    mFsMaster.mount(new AlluxioURI("/mnt"), new AlluxioURI(ufsBase), MountContext.defaults());
    AlluxioURI root = new AlluxioURI("/mnt/");
    AlluxioURI alluxioFile = new AlluxioURI("/mnt/dir1/dir2/file");
    // Create a persisted Alluxio file (but no ufs file).
    try {
        mFsMaster.createFile(alluxioFile, CreateFileContext.mergeFrom(CreateFilePOptions.newBuilder().setRecursive(true)).setWriteType(WriteType.CACHE_THROUGH));
        Assert.fail("persisted create should fail, when UFS is unavailable");
    } catch (Exception e) {
    // expected, ignore
    }
    List<FileInfo> files = mFsMaster.listStatus(root, ListStatusContext.defaults());
    Assert.assertTrue(files.isEmpty());
    try {
        // should not exist
        files = mFsMaster.listStatus(new AlluxioURI("/mnt/dir1/"), ListStatusContext.defaults());
        Assert.fail("dir should not exist, when UFS is unavailable");
    } catch (Exception e) {
    // expected, ignore
    }
    try {
        // should not exist
        mFsMaster.delete(new AlluxioURI("/mnt/dir1/"), DeleteContext.mergeFrom(DeletePOptions.newBuilder().setRecursive(true)));
        Assert.fail("cannot delete non-existing directory, when UFS is unavailable");
    } catch (Exception e) {
        // expected, ignore
        files = null;
    }
    files = mFsMaster.listStatus(new AlluxioURI("/mnt/"), ListStatusContext.defaults());
    Assert.assertTrue(files.isEmpty());
    // Stop Alluxio.
    mLocalAlluxioClusterResource.get().stopFS();
    // Create the master using the existing journal.
    try (FsMasterResource masterResource = MasterTestUtils.createLeaderFileSystemMasterFromJournal()) {
        FileSystemMaster newFsMaster = masterResource.getRegistry().get(FileSystemMaster.class);
        files = newFsMaster.listStatus(new AlluxioURI("/mnt/"), ListStatusContext.defaults());
        Assert.assertTrue(files.isEmpty());
    }
}