Examples with UfsFileStatus alluxio.underfs.UfsFileStatus used on opensource projects

Example 1 with UfsFileStatus

use of alluxio.underfs.UfsFileStatus in project alluxio by Alluxio.

the class InodeSyncStream method loadFileMetadataInternal.

/**
 * Loads metadata for the file identified by the given path from UFS into Alluxio.
 *
 * This method doesn't require any specific type of locking on inodePath. If the path needs to be
 * loaded, we will acquire a write-edge lock.
 *
 * @param rpcContext the rpc context
 * @param inodePath the path for which metadata should be loaded
 * @param resolution the UFS resolution of path
 * @param context the load metadata context
 */
static void loadFileMetadataInternal(RpcContext rpcContext, LockedInodePath inodePath, MountTable.Resolution resolution, LoadMetadataContext context, DefaultFileSystemMaster fsMaster) throws BlockInfoException, FileDoesNotExistException, InvalidPathException, FileAlreadyCompletedException, InvalidFileSizeException, IOException {
    if (inodePath.fullPathExists()) {
        return;
    }
    AlluxioURI ufsUri = resolution.getUri();
    long ufsBlockSizeByte;
    long ufsLength;
    AccessControlList acl = null;
    try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
        UnderFileSystem ufs = ufsResource.get();
        if (context.getUfsStatus() == null) {
            context.setUfsStatus(ufs.getExistingFileStatus(ufsUri.toString()));
        }
        ufsLength = ((UfsFileStatus) context.getUfsStatus()).getContentLength();
        long blockSize = ((UfsFileStatus) context.getUfsStatus()).getBlockSize();
        ufsBlockSizeByte = blockSize != UfsFileStatus.UNKNOWN_BLOCK_SIZE ? blockSize : ufs.getBlockSizeByte(ufsUri.toString());
        if (fsMaster.isAclEnabled()) {
            Pair<AccessControlList, DefaultAccessControlList> aclPair = ufs.getAclPair(ufsUri.toString());
            if (aclPair != null) {
                acl = aclPair.getFirst();
                // DefaultACL should be null, because it is a file
                if (aclPair.getSecond() != null) {
                    LOG.warn("File {} has default ACL in the UFS", inodePath.getUri());
                }
            }
        }
    }
    // Metadata loaded from UFS has no TTL set.
    CreateFileContext createFileContext = CreateFileContext.defaults();
    createFileContext.getOptions().setBlockSizeBytes(ufsBlockSizeByte);
    createFileContext.getOptions().setRecursive(context.getOptions().getCreateAncestors());
    createFileContext.getOptions().setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(context.getOptions().getCommonOptions().getTtl()).setTtlAction(context.getOptions().getCommonOptions().getTtlAction()));
    // set as through since already in UFS
    createFileContext.setWriteType(WriteType.THROUGH);
    createFileContext.setMetadataLoad(true);
    createFileContext.setOwner(context.getUfsStatus().getOwner());
    createFileContext.setGroup(context.getUfsStatus().getGroup());
    createFileContext.setXAttr(context.getUfsStatus().getXAttr());
    short ufsMode = context.getUfsStatus().getMode();
    Mode mode = new Mode(ufsMode);
    Long ufsLastModified = context.getUfsStatus().getLastModifiedTime();
    if (resolution.getShared()) {
        mode.setOtherBits(mode.getOtherBits().or(mode.getOwnerBits()));
    }
    createFileContext.getOptions().setMode(mode.toProto());
    if (acl != null) {
        createFileContext.setAcl(acl.getEntries());
    }
    if (ufsLastModified != null) {
        createFileContext.setOperationTimeMs(ufsLastModified);
    }
    try (LockedInodePath writeLockedPath = inodePath.lockFinalEdgeWrite();
        MergeJournalContext merger = new MergeJournalContext(rpcContext.getJournalContext(), writeLockedPath.getUri(), InodeSyncStream::mergeCreateComplete)) {
        // We do not want to close this wrapRpcContext because it uses elements from another context
        RpcContext wrapRpcContext = new RpcContext(rpcContext.getBlockDeletionContext(), merger, rpcContext.getOperationContext());
        fsMaster.createFileInternal(wrapRpcContext, writeLockedPath, createFileContext);
        CompleteFileContext completeContext = CompleteFileContext.mergeFrom(CompleteFilePOptions.newBuilder().setUfsLength(ufsLength)).setUfsStatus(context.getUfsStatus());
        if (ufsLastModified != null) {
            completeContext.setOperationTimeMs(ufsLastModified);
        }
        fsMaster.completeFileInternal(wrapRpcContext, writeLockedPath, completeContext);
    } catch (FileAlreadyExistsException e) {
        // This may occur if a thread created or loaded the file before we got the write lock.
        // The file already exists, so nothing needs to be loaded.
        LOG.debug("Failed to load file metadata: {}", e.toString());
    }
    // Re-traverse the path to pick up any newly created inodes.
    inodePath.traverse();
}

Example 2 with UfsFileStatus

use of alluxio.underfs.UfsFileStatus in project alluxio by Alluxio.

the class FileSystemMasterSyncMetadataTest method listStatusWithSyncMetadataAndEmptyS3Owner.

@Test
public void listStatusWithSyncMetadataAndEmptyS3Owner() throws Exception {
    AlluxioURI ufsMount = setupMockUfsS3Mount();
    short mode = ModeUtils.getUMask("0700").toShort();
    // Mock dir1 ufs path
    AlluxioURI dir1Path = ufsMount.join("dir1");
    UfsDirectoryStatus dir1Status = new UfsDirectoryStatus(dir1Path.getPath(), "", "", mode);
    Mockito.when(mUfs.getFingerprint(dir1Path.toString())).thenReturn(Fingerprint.create("s3", dir1Status).serialize());
    Mockito.when(mUfs.exists(dir1Path.toString())).thenReturn(true);
    Mockito.when(mUfs.isDirectory(dir1Path.toString())).thenReturn(true);
    Mockito.when(mUfs.isFile(dir1Path.toString())).thenReturn(false);
    Mockito.when(mUfs.getStatus(dir1Path.toString())).thenReturn(dir1Status);
    Mockito.when(mUfs.getDirectoryStatus(dir1Path.toString())).thenReturn(dir1Status);
    // Mock nested ufs path
    AlluxioURI nestedFilePath = ufsMount.join("dir1").join("file1");
    UfsFileStatus nestedFileStatus = new UfsFileStatus(nestedFilePath.getPath(), "dummy", 0, null, "", "", mode, 1024);
    Mockito.when(mUfs.getFingerprint(nestedFilePath.toString())).thenReturn(Fingerprint.create("s3", nestedFileStatus).serialize());
    Mockito.when(mUfs.getStatus(nestedFilePath.toString())).thenReturn(nestedFileStatus);
    Mockito.when(mUfs.isDirectory(nestedFilePath.toString())).thenReturn(false);
    Mockito.when(mUfs.isFile(nestedFilePath.toString())).thenReturn(true);
    Mockito.when(mUfs.getFileStatus(nestedFilePath.toString())).thenReturn(nestedFileStatus);
    Mockito.when(mUfs.exists(nestedFilePath.toString())).thenReturn(true);
    // Create directory in Alluxio only
    AlluxioURI dir1 = new AlluxioURI("/mnt/local/dir1");
    mFileSystemMaster.createDirectory(dir1, CreateDirectoryContext.defaults());
    // Mock creating the same directory and nested file in UFS out of band
    Mockito.when(mUfs.listStatus(eq(dir1Path.toString()))).thenReturn(new UfsStatus[] { new UfsFileStatus("file1", "dummy", 0, null, "", "", mode, 1024) });
    // List with sync.interval=0
    List<FileInfo> fileInfoList = mFileSystemMaster.listStatus(dir1, ListStatusContext.mergeFrom(ListStatusPOptions.newBuilder().setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setSyncIntervalMs(0).build())));
    assertEquals(1, fileInfoList.size());
    // Verify owner/group is not empty
    FileInfo mountLocalInfo = mFileSystemMaster.getFileInfo(new AlluxioURI("/mnt/local"), GetStatusContext.defaults());
    assertEquals(mountLocalInfo.getOwner(), mFileSystemMaster.getFileInfo(dir1, GetStatusContext.defaults()).getOwner());
    assertEquals(mountLocalInfo.getGroup(), mFileSystemMaster.getFileInfo(dir1, GetStatusContext.defaults()).getGroup());
    AlluxioURI file1 = new AlluxioURI("/mnt/local/dir1/file1");
    assertEquals(mountLocalInfo.getOwner(), mFileSystemMaster.getFileInfo(file1, GetStatusContext.defaults()).getOwner());
    assertEquals(mountLocalInfo.getGroup(), mFileSystemMaster.getFileInfo(file1, GetStatusContext.defaults()).getGroup());
}

Example 3 with UfsFileStatus

use of alluxio.underfs.UfsFileStatus in project alluxio by Alluxio.

the class UnderFileSystemCommonOperations method getFileStatusTest.

/**
 * Test for getting file status.
 */
@RelatedS3Operations(operations = { "upload", "getObjectMetadata" })
public void getFileStatusTest() throws IOException {
    String testFile = PathUtils.concatPath(mTopLevelTestDirectory, "testFile");
    createEmptyFile(testFile);
    UfsStatus status = mUfs.getStatus(testFile);
    if (!(status instanceof UfsFileStatus)) {
        throw new IOException("Failed to get ufs file status");
    }
}

Example 4 with UfsFileStatus

use of alluxio.underfs.UfsFileStatus in project alluxio by Alluxio.

the class HdfsUnderFileSystem method listStatus.

@Override
@Nullable
public UfsStatus[] listStatus(String path) throws IOException {
    FileStatus[] files = listStatusInternal(path);
    if (files == null) {
        return null;
    }
    UfsStatus[] rtn = new UfsStatus[files.length];
    int i = 0;
    for (FileStatus status : files) {
        // only return the relative path, to keep consistent with java.io.File.list()
        UfsStatus retStatus;
        if (!status.isDir()) {
            String contentHash = UnderFileSystemUtils.approximateContentHash(status.getLen(), status.getModificationTime());
            retStatus = new UfsFileStatus(status.getPath().getName(), contentHash, status.getLen(), status.getModificationTime(), status.getOwner(), status.getGroup(), status.getPermission().toShort(), status.getBlockSize());
        } else {
            retStatus = new UfsDirectoryStatus(status.getPath().getName(), status.getOwner(), status.getGroup(), status.getPermission().toShort(), status.getModificationTime());
        }
        rtn[i++] = retStatus;
    }
    return rtn;
}

Example 5 with UfsFileStatus

use of alluxio.underfs.UfsFileStatus in project alluxio by Alluxio.

the class LocalUnderFileSystemTest method getFileStatus.

@Test
public void getFileStatus() throws IOException {
    String file = PathUtils.concatPath(mLocalUfsRoot, getUniqueFileName());
    mLocalUfs.create(file).close();
    UfsFileStatus s = mLocalUfs.getFileStatus(file);
    assertFalse(s.isDirectory());
    assertTrue(s.isFile());
}