Examples with UnderFileSystem alluxio.underfs.UnderFileSystem used on opensource projects

Example 91 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class UfsMutableJournal method format.

@Override
public void format() throws IOException {
    LOG.info("Formatting {}", mLocation);
    try (UnderFileSystem ufs = UnderFileSystem.Factory.create(mLocation.toString(), UnderFileSystemConfiguration.defaults(ServerConfiguration.global()))) {
        if (ufs.isDirectory(mLocation.toString())) {
            for (UfsStatus p : ufs.listStatus(mLocation.toString())) {
                URI childPath;
                try {
                    childPath = URIUtils.appendPath(mLocation, p.getName());
                } catch (URISyntaxException e) {
                    throw new RuntimeException(e.getMessage());
                }
                boolean failedToDelete;
                if (p.isDirectory()) {
                    failedToDelete = !ufs.deleteDirectory(childPath.toString(), DeleteOptions.defaults().setRecursive(true));
                } else {
                    failedToDelete = !ufs.deleteFile(childPath.toString());
                }
                if (failedToDelete) {
                    throw new IOException(String.format("Failed to delete %s", childPath));
                }
            }
        } else if (!ufs.mkdirs(mLocation.toString())) {
            throw new IOException(String.format("Failed to create %s", mLocation));
        }
        // Create a breadcrumb that indicates that the journal folder has been formatted.
        try {
            UnderFileSystemUtils.touch(ufs, URIUtils.appendPath(mLocation, ServerConfiguration.getString(PropertyKey.MASTER_FORMAT_FILE_PREFIX) + System.currentTimeMillis()).toString());
        } catch (URISyntaxException e) {
            throw new RuntimeException(e.getMessage());
        }
    }
}

Example 92 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class AlluxioMasterRestServiceHandlerTest method registerMockUfs.

private void registerMockUfs() throws IOException {
    UnderFileSystemFactory underFileSystemFactoryMock = mock(UnderFileSystemFactory.class);
    when(underFileSystemFactoryMock.supportsPath(anyString(), anyObject())).thenReturn(Boolean.FALSE);
    when(underFileSystemFactoryMock.supportsPath(eq(TEST_PATH), anyObject())).thenReturn(Boolean.TRUE);
    UnderFileSystem underFileSystemMock = mock(UnderFileSystem.class);
    when(underFileSystemMock.getSpace(TEST_PATH, UnderFileSystem.SpaceType.SPACE_FREE)).thenReturn(UFS_SPACE_FREE);
    when(underFileSystemMock.getSpace(TEST_PATH, UnderFileSystem.SpaceType.SPACE_TOTAL)).thenReturn(UFS_SPACE_TOTAL);
    when(underFileSystemMock.getSpace(TEST_PATH, UnderFileSystem.SpaceType.SPACE_USED)).thenReturn(UFS_SPACE_USED);
    when(underFileSystemFactoryMock.create(eq(TEST_PATH), ArgumentMatchers.any())).thenReturn(underFileSystemMock);
    UnderFileSystemFactoryRegistry.register(underFileSystemFactoryMock);
}

Example 93 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class DailyMetadataBackupTest method before.

@Before
public void before() throws Exception {
    mRandom = new Random();
    mBackupDir = "/tmp/test/alluxio_backups";
    mMetaMaster = Mockito.mock(MetaMaster.class);
    when(mMetaMaster.backup(any(), any())).thenReturn(BackupStatus.fromProto(BackupPStatus.newBuilder().setBackupId(UUID.randomUUID().toString()).setBackupState(BackupState.Completed).setBackupUri(PathUtils.concatPath(mBackupDir, generateBackupFileName())).setBackupHost("localhost").build()));
    mUfs = Mockito.mock(UnderFileSystem.class);
    when(mUfs.getUnderFSType()).thenReturn("local");
    when(mUfs.deleteFile(any())).thenReturn(true);
    mUfsClient = Mockito.mock(UfsManager.UfsClient.class);
    when(mUfsClient.acquireUfsResource()).thenReturn(new CloseableResource<UnderFileSystem>(mUfs) {

        @Override
        public void closeResource() {
        // Noop
        }
    });
    mUfsManager = Mockito.mock(UfsManager.class);
    when(mUfsManager.getRoot()).thenReturn(mUfsClient);
    mScheduler = new ControllableScheduler();
}

Example 94 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class PersistDefinition method runTask.

@Override
public SerializableVoid runTask(PersistConfig config, SerializableVoid args, RunTaskContext context) throws Exception {
    AlluxioURI uri = new AlluxioURI(config.getFilePath());
    String ufsPath = config.getUfsPath();
    // check if the file is persisted in UFS and delete it, if we are overwriting it
    UfsManager.UfsClient ufsClient = context.getUfsManager().get(config.getMountId());
    try (CloseableResource<UnderFileSystem> ufsResource = ufsClient.acquireUfsResource()) {
        UnderFileSystem ufs = ufsResource.get();
        if (ufs == null) {
            throw new IOException("Failed to create UFS instance for " + ufsPath);
        }
        if (ufs.exists(ufsPath)) {
            if (config.isOverwrite()) {
                LOG.info("File {} is already persisted in UFS. Removing it.", config.getFilePath());
                ufs.deleteExistingFile(ufsPath);
            } else {
                throw new IOException("File " + config.getFilePath() + " is already persisted in UFS, to overwrite the file, please set the overwrite flag" + " in the config.");
            }
        }
        URIStatus uriStatus = context.getFileSystem().getStatus(uri);
        if (!uriStatus.isCompleted()) {
            throw new IOException("Cannot persist an incomplete Alluxio file: " + uri);
        }
        long bytesWritten;
        try (Closer closer = Closer.create()) {
            OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).setUpdateLastAccessTime(false).build();
            FileInStream in = closer.register(context.getFileSystem().openFile(uri, options));
            AlluxioURI dstPath = new AlluxioURI(ufsPath);
            // Create ancestor directories from top to the bottom. We cannot use recursive create
            // parents here because the permission for the ancestors can be different.
            Stack<Pair<String, String>> ancestorUfsAndAlluxioPaths = new Stack<>();
            AlluxioURI curAlluxioPath = uri.getParent();
            AlluxioURI curUfsPath = dstPath.getParent();
            // exist.
            while (!ufs.isDirectory(curUfsPath.toString()) && curAlluxioPath != null) {
                ancestorUfsAndAlluxioPaths.push(new Pair<>(curUfsPath.toString(), curAlluxioPath.toString()));
                curAlluxioPath = curAlluxioPath.getParent();
                curUfsPath = curUfsPath.getParent();
            }
            while (!ancestorUfsAndAlluxioPaths.empty()) {
                Pair<String, String> ancestorUfsAndAlluxioPath = ancestorUfsAndAlluxioPaths.pop();
                String ancestorUfsPath = ancestorUfsAndAlluxioPath.getFirst();
                String ancestorAlluxioPath = ancestorUfsAndAlluxioPath.getSecond();
                URIStatus status = context.getFileSystem().getStatus(new AlluxioURI(ancestorAlluxioPath));
                MkdirsOptions mkdirOptions = MkdirsOptions.defaults(ServerConfiguration.global()).setCreateParent(false).setOwner(status.getOwner()).setGroup(status.getGroup()).setMode(new Mode((short) status.getMode()));
                // and assume the directory is already prepared, regardless of permission matching.
                if (ufs.mkdirs(ancestorUfsPath, mkdirOptions)) {
                    List<AclEntry> allAcls = Stream.concat(status.getDefaultAcl().getEntries().stream(), status.getAcl().getEntries().stream()).collect(Collectors.toList());
                    ufs.setAclEntries(ancestorUfsPath, allAcls);
                } else if (!ufs.isDirectory(ancestorUfsPath)) {
                    throw new IOException("Failed to create " + ufsPath + " with permission " + options.toString() + " because its ancestor " + ancestorUfsPath + " is not a directory");
                }
            }
            OutputStream out = closer.register(ufs.createNonexistingFile(dstPath.toString(), CreateOptions.defaults(ServerConfiguration.global()).setOwner(uriStatus.getOwner()).setGroup(uriStatus.getGroup()).setMode(new Mode((short) uriStatus.getMode()))));
            URIStatus status = context.getFileSystem().getStatus(uri);
            List<AclEntry> allAcls = Stream.concat(status.getDefaultAcl().getEntries().stream(), status.getAcl().getEntries().stream()).collect(Collectors.toList());
            ufs.setAclEntries(dstPath.toString(), allAcls);
            bytesWritten = IOUtils.copyLarge(in, out, new byte[8 * Constants.MB]);
            incrementPersistedMetric(ufsClient.getUfsMountPointUri(), bytesWritten);
        }
        LOG.info("Persisted file {} with size {}", ufsPath, bytesWritten);
    }
    return null;
}

Example 95 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class DefaultFileSystemMaster method getDisplayMountPointInfo.

/**
 * Gets the mount point information for display from a mount information.
 *
 * @param invokeUfs if true, invoke ufs to set ufs properties
 * @param mountInfo the mount information to transform
 * @return the mount point information
 */
private MountPointInfo getDisplayMountPointInfo(MountInfo mountInfo, boolean invokeUfs) {
    MountPointInfo info = mountInfo.toDisplayMountPointInfo();
    if (!invokeUfs) {
        return info;
    }
    try (CloseableResource<UnderFileSystem> ufsResource = mUfsManager.get(mountInfo.getMountId()).acquireUfsResource()) {
        UnderFileSystem ufs = ufsResource.get();
        info.setUfsType(ufs.getUnderFSType());
        try {
            info.setUfsCapacityBytes(ufs.getSpace(info.getUfsUri(), UnderFileSystem.SpaceType.SPACE_TOTAL));
        } catch (IOException e) {
            LOG.warn("Cannot get total capacity of {}", info.getUfsUri(), e);
        }
        try {
            info.setUfsUsedBytes(ufs.getSpace(info.getUfsUri(), UnderFileSystem.SpaceType.SPACE_USED));
        } catch (IOException e) {
            LOG.warn("Cannot get used capacity of {}", info.getUfsUri(), e);
        }
    } catch (UnavailableException | NotFoundException e) {
        // We should never reach here
        LOG.error("No UFS cached for {}", info, e);
    }
    return info;
}