Examples with UnderFileSystem alluxio.underfs.UnderFileSystem used on opensource projects

Example 96 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class DefaultFileSystemMaster method mountInternal.

/**
 * Updates the mount table with the specified mount point. The mount options may be updated during
 * this method.
 *
 * @param journalContext the journal context
 * @param inodePath the Alluxio mount point
 * @param ufsPath the UFS endpoint to mount
 * @param mountId the mount id
 * @param context the mount context (may be updated)
 */
private void mountInternal(Supplier<JournalContext> journalContext, LockedInodePath inodePath, AlluxioURI ufsPath, long mountId, MountContext context) throws FileAlreadyExistsException, InvalidPathException, IOException {
    AlluxioURI alluxioPath = inodePath.getUri();
    // Adding the mount point will not create the UFS instance and thus not connect to UFS
    mUfsManager.addMount(mountId, new AlluxioURI(ufsPath.toString()), UnderFileSystemConfiguration.defaults(ServerConfiguration.global()).setReadOnly(context.getOptions().getReadOnly()).setShared(context.getOptions().getShared()).createMountSpecificConf(context.getOptions().getPropertiesMap()));
    try {
        prepareForMount(ufsPath, mountId, context);
        // Check that the alluxioPath we're creating doesn't shadow a path in the parent UFS
        MountTable.Resolution resolution = mMountTable.resolve(alluxioPath);
        try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
            String ufsResolvedPath = resolution.getUri().getPath();
            if (ufsResource.get().exists(ufsResolvedPath)) {
                throw new IOException(ExceptionMessage.MOUNT_PATH_SHADOWS_PARENT_UFS.getMessage(alluxioPath, ufsResolvedPath));
            }
        }
        // Add the mount point. This will only succeed if we are not mounting a prefix of an existing
        // mount.
        mMountTable.add(journalContext, alluxioPath, ufsPath, mountId, context.getOptions().build());
    } catch (Exception e) {
        mUfsManager.removeMount(mountId);
        throw e;
    }
}

Example 97 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class DefaultFileSystemMaster method checkConsistencyInternal.

/**
 * Checks if a path is consistent between Alluxio and the underlying storage.
 * <p>
 * A path without a backing under storage is always consistent.
 * <p>
 * A not persisted path is considered consistent if:
 * 1. It does not shadow an object in the underlying storage.
 * <p>
 * A persisted path is considered consistent if:
 * 1. An equivalent object exists for its under storage path.
 * 2. The metadata of the Alluxio and under storage object are equal.
 *
 * @param inodePath the path to check. This must exist and be read-locked
 * @return true if the path is consistent, false otherwise
 */
private boolean checkConsistencyInternal(LockedInodePath inodePath) throws InvalidPathException, IOException {
    Inode inode;
    try {
        inode = inodePath.getInode();
    } catch (FileDoesNotExistException e) {
        // already checked existence when creating the inodePath
        throw new RuntimeException(e);
    }
    MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
    try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
        UnderFileSystem ufs = ufsResource.get();
        String ufsPath = resolution.getUri().getPath();
        if (ufs == null) {
            return true;
        }
        if (!inode.isPersisted()) {
            return !ufs.exists(ufsPath);
        }
        UfsStatus ufsStatus;
        try {
            ufsStatus = ufs.getStatus(ufsPath);
        } catch (FileNotFoundException e) {
            return !inode.isPersisted();
        }
        // TODO(calvin): Evaluate which other metadata fields should be validated.
        if (inode.isDirectory()) {
            return ufsStatus.isDirectory();
        } else {
            String ufsFingerprint = Fingerprint.create(ufs.getUnderFSType(), ufsStatus).serialize();
            return ufsStatus.isFile() && (ufsFingerprint.equals(inode.asFile().getUfsFingerprint())) && ufsStatus instanceof UfsFileStatus && ((UfsFileStatus) ufsStatus).getContentLength() == inode.asFile().getLength();
        }
    }
}

Example 98 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class AbstractBackupRole method takeBackup.

/**
 * Takes a backup on local master. Note: Master state must have been suspended or locked before
 * calling this.
 *
 * @param request the backup request
 * @param entryCounter counter to receive written entry count
 * @return URI of the backup
 */
protected AlluxioURI takeBackup(BackupPRequest request, AtomicLong entryCounter) throws IOException {
    AlluxioURI backupUri;
    final Closer closer = Closer.create();
    // Acquire the UFS resource under which backup is being created.
    try (CloseableResource<UnderFileSystem> ufsResource = mUfsManager.getRoot().acquireUfsResource()) {
        // Get backup parent directory.
        String backupParentDir = request.hasTargetDirectory() ? request.getTargetDirectory() : ServerConfiguration.getString(PropertyKey.MASTER_BACKUP_DIRECTORY);
        // Get ufs resource for backup.
        UnderFileSystem ufs = ufsResource.get();
        if (request.getOptions().getLocalFileSystem() && !ufs.getUnderFSType().equals("local")) {
            // TODO(lu) Support getting UFS based on type from UfsManager
            ufs = closer.register(UnderFileSystem.Factory.create("/", UnderFileSystemConfiguration.defaults(ServerConfiguration.global())));
        }
        // Ensure parent directory for backup.
        if (!ufs.isDirectory(backupParentDir)) {
            if (!ufs.mkdirs(backupParentDir, MkdirsOptions.defaults(ServerConfiguration.global()).setCreateParent(true))) {
                throw new IOException(String.format("Failed to create directory %s", backupParentDir));
            }
        }
        // Generate backup file path.
        Instant now = Instant.now();
        String backupFileName = String.format(BackupManager.BACKUP_FILE_FORMAT, DateTimeFormatter.ISO_LOCAL_DATE.withZone(ZoneId.of("UTC")).format(now), now.toEpochMilli());
        String backupFilePath = PathUtils.concatPath(backupParentDir, backupFileName);
        // Calculate URI for the path.
        String rootUfs = ServerConfiguration.getString(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS);
        if (request.getOptions().getLocalFileSystem()) {
            rootUfs = "file:///";
        }
        backupUri = new AlluxioURI(new AlluxioURI(rootUfs), new AlluxioURI(backupFilePath));
        // Take the backup.
        try {
            try (OutputStream ufsStream = ufs.create(backupFilePath)) {
                // Create the backup from master state.
                mBackupManager.backup(ufsStream, entryCounter);
            }
            // Add a marker file indicating the file is completed.
            ufs.create(backupFilePath + ".complete").close();
        } catch (IOException e) {
            try {
                ufs.deleteExistingFile(backupFilePath);
            } catch (Exception e2) {
                LOG.error("Failed to clean up failed backup at {}", backupFilePath, e2);
                e.addSuppressed(e2);
            }
            throw new IOException(String.format("Backup failed. BackupUri: %s, LastEntryCount: %d", backupUri, entryCounter.get()), e);
        }
    } finally {
        closer.close();
    }
    return backupUri;
}

Example 99 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class UfsFallbackBlockWriteHandlerTest method before.

@Before
public void before() throws Exception {
    mFile = mTestFolder.newFile();
    mOutputStream = new FileOutputStream(mFile);
    mBlockStore = new TieredBlockStore();
    mBlockWorker = new MockBlockWorker();
    UnderFileSystem mockUfs = Mockito.mock(UnderFileSystem.class);
    UfsManager ufsManager = Mockito.mock(UfsManager.class);
    UfsManager.UfsClient ufsClient = new UfsManager.UfsClient(() -> mockUfs, AlluxioURI.EMPTY_URI);
    Mockito.when(ufsManager.get(Mockito.anyLong())).thenReturn(ufsClient);
    Mockito.when(mockUfs.createNonexistingFile(Mockito.anyString(), Mockito.any(CreateOptions.class))).thenReturn(mOutputStream).thenReturn(new FileOutputStream(mFile, true));
    mResponseObserver = Mockito.mock(StreamObserver.class);
    mWriteHandler = new UfsFallbackBlockWriteHandler(mBlockWorker, ufsManager, mResponseObserver, mUserInfo, false);
    setupResponseTrigger();
    // create a partial block in block store first
    mBlockStore.createBlock(TEST_SESSION_ID, TEST_BLOCK_ID, AllocateOptions.forCreate(CHUNK_SIZE, BlockStoreLocation.anyDirInTier(Constants.MEDIUM_MEM)));
    BlockWriter writer = mBlockStore.getBlockWriter(TEST_SESSION_ID, TEST_BLOCK_ID);
    DataBuffer buffer = newDataBuffer(PARTIAL_WRITTEN);
    mPartialChecksum = getChecksum(buffer);
    writer.append((ByteBuf) buffer.getNettyOutput());
    writer.close();
}

Example 100 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class UfsFileWriteHandler method createUfsFile.

private void createUfsFile(UfsFileWriteRequestContext context) throws IOException {
    UfsFileWriteRequest request = context.getRequest();
    Preconditions.checkState(request != null);
    Protocol.CreateUfsFileOptions createUfsFileOptions = request.getCreateUfsFileOptions();
    UfsManager.UfsClient ufsClient = mUfsManager.get(createUfsFileOptions.getMountId());
    CloseableResource<UnderFileSystem> ufsResource = ufsClient.acquireUfsResource();
    context.setUfsResource(ufsResource);
    UnderFileSystem ufs = ufsResource.get();
    CreateOptions createOptions = CreateOptions.defaults(ServerConfiguration.global()).setCreateParent(true).setOwner(createUfsFileOptions.getOwner()).setGroup(createUfsFileOptions.getGroup()).setMode(new Mode((short) createUfsFileOptions.getMode()));
    if (createUfsFileOptions.hasAcl()) {
        // This acl information will be ignored by all but HDFS implementations
        createOptions.setAcl(ProtoUtils.fromProto(createUfsFileOptions.getAcl()));
    }
    context.setOutputStream(ufs.createNonexistingFile(request.getUfsPath(), createOptions));
    context.setCreateOptions(createOptions);
    String ufsString = MetricsSystem.escape(ufsClient.getUfsMountPointUri());
    MetricKey counterKey = MetricKey.WORKER_BYTES_WRITTEN_UFS;
    MetricKey meterKey = MetricKey.WORKER_BYTES_WRITTEN_UFS_THROUGHPUT;
    context.setCounter(MetricsSystem.counterWithTags(counterKey.getName(), counterKey.isClusterAggregated(), MetricInfo.TAG_UFS, ufsString));
    context.setMeter(MetricsSystem.meterWithTags(meterKey.getName(), meterKey.isClusterAggregated(), MetricInfo.TAG_UFS, ufsString));
}