Examples with UnderFileSystem alluxio.underfs.UnderFileSystem used on opensource projects

Example 46 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class FileOutStreamAsyncWriteJobIntegrationTest method setAttributeBeforeJobScheduled.

@Test
public void setAttributeBeforeJobScheduled() throws Exception {
    PersistenceTestUtils.pauseScheduler(mLocalAlluxioClusterResource);
    URIStatus status = createAsyncFile();
    String ufsPath = status.getUfsPath();
    UnderFileSystem ufs = UnderFileSystem.Factory.create(ufsPath, ServerConfiguration.global());
    mFileSystem.setAttribute(mUri, TEST_OPTIONS);
    checkFileInAlluxio(mUri, LEN);
    checkFileNotInUnderStorage(status.getUfsPath());
    status = mFileSystem.getStatus(mUri);
    Assert.assertEquals(ModeUtils.protoToShort(TEST_OPTIONS.getMode()), status.getMode());
    Assert.assertEquals(COMMON_OPTIONS.getTtl(), status.getTtl());
    Assert.assertEquals(COMMON_OPTIONS.getTtlAction(), status.getTtlAction());
    PersistenceTestUtils.resumeScheduler(mLocalAlluxioClusterResource);
    IntegrationTestUtils.waitForPersist(mLocalAlluxioClusterResource, mUri);
    checkFileInAlluxio(mUri, LEN);
    checkFileInUnderStorage(mUri, LEN);
    status = mFileSystem.getStatus(mUri);
    Assert.assertEquals(ModeUtils.protoToShort(TEST_OPTIONS.getMode()), status.getMode());
    Assert.assertEquals(COMMON_OPTIONS.getTtl(), status.getTtl());
    Assert.assertEquals(COMMON_OPTIONS.getTtlAction(), status.getTtlAction());
    // Skip checking mode for object stores
    Assume.assumeFalse(ufs.isObjectStorage());
    Assert.assertEquals(ModeUtils.protoToShort(TEST_OPTIONS.getMode()), ufs.getFileStatus(ufsPath).getMode());
}

Example 47 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class UfsSyncIntegrationTest method checkUriStatus.

private void checkUriStatus(URIStatus uriStatus) {
    File ufsFile = new File(uriStatus.getUfsPath());
    if (uriStatus.isFolder() != ufsFile.isDirectory()) {
        Assert.fail("Directory mismatch (Alluxio isDir: " + uriStatus.isFolder() + ") (ufs isDir: " + ufsFile.isDirectory() + ") path: " + uriStatus.getPath());
    }
    if (!uriStatus.isFolder()) {
        long ufsLength = ufsFile.length();
        long alluxioLength = uriStatus.getLength();
        if (ufsLength != alluxioLength) {
            Assert.fail("Alluxio length (" + alluxioLength + ") and ufs length (" + ufsLength + ") are inconsistent. path: " + uriStatus.getPath());
        }
        // Check fingerprint.
        UnderFileSystem ufs = UnderFileSystem.Factory.create(uriStatus.getUfsPath(), ServerConfiguration.global());
        String ufsFingerprint = ufs.getFingerprint(uriStatus.getUfsPath());
        String alluxioFingerprint = uriStatus.getUfsFingerprint();
        if (!ufsFingerprint.equals(alluxioFingerprint)) {
            Assert.fail("Alluxio fingerprint (" + alluxioFingerprint + ") and ufs fingerprint (" + ufsFingerprint + ") are inconsistent. path: " + uriStatus.getPath());
        }
    }
}

Example 48 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class FileOutStreamIntegrationTest method writeInNonExistDirectory.

/**
 * Tests {@link FileOutStream#write(int)}.
 */
@Test
public void writeInNonExistDirectory() throws Exception {
    String uniqPath = PathUtils.uniqPath();
    CreateFilePOptions op = CreateFilePOptions.newBuilder().setWriteType(WritePType.CACHE_THROUGH).setRecursive(true).build();
    AlluxioURI filePath = new AlluxioURI(PathUtils.concatPath(uniqPath, "file_" + MIN_LEN + "_" + mWriteType));
    AlluxioURI parentPath = new AlluxioURI(uniqPath);
    // create a directory without a backing directory in UFS
    mFileSystem.createDirectory(parentPath, CreateDirectoryPOptions.newBuilder().setRecursive(true).setWriteType(WritePType.CACHE_THROUGH).build());
    URIStatus status = mFileSystem.getStatus(parentPath);
    String checkpointPath = status.getUfsPath();
    UnderFileSystem ufs = UnderFileSystem.Factory.create(checkpointPath, ServerConfiguration.global());
    ufs.deleteDirectory(checkpointPath);
    // write a file to a directory exists in Alluxio but not in UFS
    writeIncreasingBytesToFile(filePath, MIN_LEN, op);
    checkFileInAlluxio(filePath, MIN_LEN);
    checkFileInUnderStorage(filePath, MIN_LEN);
}

Example 49 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class DefaultFileSystemMaster method setAttributeSingleFile.

/**
 * @param inodePath the {@link LockedInodePath} to use
 * @param updateUfs whether to update the UFS with the attribute change
 * @param opTimeMs the operation time (in milliseconds)
 * @param context the method context
 */
protected void setAttributeSingleFile(RpcContext rpcContext, LockedInodePath inodePath, boolean updateUfs, long opTimeMs, SetAttributeContext context) throws FileDoesNotExistException, InvalidPathException, AccessControlException {
    Inode inode = inodePath.getInode();
    SetAttributePOptions.Builder protoOptions = context.getOptions();
    if (protoOptions.hasPinned()) {
        mInodeTree.setPinned(rpcContext, inodePath, context.getOptions().getPinned(), context.getOptions().getPinnedMediaList(), opTimeMs);
    }
    UpdateInodeEntry.Builder entry = UpdateInodeEntry.newBuilder().setId(inode.getId());
    if (protoOptions.hasReplicationMax() || protoOptions.hasReplicationMin()) {
        Integer replicationMax = protoOptions.hasReplicationMax() ? protoOptions.getReplicationMax() : null;
        Integer replicationMin = protoOptions.hasReplicationMin() ? protoOptions.getReplicationMin() : null;
        mInodeTree.setReplication(rpcContext, inodePath, replicationMax, replicationMin, opTimeMs);
    }
    // protoOptions may not have both fields set
    if (protoOptions.hasCommonOptions()) {
        FileSystemMasterCommonPOptions commonOpts = protoOptions.getCommonOptions();
        TtlAction action = commonOpts.hasTtlAction() ? commonOpts.getTtlAction() : null;
        Long ttl = commonOpts.hasTtl() ? commonOpts.getTtl() : null;
        boolean modified = false;
        if (ttl != null && inode.getTtl() != ttl) {
            entry.setTtl(ttl);
            modified = true;
        }
        if (action != null && inode.getTtlAction() != action) {
            entry.setTtlAction(ProtobufUtils.toProtobuf(action));
            modified = true;
        }
        if (modified) {
            entry.setLastModificationTimeMs(opTimeMs);
        }
    }
    if (protoOptions.hasPersisted()) {
        Preconditions.checkArgument(inode.isFile(), PreconditionMessage.PERSIST_ONLY_FOR_FILE);
        Preconditions.checkArgument(inode.asFile().isCompleted(), PreconditionMessage.FILE_TO_PERSIST_MUST_BE_COMPLETE);
        // TODO(manugoyal) figure out valid behavior in the un-persist case
        Preconditions.checkArgument(protoOptions.getPersisted(), PreconditionMessage.ERR_SET_STATE_UNPERSIST);
        if (!inode.asFile().isPersisted()) {
            entry.setPersistenceState(PersistenceState.PERSISTED.name());
            entry.setLastModificationTimeMs(context.getOperationTimeMs());
            propagatePersistedInternal(rpcContext, inodePath);
            Metrics.FILES_PERSISTED.inc();
        }
    }
    boolean ownerGroupChanged = (protoOptions.hasOwner()) || (protoOptions.hasGroup());
    boolean modeChanged = protoOptions.hasMode();
    // If the file is persisted in UFS, also update corresponding owner/group/permission.
    if ((ownerGroupChanged || modeChanged) && updateUfs && inode.isPersisted()) {
        if ((inode instanceof InodeFile) && !inode.asFile().isCompleted()) {
            LOG.debug("Alluxio does not propagate chown/chgrp/chmod to UFS for incomplete files.");
        } else {
            checkUfsMode(inodePath.getUri(), OperationType.WRITE);
            MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
            String ufsUri = resolution.getUri().toString();
            try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
                UnderFileSystem ufs = ufsResource.get();
                if (ufs.isObjectStorage()) {
                    LOG.debug("setOwner/setMode is not supported to object storage UFS via Alluxio. " + "UFS: " + ufsUri + ". This has no effect on the underlying object.");
                } else {
                    String owner = null;
                    String group = null;
                    String mode = null;
                    if (ownerGroupChanged) {
                        try {
                            owner = protoOptions.getOwner() != null ? protoOptions.getOwner() : inode.getOwner();
                            group = protoOptions.getGroup() != null ? protoOptions.getGroup() : inode.getGroup();
                            ufs.setOwner(ufsUri, owner, group);
                        } catch (IOException e) {
                            throw new AccessControlException("Could not setOwner for UFS file " + ufsUri + " . Aborting the setAttribute operation in Alluxio.", e);
                        }
                    }
                    if (modeChanged) {
                        try {
                            mode = String.valueOf(protoOptions.getMode());
                            ufs.setMode(ufsUri, ModeUtils.protoToShort(protoOptions.getMode()));
                        } catch (IOException e) {
                            throw new AccessControlException("Could not setMode for UFS file " + ufsUri + " . Aborting the setAttribute operation in Alluxio.", e);
                        }
                    }
                    // Retrieve the ufs fingerprint after the ufs changes.
                    String existingFingerprint = inode.getUfsFingerprint();
                    if (!existingFingerprint.equals(Constants.INVALID_UFS_FINGERPRINT)) {
                        // Update existing fingerprint, since contents did not change
                        Fingerprint fp = Fingerprint.parse(existingFingerprint);
                        fp.putTag(Fingerprint.Tag.OWNER, owner);
                        fp.putTag(Fingerprint.Tag.GROUP, group);
                        fp.putTag(Fingerprint.Tag.MODE, mode);
                        context.setUfsFingerprint(fp.serialize());
                    } else {
                        // Need to retrieve the fingerprint from ufs.
                        context.setUfsFingerprint(ufs.getFingerprint(ufsUri));
                    }
                }
            }
        }
    }
    if (!context.getUfsFingerprint().equals(Constants.INVALID_UFS_FINGERPRINT)) {
        entry.setUfsFingerprint(context.getUfsFingerprint());
    }
    // Only commit the set permission to inode after the propagation to UFS succeeded.
    if (protoOptions.hasOwner()) {
        entry.setOwner(protoOptions.getOwner());
    }
    if (protoOptions.hasGroup()) {
        entry.setGroup(protoOptions.getGroup());
    }
    if (modeChanged) {
        entry.setMode(ModeUtils.protoToShort(protoOptions.getMode()));
    }
    mInodeTree.updateInode(rpcContext, entry.build());
}

Example 50 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class DefaultFileSystemMaster method checkConsistencyRecursive.

private void checkConsistencyRecursive(LockedInodePath inodePath, List<AlluxioURI> inconsistentUris, boolean assertInconsistent, boolean metadataSynced) throws IOException, FileDoesNotExistException {
    Inode inode = inodePath.getInode();
    try {
        if (assertInconsistent || !checkConsistencyInternal(inodePath)) {
            inconsistentUris.add(inodePath.getUri());
            // If a dir in Alluxio is inconsistent with underlying storage,
            // we can assert the children is inconsistent.
            // If a file is inconsistent, please ignore this parameter cause it has no child node.
            assertInconsistent = true;
        }
        if (inode.isDirectory()) {
            InodeDirectory inodeDir = inode.asDirectory();
            Iterable<? extends Inode> children = mInodeStore.getChildren(inodeDir);
            for (Inode child : children) {
                try (LockedInodePath childPath = inodePath.lockChild(child, LockPattern.READ)) {
                    checkConsistencyRecursive(childPath, inconsistentUris, assertInconsistent, metadataSynced);
                }
            }
            // if the metadata has already been synced, then we could skip it.
            if (metadataSynced) {
                return;
            }
            MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
            UfsStatus[] statuses;
            try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
                UnderFileSystem ufs = ufsResource.get();
                String ufsPath = resolution.getUri().getPath();
                statuses = ufs.listStatus(ufsPath);
            }
            if (statuses != null) {
                HashSet<String> alluxioFileNames = Streams.stream(children).map(Inode::getName).collect(Collectors.toCollection(HashSet::new));
                Arrays.stream(statuses).forEach(status -> {
                    if (!alluxioFileNames.contains(status.getName())) {
                        inconsistentUris.add(inodePath.getUri().join(status.getName()));
                    }
                });
            }
        }
    } catch (InvalidPathException e) {
        LOG.debug("Path \"{}\" is invalid, has been ignored.", PathUtils.concatPath(inodePath.getUri().getPath()));
    }
}