Examples with FileSystemMasterCommonPOptions alluxio.grpc.FileSystemMasterCommonPOptions used on opensource projects

Example 1 with FileSystemMasterCommonPOptions

use of alluxio.grpc.FileSystemMasterCommonPOptions in project alluxio by Alluxio.

the class DefaultFileSystemMaster method setAttributeSingleFile.

/**
 * @param inodePath the {@link LockedInodePath} to use
 * @param updateUfs whether to update the UFS with the attribute change
 * @param opTimeMs the operation time (in milliseconds)
 * @param context the method context
 */
protected void setAttributeSingleFile(RpcContext rpcContext, LockedInodePath inodePath, boolean updateUfs, long opTimeMs, SetAttributeContext context) throws FileDoesNotExistException, InvalidPathException, AccessControlException {
    Inode inode = inodePath.getInode();
    SetAttributePOptions.Builder protoOptions = context.getOptions();
    if (protoOptions.hasPinned()) {
        mInodeTree.setPinned(rpcContext, inodePath, context.getOptions().getPinned(), context.getOptions().getPinnedMediaList(), opTimeMs);
    }
    UpdateInodeEntry.Builder entry = UpdateInodeEntry.newBuilder().setId(inode.getId());
    if (protoOptions.hasReplicationMax() || protoOptions.hasReplicationMin()) {
        Integer replicationMax = protoOptions.hasReplicationMax() ? protoOptions.getReplicationMax() : null;
        Integer replicationMin = protoOptions.hasReplicationMin() ? protoOptions.getReplicationMin() : null;
        mInodeTree.setReplication(rpcContext, inodePath, replicationMax, replicationMin, opTimeMs);
    }
    // protoOptions may not have both fields set
    if (protoOptions.hasCommonOptions()) {
        FileSystemMasterCommonPOptions commonOpts = protoOptions.getCommonOptions();
        TtlAction action = commonOpts.hasTtlAction() ? commonOpts.getTtlAction() : null;
        Long ttl = commonOpts.hasTtl() ? commonOpts.getTtl() : null;
        boolean modified = false;
        if (ttl != null && inode.getTtl() != ttl) {
            entry.setTtl(ttl);
            modified = true;
        }
        if (action != null && inode.getTtlAction() != action) {
            entry.setTtlAction(ProtobufUtils.toProtobuf(action));
            modified = true;
        }
        if (modified) {
            entry.setLastModificationTimeMs(opTimeMs);
        }
    }
    if (protoOptions.hasPersisted()) {
        Preconditions.checkArgument(inode.isFile(), PreconditionMessage.PERSIST_ONLY_FOR_FILE);
        Preconditions.checkArgument(inode.asFile().isCompleted(), PreconditionMessage.FILE_TO_PERSIST_MUST_BE_COMPLETE);
        // TODO(manugoyal) figure out valid behavior in the un-persist case
        Preconditions.checkArgument(protoOptions.getPersisted(), PreconditionMessage.ERR_SET_STATE_UNPERSIST);
        if (!inode.asFile().isPersisted()) {
            entry.setPersistenceState(PersistenceState.PERSISTED.name());
            entry.setLastModificationTimeMs(context.getOperationTimeMs());
            propagatePersistedInternal(rpcContext, inodePath);
            Metrics.FILES_PERSISTED.inc();
        }
    }
    boolean ownerGroupChanged = (protoOptions.hasOwner()) || (protoOptions.hasGroup());
    boolean modeChanged = protoOptions.hasMode();
    // If the file is persisted in UFS, also update corresponding owner/group/permission.
    if ((ownerGroupChanged || modeChanged) && updateUfs && inode.isPersisted()) {
        if ((inode instanceof InodeFile) && !inode.asFile().isCompleted()) {
            LOG.debug("Alluxio does not propagate chown/chgrp/chmod to UFS for incomplete files.");
        } else {
            checkUfsMode(inodePath.getUri(), OperationType.WRITE);
            MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
            String ufsUri = resolution.getUri().toString();
            try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
                UnderFileSystem ufs = ufsResource.get();
                if (ufs.isObjectStorage()) {
                    LOG.debug("setOwner/setMode is not supported to object storage UFS via Alluxio. " + "UFS: " + ufsUri + ". This has no effect on the underlying object.");
                } else {
                    String owner = null;
                    String group = null;
                    String mode = null;
                    if (ownerGroupChanged) {
                        try {
                            owner = protoOptions.getOwner() != null ? protoOptions.getOwner() : inode.getOwner();
                            group = protoOptions.getGroup() != null ? protoOptions.getGroup() : inode.getGroup();
                            ufs.setOwner(ufsUri, owner, group);
                        } catch (IOException e) {
                            throw new AccessControlException("Could not setOwner for UFS file " + ufsUri + " . Aborting the setAttribute operation in Alluxio.", e);
                        }
                    }
                    if (modeChanged) {
                        try {
                            mode = String.valueOf(protoOptions.getMode());
                            ufs.setMode(ufsUri, ModeUtils.protoToShort(protoOptions.getMode()));
                        } catch (IOException e) {
                            throw new AccessControlException("Could not setMode for UFS file " + ufsUri + " . Aborting the setAttribute operation in Alluxio.", e);
                        }
                    }
                    // Retrieve the ufs fingerprint after the ufs changes.
                    String existingFingerprint = inode.getUfsFingerprint();
                    if (!existingFingerprint.equals(Constants.INVALID_UFS_FINGERPRINT)) {
                        // Update existing fingerprint, since contents did not change
                        Fingerprint fp = Fingerprint.parse(existingFingerprint);
                        fp.putTag(Fingerprint.Tag.OWNER, owner);
                        fp.putTag(Fingerprint.Tag.GROUP, group);
                        fp.putTag(Fingerprint.Tag.MODE, mode);
                        context.setUfsFingerprint(fp.serialize());
                    } else {
                        // Need to retrieve the fingerprint from ufs.
                        context.setUfsFingerprint(ufs.getFingerprint(ufsUri));
                    }
                }
            }
        }
    }
    if (!context.getUfsFingerprint().equals(Constants.INVALID_UFS_FINGERPRINT)) {
        entry.setUfsFingerprint(context.getUfsFingerprint());
    }
    // Only commit the set permission to inode after the propagation to UFS succeeded.
    if (protoOptions.hasOwner()) {
        entry.setOwner(protoOptions.getOwner());
    }
    if (protoOptions.hasGroup()) {
        entry.setGroup(protoOptions.getGroup());
    }
    if (modeChanged) {
        entry.setMode(ModeUtils.protoToShort(protoOptions.getMode()));
    }
    mInodeTree.updateInode(rpcContext, entry.build());
}

Example 2 with FileSystemMasterCommonPOptions

use of alluxio.grpc.FileSystemMasterCommonPOptions in project alluxio by Alluxio.

the class DefaultFileSystemMaster method activeSyncMetadata.

/**
 * Actively sync metadata, based on a list of changed files.
 *
 * @param path the path to sync
 * @param changedFiles collection of files that are changed under the path to sync, if this is
 *        null, force sync the entire directory
 * @param executorService executor to execute the parallel incremental sync
 */
public void activeSyncMetadata(AlluxioURI path, Collection<AlluxioURI> changedFiles, ExecutorService executorService) throws IOException {
    if (changedFiles == null) {
        LOG.info("Start an active full sync of {}", path.toString());
    } else {
        LOG.info("Start an active incremental sync of {} files", changedFiles.size());
    }
    long start = System.currentTimeMillis();
    if (changedFiles != null && changedFiles.isEmpty()) {
        return;
    }
    try (RpcContext rpcContext = createRpcContext()) {
        if (changedFiles == null) {
            // full sync
            // Set sync interval to 0 to force a sync.
            FileSystemMasterCommonPOptions options = FileSystemMasterCommonPOptions.newBuilder().setSyncIntervalMs(0).build();
            LockingScheme scheme = createSyncLockingScheme(path, options, false);
            InodeSyncStream sync = new InodeSyncStream(scheme, this, rpcContext, DescendantType.ALL, options, false, false, false, false);
            if (sync.sync().equals(FAILED)) {
                LOG.debug("Active full sync on {} didn't sync any paths.", path);
            }
            long end = System.currentTimeMillis();
            LOG.info("Ended an active full sync of {} in {}ms", path.toString(), end - start);
            return;
        } else {
            // incremental sync
            Set<Callable<Void>> callables = new HashSet<>();
            for (AlluxioURI changedFile : changedFiles) {
                callables.add(() -> {
                    // Set sync interval to 0 to force a sync.
                    FileSystemMasterCommonPOptions options = FileSystemMasterCommonPOptions.newBuilder().setSyncIntervalMs(0).build();
                    LockingScheme scheme = createSyncLockingScheme(changedFile, options, false);
                    InodeSyncStream sync = new InodeSyncStream(scheme, this, rpcContext, DescendantType.ONE, options, false, false, false, false);
                    if (sync.sync().equals(FAILED)) {
                        // Use debug because this can be a noisy log
                        LOG.debug("Incremental sync on {} didn't sync any paths.", path);
                    }
                    return null;
                });
            }
            executorService.invokeAll(callables);
        }
    } catch (InterruptedException e) {
        LOG.warn("InterruptedException during active sync: {}", e.toString());
        Thread.currentThread().interrupt();
        return;
    } catch (InvalidPathException | AccessControlException e) {
        LogUtils.warnWithException(LOG, "Failed to active sync on path {}", path, e);
    }
    if (changedFiles != null) {
        long end = System.currentTimeMillis();
        LOG.info("Ended an active incremental sync of {} files in {}ms", changedFiles.size(), end - start);
    }
}

Example 3 with FileSystemMasterCommonPOptions

use of alluxio.grpc.FileSystemMasterCommonPOptions in project alluxio by Alluxio.

the class FileOutStreamTest method asyncWriteOptionPropagation.

/**
 * Tests that common options are propagated to async write request.
 */
@Test
public void asyncWriteOptionPropagation() throws Exception {
    Random rand = new Random();
    FileSystemMasterCommonPOptions commonOptions = FileSystemMasterCommonPOptions.newBuilder().setTtl(rand.nextLong()).setTtlAction(TtlAction.values()[rand.nextInt(TtlAction.values().length)]).setSyncIntervalMs(rand.nextLong()).build();
    OutStreamOptions options = new OutStreamOptions(CreateFilePOptions.newBuilder().setWriteType(WritePType.ASYNC_THROUGH).setBlockSizeBytes(BLOCK_LENGTH).setCommonOptions(commonOptions).build(), mClientContext, sConf);
    // Verify that OutStreamOptions have captured the common options properly.
    assertEquals(options.getCommonOptions(), commonOptions);
    mTestStream = createTestStream(FILE_NAME, options);
    mTestStream.write(BufferUtils.getIncreasingByteArray((int) (BLOCK_LENGTH * 1.5)));
    mTestStream.close();
    verify(mFileSystemMasterClient).completeFile(eq(FILE_NAME), any(CompleteFilePOptions.class));
    // Verify that common options for OutStreamOptions are propagated to ScheduleAsyncPersistence.
    ArgumentCaptor<CompleteFilePOptions> parameterCaptor = ArgumentCaptor.forClass(CompleteFilePOptions.class);
    verify(mFileSystemMasterClient).completeFile(eq(FILE_NAME), parameterCaptor.capture());
    assertEquals(parameterCaptor.getValue().getAsyncPersistOptions().getCommonOptions(), options.getCommonOptions());
}

Example 4 with FileSystemMasterCommonPOptions

use of alluxio.grpc.FileSystemMasterCommonPOptions in project alluxio by Alluxio.

the class UfsSyncIntegrationTest method recursiveSyncCacheDescendants.

@LocalAlluxioClusterResource.Config(confParams = { PropertyKey.Name.USER_FILE_METADATA_LOAD_TYPE, "NEVER" })
@Test
public void recursiveSyncCacheDescendants() throws Exception {
    // make nested directories/files in UFS
    new File(ufsPath("/dir1")).mkdirs();
    new File(ufsPath("/dir1/dir2")).mkdirs();
    new File(ufsPath("/dir1/dir2/dir3")).mkdirs();
    String fileA = "/dir1/dir2/dir3/fileA";
    String fileB = "/dir1/dir2/dir3/fileB";
    String fileNew = "/dir1/dir2/dir3/fileNew";
    writeUfsFile(ufsPath(fileA), 1);
    writeUfsFile(ufsPath(fileB), 1);
    FileSystemMasterCommonPOptions longinterval = FileSystemMasterCommonPOptions.newBuilder().setSyncIntervalMs(Constants.HOUR_MS).build();
    // Should not exist, since no loading or syncing
    assertFalse(mFileSystem.exists(new AlluxioURI(alluxioPath(fileA)), ExistsPOptions.newBuilder().setCommonOptions(PSYNC_NEVER).build()));
    try {
        mFileSystem.listStatus(new AlluxioURI(alluxioPath("/dir1")), ListStatusPOptions.newBuilder().setCommonOptions(PSYNC_NEVER).build());
        Assert.fail("paths are not expected to exist without sync");
    } catch (FileDoesNotExistException e) {
    // expected, continue
    }
    // recursively sync the top dir
    List<URIStatus> paths = mFileSystem.listStatus(new AlluxioURI(alluxioPath("/dir1")), ListStatusPOptions.newBuilder().setCommonOptions(PSYNC_ALWAYS).setRecursive(true).build());
    assertEquals(4, paths.size());
    // write a new UFS file
    writeUfsFile(ufsPath(fileNew), 1);
    // the new UFS file should not exist, since the sync interval is 1 hour, and an ancestor
    // already synced recently.
    assertFalse(mFileSystem.exists(new AlluxioURI(alluxioPath(fileNew)), ExistsPOptions.newBuilder().setCommonOptions(longinterval).build()));
    // newly created file should not exist
    paths = mFileSystem.listStatus(new AlluxioURI(alluxioPath("/dir1/dir2/dir3")), ListStatusPOptions.newBuilder().setCommonOptions(longinterval).build());
    assertEquals(2, paths.size());
    // create a new UFS dir
    new File(ufsPath("/dir1/dir2/dirNew")).mkdirs();
    // newly created dir should not exist, since sync interval is long, and an ancestor is
    // already synced
    assertFalse(mFileSystem.exists(new AlluxioURI(alluxioPath("/dir1/dir2/dirNew")), ExistsPOptions.newBuilder().setCommonOptions(longinterval).build()));
    // newly created dir should not exist
    paths = mFileSystem.listStatus(new AlluxioURI(alluxioPath("/dir1/dir2")), ListStatusPOptions.newBuilder().setCommonOptions(longinterval).build());
    assertEquals(1, paths.size());
    // check the original path, and verify no new files/dirs are picked up from UFS
    paths = mFileSystem.listStatus(new AlluxioURI(alluxioPath("/dir1")), ListStatusPOptions.newBuilder().setCommonOptions(longinterval).setRecursive(true).build());
    assertEquals(4, paths.size());
}

Example 5 with FileSystemMasterCommonPOptions

use of alluxio.grpc.FileSystemMasterCommonPOptions in project alluxio by Alluxio.

the class UfsSyncIntegrationTest method recursiveSync.

@LocalAlluxioClusterResource.Config(confParams = { PropertyKey.Name.USER_FILE_METADATA_LOAD_TYPE, "NEVER" })
@Test
public void recursiveSync() throws Exception {
    // make nested directories/files in UFS
    new File(ufsPath("/dir1")).mkdirs();
    new File(ufsPath("/dir1/dir2")).mkdirs();
    new File(ufsPath("/dir1/dir2/dir3")).mkdirs();
    String fileA = "/dir1/dir2/fileA";
    String fileB = "/dir1/dir2/fileB";
    String fileC = "/dir1/dir2/fileC";
    writeUfsFile(ufsPath(fileA), 1);
    writeUfsFile(ufsPath(fileB), 1);
    // Should not exist, since no loading or syncing
    assertFalse(mFileSystem.exists(new AlluxioURI(alluxioPath(fileA)), ExistsPOptions.newBuilder().setCommonOptions(FileSystemOptions.commonDefaults(mFileSystem.getConf()).toBuilder().setSyncIntervalMs(-1).build()).build()));
    try {
        mFileSystem.setAttribute(new AlluxioURI(alluxioPath("/dir1")), SetAttributePOptions.newBuilder().setRecursive(true).setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(55555).setSyncIntervalMs(-1)).build());
    } catch (FileDoesNotExistException e) {
    // expected, continue
    }
    // Enable UFS sync, before next recursive setAttribute.
    FileSystemMasterCommonPOptions ttlOption = FileSystemMasterCommonPOptions.newBuilder().setTtl(123456789).setSyncIntervalMs(0).build();
    mFileSystem.setAttribute(new AlluxioURI(alluxioPath("/dir1")), SetAttributePOptions.newBuilder().setRecursive(true).setCommonOptions(ttlOption).build());
    // Verify recursive set TTL by getting info, without sync.
    ttlOption = ttlOption.toBuilder().setSyncIntervalMs(-1).build();
    URIStatus status = mFileSystem.getStatus(new AlluxioURI(alluxioPath(fileA)));
    assertEquals(ttlOption.getTtl(), status.getTtl());
    // Add UFS fileC and remove existing UFS fileA.
    writeUfsFile(ufsPath(fileC), 1);
    assertTrue(new File(ufsPath(fileA)).delete());
    // Enable UFS sync, before next recursive setAttribute.
    ttlOption = FileSystemMasterCommonPOptions.newBuilder().setTtl(987654321).setSyncIntervalMs(0).build();
    mFileSystem.setAttribute(new AlluxioURI(alluxioPath("/dir1")), SetAttributePOptions.newBuilder().setRecursive(true).setCommonOptions(ttlOption).build());
    // Verify recursive set TTL by getting info, without sync.
    ttlOption = FileSystemMasterCommonPOptions.newBuilder().setSyncIntervalMs(-1).setTtl(987654321).build();
    status = mFileSystem.getStatus(new AlluxioURI(alluxioPath(fileB)));
    assertEquals(ttlOption.getTtl(), status.getTtl());
    // deleted UFS file should not exist.
    assertFalse(mFileSystem.exists(new AlluxioURI(alluxioPath(fileA))));
}