Examples with TtlAction alluxio.grpc.TtlAction used on opensource projects

Example 1 with TtlAction

use of alluxio.grpc.TtlAction in project alluxio by Alluxio.

the class DefaultFileSystemMaster method setAttributeSingleFile.

/**
 * @param inodePath the {@link LockedInodePath} to use
 * @param updateUfs whether to update the UFS with the attribute change
 * @param opTimeMs the operation time (in milliseconds)
 * @param context the method context
 */
protected void setAttributeSingleFile(RpcContext rpcContext, LockedInodePath inodePath, boolean updateUfs, long opTimeMs, SetAttributeContext context) throws FileDoesNotExistException, InvalidPathException, AccessControlException {
    Inode inode = inodePath.getInode();
    SetAttributePOptions.Builder protoOptions = context.getOptions();
    if (protoOptions.hasPinned()) {
        mInodeTree.setPinned(rpcContext, inodePath, context.getOptions().getPinned(), context.getOptions().getPinnedMediaList(), opTimeMs);
    }
    UpdateInodeEntry.Builder entry = UpdateInodeEntry.newBuilder().setId(inode.getId());
    if (protoOptions.hasReplicationMax() || protoOptions.hasReplicationMin()) {
        Integer replicationMax = protoOptions.hasReplicationMax() ? protoOptions.getReplicationMax() : null;
        Integer replicationMin = protoOptions.hasReplicationMin() ? protoOptions.getReplicationMin() : null;
        mInodeTree.setReplication(rpcContext, inodePath, replicationMax, replicationMin, opTimeMs);
    }
    // protoOptions may not have both fields set
    if (protoOptions.hasCommonOptions()) {
        FileSystemMasterCommonPOptions commonOpts = protoOptions.getCommonOptions();
        TtlAction action = commonOpts.hasTtlAction() ? commonOpts.getTtlAction() : null;
        Long ttl = commonOpts.hasTtl() ? commonOpts.getTtl() : null;
        boolean modified = false;
        if (ttl != null && inode.getTtl() != ttl) {
            entry.setTtl(ttl);
            modified = true;
        }
        if (action != null && inode.getTtlAction() != action) {
            entry.setTtlAction(ProtobufUtils.toProtobuf(action));
            modified = true;
        }
        if (modified) {
            entry.setLastModificationTimeMs(opTimeMs);
        }
    }
    if (protoOptions.hasPersisted()) {
        Preconditions.checkArgument(inode.isFile(), PreconditionMessage.PERSIST_ONLY_FOR_FILE);
        Preconditions.checkArgument(inode.asFile().isCompleted(), PreconditionMessage.FILE_TO_PERSIST_MUST_BE_COMPLETE);
        // TODO(manugoyal) figure out valid behavior in the un-persist case
        Preconditions.checkArgument(protoOptions.getPersisted(), PreconditionMessage.ERR_SET_STATE_UNPERSIST);
        if (!inode.asFile().isPersisted()) {
            entry.setPersistenceState(PersistenceState.PERSISTED.name());
            entry.setLastModificationTimeMs(context.getOperationTimeMs());
            propagatePersistedInternal(rpcContext, inodePath);
            Metrics.FILES_PERSISTED.inc();
        }
    }
    boolean ownerGroupChanged = (protoOptions.hasOwner()) || (protoOptions.hasGroup());
    boolean modeChanged = protoOptions.hasMode();
    // If the file is persisted in UFS, also update corresponding owner/group/permission.
    if ((ownerGroupChanged || modeChanged) && updateUfs && inode.isPersisted()) {
        if ((inode instanceof InodeFile) && !inode.asFile().isCompleted()) {
            LOG.debug("Alluxio does not propagate chown/chgrp/chmod to UFS for incomplete files.");
        } else {
            checkUfsMode(inodePath.getUri(), OperationType.WRITE);
            MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
            String ufsUri = resolution.getUri().toString();
            try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
                UnderFileSystem ufs = ufsResource.get();
                if (ufs.isObjectStorage()) {
                    LOG.debug("setOwner/setMode is not supported to object storage UFS via Alluxio. " + "UFS: " + ufsUri + ". This has no effect on the underlying object.");
                } else {
                    String owner = null;
                    String group = null;
                    String mode = null;
                    if (ownerGroupChanged) {
                        try {
                            owner = protoOptions.getOwner() != null ? protoOptions.getOwner() : inode.getOwner();
                            group = protoOptions.getGroup() != null ? protoOptions.getGroup() : inode.getGroup();
                            ufs.setOwner(ufsUri, owner, group);
                        } catch (IOException e) {
                            throw new AccessControlException("Could not setOwner for UFS file " + ufsUri + " . Aborting the setAttribute operation in Alluxio.", e);
                        }
                    }
                    if (modeChanged) {
                        try {
                            mode = String.valueOf(protoOptions.getMode());
                            ufs.setMode(ufsUri, ModeUtils.protoToShort(protoOptions.getMode()));
                        } catch (IOException e) {
                            throw new AccessControlException("Could not setMode for UFS file " + ufsUri + " . Aborting the setAttribute operation in Alluxio.", e);
                        }
                    }
                    // Retrieve the ufs fingerprint after the ufs changes.
                    String existingFingerprint = inode.getUfsFingerprint();
                    if (!existingFingerprint.equals(Constants.INVALID_UFS_FINGERPRINT)) {
                        // Update existing fingerprint, since contents did not change
                        Fingerprint fp = Fingerprint.parse(existingFingerprint);
                        fp.putTag(Fingerprint.Tag.OWNER, owner);
                        fp.putTag(Fingerprint.Tag.GROUP, group);
                        fp.putTag(Fingerprint.Tag.MODE, mode);
                        context.setUfsFingerprint(fp.serialize());
                    } else {
                        // Need to retrieve the fingerprint from ufs.
                        context.setUfsFingerprint(ufs.getFingerprint(ufsUri));
                    }
                }
            }
        }
    }
    if (!context.getUfsFingerprint().equals(Constants.INVALID_UFS_FINGERPRINT)) {
        entry.setUfsFingerprint(context.getUfsFingerprint());
    }
    // Only commit the set permission to inode after the propagation to UFS succeeded.
    if (protoOptions.hasOwner()) {
        entry.setOwner(protoOptions.getOwner());
    }
    if (protoOptions.hasGroup()) {
        entry.setGroup(protoOptions.getGroup());
    }
    if (modeChanged) {
        entry.setMode(ModeUtils.protoToShort(protoOptions.getMode()));
    }
    mInodeTree.updateInode(rpcContext, entry.build());
}

Example 2 with TtlAction

use of alluxio.grpc.TtlAction in project alluxio by Alluxio.

the class InodeTtlChecker method heartbeat.

@Override
public void heartbeat() throws InterruptedException {
    Set<TtlBucket> expiredBuckets = mTtlBuckets.getExpiredBuckets(System.currentTimeMillis());
    for (TtlBucket bucket : expiredBuckets) {
        for (Inode inode : bucket.getInodes()) {
            // Throw if interrupted.
            if (Thread.interrupted()) {
                throw new InterruptedException("InodeTtlChecker interrupted.");
            }
            AlluxioURI path = null;
            try (LockedInodePath inodePath = mInodeTree.lockFullInodePath(inode.getId(), LockPattern.READ)) {
                path = inodePath.getUri();
            } catch (FileDoesNotExistException e) {
                // The inode has already been deleted, nothing needs to be done.
                continue;
            } catch (Exception e) {
                LOG.error("Exception trying to clean up {} for ttl check: {}", inode.toString(), e.toString());
            }
            if (path != null) {
                try {
                    TtlAction ttlAction = inode.getTtlAction();
                    LOG.info("Path {} TTL has expired, performing action {}", path.getPath(), ttlAction);
                    switch(ttlAction) {
                        case FREE:
                            // parent of file
                            if (inode.isDirectory()) {
                                mFileSystemMaster.free(path, FreeContext.mergeFrom(FreePOptions.newBuilder().setForced(true).setRecursive(true)));
                            } else {
                                mFileSystemMaster.free(path, FreeContext.mergeFrom(FreePOptions.newBuilder().setForced(true)));
                            }
                            try (JournalContext journalContext = mFileSystemMaster.createJournalContext()) {
                                // Reset state
                                mInodeTree.updateInode(journalContext, UpdateInodeEntry.newBuilder().setId(inode.getId()).setTtl(Constants.NO_TTL).setTtlAction(ProtobufUtils.toProtobuf(TtlAction.DELETE)).build());
                            }
                            mTtlBuckets.remove(inode);
                            break;
                        case // Default if not set is DELETE
                        DELETE:
                            // parent of file
                            if (inode.isDirectory()) {
                                mFileSystemMaster.delete(path, DeleteContext.mergeFrom(DeletePOptions.newBuilder().setRecursive(true)));
                            } else {
                                mFileSystemMaster.delete(path, DeleteContext.defaults());
                            }
                            break;
                        default:
                            LOG.error("Unknown ttl action {}", ttlAction);
                    }
                } catch (Exception e) {
                    LOG.error("Exception trying to clean up {} for ttl check", inode.toString(), e);
                }
            }
        }
    }
    mTtlBuckets.removeBuckets(expiredBuckets);
}

Example 3 with TtlAction

use of alluxio.grpc.TtlAction in project alluxio by Alluxio.

the class FileSystemIntegrationTest method testTtlActionSetAttribute.

@LocalAlluxioClusterResource.Config(confParams = { PropertyKey.Name.USER_FILE_CREATE_TTL_ACTION, "FREE" })
@Test
public void testTtlActionSetAttribute() throws Exception {
    AlluxioURI testFile = new AlluxioURI("/test1");
    FileSystemTestUtils.createByteFile(mFileSystem, testFile, WritePType.MUST_CACHE, 512);
    TtlAction expectedAction = ServerConfiguration.getEnum(PropertyKey.USER_FILE_CREATE_TTL_ACTION, TtlAction.class);
    URIStatus stat = mFileSystem.getStatus(testFile);
    assertEquals("TTL action should be same", expectedAction, stat.getTtlAction());
    TtlAction newTtlAction = TtlAction.DELETE;
    long newTtl = 123400000;
    mFileSystem.setAttribute(testFile, SetAttributePOptions.newBuilder().setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(newTtl).build()).build());
    stat = mFileSystem.getStatus(testFile);
    assertEquals("TTL should be same", newTtl, stat.getTtl());
    assertEquals("TTL action should be same", expectedAction, stat.getTtlAction());
    mFileSystem.setAttribute(testFile, SetAttributePOptions.newBuilder().setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtlAction(newTtlAction).build()).build());
    stat = mFileSystem.getStatus(testFile);
    assertEquals("TTL should be same", newTtl, stat.getTtl());
    assertEquals("TTL action should be same", newTtlAction, stat.getTtlAction());
}

Example 4 with TtlAction

use of alluxio.grpc.TtlAction in project alluxio by Alluxio.

the class OutStreamOptionsTest method fields.

/**
 * Tests getting and setting fields.
 */
@Test
public void fields() throws Exception {
    Random random = new Random();
    long blockSize = random.nextLong();
    BlockLocationPolicy locationPolicy = new RoundRobinPolicy(mConf);
    String owner = CommonUtils.randomAlphaNumString(10);
    String group = CommonUtils.randomAlphaNumString(10);
    Mode mode = new Mode((short) random.nextInt());
    int ttl = 5;
    TtlAction ttlAction = TtlAction.FREE;
    int writeTier = random.nextInt();
    WriteType writeType = WriteType.NONE;
    mConf.set(PropertyKey.USER_FILE_CREATE_TTL, ttl);
    mConf.set(PropertyKey.USER_FILE_CREATE_TTL_ACTION, ttlAction);
    ClientContext clientContext = ClientContext.create(mConf);
    OutStreamOptions options = OutStreamOptions.defaults(clientContext);
    options.setBlockSizeBytes(blockSize);
    options.setLocationPolicy(locationPolicy);
    options.setOwner(owner);
    options.setGroup(group);
    options.setMode(mode);
    options.setWriteTier(writeTier);
    options.setWriteType(writeType);
    assertEquals(blockSize, options.getBlockSizeBytes());
    assertEquals(locationPolicy, options.getLocationPolicy());
    assertEquals(owner, options.getOwner());
    assertEquals(group, options.getGroup());
    assertEquals(mode, options.getMode());
    assertEquals(ttl, options.getCommonOptions().getTtl());
    assertEquals(ttlAction, options.getCommonOptions().getTtlAction());
    assertEquals(writeTier, options.getWriteTier());
    assertEquals(writeType.getAlluxioStorageType(), options.getAlluxioStorageType());
    assertEquals(writeType.getUnderStorageType(), options.getUnderStorageType());
}