Examples with AccessLatency diskCacheV111.util.AccessLatency used on opensource projects

Example 1 with AccessLatency

use of diskCacheV111.util.AccessLatency in project dcache by dCache.

the class ALRPStorageUnitQoSProvider method fetchRequirements.

@Override
public FileQoSRequirements fetchRequirements(FileQoSUpdate update) throws QoSException {
    FileQoSRequirements descriptor = initialize(update);
    if (descriptor == null) {
        /*
             *  Should only happen when a CLEAR CACHE LOCATION finds no locations.
             */
        return null;
    }
    FileAttributes attributes = descriptor.getAttributes();
    AccessLatency accessLatency = attributes.getAccessLatency();
    RetentionPolicy retentionPolicy = attributes.getRetentionPolicy();
    String unitKey = attributes.getStorageClass() + "@" + attributes.getHsm();
    SelectionUnit unit = poolSelectionUnit().getStorageUnit(unitKey);
    if (!(unit instanceof StorageUnit)) {
        throw new QoSException(unitKey + " does not correspond to a storage unit; " + "cannot retrieve requirements for " + descriptor.getPnfsId());
    }
    StorageUnit storageUnit = (StorageUnit) unit;
    Integer required = storageUnit.getRequiredCopies();
    List<String> onlyOneCopyPer = storageUnit.getOnlyOneCopyPer();
    if (retentionPolicy == CUSTODIAL) {
        /*
             *  REVISIT -- currently we support only one tape location.
             */
        descriptor.setRequiredTape(1);
    } else {
        descriptor.setRequiredTape(0);
    }
    if (accessLatency == ONLINE) {
        /*
             *  REVISIT -- current override of file AL based on storage unit
             *  REVISIT -- eventually we will want to override the storage unit default for a given file
             */
        descriptor.setRequiredDisk(required == null ? 1 : required);
        if (onlyOneCopyPer != null) {
            descriptor.setPartitionKeys(new HashSet<>(onlyOneCopyPer));
        }
    } else {
        descriptor.setRequiredDisk(0);
    }
    LOGGER.debug("fetchRequirements for {}, returning {}.", update, descriptor);
    return descriptor;
}

Example 2 with AccessLatency

use of diskCacheV111.util.AccessLatency in project dcache by dCache.

the class FileUpdate method getAttributes.

/**
 * @return <code>null</code> if AccessLatency is not ONLINE, or if the file
 * is not found or there are no locations for it and the message being processed is for a clear
 * cache location; otherwise the file attribute set required to process resilience.
 */
public static FileAttributes getAttributes(PnfsId pnfsId, String pool, MessageType messageType, NamespaceAccess namespace) throws CacheException {
    try {
        FileAttributes attributes = namespace.getRequiredAttributes(pnfsId);
        if (attributes == null) {
            throw new FileNotFoundCacheException(String.format("No attributes " + "returned for %s", pnfsId));
        }
        LOGGER.trace("Got required attributes for {}.", pnfsId);
        if (attributes.getLocations().isEmpty()) {
            if (messageType == CLEAR_CACHE_LOCATION) {
                LOGGER.trace("ClearCacheLocationMessage for {}; " + "no current locations; " + "file probably deleted " + "from namespace.", pnfsId);
                return null;
            }
            if (messageType != ADD_CACHE_LOCATION) {
                /*
                     * Since the scan began or the broken file reported,
                     * the file has been removed.
                     */
                throw new FileNotFoundCacheException(String.format("File no longer found: %s", pnfsId));
            }
            /*
                 *  May be due to a race between PnfsManager and resilience
                 *  to process the message into/from the namespace.
                 *
                 *  We can assume here that this is a new file, so
                 *  we just add the originating location to the attribute
                 *  location list.
                 */
            LOGGER.trace("{} has no locations yet.", pnfsId);
            Collection<String> singleLoc = new ArrayList<>();
            /*
                 *  Pool can now be <code>null</code>
                 *  but only if message type is QOS_MODIFIED.
                 */
            if (pool == null) {
                if (messageType != QOS_MODIFIED) {
                    throw new CacheException(CacheException.UNEXPECTED_SYSTEM_EXCEPTION, String.format("resilience File update for %s, " + "messageType %s, " + "has no pool location!", pnfsId, messageType));
                }
                return null;
            }
            singleLoc.add(pool);
            attributes.setLocations(singleLoc);
        }
        LOGGER.debug("After call to namespace, {} has locations {}.", pnfsId, attributes.getLocations());
        return attributes;
    } catch (FileNotFoundCacheException e) {
        LOGGER.debug("{}; {} has likely been deleted from the namespace.", e.getMessage(), pnfsId);
        return null;
    }
}

Example 3 with AccessLatency

use of diskCacheV111.util.AccessLatency in project dcache by dCache.

the class ChimeraNameSpaceProvider method createUploadPath.

@Override
public FsPath createUploadPath(Subject subject, FsPath path, FsPath rootPath, Long size, AccessLatency al, RetentionPolicy rp, String spaceToken, Set<CreateOption> options) throws CacheException {
    checkState(_uploadDirectory != null, "Upload directory is not configured.");
    try {
        /* Parent directory must exist.
             */
        ExtendedInode parentOfPath = options.contains(CreateOption.CREATE_PARENTS) ? installDirectory(subject, path.parent(), INHERIT_MODE) : lookupDirectory(subject, path.parent());
        FileAttributes attributesOfParent = !Subjects.isExemptFromNamespaceChecks(subject) ? getFileAttributesForPermissionHandler(parentOfPath) : null;
        /* File must not exist unless overwrite is enabled.
             */
        try {
            ExtendedInode inodeOfPath = parentOfPath.inodeOf(path.name(), STAT);
            if (!options.contains(CreateOption.OVERWRITE_EXISTING) || (inodeOfPath.statCache().getMode() & UnixPermission.S_TYPE) != UnixPermission.S_IFREG) {
                throw new FileExistsCacheException("File exists: " + path);
            }
            /* User must be authorized to delete existing file.
                 */
            if (!Subjects.isExemptFromNamespaceChecks(subject)) {
                FileAttributes attributesOfPath = getFileAttributesForPermissionHandler(inodeOfPath);
                if (_permissionHandler.canDeleteFile(subject, attributesOfParent, attributesOfPath) != ACCESS_ALLOWED) {
                    throw new PermissionDeniedCacheException("Access denied: " + path);
                }
            }
        } catch (FileNotFoundChimeraFsException ignored) {
        }
        /* User must be authorized to create file.
             */
        if (!Subjects.isExemptFromNamespaceChecks(subject)) {
            if (_permissionHandler.canCreateFile(subject, attributesOfParent) != ACCESS_ALLOWED) {
                throw new PermissionDeniedCacheException("Access denied: " + path);
            }
        }
        /* Attributes are inherited from real parent directory.
             */
        int mode = parentOfPath.statCache().getMode() & UnixPermission.S_PERMS;
        int gid;
        if ((mode & UnixPermission.S_ISGID) != 0) {
            gid = parentOfPath.statCache().getGid();
        } else if (Subjects.isNobody(subject) || _inheritFileOwnership) {
            gid = parentOfPath.statCache().getGid();
        } else {
            gid = Ints.checkedCast(Subjects.getPrimaryGid(subject));
        }
        int uid;
        if (Subjects.isNobody(subject) || _inheritFileOwnership) {
            uid = parentOfPath.statCache().getUid();
        } else {
            uid = Ints.checkedCast(Subjects.getUid(subject));
        }
        /* ACLs are copied from real parent to the temporary upload directory
             * such that the upload is allowed (in case write permissions rely
             * on ACLs) and such that the file will inherit the correct ACLs.
             */
        List<ACE> acl = _fs.getACL(parentOfPath);
        /* The temporary upload directory has the same tags as the real parent,
             * except target file specific properties are stored as tags local to
             * the upload directory.
             */
        Map<String, byte[]> tags = Maps.newHashMap(parentOfPath.getTags());
        if (spaceToken != null) {
            tags.put(TAG_WRITE_TOKEN, spaceToken.getBytes(UTF_8));
            /* If client provides space token to upload to, the access latency and
                 * retention policy tags of the upload directory must be disregarded.
                 */
            tags.remove(TAG_ACCESS_LATENCY);
            tags.remove(TAG_RETENTION_POLICY);
        }
        if (al != null) {
            tags.put(TAG_ACCESS_LATENCY, al.toString().getBytes(UTF_8));
        }
        if (rp != null) {
            tags.put(TAG_RETENTION_POLICY, rp.toString().getBytes(UTF_8));
        }
        if (size != null) {
            tags.put(TAG_EXPECTED_SIZE, size.toString().getBytes(UTF_8));
        }
        tags.put(TAG_PATH, path.toString().getBytes(UTF_8));
        /* Upload directory may optionally be relative to the user's root path. Whether
             * that's the case depends on if the configured upload directory is an absolute
             * or relative path.
             */
        FsPath uploadDirectory = rootPath.resolve(_uploadDirectory);
        if (_uploadSubDirectory != null) {
            uploadDirectory = uploadDirectory.chroot(String.format(_uploadSubDirectory, threadId.get()));
        }
        /* Upload directory must exist and have the right permissions.
             */
        ExtendedInode inodeOfUploadDir = installSystemDirectory(uploadDirectory, 0711, Collections.emptyList(), Collections.emptyMap());
        if (inodeOfUploadDir.statCache().getUid() != 0) {
            LOGGER.error("Owner must be root: {}", uploadDirectory);
            throw new CacheException("Owner must be root: " + uploadDirectory);
        }
        if ((inodeOfUploadDir.statCache().getMode() & UnixPermission.S_PERMS) != 0711) {
            LOGGER.error("File mode must be 0711: {}", uploadDirectory);
            throw new CacheException("File mode must be 0711: " + uploadDirectory);
        }
        /* Use cryptographically strong pseudo random UUID to create temporary upload directory.
             */
        UUID uuid = UUID.randomUUID();
        _fs.mkdir(inodeOfUploadDir, uuid.toString(), uid, gid, mode, acl, tags);
        return uploadDirectory.child(uuid.toString()).child(path.name());
    } catch (ChimeraFsException e) {
        LOGGER.error("Problem with database: {}", e.getMessage());
        throw new CacheException(CacheException.UNEXPECTED_SYSTEM_EXCEPTION, e.getMessage());
    }
}

Example 4 with AccessLatency

use of diskCacheV111.util.AccessLatency in project dcache by dCache.

the class PerformanceTest method processOperation.

private void processOperation(Operation aOp, String path) {
    try {
        switch(aOp) {
            case CREATE_ENTRY:
                provider.createFile(Subjects.ROOT, path, FileAttributes.of().uid(UID).gid(GID).mode(0664).build(), EnumSet.noneOf(FileAttribute.class));
                break;
            case PATH_TO_PNFS_ID:
                getPnfsid(path);
                break;
            case FILE_META_DATA:
                provider.getFileAttributes(Subjects.ROOT, getPnfsid(path), EnumSet.of(OWNER, OWNER_GROUP, MODE, TYPE, SIZE, CREATION_TIME, ACCESS_TIME, MODIFICATION_TIME, CHANGE_TIME));
                break;
            case DELETE_ENTRY:
                provider.deleteEntry(Subjects.ROOT, EnumSet.allOf(FileType.class), path, EnumSet.noneOf(FileAttribute.class));
                break;
            case PNFS_ID_TO_PATH:
                provider.pnfsidToPath(Subjects.ROOT, getPnfsid(path));
                break;
            case FIND:
                provider.find(Subjects.ROOT, getPnfsid(path));
                break;
            case ADD_CHECKSUM:
                provider.setFileAttributes(Subjects.ROOT, getPnfsid(path), FileAttributes.ofChecksum(CHECKSUM), EnumSet.noneOf(FileAttribute.class));
                break;
            case GET_CHECKSUMS:
                provider.getFileAttributes(Subjects.ROOT, getPnfsid(path), EnumSet.of(FileAttribute.CHECKSUM)).getChecksums();
                break;
            case SET_FILE_ATTR:
                provider.setFileAttributes(Subjects.ROOT, getPnfsid(path), FileAttributes.of().accessLatency(ONLINE).retentionPolicy(REPLICA).build(), EnumSet.noneOf(FileAttribute.class));
                break;
            case GET_FILE_ATTR:
                provider.getFileAttributes(Subjects.ROOT, getPnfsid(path), EnumSet.allOf(FileAttribute.class));
                break;
            case ADD_CACHE_LOC:
                provider.addCacheLocation(Subjects.ROOT, getPnfsid(path), CACHE_LOCATION);
                break;
            case GET_CACHE_LOC:
                provider.getCacheLocation(Subjects.ROOT, getPnfsid(path));
                break;
            case STORAGE_INFO:
                provider.getFileAttributes(Subjects.ROOT, getPnfsid(path), EnumSet.of(FileAttribute.STORAGEINFO)).getStorageInfo();
                break;
            case SET_STORAGE_INFO:
                StorageInfo info = provider.getFileAttributes(Subjects.ROOT, getPnfsid(path), EnumSet.of(FileAttribute.STORAGEINFO)).getStorageInfo();
                info.setHsm("hsm");
                info.setKey("store", "test");
                info.setKey("group", "disk");
                info.addLocation(URI.create("osm://hsm/?store=test&group=disk&bdif=1234"));
                provider.setFileAttributes(Subjects.ROOT, getPnfsid(path), FileAttributes.ofStorageInfo(info), EnumSet.noneOf(FileAttribute.class));
                break;
            case MKDIR:
                provider.createDirectory(Subjects.ROOT, path, FileAttributes.of().uid(UID).gid(GID).mode(0755).build());
                break;
            case RMDIR:
                provider.deleteEntry(Subjects.ROOT, EnumSet.of(FileType.DIR), path, EnumSet.noneOf(FileAttribute.class));
                break;
            default:
                break;
        }
    } catch (CacheException e) {
        System.err.println("Exception " + aOp.getUserInput() + " :" + e.getMessage());
    }
}

Example 5 with AccessLatency

use of diskCacheV111.util.AccessLatency in project dcache by dCache.

the class PnfsManagerTest method testDefaultALandRP.

@Test
public void testDefaultALandRP() throws ChimeraFsException {
    /*
         * this test relays on the fact that default values in PnfsManager
         * is CUSTODIAL/NEARLINE
         */
    // use back door to create the tag
    FsInode dirInode = _fs.path2inode("/pnfs/testRoot");
    _fs.createTag(dirInode, "AccessLatency");
    _fs.createTag(dirInode, "RetentionPolicy");
    String al = "ONLINE";
    String rp = "OUTPUT";
    _fs.setTag(dirInode, "AccessLatency", al.getBytes(), 0, al.getBytes().length);
    _fs.setTag(dirInode, "RetentionPolicy", rp.getBytes(), 0, rp.getBytes().length);
    PnfsCreateEntryMessage pnfsCreateEntryMessage = new PnfsCreateEntryMessage("/pnfs/testRoot/testDefaultALandRP", FileAttributes.ofFileType(REGULAR));
    _pnfsManager.createEntry(pnfsCreateEntryMessage);
    FileAttributes fileAttributes = pnfsCreateEntryMessage.getFileAttributes();
    assertEquals("AccessLatensy is not taken from the parent directory", AccessLatency.ONLINE, fileAttributes.getAccessLatency());
    assertEquals("RetentionPolicy is not taken from the parent directory", RetentionPolicy.OUTPUT, fileAttributes.getRetentionPolicy());
}