Examples with DirectoryStream org.dcache.util.list.DirectoryStream used on opensource projects

Example 1 with DirectoryStream

use of org.dcache.util.list.DirectoryStream in project dcache by dCache.

the class FileResources method getFileAttributes.

@GET
@ApiOperation(value = "Find metadata and optionally directory contents.", notes = "The method offers the possibility to list the content of a " + "directory in addition to providing metadata of a " + "specified file or directory.")
@ApiResponses({ @ApiResponse(code = 401, message = "Unauthorized"), @ApiResponse(code = 403, message = "Forbidden"), @ApiResponse(code = 404, message = "Not Found"), @ApiResponse(code = 500, message = "Internal Server Error") })
@Path("{path : .*}")
@Produces(MediaType.APPLICATION_JSON)
public JsonFileAttributes getFileAttributes(@ApiParam("Path of file or directory.") @PathParam("path") String requestPath, @ApiParam("Whether to include directory listing.") @DefaultValue("false") @QueryParam("children") boolean isList, @ApiParam("Whether to include file locality information.") @DefaultValue("false") @QueryParam("locality") boolean isLocality, @ApiParam(value = "Whether to include replica locations.") @QueryParam("locations") boolean isLocations, @ApiParam(value = "Whether to include quality of service.") @DefaultValue("false") @QueryParam("qos") boolean isQos, @ApiParam("Whether to include extended attributes.") @QueryParam("xattr") boolean isXattr, @ApiParam("Whether to include labels.") @QueryParam("labels") boolean isLabels, @ApiParam("Whether or not to list checksum values.") @QueryParam("checksum") boolean isChecksum, @ApiParam("Limit number of replies in directory listing.") @QueryParam("limit") String limit, @ApiParam("Number of entries to skip in directory listing.") @QueryParam("offset") String offset) throws CacheException {
    JsonFileAttributes fileAttributes = new JsonFileAttributes();
    Set<FileAttribute> attributes = NamespaceUtils.getRequestedAttributes(isLocality, isLocations, isQos, isChecksum, false);
    PnfsHandler handler = HandlerBuilders.roleAwarePnfsHandler(pnfsmanager);
    FsPath path = pathMapper.asDcachePath(request, requestPath, ForbiddenException::new);
    try {
        FileAttributes namespaceAttributes = handler.getFileAttributes(path, attributes);
        NamespaceUtils.chimeraToJsonAttributes(path.name(), fileAttributes, namespaceAttributes, isLocality, isLocations, isLabels, false, isXattr, isChecksum, request, poolMonitor);
        if (isQos) {
            NamespaceUtils.addQoSAttributes(fileAttributes, namespaceAttributes, request, poolMonitor, pinmanager);
        }
        // fill children list id it's a directory and listing is requested
        if (namespaceAttributes.getFileType() == FileType.DIR && isList) {
            Range<Integer> range;
            try {
                int lower = (offset == null) ? 0 : Integer.parseInt(offset);
                int ceiling = (limit == null) ? Integer.MAX_VALUE : Integer.parseInt(limit);
                if (ceiling < 0 || lower < 0) {
                    throw new BadRequestException("limit and offset can not be less than zero.");
                }
                range = (Integer.MAX_VALUE - lower < ceiling) ? Range.atLeast(lower) : Range.closedOpen(lower, lower + ceiling);
            } catch (NumberFormatException e) {
                throw new BadRequestException("limit and offset must be an integer value.");
            }
            List<JsonFileAttributes> children = new ArrayList<>();
            DirectoryStream stream = listDirectoryHandler.list(HttpServletRequests.roleAwareSubject(request), HttpServletRequests.roleAwareRestriction(request), path, null, range, attributes);
            for (DirectoryEntry entry : stream) {
                String fName = entry.getName();
                JsonFileAttributes childrenAttributes = new JsonFileAttributes();
                NamespaceUtils.chimeraToJsonAttributes(fName, childrenAttributes, entry.getFileAttributes(), isLocality, isLocations, isLabels, false, isXattr, isChecksum, request, poolMonitor);
                childrenAttributes.setFileName(fName);
                if (isQos) {
                    NamespaceUtils.addQoSAttributes(childrenAttributes, entry.getFileAttributes(), request, poolMonitor, pinmanager);
                }
                children.add(childrenAttributes);
            }
            fileAttributes.setChildren(children);
        }
    } catch (FileNotFoundCacheException e) {
        throw new NotFoundException(e);
    } catch (PermissionDeniedCacheException e) {
        if (RequestUser.isAnonymous()) {
            throw new NotAuthorizedException(e);
        } else {
            throw new ForbiddenException(e);
        }
    } catch (CacheException | InterruptedException | NoRouteToCellException ex) {
        LOG.warn(Exceptions.meaningfulMessage(ex));
        throw new InternalServerErrorException(ex);
    }
    return fileAttributes;
}

Example 2 with DirectoryStream

use of org.dcache.util.list.DirectoryStream in project dcache by dCache.

the class TargetExpansionJob method expand.

private void expand(String target, BulkJobKey key, BulkJobKey parentKey, FileAttributes attributes) throws CacheException, BulkServiceException {
    /*
         *  Fail-fast in case the job has been cancelled.
         */
    if (isTerminated()) {
        LOGGER.debug("{}, expansion job for {} {}; returning ...", loggingPrefix(), target, state.name());
        return;
    }
    LOGGER.debug("{}, expand() called for {}: key {}, parent {}.", loggingPrefix(), target, key.getJobId(), parentKey.getJobId());
    if (expansionType == ExpansionType.BREADTH_FIRST) {
        /*
             *  In breadth-first it should not matter that directories
             *  are processed as targets before their children.
             */
        checkForDirectoryTarget(target, parentKey, attributes);
    }
    try {
        LOGGER.debug("{}, listing target {}", key.getJobId(), target);
        DirectoryStream stream = getDirectoryListing(target);
        LOGGER.debug("{}, handling children", key.getJobId());
        for (DirectoryEntry entry : stream) {
            handleChildTarget(target, key, entry);
            if (isTerminated()) {
                LOGGER.debug("{}, expansion job for {} {}; returning ...", loggingPrefix(), target, state.name());
                return;
            }
        }
        LOGGER.debug("{}, finished handling children", key.getJobId());
        if (expansionType == ExpansionType.DEPTH_FIRST) {
            /*
                 *  Choosing depth-first implies the need to preserve
                 *  that order in the visiting of the nodes.  Since
                 *  submission and execution are asynchronous, it is thus
                 *  necessary to wait after a directory expansion until
                 *  all its children have completed. This guarantees these
                 *  directories will always be processed only after all
                 *  their descendants have been (necessary for, say,
                 *  deletion).
                 */
            LOGGER.debug("{}, {}, waiting for children of " + "{} to terminate.", loggingPrefix(), expansionType.name(), key.getJobId());
            completionHandler.waitForChildren(key.getJobId());
            LOGGER.debug("{}, {}, children of " + "{} have terminated.", loggingPrefix(), expansionType.name(), key.getJobId());
            /*
                 *  In depth-first it may indeed be necessary to process
                 *  directories as targets only after all their children
                 *  have terminated, so we do this here.
                 */
            checkForDirectoryTarget(target, parentKey, attributes);
        }
    } catch (InterruptedException e) {
        State state = getState();
        if (state == State.CANCELLED) {
        /*
                 *  The call to cancel will have
                 *  already notified the listener.
                 */
        } else {
            setState(State.CANCELLED);
            completionHandler.jobInterrupted(this);
        }
    }
    LOGGER.trace("{}, expand() {}, exiting ...", loggingPrefix(), target);
}

Example 3 with DirectoryStream

use of org.dcache.util.list.DirectoryStream in project dcache by dCache.

the class Storage method listSubdirectoriesRecursivelyForDelete.

/**
 * Adds transitive subdirectories of {@code dir} to {@code result}.
 *
 * @param subject    Issuer of rmdir
 * @param dir        Path to directory
 * @param attributes File attributes of {@code dir}
 * @param result     List that subdirectories are added to
 * @throws SRMAuthorizationException     if {@code subject} is not authorized to list {@code
 *                                       dir} or not authorized to list or delete any of its
 *                                       transitive subdirectories.
 * @throws SRMNonEmptyDirectoryException if {@code dir} or any of its transitive subdirectories
 *                                       contains non-directory entries.
 * @throws SRMInternalErrorException     in case of transient errors.
 * @throws SRMInvalidPathException       if {@code dir} is not a directory.
 * @throws SRMException                  in case of other errors.
 */
private void listSubdirectoriesRecursivelyForDelete(Subject subject, Restriction restriction, FsPath dir, FileAttributes attributes, List<FsPath> result) throws SRMException {
    List<DirectoryEntry> children = new ArrayList<>();
    try (DirectoryStream list = _listSource.list(subject, restriction, dir, null, Range.<Integer>all(), attributesRequiredForRmdir)) {
        for (DirectoryEntry child : list) {
            FileAttributes childAttributes = child.getFileAttributes();
            AccessType canDelete = permissionHandler.canDeleteDir(subject, attributes, childAttributes);
            if (canDelete != AccessType.ACCESS_ALLOWED) {
                throw new SRMAuthorizationException(dir + "/" + child.getName() + " (permission denied)");
            }
            if (childAttributes.getFileType() != FileType.DIR) {
                throw new SRMNonEmptyDirectoryException(dir + "/" + child.getName() + " (not empty)");
            }
            children.add(child);
        }
    } catch (NotDirCacheException e) {
        throw new SRMInvalidPathException(dir + " (not a directory)", e);
    } catch (FileNotFoundCacheException ignored) {
    // Somebody removed the directory before we could.
    } catch (PermissionDeniedCacheException e) {
        throw new SRMAuthorizationException(dir + " (permission denied)", e);
    } catch (InterruptedException e) {
        throw new SRMInternalErrorException("Operation interrupted", e);
    } catch (TimeoutCacheException e) {
        throw new SRMInternalErrorException("Name space timeout", e);
    } catch (CacheException e) {
        throw new SRMException(dir + " (" + e.getMessage() + ")");
    }
    // Result list uses post-order so directories will be deleted bottom-up.
    for (DirectoryEntry child : children) {
        FsPath path = dir.child(child.getName());
        listSubdirectoriesRecursivelyForDelete(subject, restriction, path, child.getFileAttributes(), result);
        result.add(path);
    }
}

Example 4 with DirectoryStream

use of org.dcache.util.list.DirectoryStream in project dcache by dCache.

the class LabelsResources method getFileAttributes.

@GET
@ApiOperation(value = "Find metadata and optionally virtual directory contents.", notes = "The method offers the possibility to list the content of a virtual" + "directory for labels.")
@ApiResponses({ @ApiResponse(code = 401, message = "Unauthorized"), @ApiResponse(code = 403, message = "Forbidden"), @ApiResponse(code = 404, message = "Not Found"), @ApiResponse(code = 500, message = "Internal Server Error") })
@Path("{path : .*}")
@Produces(MediaType.APPLICATION_JSON)
public JsonFileAttributes getFileAttributes(@ApiParam("Path of file or directory.") @PathParam("path") String requestPath, @ApiParam("Whether to include directory listing.") @DefaultValue("false") @QueryParam("children") boolean isList, @ApiParam("List virtual dir.") @DefaultValue("false") @QueryParam("locality") boolean isLocality, @ApiParam(value = "Whether to include replica locations.") @QueryParam("locations") boolean isLocations, @ApiParam(value = "Whether to include quality of service.") @DefaultValue("false") @QueryParam("qos") boolean isQos, @ApiParam("Whether to include extended attributes.") @QueryParam("xattr") boolean isXattr, @ApiParam("Whether to include labels.") @QueryParam("labels") boolean isLabels, @ApiParam("Limit number of replies in directory listing.") @QueryParam("limit") String limit, @ApiParam("Number of entries to skip in directory listing.") @QueryParam("offset") String offset) throws CacheException {
    JsonFileAttributes fileAttributes = new JsonFileAttributes();
    Set<FileAttribute> attributes = NamespaceUtils.getRequestedAttributes(isLocality, isLocations, false, false, false);
    FsPath path = pathMapper.asDcachePath(request, requestPath, ForbiddenException::new);
    Range<Integer> range;
    try {
        int lower = (offset == null) ? 0 : Integer.parseInt(offset);
        int ceiling = (limit == null) ? Integer.MAX_VALUE : Integer.parseInt(limit);
        if (ceiling < 0 || lower < 0) {
            throw new BadRequestException("limit and offset can not be less than zero.");
        }
        range = (Integer.MAX_VALUE - lower < ceiling) ? Range.atLeast(lower) : Range.closedOpen(lower, lower + ceiling);
    } catch (NumberFormatException e) {
        throw new BadRequestException("limit and offset must be an integer value.");
    }
    try {
        List<JsonFileAttributes> children = new ArrayList<>();
        DirectoryStream stream = listDirectoryHandler.listVirtualDirectory(HttpServletRequests.roleAwareSubject(request), HttpServletRequests.roleAwareRestriction(request), path, range, attributes);
        for (DirectoryEntry entry : stream) {
            String fPath = entry.getName();
            String fName = entry.getName();
            JsonFileAttributes childrenAttributes = new JsonFileAttributes();
            NamespaceUtils.chimeraToJsonAttributes(fPath, childrenAttributes, entry.getFileAttributes(), isLocality, isLocations, isLabels, false, isXattr, false, request, poolMonitor);
            childrenAttributes.setSourcePath(fPath);
            childrenAttributes.setFileName(fName);
            if (isQos) {
                NamespaceUtils.addQoSAttributes(childrenAttributes, entry.getFileAttributes(), request, poolMonitor, pinmanager);
            }
            children.add(childrenAttributes);
        }
        fileAttributes.setChildren(children);
    } catch (FileNotFoundCacheException e) {
        throw new NotFoundException(e);
    } catch (PermissionDeniedCacheException e) {
        if (RequestUser.isAnonymous()) {
            throw new NotAuthorizedException(e);
        } else {
            throw new ForbiddenException(e);
        }
    } catch (CacheException | InterruptedException | NoRouteToCellException ex) {
        LOGGER.warn(Exceptions.meaningfulMessage(ex));
        throw new InternalServerErrorException(ex);
    }
    return fileAttributes;
}