Examples with SpecialPathInfo org.commonjava.maven.galley.model.SpecialPathInfo used on opensource projects

Example 16 with SpecialPathInfo

use of org.commonjava.maven.galley.model.SpecialPathInfo in project indy by Commonjava.

the class PrefetchWorker method run.

@Override
public void run() {
    if (resources == null || resources.isEmpty()) {
        logger.trace("No resources for downloading");
        return;
    }
    logger.trace("Start downloading: {}", resources);
    final AtomicBoolean scheduled = new AtomicBoolean(false);
    for (Map.Entry<RemoteRepository, List<RescanableResourceWrapper>> entry : resources.entrySet()) {
        final RemoteRepository repo = entry.getKey();
        final List<RescanableResourceWrapper> res = entry.getValue();
        res.forEach(r -> {
            try {
                final String path = r.getResource().getPath();
                if (path == null || path.equals("") || path.endsWith("/") || path.endsWith(LISTING_HTML_FILE)) {
                    // If this is a rescan prefetch, we need to clear the listing cache and re-fetch from external
                    if (r.isRescan()) {
                        transfers.delete(new ConcreteResource(r.getResource().getLocation(), path, ".listing.txt"));
                    }
                    final List<RescanablePath> dirPaths = buildPaths(r.getResource(), r.isRescan());
                    logger.trace("{} is folder, will use it to schedule new Resources {}", r, dirPaths);
                    frontier.scheduleRepo(repo, dirPaths);
                    scheduled.set(true);
                } else {
                    // if repo has path masks, we need to check that first to only download path mask enabled artifacts.
                    if (PathMaskChecker.checkMask(repo, path)) {
                        // If this is a rescan prefetch, and artifact is metadata, we need to clear it and re-fetch from external
                        if (r.isRescan()) {
                            final SpecialPathInfo spi = specialPathManager.getSpecialPathInfo(r.getResource());
                            if (spi != null && spi.isMetadata()) {
                                transfers.delete(r.getResource());
                            }
                        }
                        logger.trace("{} is file", r);
                        transfers.retrieve(r.getResource());
                    } else {
                        logger.trace("Path {} in repo {} not available for path mask {}", path, repo, repo.getPathMaskPatterns());
                    }
                }
            } catch (TransferException e) {
                logger.error("Download failed during prefetch because of transfer getting failed for {}, Reason: {}", r, e.getMessage());
            }
        });
    }
    if (scheduled.get()) {
        prefetchManager.triggerWorkers();
    }
}

Example 17 with SpecialPathInfo

use of org.commonjava.maven.galley.model.SpecialPathInfo in project indy by Commonjava.

the class DefaultDownloadManager method deleteCache.

/**
 * clean just the cache (storage of groups and remote repos)
 */
private boolean deleteCache(ArtifactStore store, String path, EventMetadata eventMetadata) throws IndyWorkflowException {
    if (store.getKey().getType() == hosted) {
        SpecialPathInfo info = specialPathManager.getSpecialPathInfo(path, store.getPackageType());
        if (info == null || !info.isMetadata()) {
            return false;
        }
    }
    final Transfer item = getStorageReference(store, path == null ? ROOT_PATH : path);
    logger.trace("Delete cache, item: {}", item);
    return doDelete(item, eventMetadata);
}

Example 18 with SpecialPathInfo

use of org.commonjava.maven.galley.model.SpecialPathInfo in project galley by Commonjava.

the class TransferManagerImpl method doDelete.

private Boolean doDelete(final Transfer item, final EventMetadata eventMetadata) throws TransferException {
    if (!item.exists()) {
        return false;
    }
    Logger contentLogger = LoggerFactory.getLogger(DELETE_CONTENT_LOG);
    contentLogger.info("BEGIN: Delete {} ({})", item.getResource(), eventMetadata);
    SpecialPathInfo specialPathInfo = specialPathManager.getSpecialPathInfo(item, eventMetadata.getPackageType());
    if (specialPathInfo != null && !specialPathInfo.isDeletable()) {
        throw new TransferException("Deleting not allowed for: %s", item);
    }
    if (item.isDirectory()) {
        String[] listing;
        try {
            listing = item.list();
        } catch (final IOException e) {
            throw new TransferException("Delete failed: {}. Reason: cannot list directory due to: {}", e, item, e.getMessage());
        }
        for (final String sub : listing) {
            if (!doDelete(item.getChild(sub), eventMetadata)) {
                contentLogger.info("FAIL: Delete: {}", item.getResource());
                return false;
            }
        }
    } else {
        try {
            if (!item.delete(true, eventMetadata)) {
                throw new TransferException("Failed to delete: {}.", item);
            }
        } catch (final IOException e) {
            throw new TransferException("Failed to delete stored location: {}. Reason: {}", e, item, e.getMessage());
        }
    }
    contentLogger.info("FINISH: Delete: {}", item.getResource());
    return true;
}

Example 19 with SpecialPathInfo

use of org.commonjava.maven.galley.model.SpecialPathInfo in project indy by Commonjava.

the class ContentAccessHandler method doHead.

public Response doHead(final String packageType, final String type, final String name, final String path, final Boolean cacheOnly, final String baseUri, final HttpServletRequest request, EventMetadata eventMetadata, final Consumer<ResponseBuilder> builderModifier) {
    setContext(PACKAGE_TYPE, packageType);
    setContext(PATH, path);
    if (!PackageTypes.contains(packageType)) {
        ResponseBuilder builder = Response.status(400);
        if (builderModifier != null) {
            builderModifier.accept(builder);
        }
        return builder.build();
    }
    final StoreType st = StoreType.get(type);
    final StoreKey sk = new StoreKey(packageType, st, name);
    eventMetadata = eventMetadata.set(ContentManager.ENTRY_POINT_STORE, sk);
    setContext(CONTENT_ENTRY_POINT, sk.toString());
    Response response = null;
    if (isDirectoryPath(path, request)) {
        response = RequestUtils.redirectContentListing(packageType, type, name, path, request, builderModifier);
    } else {
        try {
            Transfer item = null;
            logger.info("Checking existence of: {}:{} (cache only? {})", sk, path, cacheOnly);
            boolean exists;
            if (Boolean.TRUE.equals(cacheOnly)) {
                logger.debug("Calling getTransfer()");
                item = contentController.getTransfer(sk, path, TransferOperation.DOWNLOAD);
                exists = item != null && item.exists();
                SpecialPathInfo spi = specialPathManager.getSpecialPathInfo(item, packageType);
                setContext(METADATA_CONTENT, Boolean.toString(spi != null && spi.isMetadata()));
                logger.debug("Got transfer reference: {}", item);
            } else {
                // Use exists for remote repo to avoid downloading file. Use getTransfer for everything else (hosted, cache-only).
                // Response will be composed of metadata by getHttpMetadata which get metadata from .http-metadata.json (because HTTP transport always writes a .http-metadata.json
                // file when it makes a request). This file stores the HTTP response status code and headers regardless exist returning true or false.
                logger.debug("Calling remote exists()");
                exists = contentController.exists(sk, path);
                logger.debug("Got remote exists: {}", exists);
            }
            if (exists) {
                HttpExchangeMetadata httpMetadata = item != null ? contentController.getHttpMetadata(item) : contentController.getHttpMetadata(sk, path);
                // For hosted / group repo, artifacts will also have metadata generated. This will fetch the item by content get method.
                if (item == null) {
                    logger.debug("Retrieving: {}:{} for existence test", sk, path);
                    item = contentController.get(sk, path, eventMetadata);
                    logger.debug("Got retrieved transfer reference: {}", item);
                }
                if (MDC.get(METADATA_CONTENT) != null) {
                    SpecialPathInfo spi = specialPathManager.getSpecialPathInfo(item, packageType);
                    setContext(METADATA_CONTENT, Boolean.toString(spi != null && spi.isMetadata()));
                }
                logger.trace("Building 200 response. Using HTTP metadata: {}", httpMetadata);
                setContext(HTTP_STATUS, String.valueOf(200));
                final ResponseBuilder builder = Response.ok();
                // restrict the npm header contentType with json to avoid some parsing error
                String contentType = packageType.equals(NPM_PKG_KEY) ? MediaType.APPLICATION_JSON : contentController.getContentType(path);
                responseHelper.setInfoHeaders(builder, item, sk, path, true, contentType, httpMetadata);
                if (!path.endsWith("/")) {
                    // Content hashing headers
                    builder.header(ApplicationHeader.md5.key(), contentDigester.digest(sk, path, new EventMetadata()).getDigests().get(ContentDigest.MD5));
                    builder.header(ApplicationHeader.sha1.key(), contentDigester.digest(sk, path, new EventMetadata()).getDigests().get(ContentDigest.SHA_1));
                }
                if (builderModifier != null) {
                    builderModifier.accept(builder);
                }
                response = builder.build();
            } else {
                logger.trace("Building 404 (or error) response...");
                if (StoreType.remote == st) {
                    final HttpExchangeMetadata metadata = contentController.getHttpMetadata(sk, path);
                    if (metadata != null) {
                        logger.trace("Using HTTP metadata to build negative response.");
                        response = responseHelper.formatResponseFromMetadata(metadata);
                    }
                }
                if (response == null) {
                    logger.debug("No HTTP metadata; building generic 404 response.");
                    setContext(HTTP_STATUS, String.valueOf(404));
                    ResponseBuilder builder = Response.status(Status.NOT_FOUND);
                    if (builderModifier != null) {
                        builderModifier.accept(builder);
                    }
                    response = builder.build();
                }
            }
        } catch (final IndyWorkflowException e) {
            logger.error(String.format("Failed to download artifact: %s from: %s. Reason: %s", path, name, e.getMessage()), e);
            response = responseHelper.formatResponse(e, builderModifier);
        }
    }
    return response;
}

Example 20 with SpecialPathInfo

use of org.commonjava.maven.galley.model.SpecialPathInfo in project indy by Commonjava.

the class ContentAccessHandler method doGet.

public Response doGet(final String packageType, final String type, final String name, String path, final String baseUri, final HttpServletRequest request, EventMetadata eventMetadata, final Consumer<ResponseBuilder> builderModifier) {
    setContext(PACKAGE_TYPE, packageType);
    setContext(PATH, path);
    if (!PackageTypes.contains(packageType)) {
        ResponseBuilder builder = Response.status(400);
        if (builderModifier != null) {
            builderModifier.accept(builder);
        }
        return builder.build();
    }
    final StoreType st = StoreType.get(type);
    final StoreKey sk = new StoreKey(packageType, st, name);
    eventMetadata = eventMetadata.set(ContentManager.ENTRY_POINT_STORE, sk);
    setContext(CONTENT_ENTRY_POINT, sk.toString());
    final AcceptInfo acceptInfo = jaxRsRequestHelper.findAccept(request, ApplicationContent.text_html);
    final String standardAccept = ApplicationContent.getStandardAccept(acceptInfo.getBaseAccept());
    Response response;
    logger.debug("GET path: '{}' (RAW: '{}')\nIn store: '{}'\nUser addMetadata header is: '{}'\nStandard addMetadata header for that is: '{}'", path, request.getPathInfo(), sk, acceptInfo.getRawAccept(), standardAccept);
    if (isDirectoryPath(path, request)) {
        response = RequestUtils.redirectContentListing(packageType, type, name, path, request, builderModifier);
    } else {
        try {
            logger.debug("START: retrieval of content: {}:{}", sk, path);
            final Transfer item = contentController.get(sk, path, eventMetadata);
            SpecialPathInfo spi = specialPathManager.getSpecialPathInfo(item, packageType);
            setContext(METADATA_CONTENT, Boolean.toString(spi != null && spi.isMetadata()));
            logger.debug("HANDLE: retrieval of content: {}:{}", sk, path);
            if (item == null) {
                return handleMissingContentQuery(sk, path, builderModifier);
            }
            boolean handleLocking = false;
            if (!item.isWriteLocked()) {
                item.lockWrite();
                handleLocking = true;
            }
            try {
                if (!item.exists()) {
                    return handleMissingContentQuery(sk, path, builderModifier);
                } else if (item.isDirectory()) {
                    logger.debug("Getting listing at: {}", path + "/");
                    response = RequestUtils.redirectContentListing(packageType, type, name, path, request, builderModifier);
                } else {
                    logger.debug("RETURNING: retrieval of content: {}:{}", sk, path);
                    // open the stream here to prevent deletion while waiting for the transfer back to the user to start...
                    addFieldToActiveSpan(TRANSFER_SIZE, item.length());
                    InputStream in = item.openInputStream(true, eventMetadata);
                    final ResponseBuilder builder = Response.ok(new TransferStreamingOutput(in, metricsManager, metricsConfig));
                    responseHelper.setInfoHeaders(builder, item, sk, path, true, contentController.getContentType(path), contentController.getHttpMetadata(item));
                    if (builderModifier != null) {
                        builderModifier.accept(builder);
                    }
                    response = builder.build();
                }
            } finally {
                if (handleLocking) {
                    item.unlock();
                }
            }
        } catch (final IOException | IndyWorkflowException e) {
            logger.error(String.format("Failed to download artifact: %s from: %s. Reason: %s", path, name, e.getMessage()), e);
            response = responseHelper.formatResponse(e, builderModifier);
        }
    }
    logger.info("RETURNING RESULT: {}:{}", sk, path);
    return response;
}