Examples with ArtifactStore org.commonjava.indy.model.core.ArtifactStore used on opensource projects

Example 21 with ArtifactStore

use of org.commonjava.indy.model.core.ArtifactStore in project indy by Commonjava.

the class ContentBrowseController method renderResult.

private ContentBrowseResult renderResult(final StoreKey key, final String requestPath, final String browseServiceUrl, final String contentServiceUrl, final UriFormatter uriFormatter, final EventMetadata eventMetadata) throws IndyWorkflowException {
    String path = requestPath.endsWith("/") ? requestPath : requestPath + "/";
    final ArtifactStore store = getStore(key);
    final List<StoreResource> listed = contentManager.list(store, path, eventMetadata);
    final Map<String, Set<String>> listingUrls = new TreeMap<>();
    final String storeBrowseUrl = uriFormatter.formatAbsolutePathTo(browseServiceUrl, key.getType().singularEndpointName(), key.getName());
    final String storeContentUrl = uriFormatter.formatAbsolutePathTo(contentServiceUrl, key.getType().singularEndpointName(), key.getName());
    if (listed != null) {
        // second pass, process the remainder.
        for (int pass = 0; pass < 2; pass++) {
            for (final ConcreteResource res : listed) {
                String p = res.getPath();
                if (pass == 0 && !p.endsWith("/")) {
                    continue;
                }
                if (p.endsWith("-") || p.endsWith("-/")) {
                    // skip npm adduser path to avoid the sensitive info showing.
                    continue;
                } else if (pass == 1) {
                    if (!p.endsWith("/")) {
                        final String dirpath = p + "/";
                        if (listingUrls.containsKey(normalize(storeBrowseUrl, dirpath))) {
                            p = dirpath;
                        }
                    } else {
                        continue;
                    }
                }
                String localUrl;
                if (p.endsWith("/")) {
                    localUrl = normalize(storeBrowseUrl, p);
                } else {
                    // So this means current path is a file not a directory, and needs to construct it to point to content api /api/content
                    localUrl = normalize(storeContentUrl, p);
                }
                Set<String> sources = listingUrls.computeIfAbsent(localUrl, k -> new HashSet<>());
                sources.add(normalize(res.getLocationUri(), res.getPath()));
            }
        }
    }
    final List<String> sources = new ArrayList<>();
    if (listed != null) {
        for (final ConcreteResource res : listed) {
            // KeyedLocation is all we use in Indy.
            logger.debug("Formatting sources URL for: {}", res);
            final KeyedLocation kl = (KeyedLocation) res.getLocation();
            final String uri = uriFormatter.formatAbsolutePathTo(browseServiceUrl, kl.getKey().getType().singularEndpointName(), kl.getKey().getName());
            if (!sources.contains(uri)) {
                logger.debug("adding source URI: '{}'", uri);
                sources.add(uri);
            }
        }
    }
    Collections.sort(sources);
    String parentPath = normalize(parentPath(path));
    if (!parentPath.endsWith("/")) {
        parentPath += "/";
    }
    final String parentUrl;
    if (parentPath.equals(path)) {
        parentPath = null;
        parentUrl = null;
    } else {
        parentUrl = uriFormatter.formatAbsolutePathTo(browseServiceUrl, key.getType().singularEndpointName(), key.getName(), parentPath);
    }
    final List<ContentBrowseResult.ListingURLResult> listingURLResults = new ArrayList<>(listingUrls.size());
    for (String localUrl : listingUrls.keySet()) {
        final String apiPath = localUrl.replace(storeBrowseUrl, "").replace(storeContentUrl, "");
        listingURLResults.add(new ContentBrowseResult.ListingURLResult(apiPath, localUrl, listingUrls.get(localUrl)));
    }
    final ContentBrowseResult result = new ContentBrowseResult();
    result.setListingUrls(listingURLResults);
    result.setParentUrl(parentUrl);
    result.setParentPath(parentPath);
    result.setPath(path);
    result.setStoreKey(key);
    result.setStoreBrowseUrl(storeBrowseUrl);
    result.setStoreContentUrl(storeContentUrl);
    result.setBaseBrowseUrl(browseServiceUrl);
    result.setBaseContentUrl(contentServiceUrl);
    result.setSources(sources);
    return result;
}

Example 22 with ArtifactStore

use of org.commonjava.indy.model.core.ArtifactStore in project indy by Commonjava.

the class IndexingContentManagerDecorator method retrieve.

@Override
public Transfer retrieve(final ArtifactStore store, final String path, final EventMetadata eventMetadata) throws IndyWorkflowException {
    if (!indexCfg.isEnabled()) {
        return delegate.retrieve(store, path, eventMetadata);
    }
    if (store == null) {
        return null;
    }
    logger.trace("Looking for indexed path: {} in: {}", path, store.getKey());
    Transfer transfer = getIndexedTransfer(store.getKey(), null, path, TransferOperation.DOWNLOAD, eventMetadata);
    if (transfer != null) {
        logger.debug("Found indexed transfer: {}. Returning.", transfer);
        return transfer;
    } else if (isAuthoritativelyMissing(store)) {
        logger.debug("Not found indexed transfer: {} and authoritative index switched on. Considering not found and return null.", transfer);
        return null;
    }
    StoreType type = store.getKey().getType();
    if (StoreType.group == type) {
        ConcreteResource resource = new ConcreteResource(LocationUtils.toLocation(store), path);
        if (nfc.isMissing(resource)) {
            logger.debug("{} is marked as missing. Returning null.", resource);
            return null;
        }
        logger.debug("No group index hits. Devolving to member store indexes.");
        KeyedLocation location = LocationUtils.toLocation(store);
        SpecialPathInfo specialPathInfo = specialPathManager.getSpecialPathInfo(location, path, store.getPackageType());
        if (specialPathInfo == null || !specialPathInfo.isMergable()) {
            if (PathMaskChecker.checkMask(store, path)) {
                transfer = getTransferFromConstituents(((Group) store).getConstituents(), resource, path, store, memberKey -> {
                    try {
                        ArtifactStore member = storeDataManager.getArtifactStore(memberKey);
                        if (member == null) {
                            logger.trace("Cannot find store for key: {}", memberKey);
                        } else {
                            return retrieve(member, path, eventMetadata);
                        }
                    } catch (IndyDataException e) {
                        logger.error(String.format("Failed to lookup store: %s (in membership of: %s). Reason: %s", memberKey, store.getKey(), e.getMessage()), e);
                    }
                    return null;
                });
                nfcForGroup(store, transfer, resource);
                return transfer;
            } else {
                return null;
            }
        } else {
            logger.debug("Merged content. Delegating to main content manager for: {} in: {}", path, store);
            transfer = delegate.retrieve(store, path, eventMetadata);
            if (!exists(transfer)) {
                Boolean metadataGenerated = (Boolean) eventMetadata.get(GROUP_METADATA_GENERATED);
                Boolean metadataExists = (Boolean) eventMetadata.get(GROUP_METADATA_EXISTS);
                if (Boolean.TRUE.equals(metadataGenerated) || Boolean.TRUE.equals(metadataExists)) {
                    // metadata generated/exists but missing due to membership change, not add to nfc so next req can retry
                    ;
                } else // don't track NFC for hosted repos
                {
                    nfc.addMissing(resource);
                }
            }
            return transfer;
        }
    }
    logger.trace("Delegating retrieve call for concrete repository: {}/{}", store, path);
    transfer = delegate.retrieve(store, path, eventMetadata);
    if (exists(transfer) && indexCfg.isEnabled()) {
        logger.debug("Got transfer from delegate: {} (will index)", transfer);
        indexManager.indexTransferIn(transfer, store.getKey());
    }
    logger.debug("Returning transfer: {}", transfer);
    return transfer;
}

Example 23 with ArtifactStore

use of org.commonjava.indy.model.core.ArtifactStore in project indy by Commonjava.

the class AuthoritativeIndexSettingManager method setAuthoritativeManager.

public void setAuthoritativeManager(@Observes final ArtifactStorePostUpdateEvent event) {
    if (indexCfg == null || !indexCfg.isAuthoritativeIndex()) {
        return;
    }
    final Collection<ArtifactStore> stores = event.getChanges();
    stores.stream().filter(store -> store.getKey().getType() == StoreType.hosted).forEach(store -> {
        final HostedRepository hosted = (HostedRepository) store;
        if (hosted.isReadonly()) {
            hosted.setAuthoritativeIndex(true);
        } else {
            hosted.setAuthoritativeIndex(false);
        }
    });
}

Example 24 with ArtifactStore

use of org.commonjava.indy.model.core.ArtifactStore in project indy by Commonjava.

the class IndexingContentManagerDecorator method store.

@Override
public Transfer store(final List<? extends ArtifactStore> stores, final StoreKey topKey, final String path, final InputStream stream, final TransferOperation op, final EventMetadata eventMetadata) throws IndyWorkflowException {
    if (!indexCfg.isEnabled()) {
        return delegate.store(stores, topKey, path, stream, op, eventMetadata);
    }
    Transfer transfer = delegate.store(stores, topKey, path, stream, op, eventMetadata);
    if (transfer != null) {
        if (indexCfg.isEnabled()) {
            indexManager.indexTransferIn(transfer, topKey);
        }
        try {
            ArtifactStore topStore = storeDataManager.getArtifactStore(topKey);
            nfc.clearMissing(new ConcreteResource(LocationUtils.toLocation(topStore), path));
            if (indexCfg.isEnabled()) {
                // We should deIndex the path for all parent groups because the new content of the path
                // may change the content index sequence based on the constituents sequence in parent groups
                indexManager.deIndexStorePath(topKey, path);
            }
        } catch (IndyDataException e) {
            Logger logger = LoggerFactory.getLogger(getClass());
            logger.error(String.format("Failed to retrieve top store: %s for NFC management. Reason: %s", topKey, e.getMessage()), e);
        }
    }
    return transfer;
}

Example 25 with ArtifactStore

use of org.commonjava.indy.model.core.ArtifactStore in project indy by Commonjava.

the class PackageTypedStorageMigrationAction method doMigrate.

private boolean doMigrate() throws IndyLifecycleException {
    Set<ArtifactStore> stores;
    try {
        stores = storeDataManager.getAllArtifactStores();
    } catch (IndyDataException e) {
        throw new IndyLifecycleException("Cannot retrieve list of repositories and groups in order to review storage locations. Reason: %s", e, e.getMessage());
    }
    File storageRoot = config.getStorageRootDirectory();
    File nfsStorageRoot = config.getNFSStorageRootDirectory();
    int migrations = 0;
    Map<File, File> unmigratedNfs = new HashMap<>();
    for (ArtifactStore store : stores) {
        File old = deprecatedStoragePath(storageRoot, store);
        File migrated = packageTypedStoragePath(storageRoot, store);
        if (old.exists()) {
            logger.info("Attempting to migrate existing storage from old directory structure: {} " + "to package-typed structure: {}", old, migrated);
            try {
                if (migrated.exists()) {
                    FileUtils.copyDirectory(old, migrated);
                    FileUtils.forceDelete(old);
                } else {
                    FileUtils.moveDirectory(old, migrated);
                }
                migrations++;
            } catch (IOException e) {
                throw new IndyLifecycleException("Failed to migrate: %s to: %s. Reason: %s", e, old, migrated);
            }
        }
        if (nfsStorageRoot != null) {
            File oldNfs = deprecatedStoragePath(nfsStorageRoot, store);
            File migratedNfs = packageTypedStoragePath(nfsStorageRoot, store);
            if (oldNfs.exists() && !migratedNfs.exists()) {
                unmigratedNfs.put(oldNfs, migratedNfs);
            }
        }
    }
    if (!unmigratedNfs.isEmpty()) {
        StringBuilder sb = new StringBuilder();
        sb.append("ERROR: Un-migrated directories detected on NFS storage!!!!");
        sb.append("\n\nThese directories still use the old <type>/<name> directory format. Indy now supports");
        sb.append("\nmultiple package types, and the storage format has changed accordingly. The new format is:");
        sb.append("\n\n    <package-type>/<type>/<name>");
        sb.append("\n\nPlease migrate these NFS directories manually. For Maven repositories:");
        sb.append("\n\n    maven/<type>/<name>");
        sb.append("\n\nFor HTTProx repositories (httprox_*):");
        sb.append("\n\n    generic-http/<type>/<name>");
        sb.append("\n\nThe following directories were detected:\n");
        unmigratedNfs.forEach((o, n) -> sb.append("\n    ").append(o).append("  =>  ").append(n));
        sb.append("\n\n");
        logger.error(sb.toString());
        throw new IndyLifecycleException("Un-migrated NFS directories detected. Indy cannot start until this has been resolved.");
    }
    return migrations > 0;
}