Examples with ArtifactStore org.commonjava.indy.model.core.ArtifactStore used on opensource projects

Example 96 with ArtifactStore

use of org.commonjava.indy.model.core.ArtifactStore in project indy by Commonjava.

the class HostedContentIndexRescanManager method indexPreRescan.

public void indexPreRescan(@Observes final ArtifactStorePreRescanEvent e) throws IndyWorkflowException {
    if (!indexConfig.isEnabled()) {
        LOGGER.debug("Content index is disabled.");
        return;
    }
    Collection<ArtifactStore> affectedRepos = e.getStores();
    for (ArtifactStore repo : affectedRepos) {
        if (repo.getType() == StoreType.hosted) {
            final HostedRepository hosted = (HostedRepository) repo;
            LOGGER.trace("Clear content index for {}", hosted.getKey());
            // Remove the content index items for the hosted which will be rescanned
            contentIndexManager.clearAllIndexedPathInStore(hosted);
            // Remove the content index items for the affected groups of the hosted which will be rescanned, note that
            // we will only cared about the items that is from this hosted only but not others(the origin key in
            // IndexedStorePath which hits this hosted)
            contentIndexManager.clearAllIndexedPathWithOriginalStore(hosted);
        }
    }
}

Example 97 with ArtifactStore

use of org.commonjava.indy.model.core.ArtifactStore in project indy by Commonjava.

the class ContentIndexWarmer method warmCaches.

public void warmCaches() {
    if (indexConfig.isWarmerEnabled()) {
        logger.info("Content index warmer enabled, will load all indexes from existed repos.");
        executor.submit(() -> {
            boolean oldAuthIdx = indexConfig.isAuthoritativeIndex();
            indexConfig.setAuthoritativeIndex(false);
            try {
                Map<StoreKey, List<Transfer>> transferMap = new ConcurrentHashMap<>();
                try {
                    List<ArtifactStore> concreteStores = storeDataManager.query().storeTypes(StoreType.hosted, StoreType.remote).getAll();
                    CountDownLatch latch = new CountDownLatch(concreteStores.size());
                    concreteStores.forEach(store -> executor.submit(() -> {
                        try {
                            List<Transfer> transfers = downloadManager.listRecursively(store.getKey(), DownloadManager.ROOT_PATH);
                            transferMap.put(store.getKey(), transfers);
                            transfers.forEach(t -> indexManager.indexTransferIn(t, store.getKey()));
                        } catch (IndyWorkflowException e) {
                            logger.warn("Failed to retrieve root directory of storage for: " + store.getKey(), e);
                        } finally {
                            latch.countDown();
                        }
                    }));
                    try {
                        latch.await();
                    } catch (InterruptedException e) {
                        logger.info("Manager thread interrupted while waiting for concrete store indexing to complete.");
                        return;
                    }
                    List<Group> groups = storeDataManager.query().storeType(Group.class).getAll();
                    CountDownLatch groupLatch = new CountDownLatch(groups.size());
                    groups.forEach(g -> executor.submit(() -> {
                        StoreKey gkey = g.getKey();
                        try {
                            List<ArtifactStore> stores = storeDataManager.query().getOrderedConcreteStoresInGroup(g.getPackageType(), g.getName());
                            stores.forEach(s -> {
                                List<Transfer> txfrs = transferMap.get(s.getKey());
                                txfrs.forEach(t -> {
                                    if (indexManager.getIndexedStoreKey(gkey, t.getPath()) == null) {
                                        indexManager.indexTransferIn(t, gkey);
                                    }
                                });
                            });
                        } catch (IndyDataException e) {
                            logger.warn("Failed to get ordered concrete stores for group: " + g.getName(), e);
                        } finally {
                            groupLatch.countDown();
                        }
                    }));
                    try {
                        groupLatch.await();
                    } catch (InterruptedException e) {
                        logger.info("Manager thread interrupted while waiting for group indexing to complete.");
                        return;
                    }
                } catch (IndyDataException e) {
                    logger.warn("Content index warm-up failed: %s", e, e.getMessage());
                }
                logger.info("Content index cache has been re-established.");
            } finally {
                indexConfig.setAuthoritativeIndex(oldAuthIdx);
            }
        });
    } else {
        logger.info("Content index warmer is not enabled.");
    }
}

Example 98 with ArtifactStore

use of org.commonjava.indy.model.core.ArtifactStore in project indy by Commonjava.

the class IndexingContentManagerDecorator method store.

@Override
@Measure
public Transfer store(final ArtifactStore store, final String path, final InputStream stream, final TransferOperation op, final EventMetadata eventMetadata) throws IndyWorkflowException {
    if (!indexCfg.isEnabled()) {
        return delegate.store(store, path, stream, op, eventMetadata);
    }
    Logger logger = LoggerFactory.getLogger(getClass());
    logger.trace("Storing: {} in: {} from indexing level", path, store.getKey());
    Transfer transfer = delegate.store(store, path, stream, op, eventMetadata);
    if (transfer != null) {
        if (indexCfg.isEnabled()) {
            logger.trace("Indexing: {} in: {}", transfer, store.getKey());
            indexManager.indexTransferIn(transfer, store.getKey());
        }
        nfc.clearMissing(new ConcreteResource(LocationUtils.toLocation(store), path));
        // may change the content index sequence based on the constituents sequence in parent groups
        if (store.getType() == StoreType.hosted) {
            final String name = String.format("ContentIndexStoreDeIndex-store(%s)-path(%s)", store.getKey(), path);
            final String context = String.format("Class: %s, method: %s, store: %s, path: %s", this.getClass().getName(), "store", store.getKey(), path);
            storeDataManager.asyncGroupAffectedBy(new StoreDataManager.ContextualTask(name, context, () -> {
                try {
                    Set<Group> groups = storeDataManager.affectedBy(Arrays.asList(store.getKey()), eventMetadata);
                    if (groups != null && !groups.isEmpty() && indexCfg.isEnabled()) {
                        groups.forEach(g -> {
                            indexManager.deIndexStorePath(g.getKey(), path);
                            nfc.clearMissing(new ConcreteResource(LocationUtils.toLocation(g), path));
                        });
                    }
                } catch (IndyDataException e) {
                    logger.error(String.format("Failed to get groups which contains: %s for NFC handling. Reason: %s", store.getKey(), e.getMessage()), e);
                }
            }));
        }
    }
    return transfer;
}

Example 99 with ArtifactStore

use of org.commonjava.indy.model.core.ArtifactStore in project indy by Commonjava.

the class IndexingContentManagerDecorator method getTransfer.

@Override
public Transfer getTransfer(final StoreKey storeKey, final String path, final TransferOperation op) throws IndyWorkflowException {
    if (!indexCfg.isEnabled()) {
        return delegate.getTransfer(storeKey, path, op);
    }
    Transfer transfer = getIndexedTransfer(storeKey, null, path, TransferOperation.DOWNLOAD, new EventMetadata());
    if (exists(transfer)) {
        logger.debug("Returning indexed transfer: {}", transfer);
        return transfer;
    }
    ArtifactStore store;
    try {
        store = storeDataManager.getArtifactStore(storeKey);
    } catch (IndyDataException e) {
        throw new IndyWorkflowException("Failed to lookup ArtifactStore: %s for NFC handling. Reason: %s", e, storeKey, e.getMessage());
    }
    if (isAuthoritativelyMissing(store)) {
        logger.debug("Not found indexed transfer: {} and authoritative index switched on. Return null.", transfer);
        return null;
    }
    ConcreteResource resource = new ConcreteResource(LocationUtils.toLocation(store), path);
    StoreType type = storeKey.getType();
    if (StoreType.group == type) {
        Group g = (Group) store;
        if (g == null) {
            throw new IndyWorkflowException("Cannot find requested group: %s", storeKey);
        }
        if (nfc.isMissing(resource)) {
            logger.debug("NFC / MISSING: {}", resource);
            return null;
        }
        logger.debug("No group index hits. Devolving to member store indexes.");
        transfer = getTransferFromConstituents(g.getConstituents(), resource, path, g, memberKey -> getTransfer(memberKey, path, op));
        nfcForGroup(store, transfer, resource);
        return transfer;
    }
    transfer = delegate.getTransfer(storeKey, path, op);
    if (exists(transfer) && indexCfg.isEnabled()) {
        logger.debug("Indexing transfer: {}", transfer);
        indexManager.indexTransferIn(transfer, storeKey);
    }
    return transfer;
}

Example 100 with ArtifactStore

use of org.commonjava.indy.model.core.ArtifactStore in project indy by Commonjava.

the class IndexingContentManagerDecorator method getTransfer.

@Override
public Transfer getTransfer(final ArtifactStore store, final String path, final TransferOperation op) throws IndyWorkflowException {
    if (!indexCfg.isEnabled()) {
        return delegate.getTransfer(store, path, op);
    }
    Transfer transfer = getIndexedTransfer(store.getKey(), null, path, TransferOperation.DOWNLOAD, new EventMetadata());
    if (exists(transfer)) {
        return transfer;
    } else if (isAuthoritativelyMissing(store)) {
        logger.info("Not found indexed transfer: {} and authoritative index switched on. Considering not found and return null.", transfer);
        return null;
    }
    ConcreteResource resource = new ConcreteResource(LocationUtils.toLocation(store), path);
    StoreType type = store.getKey().getType();
    if (StoreType.group == type) {
        if (!nfc.isMissing(resource)) {
            logger.debug("No group index hits. Devolving to member store indexes.");
            transfer = getTransferFromConstituents(((Group) store).getConstituents(), resource, path, store, memberKey -> {
                try {
                    ArtifactStore member = storeDataManager.getArtifactStore(memberKey);
                    return getTransfer(member, path, op);
                } catch (IndyDataException e) {
                    logger.error(String.format("Failed to lookup store: %s (in membership of: %s). Reason: %s", memberKey, store.getKey(), e.getMessage()), e);
                    return null;
                }
            });
            nfcForGroup(store, transfer, resource);
            return transfer;
        } else {
            logger.debug("NFC marks {} as missing. Returning null.", resource);
            return null;
        }
    }
    transfer = delegate.getTransfer(store, path, op);
    // index the transfer only if it exists, it cannot be null at this point
    if (exists(transfer) && indexCfg.isEnabled()) {
        indexManager.indexTransferIn(transfer, store.getKey());
    }
    logger.debug("Returning transfer: {}", transfer);
    return transfer;
}