Examples with TransferMetadata org.commonjava.maven.galley.io.checksum.TransferMetadata used on opensource projects

Example 6 with TransferMetadata

use of org.commonjava.maven.galley.io.checksum.TransferMetadata in project indy by Commonjava.

the class KojiMavenMetadataProviderTest method initKojiClient.

private void initKojiClient(String exchangeName, boolean verifyArtifacts) throws BindException, IOException, GalleyInitException, IndyDataException {
    StoreDataManager storeDataManager = new MemoryStoreDataManager(true);
    if (verifyArtifacts) {
        RemoteRepository verifyRepo = new RemoteRepository(MAVEN_PKG_KEY, VERIFY_REPO, server.formatUrl(VERIFY_BASEPATH));
        storeDataManager.storeArtifactStore(verifyRepo, new ChangeSummary(ChangeSummary.SYSTEM_USER, "Adding verification repo"), false, true, new EventMetadata());
        kojiConfig.setArtifactAuthorityStore(new StoreKey(MAVEN_PKG_KEY, remote, VERIFY_REPO).toString());
    }
    String resourceBase = "koji-metadata/" + exchangeName;
    configureKojiServer(server, KOJI_BASEPATH, counter, resourceBase, verifyArtifacts, VERIFY_BASEPATH);
    kojiClient = new KojiClient(kojiConfig, new MemoryPasswordManager(), Executors.newCachedThreadPool());
    GalleyCore galley = new GalleyCoreBuilder(new FileCacheProviderFactory(temp.newFolder("cache"))).withEnabledTransports(new HttpClientTransport(new HttpImpl(new org.commonjava.maven.galley.auth.MemoryPasswordManager()), new IndyObjectMapper(true), new GlobalHttpConfiguration())).build();
    DownloadManager downloadManager = new DefaultDownloadManager(storeDataManager, galley.getTransferManager(), new IndyLocationExpander(storeDataManager));
    DirectContentAccess directContentAccess = new DefaultDirectContentAccess(downloadManager, Executors.newCachedThreadPool());
    DirectContentAccess dca = new DefaultDirectContentAccess(downloadManager, Executors.newSingleThreadExecutor());
    ContentDigester contentDigester = new DefaultContentDigester(dca, new CacheHandle<String, TransferMetadata>("content-metadata", contentMetadata));
    KojiBuildAuthority buildAuthority = new KojiBuildAuthority(kojiConfig, new StandardTypeMapper(), kojiClient, storeDataManager, contentDigester, directContentAccess);
    provider = new KojiMavenMetadataProvider(this.cache, kojiClient, buildAuthority, kojiConfig);
}

Example 7 with TransferMetadata

use of org.commonjava.maven.galley.io.checksum.TransferMetadata in project indy by Commonjava.

the class DefaultContentDigester method getContentMetadata.

@Override
public synchronized TransferMetadata getContentMetadata(final Transfer transfer) {
    String cacheKey = generateCacheKey(transfer);
    logger.trace("Getting TransferMetadata for: {}", cacheKey);
    TransferMetadata metadata = metadataCache.get(cacheKey);
    if (metadata != null) {
        logger.trace("[CACHE HIT] Returning content metadata for: {}\n\n{}\n\n", cacheKey, metadata);
    } else {
        logger.trace("[CACHE MISS] Cannot find content metadata for: {}!", cacheKey);
    }
    return metadata;
}

Example 8 with TransferMetadata

use of org.commonjava.maven.galley.io.checksum.TransferMetadata in project indy by Commonjava.

the class DefaultContentDigester method digest.

public TransferMetadata digest(final StoreKey key, final String path, final EventMetadata eventMetadata) throws IndyWorkflowException {
    final Transfer transfer = directContentAccess.getTransfer(key, path);
    if (transfer == null || !transfer.exists()) {
        return new TransferMetadata(Collections.emptyMap(), 0L);
    }
    TransferMetadata meta = getContentMetadata(transfer);
    if (meta != null) {
        return meta;
    }
    String cacheKey = generateCacheKey(transfer);
    logger.debug("TransferMetadata missing for: {}. Re-reading with FORCE_CHECKSUM now to calculate it.", cacheKey);
    // FIXME: This IS NOT WORKING! It results in NPE for callers assuming a non-null TransferMetadata response.
    EventMetadata forcedEventMetadata = new EventMetadata(eventMetadata).set(FORCE_CHECKSUM, Boolean.TRUE);
    try (InputStream stream = transfer.openInputStream(false, forcedEventMetadata)) {
        // depend on ChecksummingTransferDecorator to calculate / store metadata as this gets read, using
        // the FORCE_CHECKSUM metadata key to control its generation.
        IOUtils.toByteArray(stream);
    } catch (IOException e) {
        throw new IndyWorkflowException("Failed to calculate checksums (MD5, SHA-256, etc.) for: %s. Reason: %s", e, transfer, e.getMessage());
    }
    logger.debug("Retrying TransferMetadata retrieval from cache for: {} after recalculating", cacheKey);
    return getContentMetadata(transfer);
}

Example 9 with TransferMetadata

use of org.commonjava.maven.galley.io.checksum.TransferMetadata in project indy by Commonjava.

the class FoloTrackingListener method createEntry.

private TrackedContentEntry createEntry(final TrackingKey trackingKey, final StoreKey affectedStore, final AccessChannel accessChannel, final String path, final StoreEffect effect, final EventMetadata eventMetadata) throws IndyWorkflowException {
    TrackedContentEntry entry = null;
    final Transfer txfr = downloadManager.getStorageReference(affectedStore, path);
    if (txfr != null) {
        try {
            String remoteUrl = null;
            if (StoreType.remote == affectedStore.getType()) {
                final RemoteRepository repo = (RemoteRepository) storeManager.getArtifactStore(affectedStore);
                if (repo != null) {
                    remoteUrl = UrlUtils.buildUrl(repo.getUrl(), path);
                }
            }
            TransferMetadata artifactData = contentDigester.digest(affectedStore, path, eventMetadata);
            Map<ContentDigest, String> digests = artifactData.getDigests();
            //TODO: As localUrl needs a apiBaseUrl which is from REST service context, to avoid deep propagate
            //      of it, this step will be done in REST layer. Will think better way in the future.
            entry = new TrackedContentEntry(trackingKey, affectedStore, accessChannel, remoteUrl, path, effect, artifactData.getSize(), digests.get(ContentDigest.MD5), digests.get(ContentDigest.SHA_1), digests.get(ContentDigest.SHA_256));
        } catch (final IndyDataException e) {
            throw new IndyWorkflowException("Cannot retrieve RemoteRepository: %s to calculate remote URL for: %s. Reason: %s", e, trackingKey, path, e.getMessage());
        } catch (final MalformedURLException e) {
            throw new IndyWorkflowException("Cannot format URL. Reason: %s", e, e.getMessage());
        }
    }
    return entry;
}

Example 10 with TransferMetadata

use of org.commonjava.maven.galley.io.checksum.TransferMetadata in project indy by Commonjava.

the class FoloAdminController method recalculate.

private TrackedContentEntry recalculate(final TrackedContentEntry entry) throws IndyWorkflowException {
    StoreKey affectedStore = entry.getStoreKey();
    String path = entry.getPath();
    AccessChannel channel = entry.getAccessChannel();
    Transfer transfer = contentManager.getTransfer(affectedStore, path, entry.getEffect() == StoreEffect.UPLOAD ? TransferOperation.UPLOAD : TransferOperation.DOWNLOAD);
    contentDigester.removeMetadata(transfer);
    TransferMetadata artifactData = contentDigester.digest(affectedStore, path, new EventMetadata(channel.packageType()));
    Map<ContentDigest, String> digests = artifactData.getDigests();
    return new TrackedContentEntry(entry.getTrackingKey(), affectedStore, channel, entry.getOriginUrl(), path, entry.getEffect(), artifactData.getSize(), digests.get(ContentDigest.MD5), digests.get(ContentDigest.SHA_1), digests.get(ContentDigest.SHA_256));
}