Examples with Transfer org.commonjava.maven.galley.model.Transfer used on opensource projects

Example 6 with Transfer

use of org.commonjava.maven.galley.model.Transfer in project galley by Commonjava.

the class ChecksummingTransferDecoratorTest method assertRead.

private void assertRead(final Transfer txfr, final byte[] data, final EventMetadata em, final boolean checksumFileExists, final boolean metadataConsumerContains) throws IOException {
    try (InputStream stream = txfr.openInputStream(false, em)) {
        logger.info("Reading stream");
        byte[] resultData = IOUtils.toByteArray(stream);
        logger.debug("Result is {} bytes", resultData.length);
        assertThat(Arrays.equals(resultData, data), equalTo(true));
    }
    logger.debug("Verifying .md5 file is {}", checksumFileExists ? "available" : "misssing");
    final Transfer md5Txfr = txfr.getSiblingMeta(".md5");
    assertThat(md5Txfr.exists(), equalTo(checksumFileExists));
    TransferMetadata metadata = metadataConsumer.getMetadata(txfr);
    if (metadataConsumerContains) {
        logger.debug("Verifying MD5 in metadata consumer is available");
        assertThat(metadata, notNullValue());
        String actualMd5 = metadata.getDigests().get(MD5);
        String expectedMd5 = md5Hex(data);
        logger.debug("Verifying actual MD5 content: '{}' vs. expected: '{}'", actualMd5, expectedMd5);
        assertThat(actualMd5, equalTo(expectedMd5));
    } else {
        logger.debug("Verifying MD5 in metadata consumer is missing");
        assertThat(metadata, nullValue());
    }
}

Example 7 with Transfer

use of org.commonjava.maven.galley.model.Transfer in project galley by Commonjava.

the class ChecksummingTransferDecoratorTest method forceChecksumOnReadWhenChecksumsAreDisabledForReads.

@Test
public void forceChecksumOnReadWhenChecksumsAreDisabledForReads() throws Exception {
    fixture.setDecorator(new ChecksummingTransferDecorator(Collections.<TransferOperation>emptySet(), new SpecialPathManagerImpl(), false, false, metadataConsumer, new Md5GeneratorFactory()));
    fixture.initMissingComponents();
    fixture.getCache().startReporting();
    String path = "my-path.txt";
    final Transfer txfr = fixture.getCache().getTransfer(new ConcreteResource(new SimpleLocation("test:uri"), path));
    File f = new File(temp.getRoot(), "cache/test:uri");
    f = new File(f, path);
    byte[] data = "This is a test with a bunch of data and some other stuff, in a big box sealed with chewing gum".getBytes();
    FileUtils.writeByteArrayToFile(f, data);
    logger.info("Opening transfer input stream");
    EventMetadata forceEventMetadata = new EventMetadata().set(FORCE_CHECKSUM, TRUE);
    try (InputStream stream = txfr.openInputStream(false, forceEventMetadata)) {
        logger.info("Reading stream");
        byte[] resultData = IOUtils.toByteArray(stream);
        logger.debug("Result is {} bytes", resultData.length);
        assertThat(Arrays.equals(resultData, data), equalTo(true));
    }
    final MessageDigest md = MessageDigest.getInstance("MD5");
    md.update(data);
    final byte[] digest = md.digest();
    final String digestHex = Hex.encodeHexString(digest);
    logger.debug("Verifying MD5 in metadata consumer");
    TransferMetadata metadata = metadataConsumer.getMetadata(txfr);
    assertThat(metadata, notNullValue());
    Map<ContentDigest, String> digests = metadata.getDigests();
    assertThat(digests, CoreMatchers.<Map<ContentDigest, String>>notNullValue());
    assertThat(digests.get(MD5), equalTo(digestHex));
}

Example 8 with Transfer

use of org.commonjava.maven.galley.model.Transfer in project indy by Commonjava.

the class IndexingContentManagerDecorator method getIndexedTransfer.

private Transfer getIndexedTransfer(final StoreKey storeKey, final StoreKey topKey, final String path, final TransferOperation op) throws IndyWorkflowException {
    Logger logger = LoggerFactory.getLogger(getClass());
    logger.trace("Looking for indexed path: {} in: {} (entry point: {})", path, storeKey, topKey);
    try {
        ArtifactStore store = storeDataManager.getArtifactStore(storeKey);
        if (store.isDisabled()) {
            logger.info("Content not available in index caching layer due to store disabled for {} in group {}", storeKey, topKey);
            return null;
        }
    } catch (IndyDataException e) {
        logger.error(String.format("Failed to lookup store: %s (in membership of: %s). Reason: %s", storeKey, topKey, e.getMessage()), e);
        //TODO: Need further check if it is suitable to throw a IndyWorkflowException here.
        return null;
    }
    IndexedStorePath storePath = indexManager.getIndexedStorePath(storeKey, path);
    if (storePath != null) {
        Transfer transfer = delegate.getTransfer(storeKey, path, op);
        if (transfer == null || !transfer.exists()) {
            logger.trace("Found obsolete index entry: {}. De-indexing from: {} and {}", storeKey, topKey);
            // something happened to the underlying Transfer...de-index it, and don't return it.
            indexManager.deIndexStorePath(storeKey, path);
            if (topKey != null) {
                indexManager.deIndexStorePath(topKey, path);
            }
        } else {
            logger.trace("Found it!");
            return transfer;
        }
    }
    return null;
}

Example 9 with Transfer

use of org.commonjava.maven.galley.model.Transfer in project indy by Commonjava.

the class StoreContentListener method listPaths.

private Set<String> listPaths(StoreKey key, Predicate<? super String> pathFilter) {
    Logger logger = LoggerFactory.getLogger(getClass());
    Transfer root = null;
    try {
        Set<String> paths = new HashSet<>();
        root = directContentAccess.getTransfer(key, ROOT);
        root.lockWrite();
        List<Transfer> toProcess = new ArrayList<>();
        toProcess.add(root);
        while (!toProcess.isEmpty()) {
            Transfer next = toProcess.remove(0);
            try {
                Stream.of(next.list()).forEach(filename -> {
                    Transfer t = next.getChild(filename);
                    if (t.isDirectory()) {
                        logger.debug("Adding directory path for processing: {}", t.getPath());
                        toProcess.add(t);
                    } else {
                        logger.trace("Testing file path: {}", t.getPath());
                        if (pathFilter.test(t.getPath())) {
                            logger.trace("Adding file path to results: {}", t.getPath());
                            paths.add(t.getPath());
                        } else {
                            logger.trace("Skipping file path: {}", t.getPath());
                        }
                    }
                });
            } catch (IOException e) {
                logger.error(String.format("Failed to list contents of: %s. Reason: %s", next, e), e);
            }
        }
        return paths;
    } catch (IndyWorkflowException e) {
        logger.error(String.format("Failed to retrieve root directory reference for: %s. Reason: %s", key, e), e);
    } finally {
        if (root != null) {
            root.unlock();
        }
    }
    return Collections.emptySet();
}

Example 10 with Transfer

use of org.commonjava.maven.galley.model.Transfer in project indy by Commonjava.

the class PromotionManager method getTransfersForPaths.

private List<Transfer> getTransfersForPaths(final StoreKey source, final Set<String> paths) throws IndyWorkflowException {
    final List<Transfer> contents = new ArrayList<Transfer>();
    for (final String path : paths) {
        final Transfer txfr = downloadManager.getStorageReference(source, path);
        if (txfr == null || !txfr.exists()) {
            logger.warn("Cannot promote path: '{}' from source: '{}'. It does not exist!", path, source);
            // TODO: Fail??
            continue;
        }
        contents.add(txfr);
    }
    return contents;
}