Examples with ChecksummingTransferDecorator org.commonjava.maven.galley.io.ChecksummingTransferDecorator used on opensource projects

Example 1 with ChecksummingTransferDecorator

use of org.commonjava.maven.galley.io.ChecksummingTransferDecorator in project indy by Commonjava.

the class InfinispanGalleyStorageProvider method setup.

@PostConstruct
public void setup() {
    SpecialPathInfo infoSpi = SpecialPathInfo.from(new FilePatternMatcher(".+\\.info")).setDecoratable(false).setDeletable(false).setListable(false).setPublishable(false).setRetrievable(false).setStorable(false).build();
    specialPathManager.registerSpecialPathInfo(infoSpi);
    transferDecorator = new ChecksummingTransferDecorator(Collections.singleton(TransferOperation.GENERATE), specialPathManager, new Md5GeneratorFactory(), new Sha1GeneratorFactory());
    this.cacheProvider = new GridFileSystemCacheProvider(pathGenerator, fileEventManager, transferDecorator, cacheFs);
}

Example 2 with ChecksummingTransferDecorator

use of org.commonjava.maven.galley.io.ChecksummingTransferDecorator in project galley by Commonjava.

the class ChecksummingTransferDecoratorTest method noChecksumOnReadWhenChecksumsAreDisabledForReads.

@Test
public void noChecksumOnReadWhenChecksumsAreDisabledForReads() throws Exception {
    fixture.setDecorator(new ChecksummingTransferDecorator(Collections.<TransferOperation>emptySet(), new SpecialPathManagerImpl(), false, false, metadataConsumer, new Md5GeneratorFactory()));
    fixture.initMissingComponents();
    fixture.getCache().startReporting();
    String path = "my-path.txt";
    final Transfer txfr = fixture.getCache().getTransfer(new ConcreteResource(new SimpleLocation("test:uri"), path));
    File f = new File(temp.getRoot(), "cache/test:uri");
    f = new File(f, path);
    byte[] data = "This is a test with a bunch of data and some other stuff, in a big box sealed with chewing gum".getBytes();
    FileUtils.writeByteArrayToFile(f, data);
    logger.info("Opening transfer input stream");
    EventMetadata forceEventMetadata = new EventMetadata();
    try (InputStream stream = txfr.openInputStream(false, forceEventMetadata)) {
        logger.info("Reading stream");
        byte[] resultData = IOUtils.toByteArray(stream);
        logger.debug("Result is {} bytes", resultData.length);
        assertThat(Arrays.equals(resultData, data), equalTo(true));
    }
    logger.debug("Verifying .md5 file is missing");
    final Transfer md5Txfr = txfr.getSiblingMeta(".md5");
    assertThat(md5Txfr.exists(), equalTo(false));
    logger.debug("Verifying MD5 in metadata consumer is missing");
    TransferMetadata metadata = metadataConsumer.getMetadata(txfr);
    assertThat(metadata, nullValue());
}

Example 3 with ChecksummingTransferDecorator

use of org.commonjava.maven.galley.io.ChecksummingTransferDecorator in project galley by Commonjava.

the class ChecksummingTransferDecoratorTest method customChecksumReaderFilter.

@Test
public void customChecksumReaderFilter() throws Exception {
    String path = "my-path.txt";
    fixture.setDecorator(new ChecksummingTransferDecorator(new TestDecoratorAdvisor(), new DisabledChecksummingDecoratorAdvisor(), new SpecialPathManagerImpl(), metadataConsumer, new Md5GeneratorFactory()));
    fixture.initMissingComponents();
    fixture.getCache().startReporting();
    final Transfer txfr = fixture.getCache().getTransfer(new ConcreteResource(new SimpleLocation("test:uri"), path));
    File f = new File(temp.getRoot(), "cache/test:uri");
    f = new File(f, path);
    byte[] data = "This is a test with a bunch of data and some other stuff, in a big box sealed with chewing gum".getBytes();
    FileUtils.writeByteArrayToFile(f, data);
    EventMetadata em = new EventMetadata();
    logger.debug("Reading stream with EventMetadata advice: {}", em.get(DO_CHECKSUMS));
    assertRead(txfr, data, em, false, false);
    em = new EventMetadata().set(DO_CHECKSUMS, ChecksummingDecoratorAdvisor.ChecksumAdvice.CALCULATE_NO_WRITE);
    logger.debug("Reading stream with EventMetadata advice: {}", em.get(DO_CHECKSUMS));
    assertRead(txfr, data, em, false, true);
    logger.debug("Removing checksum metadata from consumer");
    metadataConsumer.removeMetadata(txfr);
    em = new EventMetadata().set(DO_CHECKSUMS, ChecksummingDecoratorAdvisor.ChecksumAdvice.CALCULATE_AND_WRITE);
    logger.debug("Reading stream with EventMetadata advice: {}", em.get(DO_CHECKSUMS));
    assertRead(txfr, data, em, true, true);
}

Example 4 with ChecksummingTransferDecorator

use of org.commonjava.maven.galley.io.ChecksummingTransferDecorator in project galley by Commonjava.

the class ChecksummingTransferDecoratorTest method forceChecksumOnReadWhenChecksumsAreDisabledForReads.

@Test
public void forceChecksumOnReadWhenChecksumsAreDisabledForReads() throws Exception {
    fixture.setDecorator(new ChecksummingTransferDecorator(Collections.<TransferOperation>emptySet(), new SpecialPathManagerImpl(), false, false, metadataConsumer, new Md5GeneratorFactory()));
    fixture.initMissingComponents();
    fixture.getCache().startReporting();
    String path = "my-path.txt";
    final Transfer txfr = fixture.getCache().getTransfer(new ConcreteResource(new SimpleLocation("test:uri"), path));
    File f = new File(temp.getRoot(), "cache/test:uri");
    f = new File(f, path);
    byte[] data = "This is a test with a bunch of data and some other stuff, in a big box sealed with chewing gum".getBytes();
    FileUtils.writeByteArrayToFile(f, data);
    logger.info("Opening transfer input stream");
    EventMetadata forceEventMetadata = new EventMetadata().set(FORCE_CHECKSUM, TRUE);
    try (InputStream stream = txfr.openInputStream(false, forceEventMetadata)) {
        logger.info("Reading stream");
        byte[] resultData = IOUtils.toByteArray(stream);
        logger.debug("Result is {} bytes", resultData.length);
        assertThat(Arrays.equals(resultData, data), equalTo(true));
    }
    final MessageDigest md = MessageDigest.getInstance("MD5");
    md.update(data);
    final byte[] digest = md.digest();
    final String digestHex = Hex.encodeHexString(digest);
    logger.debug("Verifying MD5 in metadata consumer");
    TransferMetadata metadata = metadataConsumer.getMetadata(txfr);
    assertThat(metadata, notNullValue());
    Map<ContentDigest, String> digests = metadata.getDigests();
    assertThat(digests, CoreMatchers.<Map<ContentDigest, String>>notNullValue());
    assertThat(digests.get(MD5), equalTo(digestHex));
}

Example 5 with ChecksummingTransferDecorator

use of org.commonjava.maven.galley.io.ChecksummingTransferDecorator in project indy by Commonjava.

the class DefaultGalleyStorageProvider method setup.

@PostConstruct
public void setup() {
    SpecialPathInfo infoSpi = SpecialPathInfo.from(new FilePatternMatcher(".+\\.info")).setDecoratable(false).setDeletable(false).setListable(false).setPublishable(false).setRetrievable(false).setStorable(false).build();
    specialPathManager.registerSpecialPathInfo(infoSpi);
    ChecksummingDecoratorAdvisor readAdvisor = (transfer, op, eventMetadata) -> {
        ChecksummingDecoratorAdvisor.ChecksumAdvice result = NO_DECORATE;
        if (checksumAdvisors != null) {
            for (IndyChecksumAdvisor advisor : checksumAdvisors) {
                Optional<ChecksummingDecoratorAdvisor.ChecksumAdvice> advice = advisor.getChecksumReadAdvice(transfer, op, eventMetadata);
                if (advice.isPresent()) {
                    ChecksummingDecoratorAdvisor.ChecksumAdvice checksumAdvice = advice.get();
                    if (checksumAdvice.ordinal() > result.ordinal()) {
                        result = checksumAdvice;
                        if (checksumAdvice == CALCULATE_AND_WRITE) {
                            break;
                        }
                    }
                }
            }
        }
        logger.debug("Advising {} for {} of: {}", result, op, transfer);
        return result;
    };
    ChecksummingDecoratorAdvisor writeAdvisor = (transfer, op, eventMetadata) -> {
        ChecksummingDecoratorAdvisor.ChecksumAdvice result = NO_DECORATE;
        if (TransferOperation.GENERATE == op) {
            result = CALCULATE_AND_WRITE;
        } else if (checksumAdvisors != null) {
            for (IndyChecksumAdvisor advisor : checksumAdvisors) {
                Optional<ChecksummingDecoratorAdvisor.ChecksumAdvice> advice = advisor.getChecksumWriteAdvice(transfer, op, eventMetadata);
                if (advice.isPresent()) {
                    ChecksummingDecoratorAdvisor.ChecksumAdvice checksumAdvice = advice.get();
                    if (checksumAdvice.ordinal() > result.ordinal()) {
                        result = checksumAdvice;
                        if (checksumAdvice == CALCULATE_AND_WRITE) {
                            break;
                        }
                    }
                }
            }
        }
        logger.debug("Advising {} for {} of: {}", result, op, transfer);
        return result;
    };
    transferDecorator = new TransferDecoratorPipeline(new ChecksummingTransferDecorator(readAdvisor, writeAdvisor, specialPathManager, contentMetadataConsumer, new Md5GeneratorFactory(), new Sha1GeneratorFactory(), new Sha256GeneratorFactory()), new ContentsFilteringTransferDecorator(), new NoCacheTransferDecorator(specialPathManager));
    final File storeRoot = config.getStorageRootDirectory();
    cacheProviderFactory = new PartyLineCacheProviderFactory(storeRoot);
    final File nfsBasedir = config.getNFSStorageRootDirectory();
    if (nfsBasedir != null) {
        if (!nfsBasedir.exists()) {
            nfsBasedir.mkdirs();
        }
        // nfs root can not be created due to some security reason(like permission), will bypass FastLocal provider and use PartyLine
        if (nfsBasedir.exists()) {
            final FastLocalCacheProviderFactory fastLocalFac = new FastLocalCacheProviderFactory(storeRoot, nfsBasedir, new CacheInstanceAdapter(nfsOwnerCache), fastLocalExecutors);
            cacheProviderFactory = new RoutingCacheProviderFactory((resource) -> {
                if (resource != null) {
                    final Location loc = resource.getLocation();
                    // looking for KeyedLocation and StoreType.hosted should be faster than regex on the URI.
                    return ((loc instanceof KeyedLocation) && hosted == ((KeyedLocation) loc).getKey().getType());
                }
                return false;
            }, fastLocalFac, cacheProviderFactory);
        } else {
            logger.warn("[Indy] nfs base dir {} can not be created correctly due to some unknown reasons, will use partyline cache provider as default", nfsBasedir);
        }
    }
    // TODO: Tie this into a config file!
    transportManagerConfig = new TransportManagerConfig();
}