Examples with SpecialPathInfo org.commonjava.maven.galley.model.SpecialPathInfo used on opensource projects

Example 11 with SpecialPathInfo

use of org.commonjava.maven.galley.model.SpecialPathInfo in project galley by Commonjava.

the class TransferManagerImpl method doDelete.

private Boolean doDelete(final Transfer item, final EventMetadata eventMetadata) throws TransferException {
    if (!item.exists()) {
        return false;
    }
    Logger contentLogger = LoggerFactory.getLogger(DELETE_CONTENT_LOG);
    contentLogger.info("BEGIN: Delete {} ({})", item.getResource(), eventMetadata);
    SpecialPathInfo specialPathInfo = specialPathManager.getSpecialPathInfo(item, eventMetadata.getPackageType());
    if (specialPathInfo != null && !specialPathInfo.isDeletable()) {
        throw new TransferException("Deleting not allowed for: %s", item);
    }
    if (item.isDirectory()) {
        String[] listing;
        try {
            listing = item.list();
        } catch (final IOException e) {
            throw new TransferException("Delete failed: {}. Reason: cannot list directory due to: {}", e, item, e.getMessage());
        }
        for (final String sub : listing) {
            if (!doDelete(item.getChild(sub), eventMetadata)) {
                contentLogger.info("FAIL: Delete: {}", item.getResource());
                return false;
            }
        }
    } else {
        try {
            if (!item.delete(true, eventMetadata)) {
                throw new TransferException("Failed to delete: {}.", item);
            }
        } catch (final IOException e) {
            throw new TransferException("Failed to delete stored location: {}. Reason: {}", e, item, e.getMessage());
        }
    }
    contentLogger.info("FINISH: Delete: {}", item.getResource());
    return true;
}

Example 12 with SpecialPathInfo

use of org.commonjava.maven.galley.model.SpecialPathInfo in project galley by Commonjava.

the class TransferManagerImpl method store.

@Override
public Transfer store(final ConcreteResource resource, final InputStream stream, final EventMetadata eventMetadata) throws TransferException {
    SpecialPathInfo specialPathInfo = specialPathManager.getSpecialPathInfo(resource, eventMetadata.getPackageType());
    if (!resource.allowsStoring() || (specialPathInfo != null && !specialPathInfo.isStorable())) {
        throw new TransferException("Storing not allowed for: {}", resource);
    }
    final Transfer target = getCacheReference(resource);
    logger.info("STORE {}", target.getResource());
    OutputStream out = null;
    try {
        out = target.openOutputStream(TransferOperation.UPLOAD, true, eventMetadata);
        copy(stream, out);
    } catch (final IOException e) {
        throw new TransferException("Failed to store: {}. Reason: {}", e, resource, e.getMessage());
    } finally {
        closeQuietly(out);
    }
    return target;
}

Example 13 with SpecialPathInfo

use of org.commonjava.maven.galley.model.SpecialPathInfo in project galley by Commonjava.

the class TransferManagerImpl method doList.

private ListingResult doList(final ConcreteResource resource, final boolean suppressFailures, EventMetadata metadata) throws TransferException {
    final Transfer cachedListing = getCacheReference(resource.getChild(".listing.txt"));
    Set<String> filenames = new HashSet<String>();
    if (cachedListing.exists()) {
        InputStream stream = null;
        try {
            stream = cachedListing.openInputStream();
            filenames.addAll(IOUtils.readLines(stream, "UTF-8"));
            Logger logger = LoggerFactory.getLogger(getClass());
            logger.debug("Got cached listing:\n\n{}\n\n", filenames);
        } catch (final IOException e) {
            throw new TransferException("Failed to read listing from cached file: %s. Reason: %s", e, cachedListing, e.getMessage());
        } finally {
            closeQuietly(stream);
        }
    } else {
        final Transfer cached = getCacheReference(resource);
        if (cached.exists()) {
            if (cached.isFile()) {
                throw new TransferException("Cannot list: {}. It does not appear to be a directory.", resource);
            } else {
                try {
                    // This is fairly stupid, but we need to append '/' to the end of directories in the listing so content processors can figure
                    // out what to do with them.
                    String[] fnames = cached.list();
                    if (fnames != null && fnames.length > 0) {
                        for (String fname : fnames) {
                            final ConcreteResource child = resource.getChild(fname);
                            final Transfer childRef = getCacheReference(child);
                            if (childRef.isFile()) {
                                filenames.add(fname);
                            } else {
                                filenames.add(fname + "/");
                            }
                        }
                    }
                } catch (final IOException e) {
                    throw new TransferException("Listing failed: {}. Reason: {}", e, resource, e.getMessage());
                }
            }
        }
        if (resource.getLocation().allowsDownloading()) {
            final int timeoutSeconds = getTimeoutSeconds(resource);
            Transport transport = getTransport(resource);
            final ListingResult remoteResult = lister.list(resource, cachedListing, timeoutSeconds, transport, suppressFailures);
            if (remoteResult != null) {
                String[] remoteListing = remoteResult.getListing();
                if (remoteListing != null) {
                    final TransferDecorator decorator = cachedListing.getDecorator();
                    if (decorator != null) {
                        try {
                            Logger logger = LoggerFactory.getLogger(getClass());
                            logger.debug("Un-decorated listing:\n\n{}\n\n", remoteListing);
                            remoteListing = decorator.decorateListing(cachedListing.getParent(), remoteListing, metadata);
                        } catch (final IOException e) {
                            logger.error("Failed to decorate directory listing for: {}. Reason: {}", e, resource, e.getMessage());
                            remoteListing = null;
                        }
                    }
                }
                if (remoteListing != null) {
                    if (transport.allowsCaching()) {
                        OutputStream stream = null;
                        try {
                            Logger logger = LoggerFactory.getLogger(getClass());
                            logger.debug("Writing listing:\n\n{}\n\nto: {}", remoteListing, cachedListing);
                            stream = cachedListing.openOutputStream(TransferOperation.DOWNLOAD);
                            stream.write(join(remoteListing, "\n").getBytes("UTF-8"));
                        } catch (final IOException e) {
                            logger.debug("Failed to store directory listing for: {}. Reason: {}", e, resource, e.getMessage());
                        } finally {
                            closeQuietly(stream);
                        }
                    }
                    filenames.addAll(Arrays.asList(remoteListing));
                }
            }
        }
    }
    Logger logger = LoggerFactory.getLogger(getClass());
    logger.debug("Listing before non-listable file removal:\n\n{}\n\n", filenames);
    List<String> resultingNames = new ArrayList<String>(filenames.size());
    for (String fname : filenames) {
        ConcreteResource child = resource.getChild(fname);
        SpecialPathInfo specialPathInfo = specialPathManager.getSpecialPathInfo(child, metadata.getPackageType());
        if (specialPathInfo != null && !specialPathInfo.isListable()) {
            continue;
        }
        resultingNames.add(fname);
    }
    logger.debug("Final listing result:\n\n{}\n\n", resultingNames);
    return new ListingResult(resource, resultingNames.toArray(new String[resultingNames.size()]));
}

Example 14 with SpecialPathInfo

use of org.commonjava.maven.galley.model.SpecialPathInfo in project indy by Commonjava.

the class DefaultContentManager method getTransfer.

@Override
public Transfer getTransfer(final ArtifactStore store, final String path, final TransferOperation op) throws IndyWorkflowException {
    logger.debug("Getting transfer for: {}/{} (op: {})", store.getKey(), path, op);
    if (group == store.getKey().getType()) {
        KeyedLocation location = LocationUtils.toLocation(store);
        SpecialPathInfo spInfo = specialPathManager.getSpecialPathInfo(location, path, store.getPackageType());
        if (spInfo == null || !spInfo.isMergable()) {
            try {
                final List<ArtifactStore> allMembers = storeManager.query().packageType(store.getPackageType()).enabledState(true).getOrderedConcreteStoresInGroup(store.getName());
                logger.debug("Trying to retrieve suitable transfer for: {} in group: {} members:\n{}", path, store.getName(), allMembers);
                return getTransfer(allMembers, path, op);
            } catch (final IndyDataException e) {
                throw new IndyWorkflowException("Failed to lookup concrete members of: %s. Reason: %s", e, store, e.getMessage());
            }
        } else {
            logger.debug("Detected mergable special path: {}/{}.", store.getKey(), path);
        }
    }
    logger.debug("Retrieving storage reference (Transfer) directly for: {}/{}", store.getKey(), path);
    return downloadManager.getStorageReference(store, path, op);
}

Example 15 with SpecialPathInfo

use of org.commonjava.maven.galley.model.SpecialPathInfo in project indy by Commonjava.

the class DefaultGalleyStorageProvider method setup.

@PostConstruct
public void setup() {
    SpecialPathInfo infoSpi = SpecialPathInfo.from(new FilePatternMatcher(".+\\.info")).setDecoratable(false).setDeletable(false).setListable(false).setPublishable(false).setRetrievable(false).setStorable(false).build();
    specialPathManager.registerSpecialPathInfo(infoSpi);
    ChecksummingDecoratorAdvisor readAdvisor = (transfer, op, eventMetadata) -> {
        ChecksummingDecoratorAdvisor.ChecksumAdvice result = NO_DECORATE;
        if (checksumAdvisors != null) {
            for (IndyChecksumAdvisor advisor : checksumAdvisors) {
                Optional<ChecksummingDecoratorAdvisor.ChecksumAdvice> advice = advisor.getChecksumReadAdvice(transfer, op, eventMetadata);
                if (advice.isPresent()) {
                    ChecksummingDecoratorAdvisor.ChecksumAdvice checksumAdvice = advice.get();
                    if (checksumAdvice.ordinal() > result.ordinal()) {
                        result = checksumAdvice;
                        if (checksumAdvice == CALCULATE_AND_WRITE) {
                            break;
                        }
                    }
                }
            }
        }
        logger.debug("Advising {} for {} of: {}", result, op, transfer);
        return result;
    };
    ChecksummingDecoratorAdvisor writeAdvisor = (transfer, op, eventMetadata) -> {
        ChecksummingDecoratorAdvisor.ChecksumAdvice result = NO_DECORATE;
        if (TransferOperation.GENERATE == op) {
            result = CALCULATE_AND_WRITE;
        } else if (checksumAdvisors != null) {
            for (IndyChecksumAdvisor advisor : checksumAdvisors) {
                Optional<ChecksummingDecoratorAdvisor.ChecksumAdvice> advice = advisor.getChecksumWriteAdvice(transfer, op, eventMetadata);
                if (advice.isPresent()) {
                    ChecksummingDecoratorAdvisor.ChecksumAdvice checksumAdvice = advice.get();
                    if (checksumAdvice.ordinal() > result.ordinal()) {
                        result = checksumAdvice;
                        if (checksumAdvice == CALCULATE_AND_WRITE) {
                            break;
                        }
                    }
                }
            }
        }
        logger.debug("Advising {} for {} of: {}", result, op, transfer);
        return result;
    };
    transferDecorator = new TransferDecoratorPipeline(new ChecksummingTransferDecorator(readAdvisor, writeAdvisor, specialPathManager, contentMetadataConsumer, new Md5GeneratorFactory(), new Sha1GeneratorFactory(), new Sha256GeneratorFactory()), new ContentsFilteringTransferDecorator(), new NoCacheTransferDecorator(specialPathManager));
    final File storeRoot = config.getStorageRootDirectory();
    cacheProviderFactory = new PartyLineCacheProviderFactory(storeRoot);
    final File nfsBasedir = config.getNFSStorageRootDirectory();
    if (nfsBasedir != null) {
        if (!nfsBasedir.exists()) {
            nfsBasedir.mkdirs();
        }
        // nfs root can not be created due to some security reason(like permission), will bypass FastLocal provider and use PartyLine
        if (nfsBasedir.exists()) {
            final FastLocalCacheProviderFactory fastLocalFac = new FastLocalCacheProviderFactory(storeRoot, nfsBasedir, new CacheInstanceAdapter(nfsOwnerCache), fastLocalExecutors);
            cacheProviderFactory = new RoutingCacheProviderFactory((resource) -> {
                if (resource != null) {
                    final Location loc = resource.getLocation();
                    // looking for KeyedLocation and StoreType.hosted should be faster than regex on the URI.
                    return ((loc instanceof KeyedLocation) && hosted == ((KeyedLocation) loc).getKey().getType());
                }
                return false;
            }, fastLocalFac, cacheProviderFactory);
        } else {
            logger.warn("[Indy] nfs base dir {} can not be created correctly due to some unknown reasons, will use partyline cache provider as default", nfsBasedir);
        }
    }
    // TODO: Tie this into a config file!
    transportManagerConfig = new TransportManagerConfig();
}