Examples with KeyedLocation org.commonjava.indy.model.galley.KeyedLocation used on opensource projects

Example 16 with KeyedLocation

use of org.commonjava.indy.model.galley.KeyedLocation in project indy by Commonjava.

the class DefaultContentManager method getTransfer.

@Override
public Transfer getTransfer(final ArtifactStore store, final String path, final TransferOperation op) throws IndyWorkflowException {
    logger.debug("Getting transfer for: {}/{} (op: {})", store.getKey(), path, op);
    if (group == store.getKey().getType()) {
        KeyedLocation location = LocationUtils.toLocation(store);
        SpecialPathInfo spInfo = specialPathManager.getSpecialPathInfo(location, path, store.getPackageType());
        if (spInfo == null || !spInfo.isMergable()) {
            try {
                final List<ArtifactStore> allMembers = storeManager.query().packageType(store.getPackageType()).enabledState(true).getOrderedConcreteStoresInGroup(store.getName());
                logger.debug("Trying to retrieve suitable transfer for: {} in group: {} members:\n{}", path, store.getName(), allMembers);
                return getTransfer(allMembers, path, op);
            } catch (final IndyDataException e) {
                throw new IndyWorkflowException("Failed to lookup concrete members of: %s. Reason: %s", e, store, e.getMessage());
            }
        } else {
            logger.debug("Detected mergable special path: {}/{}.", store.getKey(), path);
        }
    }
    logger.debug("Retrieving storage reference (Transfer) directly for: {}/{}", store.getKey(), path);
    return downloadManager.getStorageReference(store, path, op);
}

Example 17 with KeyedLocation

use of org.commonjava.indy.model.galley.KeyedLocation in project indy by Commonjava.

the class FoloTrackingListener method onFileAccess.

public void onFileAccess(@Observes final FileAccessEvent event) {
    logger.debug("FILE ACCESS: {}", event);
    EventMetadata metadata = event.getEventMetadata();
    final TrackingKey trackingKey = (TrackingKey) metadata.get(FoloConstants.TRACKING_KEY);
    if (trackingKey == null) {
        logger.info("No tracking key for access to: {}", event.getTransfer());
        return;
    }
    final AccessChannel accessChannel = (AccessChannel) metadata.get(FoloConstants.ACCESS_CHANNEL);
    final Transfer transfer = event.getTransfer();
    if (transfer == null) {
        logger.info("No transfer: {}", event);
        return;
    }
    final Location location = transfer.getLocation();
    if (!(location instanceof KeyedLocation)) {
        logger.info("Not in a keyed location: {}", event.getTransfer());
        return;
    }
    try {
        final KeyedLocation keyedLocation = (KeyedLocation) location;
        if (!foloConfig.isGroupContentTracked() && keyedLocation.getKey().getType() == group) {
            logger.debug("NOT tracking content stored directly in group: {}. This content is generally aggregated metadata, and can be recalculated. Groups may not be stable in some build environments", keyedLocation.getKey());
            return;
        }
        logger.debug("Tracking report: {} += {} in {} (DOWNLOAD)", trackingKey, transfer.getPath(), keyedLocation.getKey());
        recordManager.recordArtifact(createEntry(trackingKey, keyedLocation.getKey(), accessChannel, transfer.getPath(), StoreEffect.DOWNLOAD, event.getEventMetadata()));
    } catch (final FoloContentException | IndyWorkflowException e) {
        logger.error(String.format("Failed to record download: %s. Reason: %s", transfer, e.getMessage()), e);
    }
}

Example 18 with KeyedLocation

use of org.commonjava.indy.model.galley.KeyedLocation in project indy by Commonjava.

the class FoloTrackingListener method onFileUpload.

public void onFileUpload(@Observes final FileStorageEvent event) {
    logger.debug("FILE STORAGE: {}", event);
    if (TransferOperation.UPLOAD != event.getType()) {
        logger.debug("Not a file upload from client; skipping tracking of storage");
        return;
    }
    EventMetadata metadata = event.getEventMetadata();
    final TrackingKey trackingKey = (TrackingKey) metadata.get(FoloConstants.TRACKING_KEY);
    if (trackingKey == null) {
        logger.info("No tracking key. Not recording.");
        return;
    }
    final AccessChannel accessChannel = (AccessChannel) metadata.get(FoloConstants.ACCESS_CHANNEL);
    final Transfer transfer = event.getTransfer();
    if (transfer == null) {
        logger.info("No transfer. Not recording.");
        return;
    }
    final Location location = transfer.getLocation();
    if (!(location instanceof KeyedLocation)) {
        logger.info("Invalid transfer source location: {}. Not recording.", location);
        return;
    } else if (!foloConfig.isGroupContentTracked() && ((KeyedLocation) location).getKey().getType() == group) {
        logger.debug("NOT tracking content stored directly in group: {}. This content is generally aggregated metadata, and can be recalculated. Groups may not be stable in some build environments", ((KeyedLocation) location).getKey());
        return;
    }
    final TransferOperation op = event.getType();
    StoreEffect effect = null;
    switch(op) {
        case DOWNLOAD:
            {
                effect = StoreEffect.DOWNLOAD;
                break;
            }
        case UPLOAD:
            {
                effect = StoreEffect.UPLOAD;
                break;
            }
        default:
            {
                logger.debug("Ignoring transfer operation: {} for: {}", op, transfer);
                return;
            }
    }
    try {
        final KeyedLocation keyedLocation = (KeyedLocation) location;
        logger.debug("Tracking report: {} += {} in {} ({})", trackingKey, transfer.getPath(), keyedLocation.getKey(), effect);
        recordManager.recordArtifact(createEntry(trackingKey, keyedLocation.getKey(), accessChannel, transfer.getPath(), effect, event.getEventMetadata()));
    } catch (final FoloContentException | IndyWorkflowException e) {
        logger.error(String.format("Failed to record download: %s. Reason: %s", transfer, e.getMessage()), e);
    }
}

Example 19 with KeyedLocation

use of org.commonjava.indy.model.galley.KeyedLocation in project indy by Commonjava.

the class DefaultGalleyStorageProvider method setup.

@PostConstruct
public void setup() {
    SpecialPathInfo infoSpi = SpecialPathInfo.from(new FilePatternMatcher(".+\\.info")).setDecoratable(false).setDeletable(false).setListable(false).setPublishable(false).setRetrievable(false).setStorable(false).build();
    specialPathManager.registerSpecialPathInfo(infoSpi);
    ChecksummingDecoratorAdvisor readAdvisor = (transfer, op, eventMetadata) -> {
        ChecksummingDecoratorAdvisor.ChecksumAdvice result = NO_DECORATE;
        if (checksumAdvisors != null) {
            for (IndyChecksumAdvisor advisor : checksumAdvisors) {
                Optional<ChecksummingDecoratorAdvisor.ChecksumAdvice> advice = advisor.getChecksumReadAdvice(transfer, op, eventMetadata);
                if (advice.isPresent()) {
                    ChecksummingDecoratorAdvisor.ChecksumAdvice checksumAdvice = advice.get();
                    if (checksumAdvice.ordinal() > result.ordinal()) {
                        result = checksumAdvice;
                        if (checksumAdvice == CALCULATE_AND_WRITE) {
                            break;
                        }
                    }
                }
            }
        }
        logger.debug("Advising {} for {} of: {}", result, op, transfer);
        return result;
    };
    ChecksummingDecoratorAdvisor writeAdvisor = (transfer, op, eventMetadata) -> {
        ChecksummingDecoratorAdvisor.ChecksumAdvice result = NO_DECORATE;
        if (TransferOperation.GENERATE == op) {
            result = CALCULATE_AND_WRITE;
        } else if (checksumAdvisors != null) {
            for (IndyChecksumAdvisor advisor : checksumAdvisors) {
                Optional<ChecksummingDecoratorAdvisor.ChecksumAdvice> advice = advisor.getChecksumWriteAdvice(transfer, op, eventMetadata);
                if (advice.isPresent()) {
                    ChecksummingDecoratorAdvisor.ChecksumAdvice checksumAdvice = advice.get();
                    if (checksumAdvice.ordinal() > result.ordinal()) {
                        result = checksumAdvice;
                        if (checksumAdvice == CALCULATE_AND_WRITE) {
                            break;
                        }
                    }
                }
            }
        }
        logger.debug("Advising {} for {} of: {}", result, op, transfer);
        return result;
    };
    transferDecorator = new TransferDecoratorPipeline(new ChecksummingTransferDecorator(readAdvisor, writeAdvisor, specialPathManager, contentMetadataConsumer, new Md5GeneratorFactory(), new Sha1GeneratorFactory(), new Sha256GeneratorFactory()), new ContentsFilteringTransferDecorator(), new NoCacheTransferDecorator(specialPathManager));
    final File storeRoot = config.getStorageRootDirectory();
    cacheProviderFactory = new PartyLineCacheProviderFactory(storeRoot);
    final File nfsBasedir = config.getNFSStorageRootDirectory();
    if (nfsBasedir != null) {
        if (!nfsBasedir.exists()) {
            nfsBasedir.mkdirs();
        }
        // nfs root can not be created due to some security reason(like permission), will bypass FastLocal provider and use PartyLine
        if (nfsBasedir.exists()) {
            final FastLocalCacheProviderFactory fastLocalFac = new FastLocalCacheProviderFactory(storeRoot, nfsBasedir, new CacheInstanceAdapter(nfsOwnerCache), fastLocalExecutors);
            cacheProviderFactory = new RoutingCacheProviderFactory((resource) -> {
                if (resource != null) {
                    final Location loc = resource.getLocation();
                    // looking for KeyedLocation and StoreType.hosted should be faster than regex on the URI.
                    return ((loc instanceof KeyedLocation) && hosted == ((KeyedLocation) loc).getKey().getType());
                }
                return false;
            }, fastLocalFac, cacheProviderFactory);
        } else {
            logger.warn("[Indy] nfs base dir {} can not be created correctly due to some unknown reasons, will use partyline cache provider as default", nfsBasedir);
        }
    }
    // TODO: Tie this into a config file!
    transportManagerConfig = new TransportManagerConfig();
}