Examples with Transfer org.commonjava.maven.galley.model.Transfer used on opensource projects

Example 26 with Transfer

use of org.commonjava.maven.galley.model.Transfer in project galley by Commonjava.

the class TransferManagerImpl method doBatch.

@SuppressWarnings("RedundantThrows")
private <T extends TransferBatch> T doBatch(final Set<Resource> resources, final T batch, final boolean suppressFailures, final EventMetadata eventMetadata) throws TransferException {
    logger.info("Attempting to batch-retrieve {} resources:\n  {}", resources.size(), new JoinString("\n  ", resources));
    final Set<BatchRetriever> retrievers = new HashSet<>(resources.size());
    for (final Resource resource : resources) {
        retrievers.add(new BatchRetriever(this, resource, suppressFailures, eventMetadata));
    }
    final Map<ConcreteResource, TransferException> errors = new HashMap<>();
    final Map<ConcreteResource, Transfer> transfers = new HashMap<>();
    do {
        for (final BatchRetriever retriever : retrievers) {
            batchExecutor.submit(retriever);
        }
        int count = retrievers.size();
        for (int i = 0; i < count; i++) {
            try {
                Future<BatchRetriever> pending = batchExecutor.take();
                BatchRetriever retriever = pending.get();
                final ConcreteResource resource = retriever.getLastTry();
                final TransferException error = retriever.getError();
                if (error != null) {
                    logger.warn("ERROR: {}...{}", error, resource, error.getMessage());
                    retrievers.remove(retriever);
                    if (!(error instanceof TransferLocationException)) {
                        errors.put(resource, error);
                    }
                    continue;
                }
                final Transfer transfer = retriever.getTransfer();
                if (transfer != null && transfer.exists()) {
                    transfers.put(resource, transfer);
                    retrievers.remove(retriever);
                    logger.debug("Completed: {}", resource);
                    continue;
                }
                if (!retriever.hasMoreTries()) {
                    logger.debug("Not completed, but out of tries: {}", resource);
                    retrievers.remove(retriever);
                }
            } catch (final InterruptedException e) {
                logger.error(String.format("Failed to wait for batch retrieval attempts to complete: %s", e.getMessage()), e);
                break;
            } catch (ExecutionException e) {
                logger.error(String.format("Failed to retrieve next completed retrieval: %s", e.getMessage()), e);
            }
        }
    } while (!retrievers.isEmpty());
    batch.setErrors(errors);
    batch.setTransfers(transfers);
    return batch;
}

Example 27 with Transfer

use of org.commonjava.maven.galley.model.Transfer in project galley by Commonjava.

the class TransferManagerImpl method retrieve.

@Override
public Transfer retrieve(final ConcreteResource resource, final boolean suppressFailures, final EventMetadata eventMetadata) throws TransferException {
    // logger.info( "Attempting to resolve: {}", resource );
    // TODO: Handle the case where storage isn't allowed?
    // NOTE: This would expand the notion out from simply:
    // "don't allow storing new stuff"
    // to:
    // "don't ever cache this stuff"
    Transfer target = null;
    try {
        // TODO: (see above re:storing) Handle things like local archives that really don't need to be cached...
        target = getCacheReference(resource);
        if (target.exists()) {
            logger.debug("Using cached copy of: {}", target);
            return target;
        }
        SpecialPathInfo specialPathInfo = specialPathManager.getSpecialPathInfo(resource, eventMetadata.getPackageType());
        if (!resource.allowsDownloading() || (specialPathInfo != null && !specialPathInfo.isRetrievable())) {
            logger.debug("Download not allowed for: {}. Returning null transfer.", resource);
            return null;
        }
        final Transfer retrieved = downloader.download(resource, target, getTimeoutSeconds(resource), getTransport(resource), suppressFailures, eventMetadata);
        if (retrieved != null && retrieved.exists() && !target.equals(retrieved)) {
            if (specialPathInfo == null || specialPathInfo.isCachable()) {
                cacheProvider.createAlias(retrieved.getResource(), target.getResource());
            }
        }
        if (target.exists()) {
            logger.debug("DOWNLOADED: {}", resource);
            return target;
        } else {
            logger.debug("NOT DOWNLOADED: {}", resource);
            return null;
        }
    } catch (final TransferException e) {
        fileEventManager.fire(new FileErrorEvent(target, e, eventMetadata));
        throw e;
    } catch (final IOException e) {
        final TransferException error = new TransferException("Failed to download: {}. Reason: {}", e, resource, e.getMessage());
        fileEventManager.fire(new FileErrorEvent(target, error, eventMetadata));
        throw error;
    }
}

Example 28 with Transfer

use of org.commonjava.maven.galley.model.Transfer in project galley by Commonjava.

the class PartyLineCacheProvider method getDetachedFile.

@Override
public File getDetachedFile(final ConcreteResource resource) {
    // TODO: this might be a bit heavy-handed, but we need to be sure.
    // Maybe I can improve it later.
    final Transfer txfr = getTransfer(resource);
    synchronized (txfr) {
        File f = new File(getFilePath(resource));
        // TODO: Need a better sync object!!
        synchronized (this) {
            if (resource.isRoot() && !f.isDirectory()) {
                f.mkdirs();
            }
        }
        // TODO: configurable default timeout
        final int timeoutSeconds = resource.getLocation().getAttribute(Location.CACHE_TIMEOUT_SECONDS, Integer.class, config.getDefaultTimeoutSeconds());
        if (!resource.isRoot() && f.exists() && !f.isDirectory() && config.isTimeoutProcessingEnabled() && timeoutSeconds > 0) {
            final long current = System.currentTimeMillis();
            final long lastModified = f.lastModified();
            final int tos = timeoutSeconds < Location.MIN_CACHE_TIMEOUT_SECONDS ? Location.MIN_CACHE_TIMEOUT_SECONDS : timeoutSeconds;
            final long timeout = TimeUnit.MILLISECONDS.convert(tos, TimeUnit.SECONDS);
            if (current - lastModified > timeout) {
                final File mved = new File(f.getPath() + SUFFIX_TO_DELETE);
                f.renameTo(mved);
                try {
                    logger.info("Deleting cached file: {} (moved to: {})\nTimeout: {}ms\nElapsed: {}ms\nCurrently: {}\nLast Modified: {}\nOriginal Timeout was: {}s", f, mved, timeout, (current - lastModified), new Date(current), new Date(lastModified), tos);
                    if (mved.exists()) {
                        FileUtils.forceDelete(mved);
                    }
                } catch (final IOException e) {
                    logger.error(String.format("Failed to tryDelete: %s.", f), e);
                }
            }
        }
        return f;
    }
}

Example 29 with Transfer

use of org.commonjava.maven.galley.model.Transfer in project galley by Commonjava.

the class AbstractTransferManagerTest method retrieve_cacheIfMissing.

/**
 * Test that remote content will be downloaded then cached.
 */
@Test
public void retrieve_cacheIfMissing() throws Exception {
    final String testContent = "This is a test " + System.currentTimeMillis();
    final Location loc = new SimpleLocation("file:///test-repo");
    final String path = "/path/to/test.txt";
    final ConcreteResource resource = new ConcreteResource(loc, path);
    // put in the content that we want to "download"
    getTransport().registerDownload(resource, new TestDownload(testContent.getBytes()));
    // now, use the manager to retrieve() the path...the remote content should come through here.
    Transfer transfer = getTransferManagerImpl().retrieve(resource);
    assertTransferContent(transfer, testContent);
    // now, the right content should be cached.
    // So, we'll put in some wrong content that will cause problems if the cache isn't used.
    getTransport().registerDownload(resource, new TestDownload("This is WRONG".getBytes()));
    // now, use the manager to retrieve() the path again...the cached content should come through here.
    transfer = getTransferManagerImpl().retrieve(resource);
    assertTransferContent(transfer, testContent);
}

Example 30 with Transfer

use of org.commonjava.maven.galley.model.Transfer in project galley by Commonjava.

the class HttpDownloadTest method IOExceptionDuringDownloadTransferDeletesTargetFile.

@Test
@BMRule(name = "throw IOException during writeTarget copy operation", targetClass = "HttpDownload", targetMethod = "doCopy", targetLocation = "ENTRY", condition = "!flagged(\"firstCopy\")", action = "flag(\"firstCopy\"); throw new IOException(\"BMUnit exception\");")
public void IOExceptionDuringDownloadTransferDeletesTargetFile() throws Exception {
    final String content = "This is some content " + System.currentTimeMillis() + "." + System.nanoTime();
    final String path = "/path/to/file";
    fixture.getServer().expect("GET", fixture.formatUrl(path), new ExpectationHandler() {

        int count = 0;

        @Override
        public void handle(final HttpServletRequest httpServletRequest, final HttpServletResponse httpServletResponse) throws ServletException, IOException {
            httpServletResponse.setStatus(200);
            httpServletResponse.setHeader("Content-Length", Integer.toString(content.length()));
            PrintWriter writer = httpServletResponse.getWriter();
            if (count < 1) {
                writer.write(content.substring(0, content.length() / 2));
            } else {
                writer.write(content);
            }
            count++;
        }
    });
    final String baseUri = fixture.getBaseUri();
    final SimpleHttpLocation location = new SimpleHttpLocation("test", baseUri, true, true, true, true, null);
    final Transfer transfer = fixture.getTransfer(new ConcreteResource(location, path));
    final String url = fixture.formatUrl(path);
    assertThat(transfer.exists(), equalTo(false));
    // first call, server should quit transferring halfway through the transfer
    HttpDownload dl = new HttpDownload(url, location, transfer, new HashMap<Transfer, Long>(), new EventMetadata(), fixture.getHttp(), new ObjectMapper());
    DownloadJob resultJob = dl.call();
    TransferException error = dl.getError();
    assertThat(error, notNullValue());
    error.printStackTrace();
    assertThat(resultJob, notNullValue());
    Transfer result = resultJob.getTransfer();
    assertThat(result, notNullValue());
    assertThat(result.exists(), equalTo(false));
    assertThat(transfer.exists(), equalTo(false));
    // second call should hit upstream again and succeed.
    dl = new HttpDownload(url, location, transfer, new HashMap<Transfer, Long>(), new EventMetadata(), fixture.getHttp(), new ObjectMapper());
    resultJob = dl.call();
    error = dl.getError();
    assertThat(error, nullValue());
    assertThat(resultJob, notNullValue());
    result = resultJob.getTransfer();
    assertThat(result, notNullValue());
    assertThat(result.exists(), equalTo(true));
    assertThat(transfer.exists(), equalTo(true));
    final String urlPath = fixture.getUrlPath(url);
    assertThat(fixture.getAccessesFor(urlPath), equalTo(2));
}