Examples with DownloadJob org.commonjava.maven.galley.spi.transport.DownloadJob used on opensource projects

Example 1 with DownloadJob

use of org.commonjava.maven.galley.spi.transport.DownloadJob in project galley by Commonjava.

the class DownloadHandler method joinOrStart.

private Transfer joinOrStart(final ConcreteResource resource, final Transfer target, final int timeoutSeconds, final Transport transport, final boolean suppressFailures, final EventMetadata eventMetadata) throws TransferException {
    // if the target file already exists, skip joining.
    if (target.exists()) {
        return target;
    }
    Future<DownloadJob> future;
    synchronized (pending) {
        future = pending.get(target);
        if (future == null) {
            if (transport == null) {
                return null;
            }
            final DownloadJob job = transport.createDownloadJob(resource, target, transferSizes, timeoutSeconds, eventMetadata);
            future = executor.submit(job);
            pending.put(target, future);
        }
    }
    int waitSeconds = (int) (timeoutSeconds * config.getTimeoutOverextensionFactor());
    int tries = 1;
    try {
        while (tries > 0) {
            tries--;
            try {
                final DownloadJob job = future.get(waitSeconds, TimeUnit.SECONDS);
                synchronized (pending) {
                    pending.remove(target);
                }
                final Transfer downloaded = job.getTransfer();
                if (job.getError() != null) {
                    logger.debug("NFC: Download error. Marking as missing: {}\nError was: {}", job.getError(), resource, job.getError().getMessage());
                    nfc.addMissing(resource);
                    if (!suppressFailures) {
                        throw job.getError();
                    }
                } else if (downloaded == null || !downloaded.exists()) {
                    logger.debug("NFC: Download did not complete. Marking as missing: {}", resource);
                    nfc.addMissing(resource);
                }
                return downloaded;
            } catch (final InterruptedException e) {
                if (!suppressFailures) {
                    throw new TransferException("Download interrupted: {}", e, target);
                }
            } catch (final ExecutionException e) {
                if (!suppressFailures) {
                    throw new TransferException("Download failed: {}", e, target);
                }
            } catch (final TimeoutException e) {
                Long size = transferSizes.get(target);
                if (tries > 0) {
                    continue;
                } else if (size != null && size > config.getThresholdWaitRetrySize()) {
                    logger.debug("Downloading a large file: {}. Retrying Future.get() up to {} times.", size, tries);
                    tries = (int) (size / config.getWaitRetryScalingIncrement());
                    continue;
                } else if (!suppressFailures) {
                    throw new TransferTimeoutException(target, "Timed out waiting for execution of: {}", e, target);
                }
            } catch (final TransferException e) {
                if (!suppressFailures) {
                    throw e;
                }
            } catch (final Exception e) {
                if (!suppressFailures) {
                    throw new TransferException("Download failed: {}. Reason: {}", e, resource, e.getMessage());
                }
            }
        }
    } finally {
        transferSizes.remove(target);
    }
    return null;
}

Example 2 with DownloadJob

use of org.commonjava.maven.galley.spi.transport.DownloadJob in project galley by Commonjava.

the class HttpDownloadTest method simpleRetrieveOfAvailableUrl.

@Test
public void simpleRetrieveOfAvailableUrl() throws Exception {
    final String fname = "simple-retrieval.html";
    fixture.getServer().expect(fixture.formatUrl(fname), 200, fname);
    final String baseUri = fixture.getBaseUri();
    final SimpleHttpLocation location = new SimpleHttpLocation("test", baseUri, true, true, true, true, null);
    final Transfer transfer = fixture.getTransfer(new ConcreteResource(location, fname));
    final String url = fixture.formatUrl(fname);
    Map<Transfer, Long> transferSizes = new HashMap<Transfer, Long>();
    assertThat(transfer.exists(), equalTo(false));
    final HttpDownload dl = new HttpDownload(url, location, transfer, transferSizes, new EventMetadata(), fixture.getHttp(), new ObjectMapper());
    final DownloadJob resultJob = dl.call();
    final TransferException error = dl.getError();
    assertThat(error, nullValue());
    assertThat(resultJob, notNullValue());
    final Transfer result = resultJob.getTransfer();
    assertThat(result, notNullValue());
    assertThat(result.exists(), equalTo(true));
    assertThat(transfer.exists(), equalTo(true));
    final String path = fixture.getUrlPath(url);
    assertThat(fixture.getAccessesFor(path), equalTo(1));
}

Example 3 with DownloadJob

use of org.commonjava.maven.galley.spi.transport.DownloadJob in project galley by Commonjava.

the class HttpDownloadTest method simpleRetrieveOfMissingUrl.

@Test
public void simpleRetrieveOfMissingUrl() throws Exception {
    final String fname = "simple-missing.html";
    final String baseUri = fixture.getBaseUri();
    final SimpleHttpLocation location = new SimpleHttpLocation("test", baseUri, true, true, true, true, null);
    final Transfer transfer = fixture.getTransfer(new ConcreteResource(location, fname));
    final String url = fixture.formatUrl(fname);
    Map<Transfer, Long> transferSizes = new HashMap<Transfer, Long>();
    assertThat(transfer.exists(), equalTo(false));
    final HttpDownload dl = new HttpDownload(url, location, transfer, transferSizes, new EventMetadata(), fixture.getHttp(), new ObjectMapper());
    final DownloadJob resultJob = dl.call();
    final TransferException error = dl.getError();
    assertThat(error, nullValue());
    assertThat(resultJob, notNullValue());
    final Transfer result = resultJob.getTransfer();
    assertThat(result, notNullValue());
    assertThat(result.exists(), equalTo(false));
    assertThat(transfer.exists(), equalTo(false));
    final String path = fixture.getUrlPath(url);
    assertThat(fixture.getAccessesFor(path), equalTo(1));
}

Example 4 with DownloadJob

use of org.commonjava.maven.galley.spi.transport.DownloadJob in project galley by Commonjava.

the class HttpDownloadTest method simpleRetrieveOfUrlWithError.

@Test
public void simpleRetrieveOfUrlWithError() throws Exception {
    final String fname = "simple-error.html";
    final String baseUri = fixture.getBaseUri();
    final SimpleHttpLocation location = new SimpleHttpLocation("test", baseUri, true, true, true, true, null);
    final Transfer transfer = fixture.getTransfer(new ConcreteResource(location, fname));
    final String url = fixture.formatUrl(fname);
    final String error = "Test Error.";
    final String path = fixture.getUrlPath(url);
    fixture.registerException(path, error);
    Map<Transfer, Long> transferSizes = new HashMap<Transfer, Long>();
    assertThat(transfer.exists(), equalTo(false));
    final HttpDownload dl = new HttpDownload(url, location, transfer, transferSizes, new EventMetadata(), fixture.getHttp(), new ObjectMapper());
    final DownloadJob resultJob = dl.call();
    final TransferException err = dl.getError();
    assertThat(err, notNullValue());
    assertThat(err.getMessage().contains(error), equalTo(true));
    assertThat(resultJob, notNullValue());
    final Transfer result = resultJob.getTransfer();
    assertThat(result, notNullValue());
    assertThat(transfer.exists(), equalTo(false));
    assertThat(fixture.getAccessesFor(path), equalTo(1));
}