Examples with RemoteHttpDataTransferProtocolInfo diskCacheV111.vehicles.RemoteHttpDataTransferProtocolInfo used on opensource projects

Example 1 with RemoteHttpDataTransferProtocolInfo

use of diskCacheV111.vehicles.RemoteHttpDataTransferProtocolInfo in project dcache by dCache.

the class RemoteHttpDataTransferProtocol method verifyRemoteFile.

private void verifyRemoteFile(RemoteHttpDataTransferProtocolInfo info) throws ThirdPartyTransferFailedCacheException {
    FileAttributes attributes = _channel.getFileAttributes();
    boolean isFirstAttempt = true;
    long t_max = maxRetryDuration(attributes.getSize());
    long deadline = System.currentTimeMillis() + t_max;
    try {
        while (System.currentTimeMillis() < deadline) {
            long sleepFor = Math.min(deadline - System.currentTimeMillis(), DELAY_BETWEEN_REQUESTS);
            if (!isFirstAttempt && sleepFor > 0) {
                Thread.sleep(sleepFor);
            }
            isFirstAttempt = false;
            HttpClientContext context = storeContext(new HttpClientContext());
            HttpHead head = buildHeadRequest(info, deadline);
            buildWantDigest().ifPresent(v -> head.addHeader("Want-Digest", v));
            try {
                try (CloseableHttpResponse response = _client.execute(head, context)) {
                    StatusLine status = response.getStatusLine();
                    if (status.getStatusCode() >= 300) {
                        checkThirdPartyTransferSuccessful(!info.isVerificationRequired(), "rejected HEAD: %d %s", status.getStatusCode(), status.getReasonPhrase());
                        return;
                    }
                    if (shouldRetry(response)) {
                        continue;
                    }
                    OptionalLong contentLengthHeader = contentLength(response);
                    if (contentLengthHeader.isPresent()) {
                        long contentLength = contentLengthHeader.getAsLong();
                        long fileSize = attributes.getSize();
                        checkThirdPartyTransferSuccessful(contentLength == fileSize, "HEAD Content-Length (%d) does not match file size (%d)", contentLength, fileSize);
                    } else {
                        LOGGER.debug("HEAD response did not contain Content-Length");
                    }
                    String rfc3230 = headerValue(response, "Digest");
                    checkChecksums(info, rfc3230, attributes.getChecksumsIfPresent());
                    return;
                } catch (IOException e) {
                    throw new ThirdPartyTransferFailedCacheException("failed to " + "connect to server: " + e.toString(), e);
                }
            } catch (ThirdPartyTransferFailedCacheException e) {
                List<URI> redirections = context.getRedirectLocations();
                if (redirections != null && !redirections.isEmpty()) {
                    throw new ThirdPartyTransferFailedCacheException(e.getMessage() + "; redirections " + redirections, e.getCause());
                } else {
                    throw e;
                }
            }
        }
    } catch (InterruptedException e) {
        throw new ThirdPartyTransferFailedCacheException("pool is shutting down", e);
    }
    throw new ThirdPartyTransferFailedCacheException("remote server failed " + "to provide length after " + describeDuration(GET_RETRY_DURATION, MILLISECONDS));
}

Example 2 with RemoteHttpDataTransferProtocolInfo

use of diskCacheV111.vehicles.RemoteHttpDataTransferProtocolInfo in project dcache by dCache.

the class RemoteHttpDataTransferProtocol method runIO.

@Override
public void runIO(FileAttributes attributes, RepositoryChannel channel, ProtocolInfo genericInfo, Set<? extends OpenOption> access) throws CacheException, IOException, InterruptedException {
    LOGGER.debug("info={}, attributes={},  access={}", genericInfo, attributes, access);
    RemoteHttpDataTransferProtocolInfo info = (RemoteHttpDataTransferProtocolInfo) genericInfo;
    _channel = new MoverChannel<>(access, attributes, info, channel);
    channel.optionallyAs(ChecksumChannel.class).ifPresent(c -> {
        info.getDesiredChecksum().ifPresent(t -> {
            try {
                c.addType(t);
            } catch (IOException e) {
                LOGGER.warn("Unable to calculate checksum {}: {}", t, messageOrClassName(e));
            }
        });
    });
    try {
        if (access.contains(StandardOpenOption.WRITE)) {
            receiveFile(info);
        } else {
            checkThat(!info.isVerificationRequired() || attributes.isDefined(CHECKSUM), "checksum verification failed: file has no checksum");
            sendAndCheckFile(info);
        }
    } finally {
        afterTransfer();
    }
}

Example 3 with RemoteHttpDataTransferProtocolInfo

use of diskCacheV111.vehicles.RemoteHttpDataTransferProtocolInfo in project dcache by dCache.

the class RemoteHttpDataTransferProtocol method receiveFile.

private void receiveFile(final RemoteHttpDataTransferProtocolInfo info) throws ThirdPartyTransferFailedCacheException {
    Set<Checksum> checksums;
    long deadline = System.currentTimeMillis() + GET_RETRY_DURATION;
    HttpClientContext context = storeContext(new HttpClientContext());
    try {
        try (CloseableHttpResponse response = doGet(info, context, deadline)) {
            String rfc3230 = headerValue(response, "Digest");
            checksums = Checksums.decodeRfc3230(rfc3230);
            checksums.forEach(_integrityChecker);
            HttpEntity entity = response.getEntity();
            if (entity == null) {
                throw new ThirdPartyTransferFailedCacheException("GET response contains no content");
            }
            long length = entity.getContentLength();
            if (length > 0) {
                _channel.truncate(length);
            }
            if (response.getStatusLine() != null && response.getStatusLine().getStatusCode() < 300 && length > -1) {
                _expectedTransferSize = length;
            }
            entity.writeTo(Channels.newOutputStream(_channel));
        } catch (SocketTimeoutException e) {
            String message = "socket timeout on GET (received " + describeSize(_channel.getBytesTransferred()) + " of data; " + describeSize(e.bytesTransferred) + " pending)";
            if (e.getMessage() != null) {
                message += ": " + e.getMessage();
            }
            throw new ThirdPartyTransferFailedCacheException(message, e);
        } catch (IOException e) {
            throw new ThirdPartyTransferFailedCacheException(messageOrClassName(e), e);
        } catch (InterruptedException e) {
            throw new ThirdPartyTransferFailedCacheException("pool is shutting down", e);
        }
    } catch (ThirdPartyTransferFailedCacheException e) {
        List<URI> redirections = context.getRedirectLocations();
        if (redirections != null && !redirections.isEmpty()) {
            StringBuilder message = new StringBuilder(e.getMessage());
            message.append("; redirects ").append(redirections);
            throw new ThirdPartyTransferFailedCacheException(message.toString(), e.getCause());
        } else {
            throw e;
        }
    }
    // HEAD requests.
    if (checksums.isEmpty() && info.isVerificationRequired()) {
        HttpHead head = buildHeadRequest(info, deadline);
        head.addHeader("Want-Digest", WANT_DIGEST_VALUE);
        try {
            try (CloseableHttpResponse response = _client.execute(head)) {
                String rfc3230 = headerValue(response, "Digest");
                checkThirdPartyTransferSuccessful(rfc3230 != null, "no checksums in HEAD response");
                checksums = Checksums.decodeRfc3230(rfc3230);
                checkThirdPartyTransferSuccessful(!checksums.isEmpty(), "no useful checksums in HEAD response: %s", rfc3230);
                // Ensure integrety.  If we're lucky, this won't trigger
                // rescanning the uploaded file.
                checksums.forEach(_integrityChecker);
            }
        } catch (IOException e) {
            throw new ThirdPartyTransferFailedCacheException("HEAD request failed: " + messageOrClassName(e), e);
        }
    }
}

Example 4 with RemoteHttpDataTransferProtocolInfo

use of diskCacheV111.vehicles.RemoteHttpDataTransferProtocolInfo in project dcache by dCache.

the class RemoteHttpTransferService method createMoverProtocol.

@Override
protected MoverProtocol createMoverProtocol(ProtocolInfo info) throws Exception {
    if (!(info instanceof RemoteHttpDataTransferProtocolInfo)) {
        throw new CacheException(CacheException.CANNOT_CREATE_MOVER, "Could not create third-party HTTP mover for " + info);
    }
    if (info instanceof RemoteHttpsDataTransferProtocolInfo) {
        RemoteHttpsDataTransferProtocolInfo tlsInfo = (RemoteHttpsDataTransferProtocolInfo) info;
        // connection reuse even with X.509 credentials.
        if (tlsInfo.hasCredential()) {
            X509Credential credential = tlsInfo.getCredential();
            SSLContext context = buildSSLContext(credential.getKeyManager());
            CloseableHttpClient client = createClient(context);
            return new RemoteHttpDataTransferProtocol(client) {

                @Override
                protected void afterTransfer() {
                    super.afterTransfer();
                    try {
                        client.close();
                    } catch (IOException e) {
                        LOGGER.warn("Failed to shutdown client cleanly: {}", meaningfulMessage(e));
                    }
                }
            };
        }
    }
    return new RemoteHttpDataTransferProtocol(sharedClient);
}

Example 5 with RemoteHttpDataTransferProtocolInfo

use of diskCacheV111.vehicles.RemoteHttpDataTransferProtocolInfo in project dcache by dCache.

the class Storage method performRemoteTransfer.

private String performRemoteTransfer(SRMUser srmUser, URI remoteTURL, FsPath actualFilePath, boolean store, Map<String, String> extraInfo, Long remoteCredentialId, CopyCallbacks callbacks) throws SRMException {
    DcacheUser user = asDcacheUser(srmUser);
    _log.debug("performRemoteTransfer performing {}", (store ? "store" : "restore"));
    IpProtocolInfo protocolInfo;
    InetSocketAddress remoteAddr = new InetSocketAddress(remoteTURL.getHost(), URIs.optionalPortWithDefault(remoteTURL).orElseThrow(() -> new SRMException("Unknown port number for TURL " + remoteTURL)));
    X509Credential credential = null;
    RequestCredential result = RequestCredential.getRequestCredential(remoteCredentialId);
    if (result != null) {
        credential = result.getDelegatedCredential();
    }
    switch(remoteTURL.getScheme().toLowerCase()) {
        case "gsiftp":
            if (credential == null) {
                throw new SRMAuthorizationException("Cannot authenticate " + "with remote gsiftp service; credential " + "delegation required.");
            }
            RemoteGsiftpTransferProtocolInfo gsiftpProtocolInfo = new RemoteGsiftpTransferProtocolInfo("RemoteGsiftpTransfer", 1, 1, remoteAddr, remoteTURL.toString(), getCellName(), getCellDomainName(), config.getBuffer_size(), config.getTcp_buffer_size(), credential, Optional.empty());
            gsiftpProtocolInfo.setEmode(true);
            gsiftpProtocolInfo.setNumberOfStreams(config.getParallel_streams());
            protocolInfo = gsiftpProtocolInfo;
            break;
        case "https":
            protocolInfo = new RemoteHttpsDataTransferProtocolInfo("RemoteHttpsDataTransfer", 1, 1, remoteAddr, remoteTURL.toString(), isVerifyRequired(extraInfo), httpHeaders(extraInfo), credential, Optional.empty());
            break;
        case "http":
            protocolInfo = new RemoteHttpDataTransferProtocolInfo("RemoteHttpDataTransfer", 1, 1, remoteAddr, remoteTURL.toString(), isVerifyRequired(extraInfo), httpHeaders(extraInfo), Optional.empty());
            break;
        default:
            throw new SRMException("protocol " + remoteTURL.getScheme() + " is not supported");
    }
    RemoteTransferManagerMessage request = new RemoteTransferManagerMessage(remoteTURL, actualFilePath, store, remoteCredentialId, protocolInfo);
    request.setSubject(user.getSubject());
    request.setRestriction(user.getRestriction());
    try {
        RemoteTransferManagerMessage reply = _transferManagerStub.sendAndWait(request);
        long id = reply.getId();
        _log.debug("received first RemoteGsiftpTransferManagerMessage " + "reply from transfer manager, id ={}", id);
        TransferInfo info = new TransferInfo(id, callbacks);
        _log.debug("storing info for callerId = {}", id);
        callerIdToHandler.put(id, info);
        return String.valueOf(id);
    } catch (NoRouteToCellException | TimeoutCacheException e) {
        throw new SRMInternalErrorException("Transfer manager is unavailable: " + e.getMessage(), e);
    } catch (CacheException e) {
        throw new SRMException("TransferManager error: " + e.getMessage(), e);
    } catch (InterruptedException e) {
        throw new SRMException("Request to transfer manager got interruptd", e);
    }
}