Examples with NettyTransferService org.dcache.pool.movers.NettyTransferService used on opensource projects

Example 1 with NettyTransferService

use of org.dcache.pool.movers.NettyTransferService in project dcache by dCache.

the class HttpPoolRequestHandler method doOnPut.

@Override
protected ChannelFuture doOnPut(ChannelHandlerContext context, HttpRequest request) {
    NettyTransferService<HttpProtocolInfo>.NettyMoverChannel file = null;
    Exception exception = null;
    if (isBadRequest(request)) {
        return context.newSucceededFuture();
    }
    try {
        checkContentHeader(request.headers().names(), SUPPORTED_CONTENT_HEADERS);
        file = open(request, true);
        if (!file.getIoMode().contains(StandardOpenOption.WRITE)) {
            throw new HttpException(METHOD_NOT_ALLOWED.code(), "Resource is not open for writing");
        }
        contentMd5Checksum(request).ifPresent(file::addChecksum);
        OptionalLong contentLength = contentLength(request);
        if (contentLength.isPresent()) {
            file.truncate(contentLength.getAsLong());
        } else if (file.getFileAttributes().isDefined(FileAttribute.SIZE)) {
            file.truncate(file.getFileAttributes().getSize());
        }
        file.getProtocolInfo().getWantedChecksum().ifPresent(file::addChecksumType);
        _wantedDigest = wantDigest(request).flatMap(Checksums::parseWantDigest);
        _wantedDigest.ifPresent(file::addChecksumType);
        if (is100ContinueExpected(request)) {
            context.writeAndFlush(new DefaultFullHttpResponse(HTTP_1_1, CONTINUE)).addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE);
        }
        _writeChannel = file;
        file = null;
        return null;
    } catch (Redirect e) {
        exception = e;
        return context.writeAndFlush(e.createResponse());
    } catch (HttpException e) {
        exception = e;
        return context.writeAndFlush(createErrorResponse(HttpResponseStatus.valueOf(e.getErrorCode()), e.getMessage()));
    } catch (URISyntaxException e) {
        exception = e;
        return context.writeAndFlush(createErrorResponse(BAD_REQUEST, "URI is not valid: " + e.getMessage()));
    } catch (IllegalArgumentException e) {
        exception = e;
        return context.writeAndFlush(createErrorResponse(BAD_REQUEST, e.getMessage()));
    } catch (IOException | RuntimeException e) {
        exception = e;
        return context.writeAndFlush(createErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage()));
    } finally {
        if (file != null) {
            file.release(exception);
            _files.remove(file);
        }
    }
}

Example 2 with NettyTransferService

use of org.dcache.pool.movers.NettyTransferService in project dcache by dCache.

the class HttpPoolRequestHandler method doOnGet.

/**
 * Single GET operation.
 * <p>
 * Finds the correct mover channel using the UUID in the GET. Range queries are supported. The
 * file will be sent to the remote peer in chunks to avoid server side memory issues.
 */
@Override
protected ChannelFuture doOnGet(ChannelHandlerContext context, HttpRequest request) {
    NettyTransferService<HttpProtocolInfo>.NettyMoverChannel file;
    List<HttpByteRange> ranges;
    long fileSize;
    if (isBadRequest(request)) {
        return context.newSucceededFuture();
    }
    try {
        file = open(request, false);
        if (file.getIoMode().contains(StandardOpenOption.WRITE)) {
            throw new HttpException(METHOD_NOT_ALLOWED.code(), "Resource is not open for reading");
        }
        fileSize = file.size();
        ranges = parseHttpRange(request, 0, fileSize - 1);
    } catch (Redirect e) {
        return context.writeAndFlush(e.createResponse());
    } catch (HttpException e) {
        return context.writeAndFlush(createErrorResponse(e.getErrorCode(), e.getMessage()));
    } catch (URISyntaxException e) {
        return context.writeAndFlush(createErrorResponse(BAD_REQUEST, "URI not valid: " + e.getMessage()));
    } catch (IllegalArgumentException e) {
        return context.writeAndFlush(createErrorResponse(BAD_REQUEST, e.getMessage()));
    } catch (IOException e) {
        return context.writeAndFlush(createErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage()));
    }
    Optional<String> digest = wantDigest(request).flatMap(h -> Checksums.digestHeader(h, file.getFileAttributes()));
    if (ranges == null || ranges.isEmpty()) {
        /*
             * GET for a whole file
             */
        context.write(new HttpGetResponse(fileSize, file, digest)).addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE);
        context.write(read(file, 0, fileSize - 1)).addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE);
        ChannelFuture writeAndFlush = context.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
        // Release the file immediately after supplying all of the file's content.  We're
        // assuming that the client will not make further requests against this URL.  This is
        // done to send the DoorTransferFinishedMessage in a timely fashion.
        writeAndFlush.addListener(f -> file.release());
        return writeAndFlush;
    } else if (ranges.size() == 1) {
        /* RFC 2616: 14.16. A response to a request for a single range
             * MUST NOT be sent using the multipart/byteranges media type.
             */
        HttpByteRange range = ranges.get(0);
        context.write(new HttpPartialContentResponse(range.getLower(), range.getUpper(), fileSize, digest)).addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE);
        context.write(read(file, range.getLower(), range.getUpper())).addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE);
        // client will not make further requests against this URL.
        return context.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
    } else {
        /*
             * GET for multiple ranges
             */
        long totalLen = 0;
        ByteBuf[] fragmentMarkers = new ByteBuf[ranges.size()];
        for (int i = 0; i < ranges.size(); i++) {
            HttpByteRange range = ranges.get(i);
            long upper = range.getUpper();
            long lower = range.getLower();
            totalLen += upper - lower + 1;
            ByteBuf buffer = fragmentMarkers[i] = createMultipartFragmentMarker(lower, upper, fileSize);
            totalLen += buffer.readableBytes();
        }
        ByteBuf endMarker = createMultipartEnd();
        totalLen += endMarker.readableBytes();
        context.write(new HttpMultipartResponse(digest, totalLen)).addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE);
        for (int i = 0; i < ranges.size(); i++) {
            HttpByteRange range = ranges.get(i);
            context.write(fragmentMarkers[i]).addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE);
            context.write(read(file, range.getLower(), range.getUpper())).addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE);
        }
        // client will not make further requests against this URL.
        return context.writeAndFlush(new DefaultLastHttpContent(endMarker));
    }
}

Example 3 with NettyTransferService

use of org.dcache.pool.movers.NettyTransferService in project dcache by dCache.

the class HttpPoolRequestHandler method open.

/**
 * Get the mover channel for a certain HTTP request. The mover channel is identified by UUID
 * generated upon mover start and sent back to the door as a part of the address info.
 *
 * @param request   HttpRequest that was sent by the client
 * @param exclusive True if the mover channel exclusively is to be opened in exclusive mode.
 *                  False if the mover channel can be shared with other requests.
 * @return Mover channel for specified UUID
 * @throws IllegalArgumentException Request did not include UUID or no mover channel found for
 *                                  UUID in the request
 */
private NettyTransferService<HttpProtocolInfo>.NettyMoverChannel open(HttpRequest request, boolean exclusive) throws IllegalArgumentException, URISyntaxException, Redirect {
    QueryStringDecoder queryStringDecoder = new QueryStringDecoder(request.getUri());
    Map<String, List<String>> params = queryStringDecoder.parameters();
    if (!params.containsKey(HttpTransferService.UUID_QUERY_PARAM)) {
        if (!request.getUri().equals("/favicon.ico")) {
            LOGGER.error("Received request without UUID in the query " + "string. Request-URI was {}", request.getUri());
        }
        throw new IllegalArgumentException("Query string does not include any UUID.");
    }
    List<String> uuidList = params.get(HttpTransferService.UUID_QUERY_PARAM);
    if (uuidList.isEmpty()) {
        throw new IllegalArgumentException("UUID parameter does not include any value.");
    }
    UUID uuid = UUID.fromString(uuidList.get(0));
    NettyTransferService<HttpProtocolInfo>.NettyMoverChannel file = _server.openFile(uuid, exclusive);
    if (file == null) {
        Optional<URI> referrer = buildReferrer(request, params);
        if (referrer.isPresent()) {
            throw new Redirect(referrer.get(), "Request is no longer valid");
        }
        throw new IllegalArgumentException("Request is no longer valid. " + "Please resubmit to door.");
    }
    URI uri = new URI(request.getUri());
    FsPath requestedFile = FsPath.create(uri.getPath());
    FsPath transferFile = FsPath.create(file.getProtocolInfo().getPath());
    if (!requestedFile.equals(transferFile)) {
        LOGGER.warn("Received an illegal request for file {}, while serving {}", requestedFile, transferFile);
        throw new IllegalArgumentException("The file you specified does " + "not match the UUID you specified!");
    }
    _files.add(file);
    return file;
}

Example 4 with NettyTransferService

use of org.dcache.pool.movers.NettyTransferService in project dcache by dCache.

the class XrootdPoolRequestHandler method doOnOpen.

/**
 * Obtains the right mover channel using an opaque token in the request. The mover channel is
 * wrapped by a file descriptor. The file descriptor is stored for subsequent access.
 * <p>
 * In the case that this is a write request as destination in a third party copy, a third-party
 * client is started.  The client issues login, open and read requests to the source server, and
 * writes the responses to the file descriptor.
 * <p>
 * The third-party client also sends a sync response back to the client when the transfer has
 * completed.
 */
@Override
protected XrootdResponse<OpenRequest> doOnOpen(ChannelHandlerContext ctx, OpenRequest msg) throws XrootdException {
    try {
        Map<String, String> opaqueMap = getOpaqueMap(msg.getOpaque());
        UUID uuid = getUuid(opaqueMap);
        if (uuid == null) {
            _log.info("Request to open {} contains no UUID.", msg.getPath());
            throw new XrootdException(kXR_NotAuthorized, "Request lacks the " + UUID_PREFIX + " property.");
        }
        enforceClientTlsIfDestinationRequiresItForTpc(opaqueMap);
        NettyTransferService<XrootdProtocolInfo>.NettyMoverChannel file = _server.openFile(uuid, false);
        if (file == null) {
            _log.info("No mover found for {} with UUID {}.", msg.getPath(), uuid);
            return redirectToDoor(ctx, msg, () -> {
                throw new XrootdException(kXR_NotAuthorized, UUID_PREFIX + " is no longer valid.");
            });
        }
        /*
             *  Stop any timer in case this is a reconnect.
             */
        _server.cancelReconnectTimeoutForMover(uuid);
        _log.debug("doOnOpen, called cancel on reconnect timers for {}", uuid);
        XrootdProtocolInfo protocolInfo = file.getProtocolInfo();
        try {
            FileDescriptor descriptor;
            boolean isWrite = file.getIoMode().contains(StandardOpenOption.WRITE);
            if (msg.isNew() && !isWrite) {
                throw new XrootdException(kXR_FileNotOpen, "File exists.");
            } else if (msg.isDelete() && !isWrite) {
                throw new XrootdException(kXR_Unsupported, "File exists.");
            /*
                     *  Some clients express only kXR_delete when then intend to write
                     *  so we need to consider delete as a write request here.
                     */
            } else if ((msg.isNew() || msg.isReadWrite() || msg.isDelete()) && isWrite) {
                boolean posc = (msg.getOptions() & kXR_posc) == kXR_posc || protocolInfo.getFlags().contains(XrootdProtocolInfo.Flags.POSC);
                if (opaqueMap.containsKey("tpc.src")) {
                    _log.debug("Request to open {} is as third-party destination.", msg);
                    XrootdTpcInfo tpcInfo = new XrootdTpcInfo(opaqueMap);
                    tpcInfo.setDelegatedProxy(protocolInfo.getDelegatedCredential());
                    tpcInfo.setUid(protocolInfo.getTpcUid());
                    tpcInfo.setGid(protocolInfo.getTpcGid());
                    descriptor = new TpcWriteDescriptor(file, posc, ctx, _server, opaqueMap.get("org.dcache.xrootd.client"), tpcInfo, tlsSessionInfo);
                } else {
                    descriptor = new WriteDescriptor(file, posc);
                }
            } else {
                descriptor = new ReadDescriptor(file);
            }
            FileStatus stat = msg.isRetStat() ? stat(file) : null;
            int fd = addDescriptor(descriptor);
            _redirectingDoor = protocolInfo.getDoorAddress();
            file = null;
            _hasOpenedFiles = true;
            return new OpenResponse(msg, fd, null, null, stat);
        } finally {
            if (file != null) {
                file.release();
            }
        }
    } catch (ParseException e) {
        throw new XrootdException(kXR_ArgInvalid, e.getMessage());
    } catch (IOException e) {
        throw new XrootdException(kXR_IOError, e.getMessage());
    }
}

Example 5 with NettyTransferService

use of org.dcache.pool.movers.NettyTransferService in project dcache by dCache.

the class HttpPoolRequestHandler method doOnContent.

@Override
protected ChannelFuture doOnContent(ChannelHandlerContext context, HttpContent content) {
    if (isBadRequest(content)) {
        return context.newSucceededFuture();
    }
    if (_writeChannel != null) {
        try {
            ByteBuf data = content.content();
            while (data.isReadable()) {
                data.readBytes(_writeChannel, data.readableBytes());
            }
            if (content instanceof LastHttpContent) {
                checkContentHeader(((LastHttpContent) content).trailingHeaders().names(), Collections.singletonList(CONTENT_LENGTH));
                context.channel().config().setAutoRead(false);
                NettyTransferService<HttpProtocolInfo>.NettyMoverChannel writeChannel = _writeChannel;
                _writeChannel = null;
                _files.remove(writeChannel);
                long size = writeChannel.size();
                URI location = writeChannel.getProtocolInfo().getLocation();
                ChannelPromise promise = context.newPromise();
                Futures.addCallback(writeChannel.release(), new FutureCallback<Void>() {

                    @Override
                    public void onSuccess(Void result) {
                        try {
                            Optional<String> digest = _wantedDigest.flatMap(t -> Checksums.digestHeader(t, writeChannel.getFileAttributes()));
                            context.writeAndFlush(new HttpPutResponse(size, location, digest), promise);
                        } catch (IOException e) {
                            context.writeAndFlush(createErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage()), promise);
                        }
                        context.channel().config().setAutoRead(true);
                    }

                    @Override
                    public void onFailure(Throwable t) {
                        if (t instanceof FileCorruptedCacheException) {
                            context.writeAndFlush(createErrorResponse(BAD_REQUEST, t.getMessage()), promise);
                        } else if (t instanceof CacheException) {
                            context.writeAndFlush(createErrorResponse(INTERNAL_SERVER_ERROR, t.getMessage()), promise);
                        } else {
                            context.writeAndFlush(createErrorResponse(INTERNAL_SERVER_ERROR, t.toString()), promise);
                        }
                        context.channel().config().setAutoRead(true);
                    }
                }, MoreExecutors.directExecutor());
                return promise;
            }
        } catch (OutOfDiskException e) {
            _writeChannel.release(e);
            _files.remove(_writeChannel);
            _writeChannel = null;
            return context.writeAndFlush(createErrorResponse(INSUFFICIENT_STORAGE, e.getMessage()));
        } catch (IOException e) {
            _writeChannel.release(e);
            _files.remove(_writeChannel);
            _writeChannel = null;
            return context.writeAndFlush(createErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage()));
        } catch (HttpException e) {
            _writeChannel.release(e);
            _files.remove(_writeChannel);
            _writeChannel = null;
            return context.writeAndFlush(createErrorResponse(HttpResponseStatus.valueOf(e.getErrorCode()), e.getMessage()));
        }
    }
    return null;
}