Examples with DefaultFileRegion io.netty.channel.DefaultFileRegion used on opensource projects

Example 1 with DefaultFileRegion

use of io.netty.channel.DefaultFileRegion in project netty by netty.

the class HttpStaticFileServerHandler method channelRead0.

@Override
public void channelRead0(ChannelHandlerContext ctx, FullHttpRequest request) throws Exception {
    if (!request.decoderResult().isSuccess()) {
        sendError(ctx, BAD_REQUEST);
        return;
    }
    if (request.method() != GET) {
        sendError(ctx, METHOD_NOT_ALLOWED);
        return;
    }
    final String uri = request.uri();
    final String path = sanitizeUri(uri);
    if (path == null) {
        sendError(ctx, FORBIDDEN);
        return;
    }
    File file = new File(path);
    if (file.isHidden() || !file.exists()) {
        sendError(ctx, NOT_FOUND);
        return;
    }
    if (file.isDirectory()) {
        if (uri.endsWith("/")) {
            sendListing(ctx, file, uri);
        } else {
            sendRedirect(ctx, uri + '/');
        }
        return;
    }
    if (!file.isFile()) {
        sendError(ctx, FORBIDDEN);
        return;
    }
    // Cache Validation
    String ifModifiedSince = request.headers().get(HttpHeaderNames.IF_MODIFIED_SINCE);
    if (ifModifiedSince != null && !ifModifiedSince.isEmpty()) {
        SimpleDateFormat dateFormatter = new SimpleDateFormat(HTTP_DATE_FORMAT, Locale.US);
        Date ifModifiedSinceDate = dateFormatter.parse(ifModifiedSince);
        // Only compare up to the second because the datetime format we send to the client
        // does not have milliseconds
        long ifModifiedSinceDateSeconds = ifModifiedSinceDate.getTime() / 1000;
        long fileLastModifiedSeconds = file.lastModified() / 1000;
        if (ifModifiedSinceDateSeconds == fileLastModifiedSeconds) {
            sendNotModified(ctx);
            return;
        }
    }
    RandomAccessFile raf;
    try {
        raf = new RandomAccessFile(file, "r");
    } catch (FileNotFoundException ignore) {
        sendError(ctx, NOT_FOUND);
        return;
    }
    long fileLength = raf.length();
    HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
    HttpUtil.setContentLength(response, fileLength);
    setContentTypeHeader(response, file);
    setDateAndCacheHeaders(response, file);
    if (HttpUtil.isKeepAlive(request)) {
        response.headers().set(HttpHeaderNames.CONNECTION, HttpHeaderValues.KEEP_ALIVE);
    }
    // Write the initial line and the header.
    ctx.write(response);
    // Write the content.
    ChannelFuture sendFileFuture;
    ChannelFuture lastContentFuture;
    if (ctx.pipeline().get(SslHandler.class) == null) {
        sendFileFuture = ctx.write(new DefaultFileRegion(raf.getChannel(), 0, fileLength), ctx.newProgressivePromise());
        // Write the end marker.
        lastContentFuture = ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
    } else {
        sendFileFuture = ctx.writeAndFlush(new HttpChunkedInput(new ChunkedFile(raf, 0, fileLength, 8192)), ctx.newProgressivePromise());
        // HttpChunkedInput will write the end marker (LastHttpContent) for us.
        lastContentFuture = sendFileFuture;
    }
    sendFileFuture.addListener(new ChannelProgressiveFutureListener() {

        @Override
        public void operationProgressed(ChannelProgressiveFuture future, long progress, long total) {
            if (total < 0) {
                // total unknown
                System.err.println(future.channel() + " Transfer progress: " + progress);
            } else {
                System.err.println(future.channel() + " Transfer progress: " + progress + " / " + total);
            }
        }

        @Override
        public void operationComplete(ChannelProgressiveFuture future) {
            System.err.println(future.channel() + " Transfer complete.");
        }
    });
    // Decide whether to close the connection or not.
    if (!HttpUtil.isKeepAlive(request)) {
        // Close the connection when the whole content is written out.
        lastContentFuture.addListener(ChannelFutureListener.CLOSE);
    }
}

Example 2 with DefaultFileRegion

use of io.netty.channel.DefaultFileRegion in project netty by netty.

the class FileServerHandler method channelRead0.

@Override
public void channelRead0(ChannelHandlerContext ctx, String msg) throws Exception {
    RandomAccessFile raf = null;
    long length = -1;
    try {
        raf = new RandomAccessFile(msg, "r");
        length = raf.length();
    } catch (Exception e) {
        ctx.writeAndFlush("ERR: " + e.getClass().getSimpleName() + ": " + e.getMessage() + '\n');
        return;
    } finally {
        if (length < 0 && raf != null) {
            raf.close();
        }
    }
    ctx.write("OK: " + raf.length() + '\n');
    if (ctx.pipeline().get(SslHandler.class) == null) {
        // SSL not enabled - can use zero-copy file transfer.
        ctx.write(new DefaultFileRegion(raf.getChannel(), 0, length));
    } else {
        // SSL enabled - cannot use zero-copy file transfer.
        ctx.write(new ChunkedFile(raf));
    }
    ctx.writeAndFlush("\n");
}

Example 3 with DefaultFileRegion

use of io.netty.channel.DefaultFileRegion in project netty by netty.

the class SocketFileRegionTest method testFileRegion0.

private static void testFileRegion0(ServerBootstrap sb, Bootstrap cb, boolean voidPromise, final boolean autoRead, boolean defaultFileRegion) throws Throwable {
    sb.childOption(ChannelOption.AUTO_READ, autoRead);
    cb.option(ChannelOption.AUTO_READ, autoRead);
    final int bufferSize = 1024;
    final File file = File.createTempFile("netty-", ".tmp");
    file.deleteOnExit();
    final FileOutputStream out = new FileOutputStream(file);
    final Random random = PlatformDependent.threadLocalRandom();
    // Prepend random data which will not be transferred, so that we can test non-zero start offset
    final int startOffset = random.nextInt(8192);
    for (int i = 0; i < startOffset; i++) {
        out.write(random.nextInt());
    }
    // .. and here comes the real data to transfer.
    out.write(data, bufferSize, data.length - bufferSize);
    // .. and then some extra data which is not supposed to be transferred.
    for (int i = random.nextInt(8192); i > 0; i--) {
        out.write(random.nextInt());
    }
    out.close();
    ChannelInboundHandler ch = new SimpleChannelInboundHandler<Object>() {

        @Override
        public void channelRead0(ChannelHandlerContext ctx, Object msg) throws Exception {
        }

        @Override
        public void channelReadComplete(ChannelHandlerContext ctx) throws Exception {
            if (!autoRead) {
                ctx.read();
            }
        }

        @Override
        public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
            ctx.close();
        }
    };
    TestHandler sh = new TestHandler(autoRead);
    sb.childHandler(sh);
    cb.handler(ch);
    Channel sc = sb.bind().sync().channel();
    Channel cc = cb.connect().sync().channel();
    FileRegion region = new DefaultFileRegion(new FileInputStream(file).getChannel(), startOffset, data.length - bufferSize);
    FileRegion emptyRegion = new DefaultFileRegion(new FileInputStream(file).getChannel(), 0, 0);
    if (!defaultFileRegion) {
        region = new FileRegionWrapper(region);
        emptyRegion = new FileRegionWrapper(emptyRegion);
    }
    //     https://github.com/netty/netty/issues/2964
    if (voidPromise) {
        assertEquals(cc.voidPromise(), cc.write(Unpooled.wrappedBuffer(data, 0, bufferSize), cc.voidPromise()));
        assertEquals(cc.voidPromise(), cc.write(emptyRegion, cc.voidPromise()));
        assertEquals(cc.voidPromise(), cc.writeAndFlush(region, cc.voidPromise()));
    } else {
        assertNotEquals(cc.voidPromise(), cc.write(Unpooled.wrappedBuffer(data, 0, bufferSize)));
        assertNotEquals(cc.voidPromise(), cc.write(emptyRegion));
        assertNotEquals(cc.voidPromise(), cc.writeAndFlush(region));
    }
    while (sh.counter < data.length) {
        if (sh.exception.get() != null) {
            break;
        }
        try {
            Thread.sleep(50);
        } catch (InterruptedException e) {
        // Ignore.
        }
    }
    sh.channel.close().sync();
    cc.close().sync();
    sc.close().sync();
    if (sh.exception.get() != null && !(sh.exception.get() instanceof IOException)) {
        throw sh.exception.get();
    }
    if (sh.exception.get() != null) {
        throw sh.exception.get();
    }
    // Make sure we did not receive more than we expected.
    assertThat(sh.counter, is(data.length));
}

Example 4 with DefaultFileRegion

use of io.netty.channel.DefaultFileRegion in project flink by apache.

the class TaskManagerLogHandler method respondAsLeader.

/**
	 * Response when running with leading JobManager.
	 */
@Override
protected void respondAsLeader(final ChannelHandlerContext ctx, final Routed routed, final ActorGateway jobManager) {
    if (cache == null) {
        scala.concurrent.Future<Object> portFuture = jobManager.ask(JobManagerMessages.getRequestBlobManagerPort(), timeout);
        scala.concurrent.Future<BlobCache> cacheFuture = portFuture.map(new Mapper<Object, BlobCache>() {

            @Override
            public BlobCache checkedApply(Object result) throws IOException {
                Option<String> hostOption = jobManager.actor().path().address().host();
                String host = hostOption.isDefined() ? hostOption.get() : "localhost";
                int port = (int) result;
                return new BlobCache(new InetSocketAddress(host, port), config);
            }
        }, executor);
        cache = new FlinkFuture<>(cacheFuture);
    }
    final String taskManagerID = routed.pathParams().get(TaskManagersHandler.TASK_MANAGER_ID_KEY);
    final HttpRequest request = routed.request();
    //fetch TaskManager logs if no other process is currently doing it
    if (lastRequestPending.putIfAbsent(taskManagerID, true) == null) {
        try {
            InstanceID instanceID = new InstanceID(StringUtils.hexStringToByte(taskManagerID));
            scala.concurrent.Future<JobManagerMessages.TaskManagerInstance> scalaTaskManagerFuture = jobManager.ask(new JobManagerMessages.RequestTaskManagerInstance(instanceID), timeout).mapTo(ClassTag$.MODULE$.<JobManagerMessages.TaskManagerInstance>apply(JobManagerMessages.TaskManagerInstance.class));
            Future<JobManagerMessages.TaskManagerInstance> taskManagerFuture = new FlinkFuture<>(scalaTaskManagerFuture);
            Future<BlobKey> blobKeyFuture = taskManagerFuture.thenCompose(new ApplyFunction<JobManagerMessages.TaskManagerInstance, Future<BlobKey>>() {

                @Override
                public Future<BlobKey> apply(JobManagerMessages.TaskManagerInstance value) {
                    Instance taskManager = value.instance().get();
                    if (serveLogFile) {
                        return taskManager.getTaskManagerGateway().requestTaskManagerLog(timeTimeout);
                    } else {
                        return taskManager.getTaskManagerGateway().requestTaskManagerStdout(timeTimeout);
                    }
                }
            });
            Future<String> logPathFuture = blobKeyFuture.thenCombine(cache, new BiFunction<BlobKey, BlobCache, Tuple2<BlobKey, BlobCache>>() {

                @Override
                public Tuple2<BlobKey, BlobCache> apply(BlobKey blobKey, BlobCache blobCache) {
                    return Tuple2.of(blobKey, blobCache);
                }
            }).thenComposeAsync(new ApplyFunction<Tuple2<BlobKey, BlobCache>, Future<String>>() {

                @Override
                public Future<String> apply(Tuple2<BlobKey, BlobCache> value) {
                    final BlobKey blobKey = value.f0;
                    final BlobCache blobCache = value.f1;
                    //delete previous log file, if it is different than the current one
                    HashMap<String, BlobKey> lastSubmittedFile = serveLogFile ? lastSubmittedLog : lastSubmittedStdout;
                    if (lastSubmittedFile.containsKey(taskManagerID)) {
                        if (!blobKey.equals(lastSubmittedFile.get(taskManagerID))) {
                            try {
                                blobCache.deleteGlobal(lastSubmittedFile.get(taskManagerID));
                            } catch (IOException e) {
                                return FlinkCompletableFuture.completedExceptionally(new Exception("Could not delete file for " + taskManagerID + '.', e));
                            }
                            lastSubmittedFile.put(taskManagerID, blobKey);
                        }
                    } else {
                        lastSubmittedFile.put(taskManagerID, blobKey);
                    }
                    try {
                        return FlinkCompletableFuture.completed(blobCache.getURL(blobKey).getFile());
                    } catch (IOException e) {
                        return FlinkCompletableFuture.completedExceptionally(new Exception("Could not retrieve blob for " + blobKey + '.', e));
                    }
                }
            }, executor);
            logPathFuture.exceptionally(new ApplyFunction<Throwable, Void>() {

                @Override
                public Void apply(Throwable failure) {
                    display(ctx, request, "Fetching TaskManager log failed.");
                    LOG.error("Fetching TaskManager log failed.", failure);
                    lastRequestPending.remove(taskManagerID);
                    return null;
                }
            });
            logPathFuture.thenAccept(new AcceptFunction<String>() {

                @Override
                public void accept(String filePath) {
                    File file = new File(filePath);
                    final RandomAccessFile raf;
                    try {
                        raf = new RandomAccessFile(file, "r");
                    } catch (FileNotFoundException e) {
                        display(ctx, request, "Displaying TaskManager log failed.");
                        LOG.error("Displaying TaskManager log failed.", e);
                        return;
                    }
                    long fileLength;
                    try {
                        fileLength = raf.length();
                    } catch (IOException ioe) {
                        display(ctx, request, "Displaying TaskManager log failed.");
                        LOG.error("Displaying TaskManager log failed.", ioe);
                        try {
                            raf.close();
                        } catch (IOException e) {
                            LOG.error("Could not close random access file.", e);
                        }
                        return;
                    }
                    final FileChannel fc = raf.getChannel();
                    HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
                    response.headers().set(CONTENT_TYPE, "text/plain");
                    if (HttpHeaders.isKeepAlive(request)) {
                        response.headers().set(CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
                    }
                    HttpHeaders.setContentLength(response, fileLength);
                    // write the initial line and the header.
                    ctx.write(response);
                    // write the content.
                    ChannelFuture lastContentFuture;
                    final GenericFutureListener<io.netty.util.concurrent.Future<? super Void>> completionListener = new GenericFutureListener<io.netty.util.concurrent.Future<? super Void>>() {

                        @Override
                        public void operationComplete(io.netty.util.concurrent.Future<? super Void> future) throws Exception {
                            lastRequestPending.remove(taskManagerID);
                            fc.close();
                            raf.close();
                        }
                    };
                    if (ctx.pipeline().get(SslHandler.class) == null) {
                        ctx.write(new DefaultFileRegion(fc, 0, fileLength), ctx.newProgressivePromise()).addListener(completionListener);
                        lastContentFuture = ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
                    } else {
                        try {
                            lastContentFuture = ctx.writeAndFlush(new HttpChunkedInput(new ChunkedFile(raf, 0, fileLength, 8192)), ctx.newProgressivePromise()).addListener(completionListener);
                        } catch (IOException e) {
                            display(ctx, request, "Displaying TaskManager log failed.");
                            LOG.warn("Could not write http data.", e);
                            return;
                        }
                    // HttpChunkedInput will write the end marker (LastHttpContent) for us.
                    }
                    // close the connection, if no keep-alive is needed
                    if (!HttpHeaders.isKeepAlive(request)) {
                        lastContentFuture.addListener(ChannelFutureListener.CLOSE);
                    }
                }
            });
        } catch (Exception e) {
            display(ctx, request, "Error: " + e.getMessage());
            LOG.error("Fetching TaskManager log failed.", e);
            lastRequestPending.remove(taskManagerID);
        }
    } else {
        display(ctx, request, "loading...");
    }
}

Example 5 with DefaultFileRegion

use of io.netty.channel.DefaultFileRegion in project flink by apache.

the class StaticFileServerHandler method respondAsLeader.

/**
	 * Response when running with leading JobManager.
	 */
private void respondAsLeader(ChannelHandlerContext ctx, HttpRequest request, String requestPath) throws IOException, ParseException, URISyntaxException {
    // convert to absolute path
    final File file = new File(rootPath, requestPath);
    if (!file.exists()) {
        // file does not exist. Try to load it with the classloader
        ClassLoader cl = StaticFileServerHandler.class.getClassLoader();
        try (InputStream resourceStream = cl.getResourceAsStream("web" + requestPath)) {
            boolean success = false;
            try {
                if (resourceStream != null) {
                    URL root = cl.getResource("web");
                    URL requested = cl.getResource("web" + requestPath);
                    if (root != null && requested != null) {
                        URI rootURI = new URI(root.getPath()).normalize();
                        URI requestedURI = new URI(requested.getPath()).normalize();
                        // expected scope.
                        if (!rootURI.relativize(requestedURI).equals(requestedURI)) {
                            logger.debug("Loading missing file from classloader: {}", requestPath);
                            // ensure that directory to file exists.
                            file.getParentFile().mkdirs();
                            Files.copy(resourceStream, file.toPath());
                            success = true;
                        }
                    }
                }
            } catch (Throwable t) {
                logger.error("error while responding", t);
            } finally {
                if (!success) {
                    logger.debug("Unable to load requested file {} from classloader", requestPath);
                    sendError(ctx, NOT_FOUND);
                    return;
                }
            }
        }
    }
    if (!file.exists() || file.isHidden() || file.isDirectory() || !file.isFile()) {
        sendError(ctx, NOT_FOUND);
        return;
    }
    if (!file.getCanonicalFile().toPath().startsWith(rootPath.toPath())) {
        sendError(ctx, NOT_FOUND);
        return;
    }
    // cache validation
    final String ifModifiedSince = request.headers().get(IF_MODIFIED_SINCE);
    if (ifModifiedSince != null && !ifModifiedSince.isEmpty()) {
        SimpleDateFormat dateFormatter = new SimpleDateFormat(HTTP_DATE_FORMAT, Locale.US);
        Date ifModifiedSinceDate = dateFormatter.parse(ifModifiedSince);
        // Only compare up to the second because the datetime format we send to the client
        // does not have milliseconds
        long ifModifiedSinceDateSeconds = ifModifiedSinceDate.getTime() / 1000;
        long fileLastModifiedSeconds = file.lastModified() / 1000;
        if (ifModifiedSinceDateSeconds == fileLastModifiedSeconds) {
            if (logger.isDebugEnabled()) {
                logger.debug("Responding 'NOT MODIFIED' for file '" + file.getAbsolutePath() + '\'');
            }
            sendNotModified(ctx);
            return;
        }
    }
    if (logger.isDebugEnabled()) {
        logger.debug("Responding with file '" + file.getAbsolutePath() + '\'');
    }
    // Don't need to close this manually. Netty's DefaultFileRegion will take care of it.
    final RandomAccessFile raf;
    try {
        raf = new RandomAccessFile(file, "r");
    } catch (FileNotFoundException e) {
        sendError(ctx, NOT_FOUND);
        return;
    }
    long fileLength = raf.length();
    HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
    setContentTypeHeader(response, file);
    // since the log and out files are rapidly changing, we don't want to browser to cache them
    if (!(requestPath.contains("log") || requestPath.contains("out"))) {
        setDateAndCacheHeaders(response, file);
    }
    if (HttpHeaders.isKeepAlive(request)) {
        response.headers().set(CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
    }
    HttpHeaders.setContentLength(response, fileLength);
    // write the initial line and the header.
    ctx.write(response);
    // write the content.
    ChannelFuture lastContentFuture;
    if (ctx.pipeline().get(SslHandler.class) == null) {
        ctx.write(new DefaultFileRegion(raf.getChannel(), 0, fileLength), ctx.newProgressivePromise());
        lastContentFuture = ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
    } else {
        lastContentFuture = ctx.writeAndFlush(new HttpChunkedInput(new ChunkedFile(raf, 0, fileLength, 8192)), ctx.newProgressivePromise());
    // HttpChunkedInput will write the end marker (LastHttpContent) for us.
    }
    // close the connection, if no keep-alive is needed
    if (!HttpHeaders.isKeepAlive(request)) {
        lastContentFuture.addListener(ChannelFutureListener.CLOSE);
    }
}