Examples with DefaultFileRegion org.apache.flink.shaded.netty4.io.netty.channel.DefaultFileRegion used on opensource projects

Example 1 with DefaultFileRegion

use of org.apache.flink.shaded.netty4.io.netty.channel.DefaultFileRegion in project cassandra by apache.

the class CassandraEntireSSTableStreamWriterTest method testBlockWriterOverWire.

@Test
public void testBlockWriterOverWire() throws IOException {
    StreamSession session = setupStreamingSessionForTest();
    EmbeddedChannel channel = new EmbeddedChannel();
    try (AsyncStreamingOutputPlus out = new AsyncStreamingOutputPlus(channel);
        ComponentContext context = ComponentContext.create(descriptor)) {
        CassandraEntireSSTableStreamWriter writer = new CassandraEntireSSTableStreamWriter(sstable, session, context);
        writer.write(out);
        Queue msgs = channel.outboundMessages();
        assertTrue(msgs.peek() instanceof DefaultFileRegion);
    }
}

Example 2 with DefaultFileRegion

use of org.apache.flink.shaded.netty4.io.netty.channel.DefaultFileRegion in project flink by apache.

the class HistoryServerStaticFileServerHandler method respondWithFile.

/**
 * Response when running with leading JobManager.
 */
private void respondWithFile(ChannelHandlerContext ctx, HttpRequest request, String requestPath) throws IOException, ParseException, RestHandlerException {
    // make sure we request the "index.html" in case there is a directory request
    if (requestPath.endsWith("/")) {
        requestPath = requestPath + "index.html";
    }
    if (!requestPath.contains(".")) {
        // we assume that the path ends in either .html or .js
        requestPath = requestPath + ".json";
    }
    // convert to absolute path
    final File file = new File(rootPath, requestPath);
    if (!file.exists()) {
        // file does not exist. Try to load it with the classloader
        ClassLoader cl = HistoryServerStaticFileServerHandler.class.getClassLoader();
        try (InputStream resourceStream = cl.getResourceAsStream("web" + requestPath)) {
            boolean success = false;
            try {
                if (resourceStream != null) {
                    URL root = cl.getResource("web");
                    URL requested = cl.getResource("web" + requestPath);
                    if (root != null && requested != null) {
                        URI rootURI = new URI(root.getPath()).normalize();
                        URI requestedURI = new URI(requested.getPath()).normalize();
                        // expected scope.
                        if (!rootURI.relativize(requestedURI).equals(requestedURI)) {
                            LOG.debug("Loading missing file from classloader: {}", requestPath);
                            // ensure that directory to file exists.
                            file.getParentFile().mkdirs();
                            Files.copy(resourceStream, file.toPath());
                            success = true;
                        }
                    }
                }
            } catch (Throwable t) {
                LOG.error("error while responding", t);
            } finally {
                if (!success) {
                    LOG.debug("Unable to load requested file {} from classloader", requestPath);
                    throw new NotFoundException("File not found.");
                }
            }
        }
    }
    StaticFileServerHandler.checkFileValidity(file, rootPath, LOG);
    // cache validation
    final String ifModifiedSince = request.headers().get(IF_MODIFIED_SINCE);
    if (ifModifiedSince != null && !ifModifiedSince.isEmpty()) {
        SimpleDateFormat dateFormatter = new SimpleDateFormat(StaticFileServerHandler.HTTP_DATE_FORMAT, Locale.US);
        Date ifModifiedSinceDate = dateFormatter.parse(ifModifiedSince);
        // Only compare up to the second because the datetime format we send to the client
        // does not have milliseconds
        long ifModifiedSinceDateSeconds = ifModifiedSinceDate.getTime() / 1000;
        long fileLastModifiedSeconds = file.lastModified() / 1000;
        if (ifModifiedSinceDateSeconds == fileLastModifiedSeconds) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Responding 'NOT MODIFIED' for file '" + file.getAbsolutePath() + '\'');
            }
            StaticFileServerHandler.sendNotModified(ctx);
            return;
        }
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Responding with file '" + file.getAbsolutePath() + '\'');
    }
    // Don't need to close this manually. Netty's DefaultFileRegion will take care of it.
    final RandomAccessFile raf;
    try {
        raf = new RandomAccessFile(file, "r");
    } catch (FileNotFoundException e) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Could not find file {}.", file.getAbsolutePath());
        }
        HandlerUtils.sendErrorResponse(ctx, request, new ErrorResponseBody("File not found."), NOT_FOUND, Collections.emptyMap());
        return;
    }
    try {
        long fileLength = raf.length();
        HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
        StaticFileServerHandler.setContentTypeHeader(response, file);
        // the job overview should be updated as soon as possible
        if (!requestPath.equals("/joboverview.json")) {
            StaticFileServerHandler.setDateAndCacheHeaders(response, file);
        }
        if (HttpHeaders.isKeepAlive(request)) {
            response.headers().set(CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
        }
        HttpHeaders.setContentLength(response, fileLength);
        // write the initial line and the header.
        ctx.write(response);
        // write the content.
        ChannelFuture lastContentFuture;
        if (ctx.pipeline().get(SslHandler.class) == null) {
            ctx.write(new DefaultFileRegion(raf.getChannel(), 0, fileLength), ctx.newProgressivePromise());
            lastContentFuture = ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
        } else {
            lastContentFuture = ctx.writeAndFlush(new HttpChunkedInput(new ChunkedFile(raf, 0, fileLength, 8192)), ctx.newProgressivePromise());
        // HttpChunkedInput will write the end marker (LastHttpContent) for us.
        }
        // close the connection, if no keep-alive is needed
        if (!HttpHeaders.isKeepAlive(request)) {
            lastContentFuture.addListener(ChannelFutureListener.CLOSE);
        }
    } catch (Exception e) {
        raf.close();
        LOG.error("Failed to serve file.", e);
        throw new RestHandlerException("Internal server error.", INTERNAL_SERVER_ERROR);
    }
}

Example 3 with DefaultFileRegion

use of org.apache.flink.shaded.netty4.io.netty.channel.DefaultFileRegion in project netty by netty.

the class SocketFileRegionTest method testFileRegion0.

private static void testFileRegion0(ServerBootstrap sb, Bootstrap cb, boolean voidPromise, final boolean autoRead, boolean defaultFileRegion) throws Throwable {
    sb.childOption(ChannelOption.AUTO_READ, autoRead);
    cb.option(ChannelOption.AUTO_READ, autoRead);
    final int bufferSize = 1024;
    final File file = PlatformDependent.createTempFile("netty-", ".tmp", null);
    file.deleteOnExit();
    final FileOutputStream out = new FileOutputStream(file);
    final Random random = PlatformDependent.threadLocalRandom();
    // Prepend random data which will not be transferred, so that we can test non-zero start offset
    final int startOffset = random.nextInt(8192);
    for (int i = 0; i < startOffset; i++) {
        out.write(random.nextInt());
    }
    // .. and here comes the real data to transfer.
    out.write(data, bufferSize, data.length - bufferSize);
    // .. and then some extra data which is not supposed to be transferred.
    for (int i = random.nextInt(8192); i > 0; i--) {
        out.write(random.nextInt());
    }
    out.close();
    ChannelInboundHandler ch = new SimpleChannelInboundHandler<Object>() {

        @Override
        public void channelRead0(ChannelHandlerContext ctx, Object msg) throws Exception {
        }

        @Override
        public void channelReadComplete(ChannelHandlerContext ctx) throws Exception {
            if (!autoRead) {
                ctx.read();
            }
        }

        @Override
        public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
            ctx.close();
        }
    };
    TestHandler sh = new TestHandler(autoRead);
    sb.childHandler(sh);
    cb.handler(ch);
    Channel sc = sb.bind().sync().channel();
    Channel cc = cb.connect(sc.localAddress()).sync().channel();
    FileRegion region = new DefaultFileRegion(new RandomAccessFile(file, "r").getChannel(), startOffset, data.length - bufferSize);
    FileRegion emptyRegion = new DefaultFileRegion(new RandomAccessFile(file, "r").getChannel(), 0, 0);
    if (!defaultFileRegion) {
        region = new FileRegionWrapper(region);
        emptyRegion = new FileRegionWrapper(emptyRegion);
    }
    // https://github.com/netty/netty/issues/2964
    if (voidPromise) {
        assertEquals(cc.voidPromise(), cc.write(Unpooled.wrappedBuffer(data, 0, bufferSize), cc.voidPromise()));
        assertEquals(cc.voidPromise(), cc.write(emptyRegion, cc.voidPromise()));
        assertEquals(cc.voidPromise(), cc.writeAndFlush(region, cc.voidPromise()));
    } else {
        assertNotEquals(cc.voidPromise(), cc.write(Unpooled.wrappedBuffer(data, 0, bufferSize)));
        assertNotEquals(cc.voidPromise(), cc.write(emptyRegion));
        assertNotEquals(cc.voidPromise(), cc.writeAndFlush(region));
    }
    while (sh.counter < data.length) {
        if (sh.exception.get() != null) {
            break;
        }
        try {
            Thread.sleep(50);
        } catch (InterruptedException e) {
        // Ignore.
        }
    }
    sh.channel.close().sync();
    cc.close().sync();
    sc.close().sync();
    if (sh.exception.get() != null && !(sh.exception.get() instanceof IOException)) {
        throw sh.exception.get();
    }
    if (sh.exception.get() != null) {
        throw sh.exception.get();
    }
    // Make sure we did not receive more than we expected.
    assertThat(sh.counter, is(data.length));
}

Example 4 with DefaultFileRegion

use of org.apache.flink.shaded.netty4.io.netty.channel.DefaultFileRegion in project netty by netty.

the class FileServerHandler method channelRead0.

@Override
public void channelRead0(ChannelHandlerContext ctx, String msg) throws Exception {
    RandomAccessFile raf = null;
    long length = -1;
    try {
        raf = new RandomAccessFile(msg, "r");
        length = raf.length();
    } catch (Exception e) {
        ctx.writeAndFlush("ERR: " + e.getClass().getSimpleName() + ": " + e.getMessage() + '\n');
        return;
    } finally {
        if (length < 0 && raf != null) {
            raf.close();
        }
    }
    ctx.write("OK: " + raf.length() + '\n');
    if (ctx.pipeline().get(SslHandler.class) == null) {
        // SSL not enabled - can use zero-copy file transfer.
        ctx.write(new DefaultFileRegion(raf.getChannel(), 0, length));
    } else {
        // SSL enabled - cannot use zero-copy file transfer.
        ctx.write(new ChunkedFile(raf));
    }
    ctx.writeAndFlush("\n");
}

Example 5 with DefaultFileRegion

use of org.apache.flink.shaded.netty4.io.netty.channel.DefaultFileRegion in project netty by netty.

the class HttpStaticFileServerHandler method channelRead0.

@Override
public void channelRead0(ChannelHandlerContext ctx, FullHttpRequest request) throws Exception {
    this.request = request;
    if (!request.decoderResult().isSuccess()) {
        sendError(ctx, BAD_REQUEST);
        return;
    }
    if (!GET.equals(request.method())) {
        sendError(ctx, METHOD_NOT_ALLOWED);
        return;
    }
    final boolean keepAlive = HttpUtil.isKeepAlive(request);
    final String uri = request.uri();
    final String path = sanitizeUri(uri);
    if (path == null) {
        sendError(ctx, FORBIDDEN);
        return;
    }
    File file = new File(path);
    if (file.isHidden() || !file.exists()) {
        sendError(ctx, NOT_FOUND);
        return;
    }
    if (file.isDirectory()) {
        if (uri.endsWith("/")) {
            sendListing(ctx, file, uri);
        } else {
            sendRedirect(ctx, uri + '/');
        }
        return;
    }
    if (!file.isFile()) {
        sendError(ctx, FORBIDDEN);
        return;
    }
    // Cache Validation
    String ifModifiedSince = request.headers().get(HttpHeaderNames.IF_MODIFIED_SINCE);
    if (ifModifiedSince != null && !ifModifiedSince.isEmpty()) {
        SimpleDateFormat dateFormatter = new SimpleDateFormat(HTTP_DATE_FORMAT, Locale.US);
        Date ifModifiedSinceDate = dateFormatter.parse(ifModifiedSince);
        // Only compare up to the second because the datetime format we send to the client
        // does not have milliseconds
        long ifModifiedSinceDateSeconds = ifModifiedSinceDate.getTime() / 1000;
        long fileLastModifiedSeconds = file.lastModified() / 1000;
        if (ifModifiedSinceDateSeconds == fileLastModifiedSeconds) {
            sendNotModified(ctx);
            return;
        }
    }
    RandomAccessFile raf;
    try {
        raf = new RandomAccessFile(file, "r");
    } catch (FileNotFoundException ignore) {
        sendError(ctx, NOT_FOUND);
        return;
    }
    long fileLength = raf.length();
    HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
    HttpUtil.setContentLength(response, fileLength);
    setContentTypeHeader(response, file);
    setDateAndCacheHeaders(response, file);
    if (!keepAlive) {
        response.headers().set(HttpHeaderNames.CONNECTION, HttpHeaderValues.CLOSE);
    } else if (request.protocolVersion().equals(HTTP_1_0)) {
        response.headers().set(HttpHeaderNames.CONNECTION, HttpHeaderValues.KEEP_ALIVE);
    }
    // Write the initial line and the header.
    ctx.write(response);
    // Write the content.
    ChannelFuture sendFileFuture;
    ChannelFuture lastContentFuture;
    if (ctx.pipeline().get(SslHandler.class) == null) {
        sendFileFuture = ctx.write(new DefaultFileRegion(raf.getChannel(), 0, fileLength), ctx.newProgressivePromise());
        // Write the end marker.
        lastContentFuture = ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
    } else {
        sendFileFuture = ctx.writeAndFlush(new HttpChunkedInput(new ChunkedFile(raf, 0, fileLength, 8192)), ctx.newProgressivePromise());
        // HttpChunkedInput will write the end marker (LastHttpContent) for us.
        lastContentFuture = sendFileFuture;
    }
    sendFileFuture.addListener(new ChannelProgressiveFutureListener() {

        @Override
        public void operationProgressed(ChannelProgressiveFuture future, long progress, long total) {
            if (total < 0) {
                // total unknown
                System.err.println(future.channel() + " Transfer progress: " + progress);
            } else {
                System.err.println(future.channel() + " Transfer progress: " + progress + " / " + total);
            }
        }

        @Override
        public void operationComplete(ChannelProgressiveFuture future) {
            System.err.println(future.channel() + " Transfer complete.");
        }
    });
    // Decide whether to close the connection or not.
    if (!keepAlive) {
        // Close the connection when the whole content is written out.
        lastContentFuture.addListener(ChannelFutureListener.CLOSE);
    }
}