Search in sources :

Example 6 with DefaultFileRegion

use of org.apache.flink.shaded.netty4.io.netty.channel.DefaultFileRegion in project blade by biezhi.

the class StaticFileHandler method handle.

/**
 * print static file to client
 *
 * @param webContext web context
 * @throws Exception
 */
@Override
public void handle(WebContext webContext) throws Exception {
    Request request = webContext.getRequest();
    ChannelHandlerContext ctx = webContext.getChannelHandlerContext();
    if (!HttpConst.METHOD_GET.equals(request.method())) {
        sendError(ctx, METHOD_NOT_ALLOWED);
        return;
    }
    Instant start = Instant.now();
    String uri = URLDecoder.decode(request.uri(), "UTF-8");
    String method = StringKit.padRight(request.method(), 6);
    String cleanURL = getCleanURL(request, uri);
    // webjars
    if (cleanURL.startsWith(Const.WEB_JARS)) {
        InputStream input = getResourceStreamFromJar(uri);
        writeWithJarFile(request, ctx, uri, start, cleanURL, method, input);
        return;
    }
    // jar file
    if (BladeKit.runtimeIsJAR()) {
        InputStream input = StaticFileHandler.class.getResourceAsStream(cleanURL);
        writeWithJarFile(request, ctx, uri, start, cleanURL, method, input);
        return;
    }
    // disk file
    final String path = sanitizeUri(cleanURL);
    if (path == null) {
        log403(log, method, uri);
        throw new ForbiddenException();
    }
    File file = new File(path);
    if (file.isHidden() || !file.exists()) {
        // gradle resources path
        File resourcesDirectory = getGradleResourcesDirectory();
        if (resourcesDirectory.isDirectory()) {
            file = new File(resourcesDirectory.getPath() + "/" + cleanURL.substring(1));
            if (file.isHidden() || !file.exists()) {
                throw new NotFoundException(uri);
            }
        } else {
            throw new NotFoundException(uri);
        }
    }
    if (file.isDirectory() && showFileList) {
        sendListing(ctx, uri, getFileMetas(file), cleanURL);
        return;
    }
    if (!file.isFile()) {
        sendError(ctx, FORBIDDEN);
        return;
    }
    // Cache Validation
    if (isHttp304(ctx, request, file.length(), file.lastModified())) {
        log304(log, method, uri);
        return;
    }
    HttpResponse httpResponse = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
    setContentTypeHeader(httpResponse, file);
    setDateAndCacheHeaders(httpResponse, file);
    if (request.useGZIP()) {
        File output = new File(file.getPath() + ".gz");
        IOKit.compressGZIP(file, output);
        file = output;
        setGzip(httpResponse);
    }
    RandomAccessFile raf;
    try {
        raf = new RandomAccessFile(file, "r");
    } catch (FileNotFoundException ignore) {
        sendError(ctx, NOT_FOUND);
        return;
    }
    long fileLength = raf.length();
    httpResponse.headers().set(HttpConst.CONTENT_LENGTH, fileLength);
    if (request.keepAlive()) {
        httpResponse.headers().set(HttpConst.CONNECTION, HttpConst.KEEP_ALIVE);
    }
    // Write the initial line and the header.
    ctx.write(httpResponse);
    // Write the content.
    ChannelFuture sendFileFuture;
    ChannelFuture lastContentFuture;
    if (ctx.pipeline().get(SslHandler.class) == null) {
        sendFileFuture = ctx.write(new DefaultFileRegion(raf.getChannel(), 0, fileLength), ctx.newProgressivePromise());
        // Write the end marker.
        lastContentFuture = ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
    } else {
        sendFileFuture = ctx.writeAndFlush(new HttpChunkedInput(new ChunkedFile(raf, 0, fileLength, 8192)), ctx.newProgressivePromise());
        // HttpChunkedInput will write the end marker (LastHttpContent) for us.
        lastContentFuture = sendFileFuture;
    }
    sendFileFuture.addListener(ProgressiveFutureListener.build(raf));
    // Decide whether to close the connection or not.
    if (!request.keepAlive()) {
        lastContentFuture.addListener(ChannelFutureListener.CLOSE);
    }
    log200AndCost(log, start, method, uri);
}
Also used : ChannelFuture(io.netty.channel.ChannelFuture) ForbiddenException(com.blade.exception.ForbiddenException) Instant(java.time.Instant) ChunkedFile(io.netty.handler.stream.ChunkedFile) Request(com.blade.mvc.http.Request) NotFoundException(com.blade.exception.NotFoundException) ChannelHandlerContext(io.netty.channel.ChannelHandlerContext) DefaultFileRegion(io.netty.channel.DefaultFileRegion) SslHandler(io.netty.handler.ssl.SslHandler) ChunkedFile(io.netty.handler.stream.ChunkedFile) JarFile(java.util.jar.JarFile)

Example 7 with DefaultFileRegion

use of org.apache.flink.shaded.netty4.io.netty.channel.DefaultFileRegion in project flink by apache.

the class TaskManagerLogHandler method respondAsLeader.

/**
	 * Response when running with leading JobManager.
	 */
@Override
protected void respondAsLeader(final ChannelHandlerContext ctx, final Routed routed, final ActorGateway jobManager) {
    if (cache == null) {
        scala.concurrent.Future<Object> portFuture = jobManager.ask(JobManagerMessages.getRequestBlobManagerPort(), timeout);
        scala.concurrent.Future<BlobCache> cacheFuture = portFuture.map(new Mapper<Object, BlobCache>() {

            @Override
            public BlobCache checkedApply(Object result) throws IOException {
                Option<String> hostOption = jobManager.actor().path().address().host();
                String host = hostOption.isDefined() ? hostOption.get() : "localhost";
                int port = (int) result;
                return new BlobCache(new InetSocketAddress(host, port), config);
            }
        }, executor);
        cache = new FlinkFuture<>(cacheFuture);
    }
    final String taskManagerID = routed.pathParams().get(TaskManagersHandler.TASK_MANAGER_ID_KEY);
    final HttpRequest request = routed.request();
    //fetch TaskManager logs if no other process is currently doing it
    if (lastRequestPending.putIfAbsent(taskManagerID, true) == null) {
        try {
            InstanceID instanceID = new InstanceID(StringUtils.hexStringToByte(taskManagerID));
            scala.concurrent.Future<JobManagerMessages.TaskManagerInstance> scalaTaskManagerFuture = jobManager.ask(new JobManagerMessages.RequestTaskManagerInstance(instanceID), timeout).mapTo(ClassTag$.MODULE$.<JobManagerMessages.TaskManagerInstance>apply(JobManagerMessages.TaskManagerInstance.class));
            Future<JobManagerMessages.TaskManagerInstance> taskManagerFuture = new FlinkFuture<>(scalaTaskManagerFuture);
            Future<BlobKey> blobKeyFuture = taskManagerFuture.thenCompose(new ApplyFunction<JobManagerMessages.TaskManagerInstance, Future<BlobKey>>() {

                @Override
                public Future<BlobKey> apply(JobManagerMessages.TaskManagerInstance value) {
                    Instance taskManager = value.instance().get();
                    if (serveLogFile) {
                        return taskManager.getTaskManagerGateway().requestTaskManagerLog(timeTimeout);
                    } else {
                        return taskManager.getTaskManagerGateway().requestTaskManagerStdout(timeTimeout);
                    }
                }
            });
            Future<String> logPathFuture = blobKeyFuture.thenCombine(cache, new BiFunction<BlobKey, BlobCache, Tuple2<BlobKey, BlobCache>>() {

                @Override
                public Tuple2<BlobKey, BlobCache> apply(BlobKey blobKey, BlobCache blobCache) {
                    return Tuple2.of(blobKey, blobCache);
                }
            }).thenComposeAsync(new ApplyFunction<Tuple2<BlobKey, BlobCache>, Future<String>>() {

                @Override
                public Future<String> apply(Tuple2<BlobKey, BlobCache> value) {
                    final BlobKey blobKey = value.f0;
                    final BlobCache blobCache = value.f1;
                    //delete previous log file, if it is different than the current one
                    HashMap<String, BlobKey> lastSubmittedFile = serveLogFile ? lastSubmittedLog : lastSubmittedStdout;
                    if (lastSubmittedFile.containsKey(taskManagerID)) {
                        if (!blobKey.equals(lastSubmittedFile.get(taskManagerID))) {
                            try {
                                blobCache.deleteGlobal(lastSubmittedFile.get(taskManagerID));
                            } catch (IOException e) {
                                return FlinkCompletableFuture.completedExceptionally(new Exception("Could not delete file for " + taskManagerID + '.', e));
                            }
                            lastSubmittedFile.put(taskManagerID, blobKey);
                        }
                    } else {
                        lastSubmittedFile.put(taskManagerID, blobKey);
                    }
                    try {
                        return FlinkCompletableFuture.completed(blobCache.getURL(blobKey).getFile());
                    } catch (IOException e) {
                        return FlinkCompletableFuture.completedExceptionally(new Exception("Could not retrieve blob for " + blobKey + '.', e));
                    }
                }
            }, executor);
            logPathFuture.exceptionally(new ApplyFunction<Throwable, Void>() {

                @Override
                public Void apply(Throwable failure) {
                    display(ctx, request, "Fetching TaskManager log failed.");
                    LOG.error("Fetching TaskManager log failed.", failure);
                    lastRequestPending.remove(taskManagerID);
                    return null;
                }
            });
            logPathFuture.thenAccept(new AcceptFunction<String>() {

                @Override
                public void accept(String filePath) {
                    File file = new File(filePath);
                    final RandomAccessFile raf;
                    try {
                        raf = new RandomAccessFile(file, "r");
                    } catch (FileNotFoundException e) {
                        display(ctx, request, "Displaying TaskManager log failed.");
                        LOG.error("Displaying TaskManager log failed.", e);
                        return;
                    }
                    long fileLength;
                    try {
                        fileLength = raf.length();
                    } catch (IOException ioe) {
                        display(ctx, request, "Displaying TaskManager log failed.");
                        LOG.error("Displaying TaskManager log failed.", ioe);
                        try {
                            raf.close();
                        } catch (IOException e) {
                            LOG.error("Could not close random access file.", e);
                        }
                        return;
                    }
                    final FileChannel fc = raf.getChannel();
                    HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
                    response.headers().set(CONTENT_TYPE, "text/plain");
                    if (HttpHeaders.isKeepAlive(request)) {
                        response.headers().set(CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
                    }
                    HttpHeaders.setContentLength(response, fileLength);
                    // write the initial line and the header.
                    ctx.write(response);
                    // write the content.
                    ChannelFuture lastContentFuture;
                    final GenericFutureListener<io.netty.util.concurrent.Future<? super Void>> completionListener = new GenericFutureListener<io.netty.util.concurrent.Future<? super Void>>() {

                        @Override
                        public void operationComplete(io.netty.util.concurrent.Future<? super Void> future) throws Exception {
                            lastRequestPending.remove(taskManagerID);
                            fc.close();
                            raf.close();
                        }
                    };
                    if (ctx.pipeline().get(SslHandler.class) == null) {
                        ctx.write(new DefaultFileRegion(fc, 0, fileLength), ctx.newProgressivePromise()).addListener(completionListener);
                        lastContentFuture = ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
                    } else {
                        try {
                            lastContentFuture = ctx.writeAndFlush(new HttpChunkedInput(new ChunkedFile(raf, 0, fileLength, 8192)), ctx.newProgressivePromise()).addListener(completionListener);
                        } catch (IOException e) {
                            display(ctx, request, "Displaying TaskManager log failed.");
                            LOG.warn("Could not write http data.", e);
                            return;
                        }
                    // HttpChunkedInput will write the end marker (LastHttpContent) for us.
                    }
                    // close the connection, if no keep-alive is needed
                    if (!HttpHeaders.isKeepAlive(request)) {
                        lastContentFuture.addListener(ChannelFutureListener.CLOSE);
                    }
                }
            });
        } catch (Exception e) {
            display(ctx, request, "Error: " + e.getMessage());
            LOG.error("Fetching TaskManager log failed.", e);
            lastRequestPending.remove(taskManagerID);
        }
    } else {
        display(ctx, request, "loading...");
    }
}
Also used : InstanceID(org.apache.flink.runtime.instance.InstanceID) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) InetSocketAddress(java.net.InetSocketAddress) FileNotFoundException(java.io.FileNotFoundException) FlinkFuture(org.apache.flink.runtime.concurrent.impl.FlinkFuture) BlobKey(org.apache.flink.runtime.blob.BlobKey) GenericFutureListener(io.netty.util.concurrent.GenericFutureListener) ChannelFuture(io.netty.channel.ChannelFuture) ChunkedFile(io.netty.handler.stream.ChunkedFile) JobManagerMessages(org.apache.flink.runtime.messages.JobManagerMessages) DefaultFileRegion(io.netty.channel.DefaultFileRegion) RandomAccessFile(java.io.RandomAccessFile) Option(scala.Option) RandomAccessFile(java.io.RandomAccessFile) ChunkedFile(io.netty.handler.stream.ChunkedFile) File(java.io.File) Instance(org.apache.flink.runtime.instance.Instance) BlobCache(org.apache.flink.runtime.blob.BlobCache) HttpChunkedInput(io.netty.handler.codec.http.HttpChunkedInput) HttpRequest(io.netty.handler.codec.http.HttpRequest) FileChannel(java.nio.channels.FileChannel) DefaultHttpResponse(io.netty.handler.codec.http.DefaultHttpResponse) HttpResponse(io.netty.handler.codec.http.HttpResponse) IOException(java.io.IOException) FileNotFoundException(java.io.FileNotFoundException) IOException(java.io.IOException) BiFunction(org.apache.flink.runtime.concurrent.BiFunction) Tuple2(org.apache.flink.api.java.tuple.Tuple2) DefaultHttpResponse(io.netty.handler.codec.http.DefaultHttpResponse) FlinkFuture(org.apache.flink.runtime.concurrent.impl.FlinkFuture) Future(org.apache.flink.runtime.concurrent.Future) FlinkCompletableFuture(org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture) ChannelFuture(io.netty.channel.ChannelFuture)

Example 8 with DefaultFileRegion

use of org.apache.flink.shaded.netty4.io.netty.channel.DefaultFileRegion in project flink by apache.

the class StaticFileServerHandler method respondAsLeader.

/**
	 * Response when running with leading JobManager.
	 */
private void respondAsLeader(ChannelHandlerContext ctx, HttpRequest request, String requestPath) throws IOException, ParseException, URISyntaxException {
    // convert to absolute path
    final File file = new File(rootPath, requestPath);
    if (!file.exists()) {
        // file does not exist. Try to load it with the classloader
        ClassLoader cl = StaticFileServerHandler.class.getClassLoader();
        try (InputStream resourceStream = cl.getResourceAsStream("web" + requestPath)) {
            boolean success = false;
            try {
                if (resourceStream != null) {
                    URL root = cl.getResource("web");
                    URL requested = cl.getResource("web" + requestPath);
                    if (root != null && requested != null) {
                        URI rootURI = new URI(root.getPath()).normalize();
                        URI requestedURI = new URI(requested.getPath()).normalize();
                        // expected scope.
                        if (!rootURI.relativize(requestedURI).equals(requestedURI)) {
                            logger.debug("Loading missing file from classloader: {}", requestPath);
                            // ensure that directory to file exists.
                            file.getParentFile().mkdirs();
                            Files.copy(resourceStream, file.toPath());
                            success = true;
                        }
                    }
                }
            } catch (Throwable t) {
                logger.error("error while responding", t);
            } finally {
                if (!success) {
                    logger.debug("Unable to load requested file {} from classloader", requestPath);
                    sendError(ctx, NOT_FOUND);
                    return;
                }
            }
        }
    }
    if (!file.exists() || file.isHidden() || file.isDirectory() || !file.isFile()) {
        sendError(ctx, NOT_FOUND);
        return;
    }
    if (!file.getCanonicalFile().toPath().startsWith(rootPath.toPath())) {
        sendError(ctx, NOT_FOUND);
        return;
    }
    // cache validation
    final String ifModifiedSince = request.headers().get(IF_MODIFIED_SINCE);
    if (ifModifiedSince != null && !ifModifiedSince.isEmpty()) {
        SimpleDateFormat dateFormatter = new SimpleDateFormat(HTTP_DATE_FORMAT, Locale.US);
        Date ifModifiedSinceDate = dateFormatter.parse(ifModifiedSince);
        // Only compare up to the second because the datetime format we send to the client
        // does not have milliseconds
        long ifModifiedSinceDateSeconds = ifModifiedSinceDate.getTime() / 1000;
        long fileLastModifiedSeconds = file.lastModified() / 1000;
        if (ifModifiedSinceDateSeconds == fileLastModifiedSeconds) {
            if (logger.isDebugEnabled()) {
                logger.debug("Responding 'NOT MODIFIED' for file '" + file.getAbsolutePath() + '\'');
            }
            sendNotModified(ctx);
            return;
        }
    }
    if (logger.isDebugEnabled()) {
        logger.debug("Responding with file '" + file.getAbsolutePath() + '\'');
    }
    // Don't need to close this manually. Netty's DefaultFileRegion will take care of it.
    final RandomAccessFile raf;
    try {
        raf = new RandomAccessFile(file, "r");
    } catch (FileNotFoundException e) {
        sendError(ctx, NOT_FOUND);
        return;
    }
    long fileLength = raf.length();
    HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
    setContentTypeHeader(response, file);
    // since the log and out files are rapidly changing, we don't want to browser to cache them
    if (!(requestPath.contains("log") || requestPath.contains("out"))) {
        setDateAndCacheHeaders(response, file);
    }
    if (HttpHeaders.isKeepAlive(request)) {
        response.headers().set(CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
    }
    HttpHeaders.setContentLength(response, fileLength);
    // write the initial line and the header.
    ctx.write(response);
    // write the content.
    ChannelFuture lastContentFuture;
    if (ctx.pipeline().get(SslHandler.class) == null) {
        ctx.write(new DefaultFileRegion(raf.getChannel(), 0, fileLength), ctx.newProgressivePromise());
        lastContentFuture = ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
    } else {
        lastContentFuture = ctx.writeAndFlush(new HttpChunkedInput(new ChunkedFile(raf, 0, fileLength, 8192)), ctx.newProgressivePromise());
    // HttpChunkedInput will write the end marker (LastHttpContent) for us.
    }
    // close the connection, if no keep-alive is needed
    if (!HttpHeaders.isKeepAlive(request)) {
        lastContentFuture.addListener(ChannelFutureListener.CLOSE);
    }
}
Also used : ChannelFuture(io.netty.channel.ChannelFuture) InputStream(java.io.InputStream) ChunkedFile(io.netty.handler.stream.ChunkedFile) FileNotFoundException(java.io.FileNotFoundException) DefaultHttpResponse(io.netty.handler.codec.http.DefaultHttpResponse) DefaultFullHttpResponse(io.netty.handler.codec.http.DefaultFullHttpResponse) HttpResponse(io.netty.handler.codec.http.HttpResponse) FullHttpResponse(io.netty.handler.codec.http.FullHttpResponse) DefaultFileRegion(io.netty.channel.DefaultFileRegion) URI(java.net.URI) URL(java.net.URL) Date(java.util.Date) SslHandler(io.netty.handler.ssl.SslHandler) HttpChunkedInput(io.netty.handler.codec.http.HttpChunkedInput) RandomAccessFile(java.io.RandomAccessFile) DefaultHttpResponse(io.netty.handler.codec.http.DefaultHttpResponse) RandomAccessFile(java.io.RandomAccessFile) ChunkedFile(io.netty.handler.stream.ChunkedFile) File(java.io.File) SimpleDateFormat(java.text.SimpleDateFormat)

Example 9 with DefaultFileRegion

use of org.apache.flink.shaded.netty4.io.netty.channel.DefaultFileRegion in project async-http-client by AsyncHttpClient.

the class NettyFileBody method write.

@Override
public void write(Channel channel, NettyResponseFuture<?> future) throws IOException {
    @SuppressWarnings("resource") FileChannel // netty will close the FileChannel
    fileChannel = new RandomAccessFile(file, "r").getChannel();
    boolean noZeroCopy = ChannelManager.isSslHandlerConfigured(channel.pipeline()) || config.isDisableZeroCopy();
    Object body = noZeroCopy ? new ChunkedNioFile(fileChannel, offset, length, config.getChunkedFileChunkSize()) : new DefaultFileRegion(fileChannel, offset, length);
    channel.write(body, channel.newProgressivePromise()).addListener(new WriteProgressListener(future, false, length));
    channel.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT, channel.voidPromise());
}
Also used : WriteProgressListener(org.asynchttpclient.netty.request.WriteProgressListener) RandomAccessFile(java.io.RandomAccessFile) FileChannel(java.nio.channels.FileChannel) ChunkedNioFile(io.netty.handler.stream.ChunkedNioFile) DefaultFileRegion(io.netty.channel.DefaultFileRegion)

Example 10 with DefaultFileRegion

use of org.apache.flink.shaded.netty4.io.netty.channel.DefaultFileRegion in project rocketmq by apache.

the class FileRegionEncoderTest method testEncode.

/**
 * This unit test case ensures that {@link FileRegionEncoder} indeed wraps {@link FileRegion} to
 * {@link ByteBuf}.
 * @throws IOException if there is an error.
 */
@Test
public void testEncode() throws IOException {
    FileRegionEncoder fileRegionEncoder = new FileRegionEncoder();
    EmbeddedChannel channel = new EmbeddedChannel(fileRegionEncoder);
    File file = File.createTempFile(UUID.randomUUID().toString(), ".data");
    file.deleteOnExit();
    Random random = new Random(System.currentTimeMillis());
    int dataLength = 1 << 10;
    byte[] data = new byte[dataLength];
    random.nextBytes(data);
    write(file, data);
    FileRegion fileRegion = new DefaultFileRegion(file, 0, dataLength);
    Assert.assertEquals(0, fileRegion.transfered());
    Assert.assertEquals(dataLength, fileRegion.count());
    Assert.assertTrue(channel.writeOutbound(fileRegion));
    ByteBuf out = (ByteBuf) channel.readOutbound();
    byte[] arr = new byte[out.readableBytes()];
    out.getBytes(0, arr);
    Assert.assertArrayEquals("Data should be identical", data, arr);
}
Also used : Random(java.util.Random) DefaultFileRegion(io.netty.channel.DefaultFileRegion) FileRegion(io.netty.channel.FileRegion) EmbeddedChannel(io.netty.channel.embedded.EmbeddedChannel) DefaultFileRegion(io.netty.channel.DefaultFileRegion) ByteBuf(io.netty.buffer.ByteBuf) File(java.io.File) Test(org.junit.Test)

Aggregations

DefaultFileRegion (io.netty.channel.DefaultFileRegion)14 RandomAccessFile (java.io.RandomAccessFile)11 File (java.io.File)9 IOException (java.io.IOException)8 FileNotFoundException (java.io.FileNotFoundException)7 ChunkedFile (io.netty.handler.stream.ChunkedFile)6 ChannelFuture (io.netty.channel.ChannelFuture)5 FileRegion (io.netty.channel.FileRegion)4 DefaultHttpResponse (io.netty.handler.codec.http.DefaultHttpResponse)4 HttpChunkedInput (io.netty.handler.codec.http.HttpChunkedInput)4 HttpResponse (io.netty.handler.codec.http.HttpResponse)4 URL (java.net.URL)4 FileChannel (java.nio.channels.FileChannel)4 SimpleDateFormat (java.text.SimpleDateFormat)4 Date (java.util.Date)4 Channel (io.netty.channel.Channel)3 ChannelHandlerContext (io.netty.channel.ChannelHandlerContext)3 DefaultFullHttpResponse (io.netty.handler.codec.http.DefaultFullHttpResponse)3 FullHttpResponse (io.netty.handler.codec.http.FullHttpResponse)3 SslHandler (io.netty.handler.ssl.SslHandler)3