use of org.apache.flink.shaded.netty4.io.netty.channel.DefaultFileRegion in project blade by biezhi.
the class StaticFileHandler method handle.
/**
* print static file to client
*
* @param webContext web context
* @throws Exception
*/
@Override
public void handle(WebContext webContext) throws Exception {
Request request = webContext.getRequest();
ChannelHandlerContext ctx = webContext.getChannelHandlerContext();
if (!HttpConst.METHOD_GET.equals(request.method())) {
sendError(ctx, METHOD_NOT_ALLOWED);
return;
}
Instant start = Instant.now();
String uri = URLDecoder.decode(request.uri(), "UTF-8");
String method = StringKit.padRight(request.method(), 6);
String cleanURL = getCleanURL(request, uri);
// webjars
if (cleanURL.startsWith(Const.WEB_JARS)) {
InputStream input = getResourceStreamFromJar(uri);
writeWithJarFile(request, ctx, uri, start, cleanURL, method, input);
return;
}
// jar file
if (BladeKit.runtimeIsJAR()) {
InputStream input = StaticFileHandler.class.getResourceAsStream(cleanURL);
writeWithJarFile(request, ctx, uri, start, cleanURL, method, input);
return;
}
// disk file
final String path = sanitizeUri(cleanURL);
if (path == null) {
log403(log, method, uri);
throw new ForbiddenException();
}
File file = new File(path);
if (file.isHidden() || !file.exists()) {
// gradle resources path
File resourcesDirectory = getGradleResourcesDirectory();
if (resourcesDirectory.isDirectory()) {
file = new File(resourcesDirectory.getPath() + "/" + cleanURL.substring(1));
if (file.isHidden() || !file.exists()) {
throw new NotFoundException(uri);
}
} else {
throw new NotFoundException(uri);
}
}
if (file.isDirectory() && showFileList) {
sendListing(ctx, uri, getFileMetas(file), cleanURL);
return;
}
if (!file.isFile()) {
sendError(ctx, FORBIDDEN);
return;
}
// Cache Validation
if (isHttp304(ctx, request, file.length(), file.lastModified())) {
log304(log, method, uri);
return;
}
HttpResponse httpResponse = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
setContentTypeHeader(httpResponse, file);
setDateAndCacheHeaders(httpResponse, file);
if (request.useGZIP()) {
File output = new File(file.getPath() + ".gz");
IOKit.compressGZIP(file, output);
file = output;
setGzip(httpResponse);
}
RandomAccessFile raf;
try {
raf = new RandomAccessFile(file, "r");
} catch (FileNotFoundException ignore) {
sendError(ctx, NOT_FOUND);
return;
}
long fileLength = raf.length();
httpResponse.headers().set(HttpConst.CONTENT_LENGTH, fileLength);
if (request.keepAlive()) {
httpResponse.headers().set(HttpConst.CONNECTION, HttpConst.KEEP_ALIVE);
}
// Write the initial line and the header.
ctx.write(httpResponse);
// Write the content.
ChannelFuture sendFileFuture;
ChannelFuture lastContentFuture;
if (ctx.pipeline().get(SslHandler.class) == null) {
sendFileFuture = ctx.write(new DefaultFileRegion(raf.getChannel(), 0, fileLength), ctx.newProgressivePromise());
// Write the end marker.
lastContentFuture = ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
} else {
sendFileFuture = ctx.writeAndFlush(new HttpChunkedInput(new ChunkedFile(raf, 0, fileLength, 8192)), ctx.newProgressivePromise());
// HttpChunkedInput will write the end marker (LastHttpContent) for us.
lastContentFuture = sendFileFuture;
}
sendFileFuture.addListener(ProgressiveFutureListener.build(raf));
// Decide whether to close the connection or not.
if (!request.keepAlive()) {
lastContentFuture.addListener(ChannelFutureListener.CLOSE);
}
log200AndCost(log, start, method, uri);
}
use of org.apache.flink.shaded.netty4.io.netty.channel.DefaultFileRegion in project flink by apache.
the class TaskManagerLogHandler method respondAsLeader.
/**
* Response when running with leading JobManager.
*/
@Override
protected void respondAsLeader(final ChannelHandlerContext ctx, final Routed routed, final ActorGateway jobManager) {
if (cache == null) {
scala.concurrent.Future<Object> portFuture = jobManager.ask(JobManagerMessages.getRequestBlobManagerPort(), timeout);
scala.concurrent.Future<BlobCache> cacheFuture = portFuture.map(new Mapper<Object, BlobCache>() {
@Override
public BlobCache checkedApply(Object result) throws IOException {
Option<String> hostOption = jobManager.actor().path().address().host();
String host = hostOption.isDefined() ? hostOption.get() : "localhost";
int port = (int) result;
return new BlobCache(new InetSocketAddress(host, port), config);
}
}, executor);
cache = new FlinkFuture<>(cacheFuture);
}
final String taskManagerID = routed.pathParams().get(TaskManagersHandler.TASK_MANAGER_ID_KEY);
final HttpRequest request = routed.request();
//fetch TaskManager logs if no other process is currently doing it
if (lastRequestPending.putIfAbsent(taskManagerID, true) == null) {
try {
InstanceID instanceID = new InstanceID(StringUtils.hexStringToByte(taskManagerID));
scala.concurrent.Future<JobManagerMessages.TaskManagerInstance> scalaTaskManagerFuture = jobManager.ask(new JobManagerMessages.RequestTaskManagerInstance(instanceID), timeout).mapTo(ClassTag$.MODULE$.<JobManagerMessages.TaskManagerInstance>apply(JobManagerMessages.TaskManagerInstance.class));
Future<JobManagerMessages.TaskManagerInstance> taskManagerFuture = new FlinkFuture<>(scalaTaskManagerFuture);
Future<BlobKey> blobKeyFuture = taskManagerFuture.thenCompose(new ApplyFunction<JobManagerMessages.TaskManagerInstance, Future<BlobKey>>() {
@Override
public Future<BlobKey> apply(JobManagerMessages.TaskManagerInstance value) {
Instance taskManager = value.instance().get();
if (serveLogFile) {
return taskManager.getTaskManagerGateway().requestTaskManagerLog(timeTimeout);
} else {
return taskManager.getTaskManagerGateway().requestTaskManagerStdout(timeTimeout);
}
}
});
Future<String> logPathFuture = blobKeyFuture.thenCombine(cache, new BiFunction<BlobKey, BlobCache, Tuple2<BlobKey, BlobCache>>() {
@Override
public Tuple2<BlobKey, BlobCache> apply(BlobKey blobKey, BlobCache blobCache) {
return Tuple2.of(blobKey, blobCache);
}
}).thenComposeAsync(new ApplyFunction<Tuple2<BlobKey, BlobCache>, Future<String>>() {
@Override
public Future<String> apply(Tuple2<BlobKey, BlobCache> value) {
final BlobKey blobKey = value.f0;
final BlobCache blobCache = value.f1;
//delete previous log file, if it is different than the current one
HashMap<String, BlobKey> lastSubmittedFile = serveLogFile ? lastSubmittedLog : lastSubmittedStdout;
if (lastSubmittedFile.containsKey(taskManagerID)) {
if (!blobKey.equals(lastSubmittedFile.get(taskManagerID))) {
try {
blobCache.deleteGlobal(lastSubmittedFile.get(taskManagerID));
} catch (IOException e) {
return FlinkCompletableFuture.completedExceptionally(new Exception("Could not delete file for " + taskManagerID + '.', e));
}
lastSubmittedFile.put(taskManagerID, blobKey);
}
} else {
lastSubmittedFile.put(taskManagerID, blobKey);
}
try {
return FlinkCompletableFuture.completed(blobCache.getURL(blobKey).getFile());
} catch (IOException e) {
return FlinkCompletableFuture.completedExceptionally(new Exception("Could not retrieve blob for " + blobKey + '.', e));
}
}
}, executor);
logPathFuture.exceptionally(new ApplyFunction<Throwable, Void>() {
@Override
public Void apply(Throwable failure) {
display(ctx, request, "Fetching TaskManager log failed.");
LOG.error("Fetching TaskManager log failed.", failure);
lastRequestPending.remove(taskManagerID);
return null;
}
});
logPathFuture.thenAccept(new AcceptFunction<String>() {
@Override
public void accept(String filePath) {
File file = new File(filePath);
final RandomAccessFile raf;
try {
raf = new RandomAccessFile(file, "r");
} catch (FileNotFoundException e) {
display(ctx, request, "Displaying TaskManager log failed.");
LOG.error("Displaying TaskManager log failed.", e);
return;
}
long fileLength;
try {
fileLength = raf.length();
} catch (IOException ioe) {
display(ctx, request, "Displaying TaskManager log failed.");
LOG.error("Displaying TaskManager log failed.", ioe);
try {
raf.close();
} catch (IOException e) {
LOG.error("Could not close random access file.", e);
}
return;
}
final FileChannel fc = raf.getChannel();
HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
response.headers().set(CONTENT_TYPE, "text/plain");
if (HttpHeaders.isKeepAlive(request)) {
response.headers().set(CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
}
HttpHeaders.setContentLength(response, fileLength);
// write the initial line and the header.
ctx.write(response);
// write the content.
ChannelFuture lastContentFuture;
final GenericFutureListener<io.netty.util.concurrent.Future<? super Void>> completionListener = new GenericFutureListener<io.netty.util.concurrent.Future<? super Void>>() {
@Override
public void operationComplete(io.netty.util.concurrent.Future<? super Void> future) throws Exception {
lastRequestPending.remove(taskManagerID);
fc.close();
raf.close();
}
};
if (ctx.pipeline().get(SslHandler.class) == null) {
ctx.write(new DefaultFileRegion(fc, 0, fileLength), ctx.newProgressivePromise()).addListener(completionListener);
lastContentFuture = ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
} else {
try {
lastContentFuture = ctx.writeAndFlush(new HttpChunkedInput(new ChunkedFile(raf, 0, fileLength, 8192)), ctx.newProgressivePromise()).addListener(completionListener);
} catch (IOException e) {
display(ctx, request, "Displaying TaskManager log failed.");
LOG.warn("Could not write http data.", e);
return;
}
// HttpChunkedInput will write the end marker (LastHttpContent) for us.
}
// close the connection, if no keep-alive is needed
if (!HttpHeaders.isKeepAlive(request)) {
lastContentFuture.addListener(ChannelFutureListener.CLOSE);
}
}
});
} catch (Exception e) {
display(ctx, request, "Error: " + e.getMessage());
LOG.error("Fetching TaskManager log failed.", e);
lastRequestPending.remove(taskManagerID);
}
} else {
display(ctx, request, "loading...");
}
}
use of org.apache.flink.shaded.netty4.io.netty.channel.DefaultFileRegion in project flink by apache.
the class StaticFileServerHandler method respondAsLeader.
/**
* Response when running with leading JobManager.
*/
private void respondAsLeader(ChannelHandlerContext ctx, HttpRequest request, String requestPath) throws IOException, ParseException, URISyntaxException {
// convert to absolute path
final File file = new File(rootPath, requestPath);
if (!file.exists()) {
// file does not exist. Try to load it with the classloader
ClassLoader cl = StaticFileServerHandler.class.getClassLoader();
try (InputStream resourceStream = cl.getResourceAsStream("web" + requestPath)) {
boolean success = false;
try {
if (resourceStream != null) {
URL root = cl.getResource("web");
URL requested = cl.getResource("web" + requestPath);
if (root != null && requested != null) {
URI rootURI = new URI(root.getPath()).normalize();
URI requestedURI = new URI(requested.getPath()).normalize();
// expected scope.
if (!rootURI.relativize(requestedURI).equals(requestedURI)) {
logger.debug("Loading missing file from classloader: {}", requestPath);
// ensure that directory to file exists.
file.getParentFile().mkdirs();
Files.copy(resourceStream, file.toPath());
success = true;
}
}
}
} catch (Throwable t) {
logger.error("error while responding", t);
} finally {
if (!success) {
logger.debug("Unable to load requested file {} from classloader", requestPath);
sendError(ctx, NOT_FOUND);
return;
}
}
}
}
if (!file.exists() || file.isHidden() || file.isDirectory() || !file.isFile()) {
sendError(ctx, NOT_FOUND);
return;
}
if (!file.getCanonicalFile().toPath().startsWith(rootPath.toPath())) {
sendError(ctx, NOT_FOUND);
return;
}
// cache validation
final String ifModifiedSince = request.headers().get(IF_MODIFIED_SINCE);
if (ifModifiedSince != null && !ifModifiedSince.isEmpty()) {
SimpleDateFormat dateFormatter = new SimpleDateFormat(HTTP_DATE_FORMAT, Locale.US);
Date ifModifiedSinceDate = dateFormatter.parse(ifModifiedSince);
// Only compare up to the second because the datetime format we send to the client
// does not have milliseconds
long ifModifiedSinceDateSeconds = ifModifiedSinceDate.getTime() / 1000;
long fileLastModifiedSeconds = file.lastModified() / 1000;
if (ifModifiedSinceDateSeconds == fileLastModifiedSeconds) {
if (logger.isDebugEnabled()) {
logger.debug("Responding 'NOT MODIFIED' for file '" + file.getAbsolutePath() + '\'');
}
sendNotModified(ctx);
return;
}
}
if (logger.isDebugEnabled()) {
logger.debug("Responding with file '" + file.getAbsolutePath() + '\'');
}
// Don't need to close this manually. Netty's DefaultFileRegion will take care of it.
final RandomAccessFile raf;
try {
raf = new RandomAccessFile(file, "r");
} catch (FileNotFoundException e) {
sendError(ctx, NOT_FOUND);
return;
}
long fileLength = raf.length();
HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
setContentTypeHeader(response, file);
// since the log and out files are rapidly changing, we don't want to browser to cache them
if (!(requestPath.contains("log") || requestPath.contains("out"))) {
setDateAndCacheHeaders(response, file);
}
if (HttpHeaders.isKeepAlive(request)) {
response.headers().set(CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
}
HttpHeaders.setContentLength(response, fileLength);
// write the initial line and the header.
ctx.write(response);
// write the content.
ChannelFuture lastContentFuture;
if (ctx.pipeline().get(SslHandler.class) == null) {
ctx.write(new DefaultFileRegion(raf.getChannel(), 0, fileLength), ctx.newProgressivePromise());
lastContentFuture = ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
} else {
lastContentFuture = ctx.writeAndFlush(new HttpChunkedInput(new ChunkedFile(raf, 0, fileLength, 8192)), ctx.newProgressivePromise());
// HttpChunkedInput will write the end marker (LastHttpContent) for us.
}
// close the connection, if no keep-alive is needed
if (!HttpHeaders.isKeepAlive(request)) {
lastContentFuture.addListener(ChannelFutureListener.CLOSE);
}
}
use of org.apache.flink.shaded.netty4.io.netty.channel.DefaultFileRegion in project async-http-client by AsyncHttpClient.
the class NettyFileBody method write.
@Override
public void write(Channel channel, NettyResponseFuture<?> future) throws IOException {
@SuppressWarnings("resource") FileChannel // netty will close the FileChannel
fileChannel = new RandomAccessFile(file, "r").getChannel();
boolean noZeroCopy = ChannelManager.isSslHandlerConfigured(channel.pipeline()) || config.isDisableZeroCopy();
Object body = noZeroCopy ? new ChunkedNioFile(fileChannel, offset, length, config.getChunkedFileChunkSize()) : new DefaultFileRegion(fileChannel, offset, length);
channel.write(body, channel.newProgressivePromise()).addListener(new WriteProgressListener(future, false, length));
channel.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT, channel.voidPromise());
}
use of org.apache.flink.shaded.netty4.io.netty.channel.DefaultFileRegion in project rocketmq by apache.
the class FileRegionEncoderTest method testEncode.
/**
* This unit test case ensures that {@link FileRegionEncoder} indeed wraps {@link FileRegion} to
* {@link ByteBuf}.
* @throws IOException if there is an error.
*/
@Test
public void testEncode() throws IOException {
FileRegionEncoder fileRegionEncoder = new FileRegionEncoder();
EmbeddedChannel channel = new EmbeddedChannel(fileRegionEncoder);
File file = File.createTempFile(UUID.randomUUID().toString(), ".data");
file.deleteOnExit();
Random random = new Random(System.currentTimeMillis());
int dataLength = 1 << 10;
byte[] data = new byte[dataLength];
random.nextBytes(data);
write(file, data);
FileRegion fileRegion = new DefaultFileRegion(file, 0, dataLength);
Assert.assertEquals(0, fileRegion.transfered());
Assert.assertEquals(dataLength, fileRegion.count());
Assert.assertTrue(channel.writeOutbound(fileRegion));
ByteBuf out = (ByteBuf) channel.readOutbound();
byte[] arr = new byte[out.readableBytes()];
out.getBytes(0, arr);
Assert.assertArrayEquals("Data should be identical", data, arr);
}
Aggregations