Search in sources :

Example 1 with FlinkFuture

use of org.apache.flink.runtime.concurrent.impl.FlinkFuture in project flink by apache.

the class AkkaInvocationHandler method invoke.

@Override
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
    Class<?> declaringClass = method.getDeclaringClass();
    Object result;
    if (declaringClass.equals(AkkaGateway.class) || declaringClass.equals(MainThreadExecutable.class) || declaringClass.equals(Object.class) || declaringClass.equals(StartStoppable.class) || declaringClass.equals(RpcGateway.class) || declaringClass.equals(SelfGateway.class)) {
        result = method.invoke(this, args);
    } else {
        String methodName = method.getName();
        Class<?>[] parameterTypes = method.getParameterTypes();
        Annotation[][] parameterAnnotations = method.getParameterAnnotations();
        Time futureTimeout = extractRpcTimeout(parameterAnnotations, args, timeout);
        Tuple2<Class<?>[], Object[]> filteredArguments = filterArguments(parameterTypes, parameterAnnotations, args);
        RpcInvocation rpcInvocation;
        if (isLocal) {
            rpcInvocation = new LocalRpcInvocation(methodName, filteredArguments.f0, filteredArguments.f1);
        } else {
            try {
                RemoteRpcInvocation remoteRpcInvocation = new RemoteRpcInvocation(methodName, filteredArguments.f0, filteredArguments.f1);
                if (remoteRpcInvocation.getSize() > maximumFramesize) {
                    throw new IOException("The rpc invocation size exceeds the maximum akka framesize.");
                } else {
                    rpcInvocation = remoteRpcInvocation;
                }
            } catch (IOException e) {
                LOG.warn("Could not create remote rpc invocation message. Failing rpc invocation because...", e);
                throw e;
            }
        }
        Class<?> returnType = method.getReturnType();
        if (returnType.equals(Void.TYPE)) {
            rpcEndpoint.tell(rpcInvocation, ActorRef.noSender());
            result = null;
        } else if (returnType.equals(Future.class)) {
            // execute an asynchronous call
            result = new FlinkFuture<>(Patterns.ask(rpcEndpoint, rpcInvocation, futureTimeout.toMilliseconds()));
        } else {
            // execute a synchronous call
            scala.concurrent.Future<?> scalaFuture = Patterns.ask(rpcEndpoint, rpcInvocation, futureTimeout.toMilliseconds());
            Future<?> futureResult = new FlinkFuture<>(scalaFuture);
            return futureResult.get(futureTimeout.getSize(), futureTimeout.getUnit());
        }
    }
    return result;
}
Also used : LocalRpcInvocation(org.apache.flink.runtime.rpc.akka.messages.LocalRpcInvocation) RemoteRpcInvocation(org.apache.flink.runtime.rpc.akka.messages.RemoteRpcInvocation) RpcInvocation(org.apache.flink.runtime.rpc.akka.messages.RpcInvocation) LocalRpcInvocation(org.apache.flink.runtime.rpc.akka.messages.LocalRpcInvocation) Time(org.apache.flink.api.common.time.Time) IOException(java.io.IOException) MainThreadExecutable(org.apache.flink.runtime.rpc.MainThreadExecutable) FlinkFuture(org.apache.flink.runtime.concurrent.impl.FlinkFuture) SelfGateway(org.apache.flink.runtime.rpc.SelfGateway) RemoteRpcInvocation(org.apache.flink.runtime.rpc.akka.messages.RemoteRpcInvocation) FlinkFuture(org.apache.flink.runtime.concurrent.impl.FlinkFuture) Future(org.apache.flink.runtime.concurrent.Future) StartStoppable(org.apache.flink.runtime.rpc.StartStoppable)

Example 2 with FlinkFuture

use of org.apache.flink.runtime.concurrent.impl.FlinkFuture in project flink by apache.

the class TaskManagerLogHandler method respondAsLeader.

/**
	 * Response when running with leading JobManager.
	 */
@Override
protected void respondAsLeader(final ChannelHandlerContext ctx, final Routed routed, final ActorGateway jobManager) {
    if (cache == null) {
        scala.concurrent.Future<Object> portFuture = jobManager.ask(JobManagerMessages.getRequestBlobManagerPort(), timeout);
        scala.concurrent.Future<BlobCache> cacheFuture = portFuture.map(new Mapper<Object, BlobCache>() {

            @Override
            public BlobCache checkedApply(Object result) throws IOException {
                Option<String> hostOption = jobManager.actor().path().address().host();
                String host = hostOption.isDefined() ? hostOption.get() : "localhost";
                int port = (int) result;
                return new BlobCache(new InetSocketAddress(host, port), config);
            }
        }, executor);
        cache = new FlinkFuture<>(cacheFuture);
    }
    final String taskManagerID = routed.pathParams().get(TaskManagersHandler.TASK_MANAGER_ID_KEY);
    final HttpRequest request = routed.request();
    //fetch TaskManager logs if no other process is currently doing it
    if (lastRequestPending.putIfAbsent(taskManagerID, true) == null) {
        try {
            InstanceID instanceID = new InstanceID(StringUtils.hexStringToByte(taskManagerID));
            scala.concurrent.Future<JobManagerMessages.TaskManagerInstance> scalaTaskManagerFuture = jobManager.ask(new JobManagerMessages.RequestTaskManagerInstance(instanceID), timeout).mapTo(ClassTag$.MODULE$.<JobManagerMessages.TaskManagerInstance>apply(JobManagerMessages.TaskManagerInstance.class));
            Future<JobManagerMessages.TaskManagerInstance> taskManagerFuture = new FlinkFuture<>(scalaTaskManagerFuture);
            Future<BlobKey> blobKeyFuture = taskManagerFuture.thenCompose(new ApplyFunction<JobManagerMessages.TaskManagerInstance, Future<BlobKey>>() {

                @Override
                public Future<BlobKey> apply(JobManagerMessages.TaskManagerInstance value) {
                    Instance taskManager = value.instance().get();
                    if (serveLogFile) {
                        return taskManager.getTaskManagerGateway().requestTaskManagerLog(timeTimeout);
                    } else {
                        return taskManager.getTaskManagerGateway().requestTaskManagerStdout(timeTimeout);
                    }
                }
            });
            Future<String> logPathFuture = blobKeyFuture.thenCombine(cache, new BiFunction<BlobKey, BlobCache, Tuple2<BlobKey, BlobCache>>() {

                @Override
                public Tuple2<BlobKey, BlobCache> apply(BlobKey blobKey, BlobCache blobCache) {
                    return Tuple2.of(blobKey, blobCache);
                }
            }).thenComposeAsync(new ApplyFunction<Tuple2<BlobKey, BlobCache>, Future<String>>() {

                @Override
                public Future<String> apply(Tuple2<BlobKey, BlobCache> value) {
                    final BlobKey blobKey = value.f0;
                    final BlobCache blobCache = value.f1;
                    //delete previous log file, if it is different than the current one
                    HashMap<String, BlobKey> lastSubmittedFile = serveLogFile ? lastSubmittedLog : lastSubmittedStdout;
                    if (lastSubmittedFile.containsKey(taskManagerID)) {
                        if (!blobKey.equals(lastSubmittedFile.get(taskManagerID))) {
                            try {
                                blobCache.deleteGlobal(lastSubmittedFile.get(taskManagerID));
                            } catch (IOException e) {
                                return FlinkCompletableFuture.completedExceptionally(new Exception("Could not delete file for " + taskManagerID + '.', e));
                            }
                            lastSubmittedFile.put(taskManagerID, blobKey);
                        }
                    } else {
                        lastSubmittedFile.put(taskManagerID, blobKey);
                    }
                    try {
                        return FlinkCompletableFuture.completed(blobCache.getURL(blobKey).getFile());
                    } catch (IOException e) {
                        return FlinkCompletableFuture.completedExceptionally(new Exception("Could not retrieve blob for " + blobKey + '.', e));
                    }
                }
            }, executor);
            logPathFuture.exceptionally(new ApplyFunction<Throwable, Void>() {

                @Override
                public Void apply(Throwable failure) {
                    display(ctx, request, "Fetching TaskManager log failed.");
                    LOG.error("Fetching TaskManager log failed.", failure);
                    lastRequestPending.remove(taskManagerID);
                    return null;
                }
            });
            logPathFuture.thenAccept(new AcceptFunction<String>() {

                @Override
                public void accept(String filePath) {
                    File file = new File(filePath);
                    final RandomAccessFile raf;
                    try {
                        raf = new RandomAccessFile(file, "r");
                    } catch (FileNotFoundException e) {
                        display(ctx, request, "Displaying TaskManager log failed.");
                        LOG.error("Displaying TaskManager log failed.", e);
                        return;
                    }
                    long fileLength;
                    try {
                        fileLength = raf.length();
                    } catch (IOException ioe) {
                        display(ctx, request, "Displaying TaskManager log failed.");
                        LOG.error("Displaying TaskManager log failed.", ioe);
                        try {
                            raf.close();
                        } catch (IOException e) {
                            LOG.error("Could not close random access file.", e);
                        }
                        return;
                    }
                    final FileChannel fc = raf.getChannel();
                    HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
                    response.headers().set(CONTENT_TYPE, "text/plain");
                    if (HttpHeaders.isKeepAlive(request)) {
                        response.headers().set(CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
                    }
                    HttpHeaders.setContentLength(response, fileLength);
                    // write the initial line and the header.
                    ctx.write(response);
                    // write the content.
                    ChannelFuture lastContentFuture;
                    final GenericFutureListener<io.netty.util.concurrent.Future<? super Void>> completionListener = new GenericFutureListener<io.netty.util.concurrent.Future<? super Void>>() {

                        @Override
                        public void operationComplete(io.netty.util.concurrent.Future<? super Void> future) throws Exception {
                            lastRequestPending.remove(taskManagerID);
                            fc.close();
                            raf.close();
                        }
                    };
                    if (ctx.pipeline().get(SslHandler.class) == null) {
                        ctx.write(new DefaultFileRegion(fc, 0, fileLength), ctx.newProgressivePromise()).addListener(completionListener);
                        lastContentFuture = ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
                    } else {
                        try {
                            lastContentFuture = ctx.writeAndFlush(new HttpChunkedInput(new ChunkedFile(raf, 0, fileLength, 8192)), ctx.newProgressivePromise()).addListener(completionListener);
                        } catch (IOException e) {
                            display(ctx, request, "Displaying TaskManager log failed.");
                            LOG.warn("Could not write http data.", e);
                            return;
                        }
                    // HttpChunkedInput will write the end marker (LastHttpContent) for us.
                    }
                    // close the connection, if no keep-alive is needed
                    if (!HttpHeaders.isKeepAlive(request)) {
                        lastContentFuture.addListener(ChannelFutureListener.CLOSE);
                    }
                }
            });
        } catch (Exception e) {
            display(ctx, request, "Error: " + e.getMessage());
            LOG.error("Fetching TaskManager log failed.", e);
            lastRequestPending.remove(taskManagerID);
        }
    } else {
        display(ctx, request, "loading...");
    }
}
Also used : InstanceID(org.apache.flink.runtime.instance.InstanceID) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) InetSocketAddress(java.net.InetSocketAddress) FileNotFoundException(java.io.FileNotFoundException) FlinkFuture(org.apache.flink.runtime.concurrent.impl.FlinkFuture) BlobKey(org.apache.flink.runtime.blob.BlobKey) GenericFutureListener(io.netty.util.concurrent.GenericFutureListener) ChannelFuture(io.netty.channel.ChannelFuture) ChunkedFile(io.netty.handler.stream.ChunkedFile) JobManagerMessages(org.apache.flink.runtime.messages.JobManagerMessages) DefaultFileRegion(io.netty.channel.DefaultFileRegion) RandomAccessFile(java.io.RandomAccessFile) Option(scala.Option) RandomAccessFile(java.io.RandomAccessFile) ChunkedFile(io.netty.handler.stream.ChunkedFile) File(java.io.File) Instance(org.apache.flink.runtime.instance.Instance) BlobCache(org.apache.flink.runtime.blob.BlobCache) HttpChunkedInput(io.netty.handler.codec.http.HttpChunkedInput) HttpRequest(io.netty.handler.codec.http.HttpRequest) FileChannel(java.nio.channels.FileChannel) DefaultHttpResponse(io.netty.handler.codec.http.DefaultHttpResponse) HttpResponse(io.netty.handler.codec.http.HttpResponse) IOException(java.io.IOException) FileNotFoundException(java.io.FileNotFoundException) IOException(java.io.IOException) BiFunction(org.apache.flink.runtime.concurrent.BiFunction) Tuple2(org.apache.flink.api.java.tuple.Tuple2) DefaultHttpResponse(io.netty.handler.codec.http.DefaultHttpResponse) FlinkFuture(org.apache.flink.runtime.concurrent.impl.FlinkFuture) Future(org.apache.flink.runtime.concurrent.Future) FlinkCompletableFuture(org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture) ChannelFuture(io.netty.channel.ChannelFuture)

Example 3 with FlinkFuture

use of org.apache.flink.runtime.concurrent.impl.FlinkFuture in project flink by apache.

the class AkkaRpcService method connect.

// this method does not mutate state and is thus thread-safe
@Override
public <C extends RpcGateway> Future<C> connect(final String address, final Class<C> clazz) {
    checkState(!stopped, "RpcService is stopped");
    LOG.debug("Try to connect to remote RPC endpoint with address {}. Returning a {} gateway.", address, clazz.getName());
    final ActorSelection actorSel = actorSystem.actorSelection(address);
    final scala.concurrent.Future<Object> identify = Patterns.ask(actorSel, new Identify(42), timeout.toMilliseconds());
    final scala.concurrent.Future<C> resultFuture = identify.map(new Mapper<Object, C>() {

        @Override
        public C checkedApply(Object obj) throws Exception {
            ActorIdentity actorIdentity = (ActorIdentity) obj;
            if (actorIdentity.getRef() == null) {
                throw new RpcConnectionException("Could not connect to rpc endpoint under address " + address + '.');
            } else {
                ActorRef actorRef = actorIdentity.getRef();
                final String address = AkkaUtils.getAkkaURL(actorSystem, actorRef);
                final String hostname;
                Option<String> host = actorRef.path().address().host();
                if (host.isEmpty()) {
                    hostname = "localhost";
                } else {
                    hostname = host.get();
                }
                InvocationHandler akkaInvocationHandler = new AkkaInvocationHandler(address, hostname, actorRef, timeout, maximumFramesize, null);
                // Rather than using the System ClassLoader directly, we derive the ClassLoader
                // from this class . That works better in cases where Flink runs embedded and all Flink
                // code is loaded dynamically (for example from an OSGI bundle) through a custom ClassLoader
                ClassLoader classLoader = AkkaRpcService.this.getClass().getClassLoader();
                @SuppressWarnings("unchecked") C proxy = (C) Proxy.newProxyInstance(classLoader, new Class<?>[] { clazz }, akkaInvocationHandler);
                return proxy;
            }
        }
    }, actorSystem.dispatcher());
    return new FlinkFuture<>(resultFuture);
}
Also used : ActorRef(akka.actor.ActorRef) RpcConnectionException(org.apache.flink.runtime.rpc.exceptions.RpcConnectionException) InvocationHandler(java.lang.reflect.InvocationHandler) Identify(akka.actor.Identify) RpcConnectionException(org.apache.flink.runtime.rpc.exceptions.RpcConnectionException) FlinkFuture(org.apache.flink.runtime.concurrent.impl.FlinkFuture) ActorSelection(akka.actor.ActorSelection) Option(scala.Option) ActorIdentity(akka.actor.ActorIdentity)

Aggregations

FlinkFuture (org.apache.flink.runtime.concurrent.impl.FlinkFuture)3 IOException (java.io.IOException)2 Future (org.apache.flink.runtime.concurrent.Future)2 Option (scala.Option)2 ActorIdentity (akka.actor.ActorIdentity)1 ActorRef (akka.actor.ActorRef)1 ActorSelection (akka.actor.ActorSelection)1 Identify (akka.actor.Identify)1 ChannelFuture (io.netty.channel.ChannelFuture)1 DefaultFileRegion (io.netty.channel.DefaultFileRegion)1 DefaultHttpResponse (io.netty.handler.codec.http.DefaultHttpResponse)1 HttpChunkedInput (io.netty.handler.codec.http.HttpChunkedInput)1 HttpRequest (io.netty.handler.codec.http.HttpRequest)1 HttpResponse (io.netty.handler.codec.http.HttpResponse)1 ChunkedFile (io.netty.handler.stream.ChunkedFile)1 GenericFutureListener (io.netty.util.concurrent.GenericFutureListener)1 File (java.io.File)1 FileNotFoundException (java.io.FileNotFoundException)1 RandomAccessFile (java.io.RandomAccessFile)1 InvocationHandler (java.lang.reflect.InvocationHandler)1