Examples with Tuple2 scala.Tuple2 used on opensource projects

Example 96 with Tuple2

use of scala.Tuple2 in project spark-dataflow by cloudera.

the class TransformTranslator method writeText.

private static <T> TransformEvaluator<TextIO.Write.Bound<T>> writeText() {
    return new TransformEvaluator<TextIO.Write.Bound<T>>() {

        @Override
        public void evaluate(TextIO.Write.Bound<T> transform, EvaluationContext context) {
            @SuppressWarnings("unchecked") JavaPairRDD<T, Void> last = ((JavaRDDLike<WindowedValue<T>, ?>) context.getInputRDD(transform)).map(WindowingHelpers.<T>unwindowFunction()).mapToPair(new PairFunction<T, T, Void>() {

                @Override
                public Tuple2<T, Void> call(T t) throws Exception {
                    return new Tuple2<>(t, null);
                }
            });
            ShardTemplateInformation shardTemplateInfo = new ShardTemplateInformation(transform.getNumShards(), transform.getShardTemplate(), transform.getFilenamePrefix(), transform.getFilenameSuffix());
            writeHadoopFile(last, new Configuration(), shardTemplateInfo, Text.class, NullWritable.class, TemplatedTextOutputFormat.class);
        }
    };
}

Example 97 with Tuple2

use of scala.Tuple2 in project spark-dataflow by cloudera.

the class StreamingTransformTranslator method kafka.

private static <K, V> TransformEvaluator<KafkaIO.Read.Unbound<K, V>> kafka() {
    return new TransformEvaluator<KafkaIO.Read.Unbound<K, V>>() {

        @Override
        public void evaluate(KafkaIO.Read.Unbound<K, V> transform, EvaluationContext context) {
            StreamingEvaluationContext sec = (StreamingEvaluationContext) context;
            JavaStreamingContext jssc = sec.getStreamingContext();
            Class<K> keyClazz = transform.getKeyClass();
            Class<V> valueClazz = transform.getValueClass();
            Class<? extends Decoder<K>> keyDecoderClazz = transform.getKeyDecoderClass();
            Class<? extends Decoder<V>> valueDecoderClazz = transform.getValueDecoderClass();
            Map<String, String> kafkaParams = transform.getKafkaParams();
            Set<String> topics = transform.getTopics();
            JavaPairInputDStream<K, V> inputPairStream = KafkaUtils.createDirectStream(jssc, keyClazz, valueClazz, keyDecoderClazz, valueDecoderClazz, kafkaParams, topics);
            JavaDStream<WindowedValue<KV<K, V>>> inputStream = inputPairStream.map(new Function<Tuple2<K, V>, KV<K, V>>() {

                @Override
                public KV<K, V> call(Tuple2<K, V> t2) throws Exception {
                    return KV.of(t2._1(), t2._2());
                }
            }).map(WindowingHelpers.<KV<K, V>>windowFunction());
            sec.setStream(transform, inputStream);
        }
    };
}

Example 98 with Tuple2

use of scala.Tuple2 in project flink by apache.

the class RuntimeMonitorHandlerBase method channelRead0.

@Override
protected void channelRead0(ChannelHandlerContext ctx, Routed routed) throws Exception {
    if (localJobManagerAddressFuture.isCompleted()) {
        if (localJobManagerAddress == null) {
            localJobManagerAddress = Await.result(localJobManagerAddressFuture, timeout);
        }
        Option<Tuple2<ActorGateway, Integer>> jobManager = retriever.getJobManagerGatewayAndWebPort();
        if (jobManager.isDefined()) {
            Tuple2<ActorGateway, Integer> gatewayPort = jobManager.get();
            String redirectAddress = HandlerRedirectUtils.getRedirectAddress(localJobManagerAddress, gatewayPort);
            if (redirectAddress != null) {
                HttpResponse redirect = HandlerRedirectUtils.getRedirectResponse(redirectAddress, routed.path(), httpsEnabled);
                KeepAliveWrite.flush(ctx, routed.request(), redirect);
            } else {
                respondAsLeader(ctx, routed, gatewayPort._1());
            }
        } else {
            KeepAliveWrite.flush(ctx, routed.request(), HandlerRedirectUtils.getUnavailableResponse());
        }
    } else {
        KeepAliveWrite.flush(ctx, routed.request(), HandlerRedirectUtils.getUnavailableResponse());
    }
}

Example 99 with Tuple2

use of scala.Tuple2 in project flink by apache.

the class JobManagerRetriever method notifyLeaderAddress.

@Override
public void notifyLeaderAddress(final String leaderAddress, final UUID leaderSessionID) {
    if (leaderAddress != null && !leaderAddress.equals("")) {
        try {
            final Promise<Tuple2<ActorGateway, Integer>> leaderGatewayPortPromise = new scala.concurrent.impl.Promise.DefaultPromise<>();
            synchronized (waitLock) {
                leaderGatewayPortFuture = leaderGatewayPortPromise.future();
                waitLock.notifyAll();
            }
            LOG.info("New leader reachable under {}:{}.", leaderAddress, leaderSessionID);
            AkkaUtils.getActorRefFuture(leaderAddress, actorSystem, lookupTimeout).flatMap(new Mapper<ActorRef, Future<Tuple2<ActorGateway, Object>>>() {

                @Override
                public Future<Tuple2<ActorGateway, Object>> apply(ActorRef jobManagerRef) {
                    ActorGateway leaderGateway = new AkkaActorGateway(jobManagerRef, leaderSessionID);
                    Future<Object> webMonitorPort = leaderGateway.ask(JobManagerMessages.getRequestWebMonitorPort(), timeout);
                    return Futures.successful(leaderGateway).zip(webMonitorPort);
                }
            }, actorSystem.dispatcher()).onComplete(new OnComplete<Tuple2<ActorGateway, Object>>() {

                @Override
                public void onComplete(Throwable failure, Tuple2<ActorGateway, Object> success) throws Throwable {
                    if (failure == null) {
                        if (success._2() instanceof ResponseWebMonitorPort) {
                            int webMonitorPort = ((ResponseWebMonitorPort) success._2()).port();
                            leaderGatewayPortPromise.success(new Tuple2<>(success._1(), webMonitorPort));
                        } else {
                            leaderGatewayPortPromise.failure(new Exception("Received the message " + success._2() + " as response to " + JobManagerMessages.getRequestWebMonitorPort() + ". But a message of type " + ResponseWebMonitorPort.class + " was expected."));
                        }
                    } else {
                        LOG.warn("Failed to retrieve leader gateway and port.", failure);
                        leaderGatewayPortPromise.failure(failure);
                    }
                }
            }, actorSystem.dispatcher());
        } catch (Exception e) {
            handleError(e);
        }
    }
}

Example 100 with Tuple2

use of scala.Tuple2 in project flink by apache.

the class HandlerRedirectUtilsTest method testGetRedirectAddressWithLocalAkkaPath.

@Test
public void testGetRedirectAddressWithLocalAkkaPath() throws Exception {
    ActorGateway leaderGateway = new DummyActorGateway("akka://flink/user/foobar");
    Tuple2<ActorGateway, Integer> leader = new Tuple2<>(leaderGateway, 1235);
    String redirectingAddress = HandlerRedirectUtils.getRedirectAddress(localJobManagerAddress, leader);
    Assert.assertNull(redirectingAddress);
}