Examples with Deadline org.apache.flink.api.common.time.Deadline used on opensource projects

Example 41 with Deadline

use of org.apache.flink.api.common.time.Deadline in project flink by apache.

the class AbstractQueryableStateTestBase method testMapState.

/**
 * Tests simple map state queryable state instance. Each source emits (subtaskIndex,
 * 0)..(subtaskIndex, numElements) tuples, which are then queried. The map state instance sums
 * the values up. The test succeeds after each subtask index is queried with result n*(n+1)/2.
 */
@Test
public void testMapState() throws Exception {
    final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT);
    final long numElements = 1024L;
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStateBackend(stateBackend);
    env.setParallelism(maxParallelism);
    // Very important, because cluster is shared between tests and we
    // don't explicitly check that all slots are available before
    // submitting.
    env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 1000L));
    DataStream<Tuple2<Integer, Long>> source = env.addSource(new TestAscendingValueSource(numElements));
    final MapStateDescriptor<Integer, Tuple2<Integer, Long>> mapStateDescriptor = new MapStateDescriptor<>("timon", BasicTypeInfo.INT_TYPE_INFO, source.getType());
    mapStateDescriptor.setQueryable("timon-queryable");
    source.keyBy(new KeySelector<Tuple2<Integer, Long>, Integer>() {

        private static final long serialVersionUID = 8470749712274833552L;

        @Override
        public Integer getKey(Tuple2<Integer, Long> value) {
            return value.f0;
        }
    }).process(new ProcessFunction<Tuple2<Integer, Long>, Object>() {

        private static final long serialVersionUID = -805125545438296619L;

        private transient MapState<Integer, Tuple2<Integer, Long>> mapState;

        @Override
        public void open(Configuration parameters) throws Exception {
            super.open(parameters);
            mapState = getRuntimeContext().getMapState(mapStateDescriptor);
        }

        @Override
        public void processElement(Tuple2<Integer, Long> value, Context ctx, Collector<Object> out) throws Exception {
            Tuple2<Integer, Long> v = mapState.get(value.f0);
            if (v == null) {
                v = new Tuple2<>(value.f0, 0L);
            }
            mapState.put(value.f0, new Tuple2<>(v.f0, v.f1 + value.f1));
        }
    });
    try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(deadline, clusterClient, env)) {
        final JobID jobId = autoCancellableJob.getJobId();
        final JobGraph jobGraph = autoCancellableJob.getJobGraph();
        clusterClient.submitJob(jobGraph).get();
        final long expected = numElements * (numElements + 1L) / 2L;
        for (int key = 0; key < maxParallelism; key++) {
            boolean success = false;
            while (deadline.hasTimeLeft() && !success) {
                CompletableFuture<MapState<Integer, Tuple2<Integer, Long>>> future = getKvState(deadline, client, jobId, "timon-queryable", key, BasicTypeInfo.INT_TYPE_INFO, mapStateDescriptor, false, executor);
                Tuple2<Integer, Long> value = future.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS).get(key);
                if (value != null && value.f0 != null && expected == value.f1) {
                    assertEquals("Key mismatch", key, value.f0.intValue());
                    success = true;
                } else {
                    // Retry
                    Thread.sleep(RETRY_TIMEOUT);
                }
            }
            assertTrue("Did not succeed query", success);
        }
    }
}

Example 42 with Deadline

use of org.apache.flink.api.common.time.Deadline in project flink by apache.

the class AbstractQueryableStateTestBase method testReducingState.

/**
 * Tests simple reducing state queryable state instance. Each source emits (subtaskIndex,
 * 0)..(subtaskIndex, numElements) tuples, which are then queried. The reducing state instance
 * sums these up. The test succeeds after each subtask index is queried with result n*(n+1)/2.
 */
@Test
public void testReducingState() throws Exception {
    final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT);
    final long numElements = 1024L;
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStateBackend(stateBackend);
    env.setParallelism(maxParallelism);
    // Very important, because cluster is shared between tests and we
    // don't explicitly check that all slots are available before
    // submitting.
    env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 1000L));
    DataStream<Tuple2<Integer, Long>> source = env.addSource(new TestAscendingValueSource(numElements));
    ReducingStateDescriptor<Tuple2<Integer, Long>> reducingState = new ReducingStateDescriptor<>("any", new SumReduce(), source.getType());
    source.keyBy(new KeySelector<Tuple2<Integer, Long>, Integer>() {

        private static final long serialVersionUID = 8470749712274833552L;

        @Override
        public Integer getKey(Tuple2<Integer, Long> value) {
            return value.f0;
        }
    }).asQueryableState("jungle", reducingState);
    try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(deadline, clusterClient, env)) {
        final JobID jobId = autoCancellableJob.getJobId();
        final JobGraph jobGraph = autoCancellableJob.getJobGraph();
        clusterClient.submitJob(jobGraph).get();
        final long expected = numElements * (numElements + 1L) / 2L;
        for (int key = 0; key < maxParallelism; key++) {
            boolean success = false;
            while (deadline.hasTimeLeft() && !success) {
                CompletableFuture<ReducingState<Tuple2<Integer, Long>>> future = getKvState(deadline, client, jobId, "jungle", key, BasicTypeInfo.INT_TYPE_INFO, reducingState, false, executor);
                Tuple2<Integer, Long> value = future.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS).get();
                assertEquals("Key mismatch", key, value.f0.intValue());
                if (expected == value.f1) {
                    success = true;
                } else {
                    // Retry
                    Thread.sleep(RETRY_TIMEOUT);
                }
            }
            assertTrue("Did not succeed query", success);
        }
    }
}

Example 43 with Deadline

use of org.apache.flink.api.common.time.Deadline in project flink by apache.

the class AbstractQueryableStateTestBase method testValueStateDefault.

/**
 * Tests simple value state queryable state instance with a default value set. Each source emits
 * (subtaskIndex, 0)..(subtaskIndex, numElements) tuples, the key is mapped to 1 but key 0 is
 * queried which should throw a {@link UnknownKeyOrNamespaceException} exception.
 *
 * @throws UnknownKeyOrNamespaceException thrown due querying a non-existent key
 */
@Test(expected = UnknownKeyOrNamespaceException.class)
public void testValueStateDefault() throws Throwable {
    final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT);
    final long numElements = 1024L;
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStateBackend(stateBackend);
    env.setParallelism(maxParallelism);
    // Very important, because cluster is shared between tests and we
    // don't explicitly check that all slots are available before
    // submitting.
    env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 1000L));
    DataStream<Tuple2<Integer, Long>> source = env.addSource(new TestAscendingValueSource(numElements));
    ValueStateDescriptor<Tuple2<Integer, Long>> valueState = new ValueStateDescriptor<>("any", source.getType(), Tuple2.of(0, 1337L));
    // only expose key "1"
    QueryableStateStream<Integer, Tuple2<Integer, Long>> queryableState = source.keyBy(new KeySelector<Tuple2<Integer, Long>, Integer>() {

        private static final long serialVersionUID = 4509274556892655887L;

        @Override
        public Integer getKey(Tuple2<Integer, Long> value) {
            return 1;
        }
    }).asQueryableState("hakuna", valueState);
    try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(deadline, clusterClient, env)) {
        final JobID jobId = autoCancellableJob.getJobId();
        final JobGraph jobGraph = autoCancellableJob.getJobGraph();
        clusterClient.submitJob(jobGraph).get();
        // Now query
        int key = 0;
        CompletableFuture<ValueState<Tuple2<Integer, Long>>> future = getKvState(deadline, client, jobId, queryableState.getQueryableStateName(), key, BasicTypeInfo.INT_TYPE_INFO, valueState, true, executor);
        try {
            future.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS);
        } catch (ExecutionException | CompletionException e) {
            // exception in an ExecutionException.
            throw e.getCause();
        }
    }
}

Example 44 with Deadline

use of org.apache.flink.api.common.time.Deadline in project flink by apache.

the class RecordEmitterTest method test.

@Test
public void test() throws Exception {
    TestRecordEmitter emitter = new TestRecordEmitter();
    final TimestampedValue<String> one = new TimestampedValue<>("one", 1);
    final TimestampedValue<String> two = new TimestampedValue<>("two", 2);
    final TimestampedValue<String> five = new TimestampedValue<>("five", 5);
    final TimestampedValue<String> ten = new TimestampedValue<>("ten", 10);
    final RecordEmitter.RecordQueue<TimestampedValue> queue0 = emitter.getQueue(0);
    final RecordEmitter.RecordQueue<TimestampedValue> queue1 = emitter.getQueue(1);
    queue0.put(one);
    queue0.put(five);
    queue0.put(ten);
    queue1.put(two);
    ExecutorService executor = Executors.newSingleThreadExecutor();
    executor.submit(emitter);
    Deadline dl = Deadline.fromNow(Duration.ofSeconds(10));
    while (emitter.results.size() != 4 && dl.hasTimeLeft()) {
        Thread.sleep(10);
    }
    emitter.stop();
    executor.shutdownNow();
    Assert.assertThat(emitter.results, Matchers.contains(one, five, two, ten));
}

Example 45 with Deadline

use of org.apache.flink.api.common.time.Deadline in project flink by apache.

the class LocalStandaloneKafkaResource method readMessage.

@Override
public List<String> readMessage(int expectedNumMessages, String groupId, String topic) throws IOException {
    final List<String> messages = Collections.synchronizedList(new ArrayList<>(expectedNumMessages));
    try (final AutoClosableProcess kafka = AutoClosableProcess.create(kafkaDir.resolve(Paths.get("bin", "kafka-console-consumer.sh")).toString(), "--bootstrap-server", KAFKA_ADDRESS, "--from-beginning", "--max-messages", String.valueOf(expectedNumMessages), "--topic", topic, "--consumer-property", "group.id=" + groupId).setStdoutProcessor(messages::add).runNonBlocking()) {
        final Deadline deadline = Deadline.fromNow(Duration.ofSeconds(120));
        while (deadline.hasTimeLeft() && messages.size() < expectedNumMessages) {
            try {
                LOG.info("Waiting for messages. Received {}/{}.", messages.size(), expectedNumMessages);
                Thread.sleep(500);
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                break;
            }
        }
        if (messages.size() != expectedNumMessages) {
            throw new IOException("Could not read expected number of messages.");
        }
        return messages;
    }
}