Examples with Deadline org.apache.flink.api.common.time.Deadline used on opensource projects

Example 16 with Deadline

use of org.apache.flink.api.common.time.Deadline in project flink by apache.

the class AbstractQueryableStateTestBase method testQueryNonStartedJobState.

/**
 * Similar tests as {@link #testValueState()} but before submitting the job, we already issue
 * one request which fails.
 */
@Test
public void testQueryNonStartedJobState() throws Exception {
    final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT);
    final long numElements = 1024L;
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStateBackend(stateBackend);
    env.setParallelism(maxParallelism);
    // Very important, because clusterClient is shared between tests and we
    // don't explicitly check that all slots are available before
    // submitting.
    env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 1000L));
    DataStream<Tuple2<Integer, Long>> source = env.addSource(new TestAscendingValueSource(numElements));
    ValueStateDescriptor<Tuple2<Integer, Long>> valueState = new ValueStateDescriptor<>("any", source.getType(), null);
    QueryableStateStream<Integer, Tuple2<Integer, Long>> queryableState = source.keyBy(new KeySelector<Tuple2<Integer, Long>, Integer>() {

        private static final long serialVersionUID = 7480503339992214681L;

        @Override
        public Integer getKey(Tuple2<Integer, Long> value) {
            return value.f0;
        }
    }).asQueryableState("hakuna", valueState);
    try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(deadline, clusterClient, env)) {
        final JobID jobId = autoCancellableJob.getJobId();
        final JobGraph jobGraph = autoCancellableJob.getJobGraph();
        long expected = numElements;
        // query once
        client.getKvState(autoCancellableJob.getJobId(), queryableState.getQueryableStateName(), 0, BasicTypeInfo.INT_TYPE_INFO, valueState);
        clusterClient.submitJob(jobGraph).get();
        executeValueQuery(deadline, client, jobId, "hakuna", valueState, expected);
    }
}

Example 17 with Deadline

use of org.apache.flink.api.common.time.Deadline in project flink by apache.

the class AbstractQueryableStateTestBase method testCustomKryoSerializerHandling.

/**
 * This test checks if custom Kryo serializers are loaded with correct classloader.
 */
@Test
public void testCustomKryoSerializerHandling() throws Exception {
    final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT);
    final long numElements = 1;
    final String stateName = "teriberka";
    final String customSerializerClassName = "CustomKryo";
    final URLClassLoader userClassLoader = createLoaderWithCustomKryoSerializer(customSerializerClassName);
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStateBackend(stateBackend);
    env.setParallelism(maxParallelism);
    // Very important, because cluster is shared between tests and we
    // don't explicitly check that all slots are available before
    // submitting.
    env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 1000L));
    // Custom serializer is not needed, it's used just to check if serialization works.
    env.getConfig().addDefaultKryoSerializer(Byte.class, (Serializer<?> & Serializable) createSerializer(userClassLoader));
    // Here we *force* using Kryo, to check if custom serializers are handled correctly WRT
    // classloading
    @SuppressWarnings({ "rawtypes", "unchecked" }) ValueStateDescriptor<Tuple2<Integer, Long>> valueState = new ValueStateDescriptor<>("any", new GenericTypeInfo(Tuple2.class));
    env.addSource(new TestAscendingValueSource(numElements)).keyBy(new KeySelector<Tuple2<Integer, Long>, Integer>() {

        private static final long serialVersionUID = 7662520075515707428L;

        @Override
        public Integer getKey(Tuple2<Integer, Long> value) {
            return value.f0;
        }
    }).asQueryableState(stateName, valueState);
    try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(deadline, clusterClient, env)) {
        final JobID jobId = autoCancellableJob.getJobId();
        final JobGraph jobGraph = autoCancellableJob.getJobGraph();
        jobGraph.setClasspaths(Arrays.asList(userClassLoader.getURLs()));
        clusterClient.submitJob(jobGraph).get();
        try {
            client.setUserClassLoader(userClassLoader);
            executeValueQuery(deadline, client, jobId, stateName, valueState, numElements);
        } finally {
            client.setUserClassLoader(null);
        }
    }
}

Example 18 with Deadline

use of org.apache.flink.api.common.time.Deadline in project flink by apache.

the class AbstractQueryableStateTestBase method testAggregatingState.

@Test
public void testAggregatingState() throws Exception {
    final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT);
    final long numElements = 1024L;
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStateBackend(stateBackend);
    env.setParallelism(maxParallelism);
    // Very important, because cluster is shared between tests and we
    // don't explicitly check that all slots are available before
    // submitting.
    env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 1000L));
    DataStream<Tuple2<Integer, Long>> source = env.addSource(new TestAscendingValueSource(numElements));
    final AggregatingStateDescriptor<Tuple2<Integer, Long>, String, String> aggrStateDescriptor = new AggregatingStateDescriptor<>("aggregates", new SumAggr(), String.class);
    aggrStateDescriptor.setQueryable("aggr-queryable");
    source.keyBy(new KeySelector<Tuple2<Integer, Long>, Integer>() {

        private static final long serialVersionUID = 8470749712274833552L;

        @Override
        public Integer getKey(Tuple2<Integer, Long> value) {
            return value.f0;
        }
    }).transform("TestAggregatingOperator", BasicTypeInfo.STRING_TYPE_INFO, new AggregatingTestOperator(aggrStateDescriptor));
    try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(deadline, clusterClient, env)) {
        final JobID jobId = autoCancellableJob.getJobId();
        final JobGraph jobGraph = autoCancellableJob.getJobGraph();
        clusterClient.submitJob(jobGraph).get();
        for (int key = 0; key < maxParallelism; key++) {
            boolean success = false;
            while (deadline.hasTimeLeft() && !success) {
                CompletableFuture<AggregatingState<Tuple2<Integer, Long>, String>> future = getKvState(deadline, client, jobId, "aggr-queryable", key, BasicTypeInfo.INT_TYPE_INFO, aggrStateDescriptor, false, executor);
                String value = future.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS).get();
                if (Long.parseLong(value) == numElements * (numElements + 1L) / 2L) {
                    success = true;
                } else {
                    // Retry
                    Thread.sleep(RETRY_TIMEOUT);
                }
            }
            assertTrue("Did not succeed query", success);
        }
    }
}

Example 19 with Deadline

use of org.apache.flink.api.common.time.Deadline in project flink by apache.

the class AbstractQueryableStateTestBase method testListState.

/**
 * Tests simple list state queryable state instance. Each source emits (subtaskIndex,
 * 0)..(subtaskIndex, numElements) tuples, which are then queried. The list state instance add
 * the values to the list. The test succeeds after each subtask index is queried and the list
 * contains the correct number of distinct elements.
 */
@Test
public void testListState() throws Exception {
    final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT);
    final long numElements = 1024L;
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStateBackend(stateBackend);
    env.setParallelism(maxParallelism);
    // Very important, because cluster is shared between tests and we
    // don't explicitly check that all slots are available before
    // submitting.
    env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 1000L));
    DataStream<Tuple2<Integer, Long>> source = env.addSource(new TestAscendingValueSource(numElements));
    final ListStateDescriptor<Long> listStateDescriptor = new ListStateDescriptor<Long>("list", BasicTypeInfo.LONG_TYPE_INFO);
    listStateDescriptor.setQueryable("list-queryable");
    source.keyBy(new KeySelector<Tuple2<Integer, Long>, Integer>() {

        private static final long serialVersionUID = 8470749712274833552L;

        @Override
        public Integer getKey(Tuple2<Integer, Long> value) {
            return value.f0;
        }
    }).process(new ProcessFunction<Tuple2<Integer, Long>, Object>() {

        private static final long serialVersionUID = -805125545438296619L;

        private transient ListState<Long> listState;

        @Override
        public void open(Configuration parameters) throws Exception {
            super.open(parameters);
            listState = getRuntimeContext().getListState(listStateDescriptor);
        }

        @Override
        public void processElement(Tuple2<Integer, Long> value, Context ctx, Collector<Object> out) throws Exception {
            listState.add(value.f1);
        }
    });
    try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(deadline, clusterClient, env)) {
        final JobID jobId = autoCancellableJob.getJobId();
        final JobGraph jobGraph = autoCancellableJob.getJobGraph();
        clusterClient.submitJob(jobGraph).get();
        final Map<Integer, Set<Long>> results = new HashMap<>();
        for (int key = 0; key < maxParallelism; key++) {
            boolean success = false;
            while (deadline.hasTimeLeft() && !success) {
                final CompletableFuture<ListState<Long>> future = getKvState(deadline, client, jobId, "list-queryable", key, BasicTypeInfo.INT_TYPE_INFO, listStateDescriptor, false, executor);
                Iterable<Long> value = future.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS).get();
                Set<Long> res = new HashSet<>();
                for (Long v : value) {
                    res.add(v);
                }
                // the source starts at 0, so +1
                if (res.size() == numElements + 1L) {
                    success = true;
                    results.put(key, res);
                } else {
                    // Retry
                    Thread.sleep(RETRY_TIMEOUT);
                }
            }
            assertTrue("Did not succeed query", success);
        }
        for (int key = 0; key < maxParallelism; key++) {
            Set<Long> values = results.get(key);
            for (long i = 0L; i <= numElements; i++) {
                assertTrue(values.contains(i));
            }
        }
    }
}

Example 20 with Deadline

use of org.apache.flink.api.common.time.Deadline in project flink by apache.

the class JobMasterStopWithSavepointITCase method throwingExceptionOnCallbackWithRestartsHelper.

private void throwingExceptionOnCallbackWithRestartsHelper(final boolean terminate) throws Exception {
    final Deadline deadline = Deadline.fromNow(Duration.ofSeconds(15));
    final int numberOfCheckpointsToExpect = 10;
    numberOfRestarts = new CountDownLatch(2);
    checkpointsToWaitFor = new CountDownLatch(numberOfCheckpointsToExpect);
    setUpJobGraph(ExceptionOnCallbackStreamTask.class, RestartStrategies.fixedDelayRestart(15, Time.milliseconds(10)));
    assertThat(getJobStatus(), equalTo(JobStatus.RUNNING));
    try {
        stopWithSavepoint(terminate).get(50, TimeUnit.MILLISECONDS);
        fail();
    } catch (Exception e) {
    // expected
    }
    // wait until we restart at least 2 times and until we see at least 10 checkpoints.
    assertTrue(numberOfRestarts.await(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS));
    assertTrue(checkpointsToWaitFor.await(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS));
    // verifying that we actually received a synchronous checkpoint
    assertTrue(syncSavepointId.get() > 0);
    assertThat(getJobStatus(), equalTo(JobStatus.RUNNING));
    // make sure that we saw the synchronous savepoint and
    // that after that we saw more checkpoints due to restarts.
    final long syncSavepoint = syncSavepointId.get();
    assertTrue(syncSavepoint > 0 && syncSavepoint < numberOfCheckpointsToExpect);
    clusterClient.cancel(jobGraph.getJobID()).get();
    assertThat(getJobStatus(), either(equalTo(JobStatus.CANCELLING)).or(equalTo(JobStatus.CANCELED)));
}