Examples with TestingClusterClient org.apache.flink.kubernetes.operator.TestingClusterClient used on opensource projects

Example 1 with TestingClusterClient

use of org.apache.flink.kubernetes.operator.TestingClusterClient in project flink-kubernetes-operator by apache.

the class FlinkServiceTest method testCancelJobWithStatelessUpgradeMode.

@Test
public void testCancelJobWithStatelessUpgradeMode() throws Exception {
    final TestingClusterClient<String> testingClusterClient = new TestingClusterClient<>(configuration, TestUtils.TEST_DEPLOYMENT_NAME);
    final CompletableFuture<JobID> cancelFuture = new CompletableFuture<>();
    testingClusterClient.setCancelFunction(jobID -> {
        cancelFuture.complete(jobID);
        return CompletableFuture.completedFuture(Acknowledge.get());
    });
    final FlinkService flinkService = createFlinkService(testingClusterClient);
    JobID jobID = JobID.generate();
    FlinkDeployment deployment = TestUtils.buildApplicationCluster();
    JobStatus jobStatus = deployment.getStatus().getJobStatus();
    jobStatus.setJobId(jobID.toHexString());
    ReconciliationUtils.updateForSpecReconciliationSuccess(deployment, JobState.RUNNING);
    deployment.getStatus().setJobManagerDeploymentStatus(JobManagerDeploymentStatus.READY);
    deployment.getStatus().getJobStatus().setState("RUNNING");
    flinkService.cancelJob(deployment, UpgradeMode.STATELESS);
    assertTrue(cancelFuture.isDone());
    assertEquals(jobID, cancelFuture.get());
    assertNull(jobStatus.getSavepointInfo().getLastSavepoint());
}

Example 2 with TestingClusterClient

use of org.apache.flink.kubernetes.operator.TestingClusterClient in project flink-kubernetes-operator by apache.

the class FlinkServiceTest method testGetLastCheckpoint.

@Test
public void testGetLastCheckpoint() throws Exception {
    ObjectMapper objectMapper = RestMapperUtils.getStrictObjectMapper();
    var testingClusterClient = new TestingClusterClient<>(configuration, TestUtils.TEST_DEPLOYMENT_NAME);
    String responseWithHistory = "{\"counts\":{\"restored\":1,\"total\":79,\"in_progress\":0,\"completed\":69,\"failed\":10},\"summary\":{\"checkpointed_size\":{\"min\":23928,\"max\":53670,\"avg\":28551,\"p50\":28239,\"p90\":28563,\"p95\":28635,\"p99\":53670,\"p999\":53670},\"state_size\":{\"min\":23928,\"max\":53670,\"avg\":28551,\"p50\":28239,\"p90\":28563,\"p95\":28635,\"p99\":53670,\"p999\":53670},\"end_to_end_duration\":{\"min\":14,\"max\":117,\"avg\":24,\"p50\":22,\"p90\":32,\"p95\":40.5,\"p99\":117,\"p999\":117},\"alignment_buffered\":{\"min\":0,\"max\":0,\"avg\":0,\"p50\":0,\"p90\":0,\"p95\":0,\"p99\":0,\"p999\":0},\"processed_data\":{\"min\":0,\"max\":1274,\"avg\":280,\"p50\":112,\"p90\":840,\"p95\":1071,\"p99\":1274,\"p999\":1274},\"persisted_data\":{\"min\":0,\"max\":0,\"avg\":0,\"p50\":0,\"p90\":0,\"p95\":0,\"p99\":0,\"p999\":0}},\"latest\":{\"completed\":{\"className\":\"completed\",\"id\":96,\"status\":\"COMPLETED\",\"is_savepoint\":false,\"trigger_timestamp\":1653212837604,\"latest_ack_timestamp\":1653212837621,\"checkpointed_size\":28437,\"state_size\":28437,\"end_to_end_duration\":17,\"alignment_buffered\":0,\"processed_data\":560,\"persisted_data\":0,\"num_subtasks\":4,\"num_acknowledged_subtasks\":4,\"checkpoint_type\":\"CHECKPOINT\",\"tasks\":{},\"external_path\":\"file:/flink-data/checkpoints/00000000000000000000000000000000/chk-96\",\"discarded\":false},\"savepoint\":{\"className\":\"completed\",\"id\":51,\"status\":\"COMPLETED\",\"is_savepoint\":true,\"trigger_timestamp\":1653212748176,\"latest_ack_timestamp\":1653212748233,\"checkpointed_size\":53670,\"state_size\":53670,\"end_to_end_duration\":57,\"alignment_buffered\":0,\"processed_data\":483,\"persisted_data\":0,\"num_subtasks\":4,\"num_acknowledged_subtasks\":4,\"checkpoint_type\":\"SAVEPOINT\",\"tasks\":{},\"external_path\":\"file:/flink-data/savepoints/savepoint-000000-e8ea2482ce4f\",\"discarded\":false},\"failed\":null,\"restored\":{\"id\":27,\"restore_timestamp\":1653212683022,\"is_savepoint\":true,\"external_path\":\"file:/flink-data/savepoints/savepoint-000000-5930e5326ca7\"}},\"history\":[{\"className\":\"completed\",\"id\":96,\"status\":\"COMPLETED\",\"is_savepoint\":false,\"trigger_timestamp\":1653212837604,\"latest_ack_timestamp\":1653212837621,\"checkpointed_size\":28437,\"state_size\":28437,\"end_to_end_duration\":17,\"alignment_buffered\":0,\"processed_data\":560,\"persisted_data\":0,\"num_subtasks\":4,\"num_acknowledged_subtasks\":4,\"checkpoint_type\":\"CHECKPOINT\",\"tasks\":{},\"external_path\":\"file:/flink-data/checkpoints/00000000000000000000000000000000/chk-96\",\"discarded\":false},{\"className\":\"completed\",\"id\":95,\"status\":\"COMPLETED\",\"is_savepoint\":false,\"trigger_timestamp\":1653212835603,\"latest_ack_timestamp\":1653212835622,\"checkpointed_size\":28473,\"state_size\":28473,\"end_to_end_duration\":19,\"alignment_buffered\":0,\"processed_data\":42,\"persisted_data\":0,\"num_subtasks\":4,\"num_acknowledged_subtasks\":4,\"checkpoint_type\":\"CHECKPOINT\",\"tasks\":{},\"external_path\":\"file:/flink-data/checkpoints/00000000000000000000000000000000/chk-95\",\"discarded\":true},{\"className\":\"completed\",\"id\":94,\"status\":\"COMPLETED\",\"is_savepoint\":false,\"trigger_timestamp\":1653212833603,\"latest_ack_timestamp\":1653212833623,\"checkpointed_size\":27969,\"state_size\":27969,\"end_to_end_duration\":20,\"alignment_buffered\":0,\"processed_data\":28,\"persisted_data\":0,\"num_subtasks\":4,\"num_acknowledged_subtasks\":4,\"checkpoint_type\":\"CHECKPOINT\",\"tasks\":{},\"external_path\":\"file:/flink-data/checkpoints/00000000000000000000000000000000/chk-94\",\"discarded\":true},{\"className\":\"completed\",\"id\":93,\"status\":\"COMPLETED\",\"is_savepoint\":false,\"trigger_timestamp\":1653212831603,\"latest_ack_timestamp\":1653212831621,\"checkpointed_size\":28113,\"state_size\":28113,\"end_to_end_duration\":18,\"alignment_buffered\":0,\"processed_data\":138,\"persisted_data\":0,\"num_subtasks\":4,\"num_acknowledged_subtasks\":4,\"checkpoint_type\":\"CHECKPOINT\",\"tasks\":{},\"external_path\":\"file:/flink-data/checkpoints/00000000000000000000000000000000/chk-93\",\"discarded\":true},{\"className\":\"completed\",\"id\":92,\"status\":\"COMPLETED\",\"is_savepoint\":false,\"trigger_timestamp\":1653212829603,\"latest_ack_timestamp\":1653212829621,\"checkpointed_size\":28293,\"state_size\":28293,\"end_to_end_duration\":18,\"alignment_buffered\":0,\"processed_data\":196,\"persisted_data\":0,\"num_subtasks\":4,\"num_acknowledged_subtasks\":4,\"checkpoint_type\":\"CHECKPOINT\",\"tasks\":{},\"external_path\":\"file:/flink-data/checkpoints/00000000000000000000000000000000/chk-92\",\"discarded\":true},{\"className\":\"completed\",\"id\":91,\"status\":\"COMPLETED\",\"is_savepoint\":false,\"trigger_timestamp\":1653212827603,\"latest_ack_timestamp\":1653212827629,\"checkpointed_size\":27969,\"state_size\":27969,\"end_to_end_duration\":26,\"alignment_buffered\":0,\"processed_data\":0,\"persisted_data\":0,\"num_subtasks\":4,\"num_acknowledged_subtasks\":4,\"checkpoint_type\":\"CHECKPOINT\",\"tasks\":{},\"external_path\":\"file:/flink-data/checkpoints/00000000000000000000000000000000/chk-91\",\"discarded\":true},{\"className\":\"completed\",\"id\":90,\"status\":\"COMPLETED\",\"is_savepoint\":false,\"trigger_timestamp\":1653212825603,\"latest_ack_timestamp\":1653212825641,\"checkpointed_size\":27735,\"state_size\":27735,\"end_to_end_duration\":38,\"alignment_buffered\":0,\"processed_data\":0,\"persisted_data\":0,\"num_subtasks\":4,\"num_acknowledged_subtasks\":4,\"checkpoint_type\":\"CHECKPOINT\",\"tasks\":{},\"external_path\":\"file:/flink-data/checkpoints/00000000000000000000000000000000/chk-90\",\"discarded\":true},{\"className\":\"completed\",\"id\":89,\"status\":\"COMPLETED\",\"is_savepoint\":false,\"trigger_timestamp\":1653212823603,\"latest_ack_timestamp\":1653212823618,\"checkpointed_size\":28545,\"state_size\":28545,\"end_to_end_duration\":15,\"alignment_buffered\":0,\"processed_data\":364,\"persisted_data\":0,\"num_subtasks\":4,\"num_acknowledged_subtasks\":4,\"checkpoint_type\":\"CHECKPOINT\",\"tasks\":{},\"external_path\":\"file:/flink-data/checkpoints/00000000000000000000000000000000/chk-89\",\"discarded\":true},{\"className\":\"completed\",\"id\":88,\"status\":\"COMPLETED\",\"is_savepoint\":false,\"trigger_timestamp\":1653212821603,\"latest_ack_timestamp\":1653212821619,\"checkpointed_size\":28275,\"state_size\":28275,\"end_to_end_duration\":16,\"alignment_buffered\":0,\"processed_data\":0,\"persisted_data\":0,\"num_subtasks\":4,\"num_acknowledged_subtasks\":4,\"checkpoint_type\":\"CHECKPOINT\",\"tasks\":{},\"external_path\":\"file:/flink-data/checkpoints/00000000000000000000000000000000/chk-88\",\"discarded\":true},{\"className\":\"completed\",\"id\":87,\"status\":\"COMPLETED\",\"is_savepoint\":false,\"trigger_timestamp\":1653212819604,\"latest_ack_timestamp\":1653212819622,\"checkpointed_size\":28518,\"state_size\":28518,\"end_to_end_duration\":18,\"alignment_buffered\":0,\"processed_data\":0,\"persisted_data\":0,\"num_subtasks\":4,\"num_acknowledged_subtasks\":4,\"checkpoint_type\":\"CHECKPOINT\",\"tasks\":{},\"external_path\":\"file:/flink-data/checkpoints/00000000000000000000000000000000/chk-87\",\"discarded\":true}]}";
    String responseWithoutHistory = "{\"counts\":{\"restored\":1,\"total\":79,\"in_progress\":0,\"completed\":69,\"failed\":10},\"summary\":{\"checkpointed_size\":{\"min\":23928,\"max\":53670,\"avg\":28551,\"p50\":28239,\"p90\":28563,\"p95\":28635,\"p99\":53670,\"p999\":53670},\"state_size\":{\"min\":23928,\"max\":53670,\"avg\":28551,\"p50\":28239,\"p90\":28563,\"p95\":28635,\"p99\":53670,\"p999\":53670},\"end_to_end_duration\":{\"min\":14,\"max\":117,\"avg\":24,\"p50\":22,\"p90\":32,\"p95\":40.5,\"p99\":117,\"p999\":117},\"alignment_buffered\":{\"min\":0,\"max\":0,\"avg\":0,\"p50\":0,\"p90\":0,\"p95\":0,\"p99\":0,\"p999\":0},\"processed_data\":{\"min\":0,\"max\":1274,\"avg\":280,\"p50\":112,\"p90\":840,\"p95\":1071,\"p99\":1274,\"p999\":1274},\"persisted_data\":{\"min\":0,\"max\":0,\"avg\":0,\"p50\":0,\"p90\":0,\"p95\":0,\"p99\":0,\"p999\":0}},\"latest\":{\"completed\":null,\"savepoint\":null,\"failed\":null,\"restored\":{\"id\":27,\"restore_timestamp\":1653212683022,\"is_savepoint\":true,\"external_path\":\"file:/flink-data/savepoints/savepoint-000000-5930e5326ca7\"}},\"history\":[]}";
    String responseWithoutHistoryInternal = "{\"counts\":{\"restored\":1,\"total\":79,\"in_progress\":0,\"completed\":69,\"failed\":10},\"summary\":{\"checkpointed_size\":{\"min\":23928,\"max\":53670,\"avg\":28551,\"p50\":28239,\"p90\":28563,\"p95\":28635,\"p99\":53670,\"p999\":53670},\"state_size\":{\"min\":23928,\"max\":53670,\"avg\":28551,\"p50\":28239,\"p90\":28563,\"p95\":28635,\"p99\":53670,\"p999\":53670},\"end_to_end_duration\":{\"min\":14,\"max\":117,\"avg\":24,\"p50\":22,\"p90\":32,\"p95\":40.5,\"p99\":117,\"p999\":117},\"alignment_buffered\":{\"min\":0,\"max\":0,\"avg\":0,\"p50\":0,\"p90\":0,\"p95\":0,\"p99\":0,\"p999\":0},\"processed_data\":{\"min\":0,\"max\":1274,\"avg\":280,\"p50\":112,\"p90\":840,\"p95\":1071,\"p99\":1274,\"p999\":1274},\"persisted_data\":{\"min\":0,\"max\":0,\"avg\":0,\"p50\":0,\"p90\":0,\"p95\":0,\"p99\":0,\"p999\":0}},\"latest\":{\"completed\":null,\"savepoint\":null,\"failed\":null,\"restored\":{\"id\":27,\"restore_timestamp\":1653212683022,\"is_savepoint\":true,\"external_path\":\"<checkpoint-not-externally-addressable>\"}},\"history\":[]}";
    var responseContainer = new ArrayList<CheckpointHistoryWrapper>();
    testingClusterClient.setRequestProcessor((headers, parameters, requestBody) -> {
        if (headers instanceof CustomCheckpointingStatisticsHeaders) {
            return CompletableFuture.completedFuture(responseContainer.get(0));
        }
        fail("unknown request");
        return null;
    });
    var flinkService = createFlinkService(testingClusterClient);
    responseContainer.add(objectMapper.readValue(responseWithHistory, CheckpointHistoryWrapper.class));
    var checkpointOpt = flinkService.getLastCheckpoint(new JobID(), new Configuration());
    assertEquals("file:/flink-data/checkpoints/00000000000000000000000000000000/chk-96", checkpointOpt.get().getLocation());
    responseContainer.set(0, objectMapper.readValue(responseWithoutHistory, CheckpointHistoryWrapper.class));
    checkpointOpt = flinkService.getLastCheckpoint(new JobID(), new Configuration());
    assertEquals("file:/flink-data/savepoints/savepoint-000000-5930e5326ca7", checkpointOpt.get().getLocation());
    responseContainer.set(0, objectMapper.readValue(responseWithoutHistoryInternal, CheckpointHistoryWrapper.class));
    try {
        flinkService.getLastCheckpoint(new JobID(), new Configuration());
        fail();
    } catch (DeploymentFailedException dpe) {
    }
}

Example 3 with TestingClusterClient

use of org.apache.flink.kubernetes.operator.TestingClusterClient in project flink-kubernetes-operator by apache.

the class FlinkServiceTest method testTriggerSavepoint.

@Test
public void testTriggerSavepoint() throws Exception {
    final TestingClusterClient<String> testingClusterClient = new TestingClusterClient<>(configuration, TestUtils.TEST_DEPLOYMENT_NAME);
    final CompletableFuture<Tuple3<JobID, String, Boolean>> triggerSavepointFuture = new CompletableFuture<>();
    final String savepointPath = "file:///path/of/svp";
    configuration.set(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointPath);
    testingClusterClient.setRequestProcessor((headers, parameters, requestBody) -> {
        triggerSavepointFuture.complete(new Tuple3<>(((SavepointTriggerMessageParameters) parameters).jobID.getValue(), ((SavepointTriggerRequestBody) requestBody).getTargetDirectory().get(), ((SavepointTriggerRequestBody) requestBody).isCancelJob()));
        return CompletableFuture.completedFuture(new TriggerResponse(new TriggerId()));
    });
    final FlinkService flinkService = createFlinkService(testingClusterClient);
    final JobID jobID = JobID.generate();
    final FlinkDeployment flinkDeployment = TestUtils.buildApplicationCluster();
    ReconciliationUtils.updateForSpecReconciliationSuccess(flinkDeployment, JobState.RUNNING);
    JobStatus jobStatus = new JobStatus();
    jobStatus.setJobId(jobID.toString());
    flinkDeployment.getStatus().setJobStatus(jobStatus);
    flinkService.triggerSavepoint(flinkDeployment.getStatus().getJobStatus().getJobId(), SavepointTriggerType.MANUAL, flinkDeployment.getStatus().getJobStatus().getSavepointInfo(), configuration);
    assertTrue(triggerSavepointFuture.isDone());
    assertEquals(jobID, triggerSavepointFuture.get().f0);
    assertEquals(savepointPath, triggerSavepointFuture.get().f1);
    assertFalse(triggerSavepointFuture.get().f2);
}

Example 4 with TestingClusterClient

use of org.apache.flink.kubernetes.operator.TestingClusterClient in project flink-kubernetes-operator by apache.

the class FlinkServiceTest method testCancelJobWithLastStateUpgradeMode.

@Test
public void testCancelJobWithLastStateUpgradeMode() throws Exception {
    FlinkDeployment deployment = TestUtils.buildApplicationCluster();
    ReconciliationUtils.updateForSpecReconciliationSuccess(deployment, JobState.RUNNING);
    final TestingClusterClient<String> testingClusterClient = new TestingClusterClient<>(configuration, TestUtils.TEST_DEPLOYMENT_NAME);
    final FlinkService flinkService = createFlinkService(testingClusterClient);
    client.apps().deployments().inNamespace(TestUtils.TEST_NAMESPACE).create(createTestingDeployment());
    assertNotNull(client.apps().deployments().inNamespace(TestUtils.TEST_NAMESPACE).withName(TestUtils.TEST_DEPLOYMENT_NAME).get());
    JobID jobID = JobID.generate();
    JobStatus jobStatus = deployment.getStatus().getJobStatus();
    jobStatus.setJobId(jobID.toHexString());
    flinkService.cancelJob(deployment, UpgradeMode.LAST_STATE);
    assertNull(jobStatus.getSavepointInfo().getLastSavepoint());
    assertNull(client.apps().deployments().inNamespace(TestUtils.TEST_NAMESPACE).withName(TestUtils.TEST_DEPLOYMENT_NAME).get());
}

Example 5 with TestingClusterClient

use of org.apache.flink.kubernetes.operator.TestingClusterClient in project flink-kubernetes-operator by apache.

the class FlinkServiceTest method testCancelJobWithSavepointUpgradeMode.

@Test
public void testCancelJobWithSavepointUpgradeMode() throws Exception {
    final TestingClusterClient<String> testingClusterClient = new TestingClusterClient<>(configuration, TestUtils.TEST_DEPLOYMENT_NAME);
    final CompletableFuture<Tuple3<JobID, Boolean, String>> stopWithSavepointFuture = new CompletableFuture<>();
    final String savepointPath = "file:///path/of/svp-1";
    configuration.set(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointPath);
    testingClusterClient.setStopWithSavepointFunction((jobID, advanceToEndOfEventTime, savepointDir) -> {
        stopWithSavepointFuture.complete(new Tuple3<>(jobID, advanceToEndOfEventTime, savepointDir));
        return CompletableFuture.completedFuture(savepointPath);
    });
    final FlinkService flinkService = createFlinkService(testingClusterClient);
    JobID jobID = JobID.generate();
    FlinkDeployment deployment = TestUtils.buildApplicationCluster();
    deployment.getSpec().getFlinkConfiguration().put(CheckpointingOptions.SAVEPOINT_DIRECTORY.key(), savepointPath);
    deployment.getStatus().setJobManagerDeploymentStatus(JobManagerDeploymentStatus.READY);
    JobStatus jobStatus = deployment.getStatus().getJobStatus();
    jobStatus.setJobId(jobID.toHexString());
    jobStatus.setState(org.apache.flink.api.common.JobStatus.RUNNING.name());
    ReconciliationUtils.updateForSpecReconciliationSuccess(deployment, JobState.RUNNING);
    flinkService.cancelJob(deployment, UpgradeMode.SAVEPOINT);
    assertTrue(stopWithSavepointFuture.isDone());
    assertEquals(jobID, stopWithSavepointFuture.get().f0);
    assertFalse(stopWithSavepointFuture.get().f1);
    assertEquals(savepointPath, stopWithSavepointFuture.get().f2);
    assertEquals(savepointPath, jobStatus.getSavepointInfo().getLastSavepoint().getLocation());
}