Examples with FlinkDeployment org.apache.flink.kubernetes.operator.crd.FlinkDeployment used on opensource projects

Example 61 with FlinkDeployment

use of org.apache.flink.kubernetes.operator.crd.FlinkDeployment in project flink-kubernetes-operator by apache.

the class FlinkDeploymentControllerTest method verifyBasicReconcileLoop.

@Test
public void verifyBasicReconcileLoop() {
    FlinkDeployment appCluster = TestUtils.buildApplicationCluster();
    UpdateControl<FlinkDeployment> updateControl;
    updateControl = testController.reconcile(appCluster, TestUtils.createEmptyContext());
    assertTrue(updateControl.isUpdateStatus());
    assertEquals(JobManagerDeploymentStatus.DEPLOYING.toUpdateControl(appCluster, operatorConfiguration).getScheduleDelay(), updateControl.getScheduleDelay());
    // Validate reconciliation status
    ReconciliationStatus reconciliationStatus = appCluster.getStatus().getReconciliationStatus();
    assertTrue(reconciliationStatus.isSuccess());
    assertNull(reconciliationStatus.getError());
    assertEquals(appCluster.getSpec(), reconciliationStatus.getLastReconciledSpec());
    updateControl = testController.reconcile(appCluster, context);
    assertTrue(updateControl.isUpdateStatus());
    assertEquals(JobManagerDeploymentStatus.DEPLOYED_NOT_READY.toUpdateControl(appCluster, operatorConfiguration).getScheduleDelay(), updateControl.getScheduleDelay());
    updateControl = testController.reconcile(appCluster, context);
    assertTrue(updateControl.isUpdateStatus());
    assertEquals(JobManagerDeploymentStatus.READY.toUpdateControl(appCluster, operatorConfiguration).getScheduleDelay(), updateControl.getScheduleDelay());
    // Validate job status
    JobStatus jobStatus = appCluster.getStatus().getJobStatus();
    JobStatusMessage expectedJobStatus = flinkService.listJobs().get(0).f1;
    assertEquals(expectedJobStatus.getJobId().toHexString(), jobStatus.getJobId());
    assertEquals(expectedJobStatus.getJobName(), jobStatus.getJobName());
    assertEquals(expectedJobStatus.getJobState().toString(), jobStatus.getState());
    // Send in invalid update
    appCluster = ReconciliationUtils.clone(appCluster);
    appCluster.getSpec().setJob(null);
    updateControl = testController.reconcile(appCluster, context);
    assertTrue(updateControl.isUpdateStatus());
    assertFalse(updateControl.getScheduleDelay().isPresent());
    reconciliationStatus = appCluster.getStatus().getReconciliationStatus();
    assertFalse(reconciliationStatus.isSuccess());
    assertEquals("Cannot switch from job to session cluster", reconciliationStatus.getError());
    assertNotNull(reconciliationStatus.getLastReconciledSpec().getJob());
    // Validate job status correct even with error
    jobStatus = appCluster.getStatus().getJobStatus();
    expectedJobStatus = flinkService.listJobs().get(0).f1;
    assertEquals(expectedJobStatus.getJobId().toHexString(), jobStatus.getJobId());
    assertEquals(expectedJobStatus.getJobName(), jobStatus.getJobName());
    assertEquals(expectedJobStatus.getJobState().toString(), jobStatus.getState());
}

Example 62 with FlinkDeployment

use of org.apache.flink.kubernetes.operator.crd.FlinkDeployment in project flink-kubernetes-operator by apache.

the class JobReconcilerTest method testUpgradeModeChangeFromSavepointToLastState.

@Test
public void testUpgradeModeChangeFromSavepointToLastState() throws Exception {
    final String expectedSavepointPath = "savepoint_0";
    final Context context = TestUtils.createContextWithReadyJobManagerDeployment();
    final TestingFlinkService flinkService = new TestingFlinkService();
    final JobReconciler reconciler = new JobReconciler(null, flinkService, operatorConfiguration);
    final FlinkDeployment deployment = TestUtils.buildApplicationCluster();
    final Configuration config = FlinkUtils.getEffectiveConfig(deployment, new Configuration());
    reconciler.reconcile("test", deployment, context, config);
    List<Tuple2<String, JobStatusMessage>> runningJobs = flinkService.listJobs();
    verifyAndSetRunningJobsToStatus(deployment, runningJobs);
    // Suspend FlinkDeployment with savepoint upgrade mode
    deployment.getSpec().getJob().setUpgradeMode(UpgradeMode.SAVEPOINT);
    deployment.getSpec().getJob().setState(JobState.SUSPENDED);
    deployment.getSpec().setImage("new-image-1");
    reconciler.reconcile("test", deployment, context, config);
    assertEquals(0, flinkService.listJobs().size());
    assertTrue(JobState.SUSPENDED.name().equalsIgnoreCase(deployment.getStatus().getJobStatus().getState()));
    assertEquals(expectedSavepointPath, deployment.getStatus().getJobStatus().getSavepointInfo().getLastSavepoint().getLocation());
    // Resume FlinkDeployment with last-state upgrade mode
    deployment.getStatus().getReconciliationStatus().getLastReconciledSpec().getJob().setState(JobState.SUSPENDED);
    deployment.getSpec().getJob().setUpgradeMode(UpgradeMode.LAST_STATE);
    deployment.getSpec().getJob().setState(JobState.RUNNING);
    deployment.getSpec().setImage("new-image-2");
    reconciler.reconcile("test", deployment, context, config);
    runningJobs = flinkService.listJobs();
    assertEquals(expectedSavepointPath, config.get(SavepointConfigOptions.SAVEPOINT_PATH));
    assertEquals(1, runningJobs.size());
    assertEquals(expectedSavepointPath, runningJobs.get(0).f0);
}

Example 63 with FlinkDeployment

use of org.apache.flink.kubernetes.operator.crd.FlinkDeployment in project flink-kubernetes-operator by apache.

the class JobReconcilerTest method testUpgrade.

@Test
public void testUpgrade() throws Exception {
    Context context = TestUtils.createContextWithReadyJobManagerDeployment();
    TestingFlinkService flinkService = new TestingFlinkService();
    JobReconciler reconciler = new JobReconciler(null, flinkService, operatorConfiguration);
    FlinkDeployment deployment = TestUtils.buildApplicationCluster();
    Configuration config = FlinkUtils.getEffectiveConfig(deployment, new Configuration());
    reconciler.reconcile("test", deployment, context, config);
    List<Tuple2<String, JobStatusMessage>> runningJobs = flinkService.listJobs();
    verifyAndSetRunningJobsToStatus(deployment, runningJobs);
    // Test stateless upgrade
    FlinkDeployment statelessUpgrade = ReconciliationUtils.clone(deployment);
    statelessUpgrade.getSpec().getJob().setUpgradeMode(UpgradeMode.STATELESS);
    statelessUpgrade.getSpec().getFlinkConfiguration().put("new", "conf");
    reconciler.reconcile("test", statelessUpgrade, context, config);
    runningJobs = flinkService.listJobs();
    assertEquals(1, runningJobs.size());
    assertNull(runningJobs.get(0).f0);
    deployment.getStatus().getJobStatus().setJobId(runningJobs.get(0).f1.getJobId().toHexString());
    // Test stateful upgrade
    FlinkDeployment statefulUpgrade = ReconciliationUtils.clone(deployment);
    statefulUpgrade.getSpec().getJob().setUpgradeMode(UpgradeMode.SAVEPOINT);
    statefulUpgrade.getSpec().getFlinkConfiguration().put("new", "conf2");
    reconciler.reconcile("test", statefulUpgrade, context, new Configuration(config));
    runningJobs = flinkService.listJobs();
    assertEquals(1, runningJobs.size());
    assertEquals("savepoint_0", runningJobs.get(0).f0);
}

Example 64 with FlinkDeployment

use of org.apache.flink.kubernetes.operator.crd.FlinkDeployment in project flink-kubernetes-operator by apache.

the class DeploymentValidatorTest method testSuccess.

private void testSuccess(Consumer<FlinkDeployment> deploymentModifier) {
    FlinkDeployment deployment = TestUtils.buildApplicationCluster();
    deploymentModifier.accept(deployment);
    validator.validate(deployment).ifPresent(Assert::fail);
}

Example 65 with FlinkDeployment

use of org.apache.flink.kubernetes.operator.crd.FlinkDeployment in project flink-kubernetes-operator by apache.

the class Observer method observeJmDeployment.

private void observeJmDeployment(FlinkDeployment flinkApp, Context context, Configuration effectiveConfig) {
    FlinkDeploymentStatus deploymentStatus = flinkApp.getStatus();
    JobManagerDeploymentStatus previousJmStatus = deploymentStatus.getJobManagerDeploymentStatus();
    if (JobManagerDeploymentStatus.READY == previousJmStatus) {
        return;
    }
    if (JobManagerDeploymentStatus.DEPLOYED_NOT_READY == previousJmStatus) {
        deploymentStatus.setJobManagerDeploymentStatus(JobManagerDeploymentStatus.READY);
        return;
    }
    Optional<Deployment> deployment = context.getSecondaryResource(Deployment.class);
    if (deployment.isPresent()) {
        DeploymentStatus status = deployment.get().getStatus();
        DeploymentSpec spec = deployment.get().getSpec();
        if (status != null && status.getAvailableReplicas() != null && spec.getReplicas().intValue() == status.getReplicas() && spec.getReplicas().intValue() == status.getAvailableReplicas() && flinkService.isJobManagerPortReady(effectiveConfig)) {
            // typically it takes a few seconds for the REST server to be ready
            LOG.info("JobManager deployment {} in namespace {} port ready, waiting for the REST API...", flinkApp.getMetadata().getName(), flinkApp.getMetadata().getNamespace());
            deploymentStatus.setJobManagerDeploymentStatus(JobManagerDeploymentStatus.DEPLOYED_NOT_READY);
            return;
        }
        LOG.info("JobManager deployment {} in namespace {} exists but not ready yet, status {}", flinkApp.getMetadata().getName(), flinkApp.getMetadata().getNamespace(), status);
        List<DeploymentCondition> conditions = status.getConditions();
        for (DeploymentCondition dc : conditions) {
            if ("FailedCreate".equals(dc.getReason()) && "ReplicaFailure".equals(dc.getType())) {
                // throw only when not already in error status to allow for spec update
                if (!JobManagerDeploymentStatus.ERROR.equals(deploymentStatus.getJobManagerDeploymentStatus())) {
                    throw new DeploymentFailedException(DeploymentFailedException.COMPONENT_JOBMANAGER, dc);
                }
                return;
            }
        }
        deploymentStatus.setJobManagerDeploymentStatus(JobManagerDeploymentStatus.DEPLOYING);
        return;
    }
    deploymentStatus.setJobManagerDeploymentStatus(JobManagerDeploymentStatus.MISSING);
}