Examples with JobDescriptor com.netflix.titus.api.jobmanager.model.job.JobDescriptor used on opensource projects

Example 16 with JobDescriptor

use of com.netflix.titus.api.jobmanager.model.job.JobDescriptor in project titus-control-plane by Netflix.

the class JobRuntimePredictionSanitizer method capPredictionToRuntimeLimit.

/**
 * Use the prediction when available and shorter than the runtime limit, otherwise the runtime limit becomes
 * the prediction if within {@link JobRuntimePredictionConfiguration#getMaxOpportunisticRuntimeLimitMs()}
 */
@SuppressWarnings("unchecked")
private JobDescriptor capPredictionToRuntimeLimit(JobDescriptor jobDescriptor) {
    // non-batch jobs have been filtered before this point, it is safe to cast
    BatchJobExt extensions = ((JobDescriptor<BatchJobExt>) jobDescriptor).getExtensions();
    long runtimeLimitMs = extensions.getRuntimeLimitMs();
    if (runtimeLimitMs <= 0 || runtimeLimitMs > configuration.getMaxOpportunisticRuntimeLimitMs()) {
        // no runtime limit or too high to be used, noop
        return jobDescriptor;
    }
    return JobFunctions.getJobRuntimePrediction(jobDescriptor).filter(prediction -> runtimeLimitMs > prediction.toMillis()).map(ignored -> jobDescriptor).orElseGet(() -> JobFunctions.appendJobDescriptorAttributes(jobDescriptor, ImmutableMap.<String, String>builder().put(JOB_ATTRIBUTES_RUNTIME_PREDICTION_SEC, Double.toString(runtimeLimitMs / 1000.0)).put(JOB_ATTRIBUTES_RUNTIME_PREDICTION_CONFIDENCE, Double.toString(1.0)).build()));
}

Example 17 with JobDescriptor

use of com.netflix.titus.api.jobmanager.model.job.JobDescriptor in project titus-control-plane by Netflix.

the class EmbeddedTitusGateway method boot.

public EmbeddedTitusGateway boot() {
    Stopwatch timer = Stopwatch.createStarted();
    logger.info("Starting Titus Gateway");
    injector = InjectorBuilder.fromModules(newJettyModule(), new ArchaiusModule() {

        @Override
        protected void configureArchaius() {
            config.setProperty("titus.masterClient.masterIp", masterGrpcHost);
            if (embeddedTitusMaster == null) {
                config.setProperty("titus.masterClient.masterGrpcPort", masterGrpcPort);
                config.setProperty("titus.masterClient.masterHttpPort", masterHttpPort);
            } else {
                // In the embedded mode, master cannot run jetty, so we set only GRPC port.
                config.setProperty("titus.masterClient.masterGrpcPort", embeddedTitusMaster.getGrpcPort());
                config.setProperty("titus.masterClient.masterHttpPort", "0");
            }
            bindApplicationConfigurationOverride().toInstance(config);
        }
    }, Modules.override(new TitusGatewayModule(enableREST)).with(new AbstractModule() {

        @Override
        protected void configure() {
            if (store != null) {
                bind(JobStore.class).toInstance(store);
            }
            bind(new TypeLiteral<AdmissionValidator<JobDescriptor>>() {
            }).toInstance(validator);
        }

        @Provides
        @Singleton
        public AdmissionSanitizer<JobDescriptor> getJobSanitizer(TitusValidatorConfiguration configuration) {
            return new AggregatingSanitizer(configuration, Collections.singletonList(jobSanitizer));
        }
    })).createInjector();
    logger.info("Embedded TitusGateway started in {}ms", timer.elapsed(TimeUnit.MILLISECONDS));
    return this;
}

Example 18 with JobDescriptor

use of com.netflix.titus.api.jobmanager.model.job.JobDescriptor in project titus-control-plane by Netflix.

the class JobRuntimePredictionSanitizerTest method errorsCauseSanitizationToBeSkipped.

@Test
public void errorsCauseSanitizationToBeSkipped() {
    JobDescriptor<BatchJobExt> noRuntimeLimit = jobDescriptor.but(jd -> jd.getExtensions().toBuilder().withRuntimeLimitMs(0));
    JobRuntimePredictionClient errorClient = mock(JobRuntimePredictionClient.class);
    when(errorClient.getRuntimePredictions(any())).thenReturn(Mono.error(TitusServiceException.internal("bad request")));
    AdmissionSanitizer<JobDescriptor> sanitizer = new JobRuntimePredictionSanitizer(errorClient, JobRuntimePredictionSelectors.best(), config, TitusRuntimes.test());
    StepVerifier.create(sanitizer.sanitizeAndApply(noRuntimeLimit)).assertNext(result -> assertThat(((JobDescriptor<?>) result).getAttributes()).containsEntry(JOB_ATTRIBUTES_SANITIZATION_SKIPPED_RUNTIME_PREDICTION, "true").doesNotContainKeys(JOB_ATTRIBUTES_RUNTIME_PREDICTION_CONFIDENCE, JOB_ATTRIBUTES_RUNTIME_PREDICTION_SEC, JOB_ATTRIBUTES_RUNTIME_PREDICTION_MODEL_ID, JOB_ATTRIBUTES_RUNTIME_PREDICTION_VERSION)).verifyComplete();
}

Example 19 with JobDescriptor

use of com.netflix.titus.api.jobmanager.model.job.JobDescriptor in project titus-control-plane by Netflix.

the class JobRuntimePredictionSanitizerTest method predictionsAreCappedToRuntimeLimit.

@Test
public void predictionsAreCappedToRuntimeLimit() {
    JobRuntimePredictionClient client = mock(JobRuntimePredictionClient.class);
    JobRuntimePredictions predictions = new JobRuntimePredictions("v1", modelId, new TreeSet<>(Arrays.asList(new JobRuntimePrediction(0.05, 10.1), new JobRuntimePrediction(0.25, 30.2), new JobRuntimePrediction(0.90, 90.5))));
    when(client.getRuntimePredictions(any())).thenReturn(Mono.just(predictions));
    AdmissionSanitizer<JobDescriptor> sanitizer = new JobRuntimePredictionSanitizer(client, JobRuntimePredictionSelectors.best(), config, TitusRuntimes.test());
    StepVerifier.create(sanitizer.sanitizeAndApply(jobDescriptor)).assertNext(result -> assertThat(((JobDescriptor<?>) result).getAttributes()).containsEntry(JOB_ATTRIBUTES_RUNTIME_PREDICTION_SEC, "60.0").containsEntry(JOB_ATTRIBUTES_RUNTIME_PREDICTION_CONFIDENCE, "1.0").containsEntry(JOB_ATTRIBUTES_RUNTIME_PREDICTION_MODEL_ID, modelId).containsEntry(JOB_ATTRIBUTES_RUNTIME_PREDICTION_VERSION, "v1").containsEntry(JOB_ATTRIBUTES_RUNTIME_PREDICTION_AVAILABLE, "0.05=10.1;0.25=30.2;0.9=90.5")).verifyComplete();
}

Example 20 with JobDescriptor

use of com.netflix.titus.api.jobmanager.model.job.JobDescriptor in project titus-control-plane by Netflix.

the class JobRuntimePredictionSanitizerTest method emptyPredictionsCauseSanitizationToBeSkipped.

@Test
public void emptyPredictionsCauseSanitizationToBeSkipped() {
    JobDescriptor<BatchJobExt> noRuntimeLimit = jobDescriptor.but(jd -> jd.getExtensions().toBuilder().withRuntimeLimitMs(0));
    JobRuntimePredictionClient client = mock(JobRuntimePredictionClient.class);
    JobRuntimePredictions predictions = new JobRuntimePredictions("v1", modelId, new TreeSet<>());
    when(client.getRuntimePredictions(any())).thenReturn(Mono.just(predictions));
    AdmissionSanitizer<JobDescriptor> sanitizer = new JobRuntimePredictionSanitizer(client, JobRuntimePredictionSelectors.best(), config, TitusRuntimes.test());
    StepVerifier.create(sanitizer.sanitizeAndApply(noRuntimeLimit)).assertNext(result -> assertThat(((JobDescriptor<?>) result).getAttributes()).containsEntry(JOB_ATTRIBUTES_SANITIZATION_SKIPPED_RUNTIME_PREDICTION, "true").containsEntry(JOB_ATTRIBUTES_RUNTIME_PREDICTION_MODEL_ID, modelId).containsEntry(JOB_ATTRIBUTES_RUNTIME_PREDICTION_VERSION, "v1").doesNotContainKeys(JOB_ATTRIBUTES_RUNTIME_PREDICTION_SEC, JOB_ATTRIBUTES_RUNTIME_PREDICTION_CONFIDENCE, JOB_ATTRIBUTES_RUNTIME_PREDICTION_AVAILABLE)).verifyComplete();
}