Examples with Pair com.netflix.titus.common.util.tuple.Pair used on opensource projects

Example 21 with Pair

use of com.netflix.titus.common.util.tuple.Pair in project titus-control-plane by Netflix.

the class TaskMigrationDeschedulerTest method testAgentInstanceRequiredMigration.

@Test
public void testAgentInstanceRequiredMigration() {
    Task job1Task0 = jobOperations.getTasks("job1").get(0);
    relocationConnectorStubs.place("active1", job1Task0);
    relocationConnectorStubs.setQuota("job1", 1);
    job1Task0 = jobOperations.findTaskById(job1Task0.getId()).get().getRight();
    relocationConnectorStubs.markNodeRelocationRequired(job1Task0.getTaskContext().get(TaskAttributes.TASK_ATTRIBUTES_AGENT_INSTANCE_ID));
    Optional<Pair<TitusNode, List<Task>>> results = newDescheduler(Collections.emptyMap()).nextBestMatch();
    assertThat(results).isNotEmpty();
}

Example 22 with Pair

use of com.netflix.titus.common.util.tuple.Pair in project titus-control-plane by Netflix.

the class TaskMigrationDeschedulerTest method testSelfMigration.

@Test
public void testSelfMigration() {
    Task job1Task0 = jobOperations.getTasks("job1").get(0);
    relocationConnectorStubs.place("removable1", job1Task0);
    relocationConnectorStubs.setQuota("job1", 1);
    TaskRelocationPlan job1Task0Plan = TaskRelocationPlan.newBuilder().withTaskId(job1Task0.getId()).withRelocationTime(Long.MAX_VALUE / 2).build();
    Optional<Pair<TitusNode, List<Task>>> results = newDescheduler(Collections.singletonMap(job1Task0.getId(), job1Task0Plan)).nextBestMatch();
    assertThat(results).isEmpty();
}

Example 23 with Pair

use of com.netflix.titus.common.util.tuple.Pair in project titus-control-plane by Netflix.

the class DefaultDeschedulerService method deschedule.

@Override
public List<DeschedulingResult> deschedule(Map<String, TaskRelocationPlan> plannedAheadTaskRelocationPlans) {
    List<Pair<Job, List<Task>>> allJobsAndTasks = jobOperations.getJobsAndTasks();
    Map<String, Job<?>> jobs = allJobsAndTasks.stream().map(Pair::getLeft).collect(Collectors.toMap(Job::getId, j -> j));
    Map<String, Task> tasksById = allJobsAndTasks.stream().flatMap(p -> p.getRight().stream()).collect(Collectors.toMap(Task::getId, t -> t));
    EvacuatedAgentsAllocationTracker evacuatedAgentsAllocationTracker = new EvacuatedAgentsAllocationTracker(nodeDataResolver.resolve(), tasksById);
    EvictionQuotaTracker evictionQuotaTracker = new EvictionQuotaTracker(evictionOperations, jobs);
    TaskMigrationDescheduler taskMigrationDescheduler = new TaskMigrationDescheduler(plannedAheadTaskRelocationPlans, evacuatedAgentsAllocationTracker, evictionQuotaTracker, evictionConfiguration, jobs, tasksById, titusRuntime);
    Map<String, DeschedulingResult> requestedImmediateEvictions = taskMigrationDescheduler.findAllImmediateEvictions();
    Map<String, DeschedulingResult> requestedEvictions = taskMigrationDescheduler.findRequestedJobOrTaskMigrations();
    Map<String, DeschedulingResult> allRequestedEvictions = CollectionsExt.merge(requestedImmediateEvictions, requestedEvictions);
    Map<String, DeschedulingResult> regularEvictions = new HashMap<>();
    Optional<Pair<TitusNode, List<Task>>> bestMatch;
    while ((bestMatch = taskMigrationDescheduler.nextBestMatch()).isPresent()) {
        TitusNode agent = bestMatch.get().getLeft();
        List<Task> tasks = bestMatch.get().getRight();
        tasks.forEach(task -> {
            if (!allRequestedEvictions.containsKey(task.getId())) {
                Optional<TaskRelocationPlan> relocationPlanForTask = getRelocationPlanForTask(agent, task, plannedAheadTaskRelocationPlans);
                relocationPlanForTask.ifPresent(rp -> regularEvictions.put(task.getId(), DeschedulingResult.newBuilder().withTask(task).withAgentInstance(agent).withTaskRelocationPlan(rp).build()));
            }
        });
    }
    // Find eviction which could not be scheduled now.
    for (Task task : tasksById.values()) {
        if (allRequestedEvictions.containsKey(task.getId()) || regularEvictions.containsKey(task.getId())) {
            continue;
        }
        if (evacuatedAgentsAllocationTracker.isEvacuated(task)) {
            DeschedulingFailure failure = taskMigrationDescheduler.getDeschedulingFailure(task);
            TaskRelocationPlan relocationPlan = plannedAheadTaskRelocationPlans.get(task.getId());
            if (relocationPlan == null) {
                relocationPlan = newNotDelayedRelocationPlan(task, false);
            }
            TitusNode agent = evacuatedAgentsAllocationTracker.getRemovableAgent(task);
            regularEvictions.put(task.getId(), DeschedulingResult.newBuilder().withTask(task).withAgentInstance(agent).withTaskRelocationPlan(relocationPlan).withFailure(failure).build());
        }
    }
    return CollectionsExt.merge(new ArrayList<>(allRequestedEvictions.values()), new ArrayList<>(regularEvictions.values()));
}

Example 24 with Pair

use of com.netflix.titus.common.util.tuple.Pair in project titus-control-plane by Netflix.

the class ReactorMapWithStateTransformer method apply.

@Override
public Publisher<R> apply(Flux<T> source) {
    return Flux.create(sink -> {
        AtomicReference<S> lastState = new AtomicReference<>(zeroSupplier.get());
        Flux<Either<T, Function<S, Pair<R, S>>>> sourceEither = source.map(Either::ofValue);
        Flux<Either<T, Function<S, Pair<R, S>>>> cleanupEither = cleanupActions.map(Either::ofError);
        Disposable subscription = Flux.merge(sourceEither, cleanupEither).subscribe(next -> {
            Pair<R, S> result;
            if (next.hasValue()) {
                try {
                    result = transformer.apply(next.getValue(), lastState.get());
                } catch (Throwable e) {
                    sink.error(e);
                    return;
                }
            } else {
                try {
                    Function<S, Pair<R, S>> action = next.getError();
                    result = action.apply(lastState.get());
                } catch (Throwable e) {
                    sink.error(e);
                    return;
                }
            }
            lastState.set(result.getRight());
            sink.next(result.getLeft());
        }, sink::error, sink::complete);
        sink.onCancel(subscription);
    });
}

Example 25 with Pair

use of com.netflix.titus.common.util.tuple.Pair in project titus-control-plane by Netflix.

the class CassandraJobStore method retrieveJobs.

@Override
public Observable<Pair<List<Job<?>>, Integer>> retrieveJobs() {
    Observable result = Observable.fromCallable(() -> {
        List<String> jobIds = activeJobIdsBucketManager.getItems();
        return jobIds.stream().map(retrieveActiveJobStatement::bind).map(this::execute).collect(Collectors.toList());
    }).flatMap(observables -> Observable.merge(observables, getConcurrencyLimit()).flatMapIterable(resultSet -> {
        List<Row> allRows = resultSet.all();
        if (allRows.isEmpty()) {
            logger.debug("Job id with no record");
            return Collections.emptyList();
        }
        return allRows.stream().map(row -> row.getString(0)).map(value -> {
            String effectiveValue;
            if (fitBadDataInjection.isPresent()) {
                effectiveValue = fitBadDataInjection.get().afterImmediate(JobStoreFitAction.ErrorKind.CorruptedRawJobRecords.name(), value);
            } else {
                effectiveValue = value;
            }
            Job<?> job;
            try {
                job = deserializeJob(effectiveValue);
            } catch (Exception e) {
                logger.error("Cannot map serialized job data to Job class: {}", effectiveValue, e);
                return Either.ofError(e);
            }
            if (job.getJobDescriptor().getDisruptionBudget() == null) {
                titusRuntime.getCodeInvariants().inconsistent("jobWithNoDisruptionBudget: jobId=%s", job.getId());
                job = JobFunctions.changeDisruptionBudget(job, DisruptionBudget.none());
            }
            // TODO Remove this code when there are no more jobs with missing migration data (caused by a bug in ServiceJobExt builder).
            if (job.getJobDescriptor().getExtensions() instanceof ServiceJobExt) {
                Job<ServiceJobExt> serviceJob = (Job<ServiceJobExt>) job;
                ServiceJobExt ext = serviceJob.getJobDescriptor().getExtensions();
                if (ext.getMigrationPolicy() == null) {
                    titusRuntime.getCodePointTracker().markReachable("Corrupted task migration record in Cassandra: " + job.getId());
                    ServiceJobExt fixedExt = ext.toBuilder().withMigrationPolicy(SystemDefaultMigrationPolicy.newBuilder().build()).build();
                    logger.warn("Service job with no migration policy defined. Setting system default: {}", job.getId());
                    job = serviceJob.toBuilder().withJobDescriptor(serviceJob.getJobDescriptor().toBuilder().withExtensions(fixedExt).build()).build();
                }
            }
            if (!fitBadDataInjection.isPresent()) {
                return Either.ofValue(job);
            }
            Job<?> effectiveJob = fitBadDataInjection.get().afterImmediate(JobStoreFitAction.ErrorKind.CorruptedJobRecords.name(), job);
            return Either.ofValue(effectiveJob);
        }).collect(Collectors.toList());
    })).toList().map(everything -> {
        List<Job> goodJobs = (List<Job>) everything.stream().filter(Either::hasValue).map(Either::getValue).collect(Collectors.toList());
        int errors = everything.size() - goodJobs.size();
        return Pair.of(goodJobs, errors);
    });
    return result;
}