Examples with Pair com.netflix.titus.common.util.tuple.Pair used on opensource projects

Example 31 with Pair

use of com.netflix.titus.common.util.tuple.Pair in project titus-control-plane by Netflix.

the class TaskMigrationDescheduler method computeFitness.

private Pair<Double, List<Task>> computeFitness(TitusNode agent) {
    List<Task> tasks = evacuatedAgentsAllocationTracker.getTasksOnAgent(agent.getId());
    if (tasks.isEmpty()) {
        return FITNESS_RESULT_NONE;
    }
    boolean systemWindowOpen = evictionQuotaTracker.isSystemDisruptionWindowOpen();
    long availableTerminationLimit;
    if (systemWindowOpen) {
        availableTerminationLimit = Math.min(tasks.size(), evictionQuotaTracker.getSystemEvictionQuota());
        if (availableTerminationLimit <= 0) {
            return FITNESS_RESULT_NONE;
        }
    } else {
        // system window is closed, we'll need to inspect all eligible jobs during closed window
        availableTerminationLimit = tasks.size();
    }
    Map<String, List<Task>> chosen = new HashMap<>();
    List<Task> chosenList = new ArrayList<>();
    for (Task task : tasks) {
        if (canTerminate(task)) {
            String jobId = task.getJobId();
            Job<?> job = jobsById.get(jobId);
            // if window is closed, then only pick up jobs that are exempt
            boolean continueWithJobQuotaCheck = systemWindowOpen || isJobExemptFromSystemDisruptionWindow(job);
            if (continueWithJobQuotaCheck) {
                // applying job eviction quota
                long quota = evictionQuotaTracker.getJobEvictionQuota(jobId);
                long used = chosen.getOrDefault(jobId, Collections.emptyList()).size();
                if ((quota - used) > 0) {
                    chosen.computeIfAbsent(jobId, jid -> new ArrayList<>()).add(task);
                    chosenList.add(task);
                    if (availableTerminationLimit <= chosenList.size()) {
                        break;
                    }
                }
            }
        }
    }
    if (chosenList.size() == 0) {
        return FITNESS_RESULT_NONE;
    }
    int leftOnAgent = tasks.size() - chosenList.size();
    double fitness = Math.max(FITNESS_PERFECT - leftOnAgent * TASK_ON_AGENT_PENALTY, 0.01);
    return Pair.of(fitness, chosenList);
}

Example 32 with Pair

use of com.netflix.titus.common.util.tuple.Pair in project titus-control-plane by Netflix.

the class GrpcJobReplicatorEventStreamTest method testCacheTaskMove.

@Test
public void testCacheTaskMove() {
    Pair<Job, List<Task>> pair = jobServiceStub.createJobAndTasks(SERVICE_JOB);
    Job target = jobServiceStub.createJob(SERVICE_JOB);
    Task task = pair.getRight().get(0);
    String sourceJobId = pair.getLeft().getId();
    String targetJobId = target.getId();
    List<ReplicatorEvent<JobSnapshot, JobManagerEvent<?>>> events = new ArrayList<>();
    newConnectVerifier().assertNext(next -> assertThat(next.getSnapshot().getTaskMap().values()).allSatisfy(t -> assertThat(t.getStatus().getState()).isEqualTo(TaskState.Accepted))).then(() -> jobServiceStub.moveTaskToState(task, TaskState.Started)).assertNext(next -> {
        JobSnapshot snapshot = next.getSnapshot();
        Optional<Pair<Job<?>, Task>> taskOpt = snapshot.findTaskById(task.getId());
        assertThat(taskOpt).isPresent();
        assertThat(taskOpt.get().getRight().getStatus().getState()).isEqualTo(TaskState.Started);
        assertThat(snapshot.getTasks(sourceJobId)).containsKey(task.getId());
    }).then(() -> jobServiceStub.getJobOperations().moveServiceTask(sourceJobId, targetJobId, task.getId(), CallMetadata.newBuilder().withCallerId("Test").withCallReason("testing").build()).test().awaitTerminalEvent().assertNoErrors()).recordWith(() -> events).thenConsumeWhile(next -> {
        JobManagerEvent<?> trigger = next.getTrigger();
        if (!(trigger instanceof TaskUpdateEvent)) {
            return true;
        }
        TaskUpdateEvent taskUpdateEvent = (TaskUpdateEvent) trigger;
        return !taskUpdateEvent.isMovedFromAnotherJob();
    }).thenCancel().verify();
    assertThat(events).hasSize(3);
    events.stream().map(ReplicatorEvent::getTrigger).forEach(jobManagerEvent -> {
        if (jobManagerEvent instanceof JobUpdateEvent) {
            JobUpdateEvent jobUpdateEvent = (JobUpdateEvent) jobManagerEvent;
            String eventJobId = jobUpdateEvent.getCurrent().getId();
            assertThat(eventJobId).isIn(sourceJobId, targetJobId);
        } else if (jobManagerEvent instanceof TaskUpdateEvent) {
            TaskUpdateEvent taskUpdateEvent = (TaskUpdateEvent) jobManagerEvent;
            assertThat(taskUpdateEvent.isMovedFromAnotherJob()).isTrue();
            assertThat(taskUpdateEvent.getCurrentJob().getId()).isEqualTo(targetJobId);
            assertThat(taskUpdateEvent.getCurrent().getJobId()).isEqualTo(targetJobId);
            assertThat(taskUpdateEvent.getCurrent().getTaskContext().get(TaskAttributes.TASK_ATTRIBUTES_MOVED_FROM_JOB)).isEqualTo(sourceJobId);
        } else {
            fail("Unexpected event type: %s", jobManagerEvent);
        }
    });
}

Example 33 with Pair

use of com.netflix.titus.common.util.tuple.Pair in project titus-control-plane by Netflix.

the class ReservationUsageCalculator method buildUsage.

public Map<String, ReservationUsage> buildUsage() {
    Map<String, ResourceAccumulator> accumulatorMap = new HashMap<>();
    List<Pair<Job, List<Task>>> jobsAndTasks = jobOperations.getJobsAndTasks();
    Set<String> capacityGroupNames = capacityManagementService.getApplicationSLAs().stream().map(ApplicationSLA::getAppName).collect(Collectors.toSet());
    for (Pair<Job, List<Task>> jobAndTasks : jobsAndTasks) {
        Job job = jobAndTasks.getLeft();
        String capacityGroup = capacityGroupNames.contains(job.getJobDescriptor().getCapacityGroup()) ? job.getJobDescriptor().getCapacityGroup() : ApplicationSlaManagementService.DEFAULT_APPLICATION;
        ResourceAccumulator accumulator = accumulatorMap.computeIfAbsent(capacityGroup, cp -> new ResourceAccumulator());
        processJob(accumulator, jobAndTasks);
    }
    Map<String, ReservationUsage> result = new HashMap<>();
    accumulatorMap.forEach((capacityGroup, accumulator) -> result.put(capacityGroup, accumulator.toReservationUsage()));
    capacityManagementService.getApplicationSLAs().forEach(capacityGroup -> {
        if (!result.containsKey(capacityGroup.getAppName())) {
            result.put(capacityGroup.getAppName(), ReservationUsage.none());
        }
    });
    return result;
}

Example 34 with Pair

use of com.netflix.titus.common.util.tuple.Pair in project titus-control-plane by Netflix.

the class TaskTerminationExecutor method checkTaskIsRunningOrThrowAnException.

private Pair<Job<?>, Task> checkTaskIsRunningOrThrowAnException(String taskId) {
    Optional<Pair<Job<?>, Task>> jobAndTask = jobOperations.findTaskById(taskId);
    if (!jobAndTask.isPresent()) {
        throw EvictionException.taskNotFound(taskId);
    }
    Task task = jobAndTask.get().getRight();
    TaskState state = task.getStatus().getState();
    if (state == TaskState.Accepted) {
        throw EvictionException.taskNotScheduledYet(task);
    }
    if (!TaskState.isBefore(state, TaskState.KillInitiated)) {
        throw EvictionException.taskAlreadyStopped(task);
    }
    return jobAndTask.get();
}

Example 35 with Pair

use of com.netflix.titus.common.util.tuple.Pair in project titus-control-plane by Netflix.

the class TaskMigrationDeschedulerTest method testSelfMigrationAfterDeadline.

@Test
public void testSelfMigrationAfterDeadline() {
    Task job1Task0 = jobOperations.getTasks("job1").get(0);
    relocationConnectorStubs.place("removable1", job1Task0);
    relocationConnectorStubs.setQuota("job1", 1);
    TaskRelocationPlan job1Task0Plan = TaskRelocationPlan.newBuilder().withTaskId(job1Task0.getId()).withRelocationTime(clock.wallTime() - 1).build();
    Optional<Pair<TitusNode, List<Task>>> results = newDescheduler(Collections.singletonMap(job1Task0.getId(), job1Task0Plan)).nextBestMatch();
    assertThat(results).isNotEmpty();
}