Examples with ResourceCounter org.apache.flink.runtime.util.ResourceCounter used on opensource projects

Example 1 with ResourceCounter

use of org.apache.flink.runtime.util.ResourceCounter in project flink by apache.

the class BiDirectionalResourceToRequirementMapping method internalDecrementCount.

private static void internalDecrementCount(Map<ResourceProfile, ResourceCounter> primaryMap, ResourceProfile primaryKey, ResourceProfile secondaryKey, int decrement) {
    primaryMap.compute(primaryKey, (resourceProfile, resourceCounter) -> {
        Preconditions.checkState(resourceCounter != null, "Attempting to decrement count of %s->%s, but primary key was unknown.", resourceProfile, secondaryKey);
        final ResourceCounter newCounter = resourceCounter.subtract(secondaryKey, decrement);
        return newCounter.isEmpty() ? null : newCounter;
    });
}

Example 2 with ResourceCounter

use of org.apache.flink.runtime.util.ResourceCounter in project flink by apache.

the class JobScopedResourceTracker method tryAssigningExcessSlots.

private void tryAssigningExcessSlots() {
    if (LOG.isTraceEnabled()) {
        LOG.trace("There are {} excess resources for job {} before re-assignment.", excessResources.getTotalResourceCount(), jobId);
    }
    ResourceCounter assignedResources = ResourceCounter.empty();
    for (Map.Entry<ResourceProfile, Integer> excessResource : excessResources.getResourcesWithCount()) {
        for (int i = 0; i < excessResource.getValue(); i++) {
            final ResourceProfile resourceProfile = excessResource.getKey();
            final Optional<ResourceProfile> matchingRequirement = findMatchingRequirement(resourceProfile);
            if (matchingRequirement.isPresent()) {
                resourceToRequirementMapping.incrementCount(matchingRequirement.get(), resourceProfile, 1);
                assignedResources = assignedResources.add(resourceProfile, 1);
            } else {
                break;
            }
        }
    }
    for (Map.Entry<ResourceProfile, Integer> assignedResource : assignedResources.getResourcesWithCount()) {
        excessResources = excessResources.subtract(assignedResource.getKey(), assignedResource.getValue());
    }
    if (LOG.isTraceEnabled()) {
        LOG.trace("There are {} excess resources for job {} after re-assignment.", excessResources.getTotalResourceCount(), jobId);
    }
}

Example 3 with ResourceCounter

use of org.apache.flink.runtime.util.ResourceCounter in project flink by apache.

the class DefaultDeclarativeSlotPool method freeAndReleaseSlots.

private ResourceCounter freeAndReleaseSlots(Collection<AllocatedSlot> currentlyReservedSlots, Collection<AllocatedSlot> slots, Exception cause) {
    ResourceCounter previouslyFulfilledRequirements = getFulfilledRequirements(currentlyReservedSlots);
    releasePayload(currentlyReservedSlots, cause);
    releaseSlots(slots, cause);
    return previouslyFulfilledRequirements;
}

Example 4 with ResourceCounter

use of org.apache.flink.runtime.util.ResourceCounter in project flink by apache.

the class DefaultDeclarativeSlotPool method releaseIdleSlots.

@Override
public void releaseIdleSlots(long currentTimeMillis) {
    final Collection<AllocatedSlotPool.FreeSlotInfo> freeSlotsInformation = slotPool.getFreeSlotsInformation();
    ResourceCounter excessResources = fulfilledResourceRequirements.subtract(totalResourceRequirements);
    final Iterator<AllocatedSlotPool.FreeSlotInfo> freeSlotIterator = freeSlotsInformation.iterator();
    final Collection<AllocatedSlot> slotsToReturnToOwner = new ArrayList<>();
    while (!excessResources.isEmpty() && freeSlotIterator.hasNext()) {
        final AllocatedSlotPool.FreeSlotInfo idleSlot = freeSlotIterator.next();
        if (currentTimeMillis >= idleSlot.getFreeSince() + idleSlotTimeout.toMilliseconds()) {
            final ResourceProfile matchingProfile = getMatchingResourceProfile(idleSlot.getAllocationId());
            if (excessResources.containsResource(matchingProfile)) {
                excessResources = excessResources.subtract(matchingProfile, 1);
                final Optional<AllocatedSlot> removedSlot = slotPool.removeSlot(idleSlot.getAllocationId());
                final AllocatedSlot allocatedSlot = removedSlot.orElseThrow(() -> new IllegalStateException(String.format("Could not find slot for allocation id %s.", idleSlot.getAllocationId())));
                slotsToReturnToOwner.add(allocatedSlot);
            }
        }
    }
    releaseSlots(slotsToReturnToOwner, new FlinkException("Returning idle slots to their owners."));
    LOG.debug("Idle slots have been returned; new total acquired resources: {}", fulfilledResourceRequirements);
}

Example 5 with ResourceCounter

use of org.apache.flink.runtime.util.ResourceCounter in project flink by apache.

the class DefaultDeclarativeSlotPoolTest method testReleaseSlotOnlyReturnsFulfilledRequirementsOfReservedSlots.

@Test
public void testReleaseSlotOnlyReturnsFulfilledRequirementsOfReservedSlots() {
    withSlotPoolContainingOneTaskManagerWithTwoSlotsWithUniqueResourceProfiles((slotPool, freeSlot, slotToReserve, ignored) -> {
        slotPool.reserveFreeSlot(slotToReserve.getAllocationId(), slotToReserve.getResourceProfile()).tryAssignPayload(new TestingPhysicalSlotPayload());
        final ResourceCounter fulfilledRequirementsOfFreeSlot = slotPool.releaseSlot(freeSlot.getAllocationId(), new FlinkException("Test failure"));
        final ResourceCounter fulfilledRequirementsOfReservedSlot = slotPool.releaseSlot(slotToReserve.getAllocationId(), new FlinkException("Test failure"));
        assertThat(fulfilledRequirementsOfFreeSlot.getResources(), is(empty()));
        assertThat(fulfilledRequirementsOfReservedSlot.getResourceCount(slotToReserve.getResourceProfile()), is(1));
    });
}