Examples with SlotSharingGroupAssignment org.apache.flink.runtime.instance.SlotSharingGroupAssignment used on opensource projects

Example 1 with SlotSharingGroupAssignment

use of org.apache.flink.runtime.instance.SlotSharingGroupAssignment in project flink by apache.

the class Scheduler method scheduleTask.

/**
	 * Returns either a {@link SimpleSlot}, or a {@link Future}.
	 */
private Object scheduleTask(ScheduledUnit task, boolean queueIfNoResource) throws NoResourceAvailableException {
    if (task == null) {
        throw new NullPointerException();
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Scheduling task " + task);
    }
    final ExecutionVertex vertex = task.getTaskToExecute().getVertex();
    final Iterable<TaskManagerLocation> preferredLocations = vertex.getPreferredLocationsBasedOnInputs();
    final boolean forceExternalLocation = false && preferredLocations != null && preferredLocations.iterator().hasNext();
    synchronized (globalLock) {
        SlotSharingGroup sharingUnit = task.getSlotSharingGroup();
        if (sharingUnit != null) {
            if (queueIfNoResource) {
                throw new IllegalArgumentException("A task with a vertex sharing group was scheduled in a queued fashion.");
            }
            final SlotSharingGroupAssignment assignment = sharingUnit.getTaskAssignment();
            final CoLocationConstraint constraint = task.getLocationConstraint();
            // sanity check that we do not use an externally forced location and a co-location constraint together
            if (constraint != null && forceExternalLocation) {
                throw new IllegalArgumentException("The scheduling cannot be constrained simultaneously by a " + "co-location constraint and an external location constraint.");
            }
            // get a slot from the group, if the group has one for us (and can fulfill the constraint)
            final SimpleSlot slotFromGroup;
            if (constraint == null) {
                slotFromGroup = assignment.getSlotForTask(vertex);
            } else {
                slotFromGroup = assignment.getSlotForTask(vertex, constraint);
            }
            SimpleSlot newSlot = null;
            SimpleSlot toUse = null;
            // the following needs to make sure any allocated slot is released in case of an error
            try {
                // any slot that is local, or where the assignment was unconstrained is good!
                if (slotFromGroup != null && slotFromGroup.getLocality() != Locality.NON_LOCAL) {
                    // the location, because we are quite happy with the slot
                    if (constraint != null && !constraint.isAssigned()) {
                        constraint.lockLocation();
                    }
                    updateLocalityCounters(slotFromGroup, vertex);
                    return slotFromGroup;
                }
                // the group did not have a local slot for us. see if we can one (or a better one)
                // our location preference is either determined by the location constraint, or by the
                // vertex's preferred locations
                final Iterable<TaskManagerLocation> locations;
                final boolean localOnly;
                if (constraint != null && constraint.isAssigned()) {
                    locations = Collections.singleton(constraint.getLocation());
                    localOnly = true;
                } else {
                    locations = vertex.getPreferredLocationsBasedOnInputs();
                    localOnly = forceExternalLocation;
                }
                newSlot = getNewSlotForSharingGroup(vertex, locations, assignment, constraint, localOnly);
                if (newSlot == null) {
                    if (slotFromGroup == null) {
                        if (constraint != null && constraint.isAssigned()) {
                            // nothing is available on the node where the co-location constraint forces us to
                            throw new NoResourceAvailableException("Could not allocate a slot on instance " + constraint.getLocation() + ", as required by the co-location constraint.");
                        } else if (forceExternalLocation) {
                            // could not satisfy the external location constraint
                            String hosts = getHostnamesFromInstances(preferredLocations);
                            throw new NoResourceAvailableException("Could not schedule task " + vertex + " to any of the required hosts: " + hosts);
                        } else {
                            // simply nothing is available
                            throw new NoResourceAvailableException(task, getNumberOfAvailableInstances(), getTotalNumberOfSlots(), getNumberOfAvailableSlots());
                        }
                    } else {
                        // got a non-local from the group, and no new one, so we use the non-local
                        // slot from the sharing group
                        toUse = slotFromGroup;
                    }
                } else if (slotFromGroup == null || !slotFromGroup.isAlive() || newSlot.getLocality() == Locality.LOCAL) {
                    // then we use the new slot
                    if (slotFromGroup != null) {
                        slotFromGroup.releaseSlot();
                    }
                    toUse = newSlot;
                } else {
                    // both are available and usable. neither is local. in that case, we may
                    // as well use the slot from the sharing group, to minimize the number of
                    // instances that the job occupies
                    newSlot.releaseSlot();
                    toUse = slotFromGroup;
                }
                // the location, because we are going to use that slot
                if (constraint != null && !constraint.isAssigned()) {
                    constraint.lockLocation();
                }
                updateLocalityCounters(toUse, vertex);
            } catch (NoResourceAvailableException e) {
                throw e;
            } catch (Throwable t) {
                if (slotFromGroup != null) {
                    slotFromGroup.releaseSlot();
                }
                if (newSlot != null) {
                    newSlot.releaseSlot();
                }
                ExceptionUtils.rethrow(t, "An error occurred while allocating a slot in a sharing group");
            }
            return toUse;
        } else {
            // 2) === schedule without hints and sharing ===
            SimpleSlot slot = getFreeSlotForTask(vertex, preferredLocations, forceExternalLocation);
            if (slot != null) {
                updateLocalityCounters(slot, vertex);
                return slot;
            } else {
                // no resource available now, so queue the request
                if (queueIfNoResource) {
                    CompletableFuture<SimpleSlot> future = new FlinkCompletableFuture<>();
                    this.taskQueue.add(new QueuedTask(task, future));
                    return future;
                } else if (forceExternalLocation) {
                    String hosts = getHostnamesFromInstances(preferredLocations);
                    throw new NoResourceAvailableException("Could not schedule task " + vertex + " to any of the required hosts: " + hosts);
                } else {
                    throw new NoResourceAvailableException(getNumberOfAvailableInstances(), getTotalNumberOfSlots(), getNumberOfAvailableSlots());
                }
            }
        }
    }
}

Example 2 with SlotSharingGroupAssignment

use of org.apache.flink.runtime.instance.SlotSharingGroupAssignment in project flink by apache.

the class CoLocationConstraintTest method testAssignSlotAndLockLocation.

@Test
public void testAssignSlotAndLockLocation() {
    try {
        JobID jid = new JobID();
        JobVertex vertex = new JobVertex("vertex");
        vertex.setParallelism(1);
        SlotSharingGroup sharingGroup = new SlotSharingGroup(vertex.getID());
        SlotSharingGroupAssignment assignment = sharingGroup.getTaskAssignment();
        CoLocationGroup constraintGroup = new CoLocationGroup(vertex);
        CoLocationConstraint constraint = constraintGroup.getLocationConstraint(0);
        // constraint is completely unassigned
        assertFalse(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        Instance instance1 = SchedulerTestUtils.getRandomInstance(2);
        Instance instance2 = SchedulerTestUtils.getRandomInstance(2);
        SharedSlot slot1_1 = instance1.allocateSharedSlot(jid, assignment);
        SharedSlot slot1_2 = instance1.allocateSharedSlot(jid, assignment);
        SharedSlot slot2_1 = instance2.allocateSharedSlot(jid, assignment);
        SharedSlot slot2_2 = instance2.allocateSharedSlot(jid, assignment);
        // constraint is still completely unassigned
        assertFalse(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        // set the slot, but do not lock the location yet
        constraint.setSharedSlot(slot1_1);
        assertFalse(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        // try to get the location
        try {
            constraint.getLocation();
            fail("should throw an IllegalStateException");
        } catch (IllegalStateException e) {
        // as expected
        } catch (Exception e) {
            fail("wrong exception, should be IllegalStateException");
        }
        // check that we can reassign the slot as long as the location is not locked
        constraint.setSharedSlot(slot2_1);
        // the previous slot should have been released now
        assertTrue(slot1_1.isReleased());
        // still the location is not assigned
        assertFalse(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        // we can do an identity re-assign
        constraint.setSharedSlot(slot2_1);
        assertFalse(slot2_1.isReleased());
        // still the location is not assigned
        assertFalse(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        constraint.lockLocation();
        // now, the location is assigned and we have a location
        assertTrue(constraint.isAssigned());
        assertTrue(constraint.isAssignedAndAlive());
        assertEquals(instance2.getTaskManagerLocation(), constraint.getLocation());
        // release the slot
        slot2_1.releaseSlot();
        // we should still have a location
        assertTrue(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        assertEquals(instance2.getTaskManagerLocation(), constraint.getLocation());
        // we can not assign a different location
        try {
            constraint.setSharedSlot(slot1_2);
            fail("should throw an IllegalArgumentException");
        } catch (IllegalArgumentException e) {
        // as expected
        } catch (Exception e) {
            fail("wrong exception, should be IllegalArgumentException");
        }
        // assign a new slot with the same location
        constraint.setSharedSlot(slot2_2);
        assertTrue(constraint.isAssigned());
        assertTrue(constraint.isAssignedAndAlive());
        assertEquals(instance2.getTaskManagerLocation(), constraint.getLocation());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}