Examples with AbstractID org.apache.flink.util.AbstractID used on opensource projects

Example 1 with AbstractID

use of org.apache.flink.util.AbstractID in project flink by apache.

the class SlotSharingGroupAssignment method internalDisposeEmptySharedSlot.

/**
	 * 
	 * <p><b>NOTE: This method must be called from within a scope that holds the lock.</b></p>
	 */
private void internalDisposeEmptySharedSlot(SharedSlot sharedSlot) {
    // sanity check
    if (sharedSlot.isAlive() | !sharedSlot.getSubSlots().isEmpty()) {
        throw new IllegalArgumentException();
    }
    final SharedSlot parent = sharedSlot.getParent();
    final AbstractID groupID = sharedSlot.getGroupID();
    if (parent == null) {
        // root slot, return to the instance.
        sharedSlot.getOwner().returnAllocatedSlot(sharedSlot);
        // also, make sure we remove this slot from everywhere
        allSlots.remove(sharedSlot);
        removeSlotFromAllEntries(availableSlotsPerJid, sharedSlot);
    } else if (groupID != null) {
        if (sharedSlot.markReleased()) {
            LOG.debug("Internally dispose empty shared slot {}.", sharedSlot);
            int parentRemaining = parent.removeDisposedChildSlot(sharedSlot);
            if (parentRemaining > 0) {
                // the parent becomes available for the group again
                Map<ResourceID, List<SharedSlot>> slotsForGroup = availableSlotsPerJid.get(groupID);
                // sanity check
                if (slotsForGroup == null) {
                    throw new IllegalStateException("Trying to return a slot for group " + groupID + " when available slots indicated that all slots were available.");
                }
                putIntoMultiMap(slotsForGroup, parent.getTaskManagerID(), parent);
            } else {
                // this was the last child of the parent. release the parent.
                parent.markCancelled();
                internalDisposeEmptySharedSlot(parent);
            }
        }
    } else {
        throw new IllegalStateException("Found a shared slot that is neither a root slot, nor associated with a vertex group.");
    }
}

Example 2 with AbstractID

use of org.apache.flink.util.AbstractID in project flink by apache.

the class SharedSlotsTest method allocateAndReleaseEmptySlot.

@Test
public void allocateAndReleaseEmptySlot() {
    try {
        JobID jobId = new JobID();
        JobVertexID vertexId = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(vertexId);
        SlotSharingGroupAssignment assignment = sharingGroup.getTaskAssignment();
        assertEquals(0, assignment.getNumberOfSlots());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vertexId));
        Instance instance = SchedulerTestUtils.getRandomInstance(2);
        assertEquals(2, instance.getTotalNumberOfSlots());
        assertEquals(0, instance.getNumberOfAllocatedSlots());
        assertEquals(2, instance.getNumberOfAvailableSlots());
        // allocate a shared slot
        SharedSlot slot = instance.allocateSharedSlot(jobId, assignment);
        assertEquals(2, instance.getTotalNumberOfSlots());
        assertEquals(1, instance.getNumberOfAllocatedSlots());
        assertEquals(1, instance.getNumberOfAvailableSlots());
        // check that the new slot is fresh
        assertTrue(slot.isAlive());
        assertFalse(slot.isCanceled());
        assertFalse(slot.isReleased());
        assertEquals(0, slot.getNumberLeaves());
        assertFalse(slot.hasChildren());
        assertTrue(slot.isRootAndEmpty());
        assertNotNull(slot.toString());
        assertTrue(slot.getSubSlots().isEmpty());
        assertEquals(0, slot.getSlotNumber());
        assertEquals(0, slot.getRootSlotNumber());
        // release the slot immediately.
        slot.releaseSlot();
        assertTrue(slot.isCanceled());
        assertTrue(slot.isReleased());
        // the slot sharing group and instance should not
        assertEquals(2, instance.getTotalNumberOfSlots());
        assertEquals(0, instance.getNumberOfAllocatedSlots());
        assertEquals(2, instance.getNumberOfAvailableSlots());
        assertEquals(0, assignment.getNumberOfSlots());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vertexId));
        // we should not be able to allocate any children from this released slot
        assertNull(slot.allocateSharedSlot(new AbstractID()));
        assertNull(slot.allocateSubSlot(new AbstractID()));
        // we cannot add this slot to the assignment group
        assertNull(assignment.addSharedSlotAndAllocateSubSlot(slot, Locality.NON_LOCAL, vertexId));
        assertEquals(0, assignment.getNumberOfSlots());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 3 with AbstractID

use of org.apache.flink.util.AbstractID in project flink by apache.

the class CoLocationConstraintTest method testCreateConstraints.

@Test
public void testCreateConstraints() {
    try {
        JobVertexID id1 = new JobVertexID();
        JobVertexID id2 = new JobVertexID();
        JobVertex vertex1 = new JobVertex("vertex1", id1);
        vertex1.setParallelism(2);
        JobVertex vertex2 = new JobVertex("vertex2", id2);
        vertex2.setParallelism(3);
        CoLocationGroup group = new CoLocationGroup(vertex1, vertex2);
        AbstractID groupId = group.getId();
        assertNotNull(groupId);
        CoLocationConstraint constraint1 = group.getLocationConstraint(0);
        CoLocationConstraint constraint2 = group.getLocationConstraint(1);
        CoLocationConstraint constraint3 = group.getLocationConstraint(2);
        assertFalse(constraint1 == constraint2);
        assertFalse(constraint1 == constraint3);
        assertFalse(constraint2 == constraint3);
        assertEquals(groupId, constraint1.getGroupId());
        assertEquals(groupId, constraint2.getGroupId());
        assertEquals(groupId, constraint3.getGroupId());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 4 with AbstractID

use of org.apache.flink.util.AbstractID in project flink by apache.

the class RocksDBStateBackend method ensureRocksDBIsLoaded.

// ------------------------------------------------------------------------
//  static library loading utilities
// ------------------------------------------------------------------------
private void ensureRocksDBIsLoaded(String tempDirectory) throws IOException {
    synchronized (RocksDBStateBackend.class) {
        if (!rocksDbInitialized) {
            final File tempDirParent = new File(tempDirectory).getAbsoluteFile();
            LOG.info("Attempting to load RocksDB native library and store it under '{}'", tempDirParent);
            Throwable lastException = null;
            for (int attempt = 1; attempt <= ROCKSDB_LIB_LOADING_ATTEMPTS; attempt++) {
                try {
                    // when multiple instances of this class and RocksDB exist in different
                    // class loaders, then we can see the following exception:
                    // "java.lang.UnsatisfiedLinkError: Native Library /path/to/temp/dir/librocksdbjni-linux64.so
                    // already loaded in another class loader"
                    // to avoid that, we need to add a random element to the library file path
                    // (I know, seems like an unnecessary hack, since the JVM obviously can handle multiple
                    //  instances of the same JNI library being loaded in different class loaders, but
                    //  apparently not when coming from the same file path, so there we go)
                    final File rocksLibFolder = new File(tempDirParent, "rocksdb-lib-" + new AbstractID());
                    // make sure the temp path exists
                    LOG.debug("Attempting to create RocksDB native library folder {}", rocksLibFolder);
                    // noinspection ResultOfMethodCallIgnored
                    rocksLibFolder.mkdirs();
                    // explicitly load the JNI dependency if it has not been loaded before
                    NativeLibraryLoader.getInstance().loadLibrary(rocksLibFolder.getAbsolutePath());
                    // this initialization here should validate that the loading succeeded
                    RocksDB.loadLibrary();
                    // seems to have worked
                    LOG.info("Successfully loaded RocksDB native library");
                    rocksDbInitialized = true;
                    return;
                } catch (Throwable t) {
                    lastException = t;
                    LOG.debug("RocksDB JNI library loading attempt {} failed", attempt, t);
                    // try to force RocksDB to attempt reloading the library
                    try {
                        resetRocksDBLoadedFlag();
                    } catch (Throwable tt) {
                        LOG.debug("Failed to reset 'initialized' flag in RocksDB native code loader", tt);
                    }
                }
            }
            throw new IOException("Could not load the native RocksDB library", lastException);
        }
    }
}

Example 5 with AbstractID

use of org.apache.flink.util.AbstractID in project flink by apache.

the class OperatorGroupTest method testVariables.

@Test
public void testVariables() {
    MetricRegistry registry = new MetricRegistry(MetricRegistryConfiguration.defaultMetricRegistryConfiguration());
    JobID jid = new JobID();
    AbstractID tid = new AbstractID();
    AbstractID eid = new AbstractID();
    TaskManagerMetricGroup tmGroup = new TaskManagerMetricGroup(registry, "theHostName", "test-tm-id");
    TaskManagerJobMetricGroup jmGroup = new TaskManagerJobMetricGroup(registry, tmGroup, jid, "myJobName");
    TaskMetricGroup taskGroup = new TaskMetricGroup(registry, jmGroup, tid, eid, "aTaskName", 11, 0);
    OperatorMetricGroup opGroup = new OperatorMetricGroup(registry, taskGroup, "myOpName");
    Map<String, String> variables = opGroup.getAllVariables();
    testVariable(variables, ScopeFormat.SCOPE_HOST, "theHostName");
    testVariable(variables, ScopeFormat.SCOPE_TASKMANAGER_ID, "test-tm-id");
    testVariable(variables, ScopeFormat.SCOPE_JOB_ID, jid.toString());
    testVariable(variables, ScopeFormat.SCOPE_JOB_NAME, "myJobName");
    testVariable(variables, ScopeFormat.SCOPE_TASK_VERTEX_ID, tid.toString());
    testVariable(variables, ScopeFormat.SCOPE_TASK_NAME, "aTaskName");
    testVariable(variables, ScopeFormat.SCOPE_TASK_ATTEMPT_ID, eid.toString());
    testVariable(variables, ScopeFormat.SCOPE_TASK_SUBTASK_INDEX, "11");
    testVariable(variables, ScopeFormat.SCOPE_TASK_ATTEMPT_NUM, "0");
    testVariable(variables, ScopeFormat.SCOPE_OPERATOR_NAME, "myOpName");
    registry.shutdown();
}