Examples with AbstractID org.apache.flink.util.AbstractID used on opensource projects

Example 11 with AbstractID

use of org.apache.flink.util.AbstractID in project flink by apache.

the class StatsDReporterTest method testAddingMetrics.

/**
	 * Tests that the registered metrics' names don't contain invalid characters.
	 */
@Test
public void testAddingMetrics() throws NoSuchFieldException, IllegalAccessException {
    Configuration configuration = new Configuration();
    String taskName = "testTask";
    String jobName = "testJob:-!ax..?";
    String hostname = "local::host:";
    String taskManagerId = "tas:kMana::ger";
    String counterName = "testCounter";
    configuration.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test");
    configuration.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, "org.apache.flink.metrics.statsd.StatsDReporterTest$TestingStatsDReporter");
    configuration.setString(ConfigConstants.METRICS_SCOPE_NAMING_TASK, "<host>.<tm_id>.<job_name>");
    configuration.setString(ConfigConstants.METRICS_SCOPE_DELIMITER, "_");
    MetricRegistry metricRegistry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(configuration));
    char delimiter = metricRegistry.getDelimiter();
    TaskManagerMetricGroup tmMetricGroup = new TaskManagerMetricGroup(metricRegistry, hostname, taskManagerId);
    TaskManagerJobMetricGroup tmJobMetricGroup = new TaskManagerJobMetricGroup(metricRegistry, tmMetricGroup, new JobID(), jobName);
    TaskMetricGroup taskMetricGroup = new TaskMetricGroup(metricRegistry, tmJobMetricGroup, new AbstractID(), new AbstractID(), taskName, 0, 0);
    SimpleCounter myCounter = new SimpleCounter();
    taskMetricGroup.counter(counterName, myCounter);
    List<MetricReporter> reporters = metricRegistry.getReporters();
    assertTrue(reporters.size() == 1);
    MetricReporter metricReporter = reporters.get(0);
    assertTrue("Reporter should be of type StatsDReporter", metricReporter instanceof StatsDReporter);
    TestingStatsDReporter reporter = (TestingStatsDReporter) metricReporter;
    Map<Counter, String> counters = reporter.getCounters();
    assertTrue(counters.containsKey(myCounter));
    String expectedCounterName = reporter.filterCharacters(hostname) + delimiter + reporter.filterCharacters(taskManagerId) + delimiter + reporter.filterCharacters(jobName) + delimiter + reporter.filterCharacters(counterName);
    assertEquals(expectedCounterName, counters.get(myCounter));
    metricRegistry.shutdown();
}

Example 12 with AbstractID

use of org.apache.flink.util.AbstractID in project flink by apache.

the class ScheduledDropwizardReporterTest method testAddingMetrics.

/**
	 * Tests that the registered metrics' names don't contain invalid characters.
	 */
@Test
public void testAddingMetrics() throws NoSuchFieldException, IllegalAccessException {
    Configuration configuration = new Configuration();
    String taskName = "test\"Ta\"..sk";
    String jobName = "testJ\"ob:-!ax..?";
    String hostname = "loc<>al\"::host\".:";
    String taskManagerId = "tas:kMana::ger";
    String counterName = "testCounter";
    configuration.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test");
    configuration.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, "org.apache.flink.dropwizard.ScheduledDropwizardReporterTest$TestingScheduledDropwizardReporter");
    configuration.setString(ConfigConstants.METRICS_SCOPE_NAMING_TASK, "<host>.<tm_id>.<job_name>");
    configuration.setString(ConfigConstants.METRICS_SCOPE_DELIMITER, "_");
    MetricRegistryConfiguration metricRegistryConfiguration = MetricRegistryConfiguration.fromConfiguration(configuration);
    MetricRegistry metricRegistry = new MetricRegistry(metricRegistryConfiguration);
    char delimiter = metricRegistry.getDelimiter();
    TaskManagerMetricGroup tmMetricGroup = new TaskManagerMetricGroup(metricRegistry, hostname, taskManagerId);
    TaskManagerJobMetricGroup tmJobMetricGroup = new TaskManagerJobMetricGroup(metricRegistry, tmMetricGroup, new JobID(), jobName);
    TaskMetricGroup taskMetricGroup = new TaskMetricGroup(metricRegistry, tmJobMetricGroup, new AbstractID(), new AbstractID(), taskName, 0, 0);
    SimpleCounter myCounter = new SimpleCounter();
    com.codahale.metrics.Meter dropwizardMeter = new com.codahale.metrics.Meter();
    DropwizardMeterWrapper meterWrapper = new DropwizardMeterWrapper(dropwizardMeter);
    taskMetricGroup.counter(counterName, myCounter);
    taskMetricGroup.meter("meter", meterWrapper);
    List<MetricReporter> reporters = metricRegistry.getReporters();
    assertTrue(reporters.size() == 1);
    MetricReporter metricReporter = reporters.get(0);
    assertTrue("Reporter should be of type ScheduledDropwizardReporter", metricReporter instanceof ScheduledDropwizardReporter);
    TestingScheduledDropwizardReporter reporter = (TestingScheduledDropwizardReporter) metricReporter;
    Map<Counter, String> counters = reporter.getCounters();
    assertTrue(counters.containsKey(myCounter));
    Map<Meter, String> meters = reporter.getMeters();
    assertTrue(meters.containsKey(meterWrapper));
    String expectedCounterName = reporter.filterCharacters(hostname) + delimiter + reporter.filterCharacters(taskManagerId) + delimiter + reporter.filterCharacters(jobName) + delimiter + reporter.filterCharacters(counterName);
    assertEquals(expectedCounterName, counters.get(myCounter));
    metricRegistry.shutdown();
}

Example 13 with AbstractID

use of org.apache.flink.util.AbstractID in project flink by apache.

the class SlotSharingGroupAssignment method releaseSimpleSlot.

// ------------------------------------------------------------------------
//  Slot releasing
// ------------------------------------------------------------------------
/**
	 * Releases the simple slot from the assignment group.
	 * 
	 * @param simpleSlot The SimpleSlot to be released
	 */
void releaseSimpleSlot(SimpleSlot simpleSlot) {
    synchronized (lock) {
        // that the releasing is in progress
        if (simpleSlot.markCancelled()) {
            // sanity checks
            if (simpleSlot.isAlive()) {
                throw new IllegalStateException("slot is still alive");
            }
            // check whether the slot is already released
            if (simpleSlot.markReleased()) {
                LOG.debug("Release simple slot {}.", simpleSlot);
                AbstractID groupID = simpleSlot.getGroupID();
                SharedSlot parent = simpleSlot.getParent();
                // if we have a group ID, then our parent slot is tracked here
                if (groupID != null && !allSlots.contains(parent)) {
                    throw new IllegalArgumentException("Slot was not associated with this SlotSharingGroup before.");
                }
                int parentRemaining = parent.removeDisposedChildSlot(simpleSlot);
                if (parentRemaining > 0) {
                    if (groupID != null) {
                        // if we have a group ID, then our parent becomes available
                        // for that group again. otherwise, the slot is part of a
                        // co-location group and nothing becomes immediately available
                        Map<ResourceID, List<SharedSlot>> slotsForJid = availableSlotsPerJid.get(groupID);
                        // sanity check
                        if (slotsForJid == null) {
                            throw new IllegalStateException("Trying to return a slot for group " + groupID + " when available slots indicated that all slots were available.");
                        }
                        putIntoMultiMap(slotsForJid, parent.getTaskManagerID(), parent);
                    }
                } else {
                    // the parent shared slot is now empty and can be released
                    parent.markCancelled();
                    internalDisposeEmptySharedSlot(parent);
                }
            }
        }
    }
}

Example 14 with AbstractID

use of org.apache.flink.util.AbstractID in project flink by apache.

the class SlotSharingGroupAssignment method addSharedSlotAndAllocateSubSlot.

private SimpleSlot addSharedSlotAndAllocateSubSlot(SharedSlot sharedSlot, Locality locality, JobVertexID groupId, CoLocationConstraint constraint) {
    // sanity checks
    if (!sharedSlot.isRootAndEmpty()) {
        throw new IllegalArgumentException("The given slot is not an empty root slot.");
    }
    final ResourceID location = sharedSlot.getTaskManagerID();
    synchronized (lock) {
        // early out in case that the slot died (instance disappeared)
        if (!sharedSlot.isAlive()) {
            return null;
        }
        // add to the total bookkeeping
        if (!allSlots.add(sharedSlot)) {
            throw new IllegalArgumentException("Slot was already contained in the assignment group");
        }
        SimpleSlot subSlot;
        AbstractID groupIdForMap;
        if (constraint == null) {
            // allocate us a sub slot to return
            subSlot = sharedSlot.allocateSubSlot(groupId);
            groupIdForMap = groupId;
        } else {
            // sanity check
            if (constraint.isAssignedAndAlive()) {
                throw new IllegalStateException("Trying to add a shared slot to a co-location constraint that has a life slot.");
            }
            // we need a co-location slot --> a SimpleSlot nested in a SharedSlot to
            //                                host other co-located tasks
            SharedSlot constraintGroupSlot = sharedSlot.allocateSharedSlot(constraint.getGroupId());
            groupIdForMap = constraint.getGroupId();
            if (constraintGroupSlot != null) {
                // the sub-slots in the co-location constraint slot have no own group IDs
                subSlot = constraintGroupSlot.allocateSubSlot(null);
                if (subSlot != null) {
                    // all went well, we can give the constraint its slot
                    constraint.setSharedSlot(constraintGroupSlot);
                // NOTE: Do not lock the location constraint, because we don't yet know whether we will
                // take the slot here
                } else {
                    // if we could not create a sub slot, release the co-location slot
                    // note that this does implicitly release the slot we have just added
                    // as well, because we release its last child slot. That is expected
                    // and desired.
                    constraintGroupSlot.releaseSlot();
                }
            } else {
                // this should not happen, as we are under the lock that also
                // guards slot disposals. Keep the check to be on the safe side
                subSlot = null;
            }
        }
        if (subSlot != null) {
            // preserve the locality information
            subSlot.setLocality(locality);
            // let the other groups know that this slot exists and that they
            // can place a task into this slot.
            boolean entryForNewJidExists = false;
            for (Map.Entry<AbstractID, Map<ResourceID, List<SharedSlot>>> entry : availableSlotsPerJid.entrySet()) {
                // there is already an entry for this groupID
                if (entry.getKey().equals(groupIdForMap)) {
                    entryForNewJidExists = true;
                    continue;
                }
                Map<ResourceID, List<SharedSlot>> available = entry.getValue();
                putIntoMultiMap(available, location, sharedSlot);
            }
            // make sure an empty entry exists for this group, if no other entry exists
            if (!entryForNewJidExists) {
                availableSlotsPerJid.put(groupIdForMap, new LinkedHashMap<ResourceID, List<SharedSlot>>());
            }
            return subSlot;
        } else {
            // This should be a rare case, since this method is called with a fresh slot.
            return null;
        }
    }
// end synchronized (lock)
}

Example 15 with AbstractID

use of org.apache.flink.util.AbstractID in project flink by apache.

the class SlotSharingGroupAssignment method removeSlotFromAllEntries.

private static void removeSlotFromAllEntries(Map<AbstractID, Map<ResourceID, List<SharedSlot>>> availableSlots, SharedSlot slot) {
    final ResourceID taskManagerId = slot.getTaskManagerID();
    for (Map.Entry<AbstractID, Map<ResourceID, List<SharedSlot>>> entry : availableSlots.entrySet()) {
        Map<ResourceID, List<SharedSlot>> map = entry.getValue();
        List<SharedSlot> list = map.get(taskManagerId);
        if (list != null) {
            list.remove(slot);
            if (list.isEmpty()) {
                map.remove(taskManagerId);
            }
        }
    }
}