use of java.util.function.Consumer in project pravega by pravega.
the class ContainerReadIndexTests method testCacheEviction.
/**
* Tests the ability to evict entries from the ReadIndex under various conditions:
* * If an entry is aged out
* * If an entry is pushed out because of cache space pressure.
* <p>
* This also verifies that certain entries, such as RedirectReadIndexEntries and entries after the Storage Offset are
* not removed.
* <p>
* The way this test goes is as follows (it's pretty subtle, because there aren't many ways to hook into the ReadIndex and see what it's doing)
* 1. It creates a bunch of segments, and populates them in storage (each) up to offset N/2-1 (this is called pre-storage)
* 2. It populates the ReadIndex for each of those segments from offset N/2 to offset N-1 (this is called post-storage)
* 3. It loads all the data from Storage into the ReadIndex, in entries of size equal to those already loaded in step #2.
* 3a. At this point, all the entries added in step #2 have Generations 0..A/4-1, and step #3 have generations A/4..A-1
* 4. Append more data at the end. This forces the generation to increase to 1.25A.
* 4a. Nothing should be evicted from the cache now, since the earliest items are all post-storage.
* 5. We 'touch' (read) the first 1/3 of pre-storage entries (offsets 0..N/4).
* 5a. At this point, those entries (offsets 0..N/6) will have the newest generations (1.25A..1.5A)
* 6. We append more data (equivalent to the data we touched)
* 6a. Nothing should be evicted, since those generations that were just eligible for removal were touched and bumped up.
* 7. We forcefully increase the current generation by 1 (without touching the ReadIndex)
* 7a. At this point, we expect all the pre-storage items, except the touched ones, to be evicted. This is generations 0.25A-0.75A.
* 8. Update the metadata and indicate that all the post-storage entries are now pre-storage and bump the generation by 0.75A.
* 8a. At this point, we expect all former post-storage items and pre-storage items to be evicted (in this order).
* <p>
* The final order of eviction (in terms of offsets, for each segment), is:
* * 0.25N-0.75N, 0.75N..N, N..1.25N, 0..0.25N, 1.25N..1.5N (remember that we added quite a bunch of items after the initial run).
*/
@Test
@SuppressWarnings("checkstyle:CyclomaticComplexity")
public void testCacheEviction() throws Exception {
// Create a CachePolicy with a set number of generations and a known max size.
// Each generation contains exactly one entry, so the number of generations is also the number of entries.
final int appendSize = 100;
// This also doubles as number of generations (each generation, we add one append for each segment).
final int entriesPerSegment = 100;
final int cacheMaxSize = SEGMENT_COUNT * entriesPerSegment * appendSize;
// 25% of the entries are beyond the StorageOffset
final int postStorageEntryCount = entriesPerSegment / 4;
// 75% of the entries are before the StorageOffset.
final int preStorageEntryCount = entriesPerSegment - postStorageEntryCount;
CachePolicy cachePolicy = new CachePolicy(cacheMaxSize, Duration.ofMillis(1000 * 2 * entriesPerSegment), Duration.ofMillis(1000));
// To properly test this, we want predictable storage reads.
ReadIndexConfig config = ConfigHelpers.withInfiniteCachePolicy(ReadIndexConfig.builder().with(ReadIndexConfig.STORAGE_READ_ALIGNMENT, appendSize)).build();
ArrayList<CacheKey> removedKeys = new ArrayList<>();
@Cleanup TestContext context = new TestContext(config, cachePolicy);
// Record every cache removal.
context.cacheFactory.cache.removeCallback = removedKeys::add;
// Create the segments (metadata + storage).
ArrayList<Long> segmentIds = createSegments(context);
createSegmentsInStorage(context);
// Populate the Storage with appropriate data.
byte[] preStorageData = new byte[preStorageEntryCount * appendSize];
for (long segmentId : segmentIds) {
UpdateableSegmentMetadata sm = context.metadata.getStreamSegmentMetadata(segmentId);
val handle = context.storage.openWrite(sm.getName()).join();
context.storage.write(handle, 0, new ByteArrayInputStream(preStorageData), preStorageData.length, TIMEOUT).join();
sm.setStorageLength(preStorageData.length);
sm.setLength(preStorageData.length);
}
// Callback that appends one entry at the end of the given segment id.
Consumer<Long> appendOneEntry = segmentId -> {
UpdateableSegmentMetadata sm = context.metadata.getStreamSegmentMetadata(segmentId);
byte[] data = new byte[appendSize];
long offset = sm.getLength();
sm.setLength(offset + data.length);
try {
context.readIndex.append(segmentId, offset, data);
} catch (StreamSegmentNotExistsException ex) {
throw new CompletionException(ex);
}
};
// Populate the ReadIndex with the Append entries (post-StorageOffset)
for (int i = 0; i < postStorageEntryCount; i++) {
segmentIds.forEach(appendOneEntry);
// Each time we make a round of appends (one per segment), we increment the generation in the CacheManager.
context.cacheManager.applyCachePolicy();
}
// Read all the data from Storage, making sure we carefully associate them with the proper generation.
for (int i = 0; i < preStorageEntryCount; i++) {
long offset = i * appendSize;
for (long segmentId : segmentIds) {
@Cleanup ReadResult result = context.readIndex.read(segmentId, offset, appendSize, TIMEOUT);
ReadResultEntry resultEntry = result.next();
Assert.assertEquals("Unexpected type of ReadResultEntry when trying to load up data into the ReadIndex Cache.", ReadResultEntryType.Storage, resultEntry.getType());
resultEntry.requestContent(TIMEOUT);
ReadResultEntryContents contents = resultEntry.getContent().get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
Assert.assertFalse("Not expecting more data to be available for reading.", result.hasNext());
Assert.assertEquals("Unexpected ReadResultEntry length when trying to load up data into the ReadIndex Cache.", appendSize, contents.getLength());
}
context.cacheManager.applyCachePolicy();
}
Assert.assertEquals("Not expecting any removed Cache entries at this point (cache is not full).", 0, removedKeys.size());
// Append more data (equivalent to all post-storage entries), and verify that NO entries are being evicted (we cannot evict post-storage entries).
for (int i = 0; i < postStorageEntryCount; i++) {
segmentIds.forEach(appendOneEntry);
context.cacheManager.applyCachePolicy();
}
Assert.assertEquals("Not expecting any removed Cache entries at this point (only eligible entries were post-storage).", 0, removedKeys.size());
// 'Touch' the first few entries read from storage. This should move them to the back of the queue (they won't be the first ones to be evicted).
int touchCount = preStorageEntryCount / 3;
for (int i = 0; i < touchCount; i++) {
long offset = i * appendSize;
for (long segmentId : segmentIds) {
@Cleanup ReadResult result = context.readIndex.read(segmentId, offset, appendSize, TIMEOUT);
ReadResultEntry resultEntry = result.next();
Assert.assertEquals("Unexpected type of ReadResultEntry when trying to load up data into the ReadIndex Cache.", ReadResultEntryType.Cache, resultEntry.getType());
}
}
// Append more data (equivalent to the amount of data we 'touched'), and verify that the entries we just touched are not being removed..
for (int i = 0; i < touchCount; i++) {
segmentIds.forEach(appendOneEntry);
context.cacheManager.applyCachePolicy();
}
Assert.assertEquals("Not expecting any removed Cache entries at this point (we touched old entries and they now have the newest generation).", 0, removedKeys.size());
// Increment the generations so that we are caught up to just before the generation where the "touched" items now live.
context.cacheManager.applyCachePolicy();
// We expect all but the 'touchCount' pre-Storage entries to be removed.
int expectedRemovalCount = (preStorageEntryCount - touchCount) * SEGMENT_COUNT;
Assert.assertEquals("Unexpected number of removed entries after having forced out all pre-storage entries.", expectedRemovalCount, removedKeys.size());
// Now update the metadata and indicate that all the post-storage data has been moved to storage.
segmentIds.forEach(segmentId -> {
UpdateableSegmentMetadata sm = context.metadata.getStreamSegmentMetadata(segmentId);
sm.setStorageLength(sm.getLength());
});
// We add one artificial entry, which we'll be touching forever and ever; this forces the CacheManager to
// update its current generation every time. We will be ignoring this entry for our test.
SegmentMetadata readSegment = context.metadata.getStreamSegmentMetadata(segmentIds.get(0));
appendOneEntry.accept(readSegment.getId());
// Now evict everything (whether by size of by aging out).
for (int i = 0; i < cachePolicy.getMaxGenerations(); i++) {
@Cleanup ReadResult result = context.readIndex.read(readSegment.getId(), readSegment.getLength() - appendSize, appendSize, TIMEOUT);
result.next();
context.cacheManager.applyCachePolicy();
}
int expectedRemovalCountPerSegment = entriesPerSegment + touchCount + postStorageEntryCount;
int expectedTotalRemovalCount = SEGMENT_COUNT * expectedRemovalCountPerSegment;
Assert.assertEquals("Unexpected number of removed entries after having forced out all the entries.", expectedTotalRemovalCount, removedKeys.size());
// Finally, verify that the evicted items are in the correct order (for each segment). See this test's description for details.
for (long segmentId : segmentIds) {
List<CacheKey> segmentRemovedKeys = removedKeys.stream().filter(key -> key.getStreamSegmentId() == segmentId).collect(Collectors.toList());
Assert.assertEquals("Unexpected number of removed entries for segment " + segmentId, expectedRemovalCountPerSegment, segmentRemovedKeys.size());
// The correct order of eviction (N=entriesPerSegment) is: 0.25N-0.75N, 0.75N..N, N..1.25N, 0..0.25N, 1.25N..1.5N.
// This is equivalent to the following tests
// 0.25N-1.25N
checkOffsets(segmentRemovedKeys, segmentId, 0, entriesPerSegment, entriesPerSegment * appendSize / 4, appendSize);
// 0..0.25N
checkOffsets(segmentRemovedKeys, segmentId, entriesPerSegment, entriesPerSegment / 4, 0, appendSize);
// 1.25N..1.5N
checkOffsets(segmentRemovedKeys, segmentId, entriesPerSegment + entriesPerSegment / 4, entriesPerSegment / 4, (int) (entriesPerSegment * appendSize * 1.25), appendSize);
}
}
use of java.util.function.Consumer in project Terasology by MovingBlocks.
the class NUIEditorMenuTreeBuilder method createAddContextMenu.
public MenuTree createAddContextMenu(JsonTree node) {
MenuTree addTree = new MenuTree(OPTION_ADD_EXTENDED);
JsonTreeValue.Type type = node.getValue().getType();
if (type == JsonTreeValue.Type.ARRAY) {
// Add generic item addition options.
addTree.addOption("Boolean value", n -> {
JsonTree child = new JsonTree(new JsonTreeValue(null, false, JsonTreeValue.Type.VALUE));
n.addChild(child);
for (Consumer<JsonTree> listener : addContextMenuListeners) {
listener.accept(child);
}
}, node);
addTree.addOption("Number value", n -> {
JsonTree child = new JsonTree((new JsonTreeValue(null, 0.0f, JsonTreeValue.Type.VALUE)));
n.addChild(child);
for (Consumer<JsonTree> listener : addContextMenuListeners) {
listener.accept(n);
}
}, node);
addTree.addOption("String value", n -> {
JsonTree child = new JsonTree((new JsonTreeValue(null, "", JsonTreeValue.Type.VALUE)));
n.addChild(child);
for (Consumer<JsonTree> listener : addContextMenuListeners) {
listener.accept(child);
}
}, node);
} else if (type == JsonTreeValue.Type.OBJECT) {
// Add a generic key/value pair addition option.
addTree.addOption("Key/value pair", n -> {
JsonTree child = new JsonTree((new JsonTreeValue("", "", JsonTreeValue.Type.KEY_VALUE_PAIR)));
n.addChild(child);
for (Consumer<JsonTree> listener : addContextMenuListeners) {
listener.accept(child);
}
}, node);
populateContextMenu(node, addTree, false);
}
return addTree;
}
use of java.util.function.Consumer in project kie-wb-common by kiegroup.
the class ToolboxTextTooltipTest method testUseText.
@Test
@SuppressWarnings("unchecked")
public void testUseText() {
final Consumer textConsumer = mock(Consumer.class);
ToolboxTextTooltip cascade = tested.withText(textConsumer);
assertEquals(tested, cascade);
verify(tooltip, times(1)).withText(eq(textConsumer));
}
use of java.util.function.Consumer in project kie-wb-common by kiegroup.
the class ProjectDiagramEditorTest method testLoadContentError.
@Test
public void testLoadContentError() {
ArgumentCaptor<ServiceCallback> callbackArgumentCaptor = forClass(ServiceCallback.class);
tested.loadContent();
verify(projectDiagramServices, times(1)).getByPath(eq(path), callbackArgumentCaptor.capture());
callbackArgumentCaptor.getValue().onError(new ClientRuntimeError(new DefinitionNotFoundException()));
verify(placeManager, times(1)).forceClosePlace(any(PathPlaceRequest.class));
ArgumentCaptor<Consumer> consumerArgumentCaptor = forClass(Consumer.class);
verify(diagramClientErrorHandler, times(1)).handleError(any(ClientRuntimeError.class), consumerArgumentCaptor.capture());
consumerArgumentCaptor.getValue().accept("error message");
verify(errorPopupPresenter, times(1)).showMessage("error message");
}
use of java.util.function.Consumer in project kie-wb-common by kiegroup.
the class GraphValidatorImpl method validate.
/**
* Performs the validation for the <code>graph</code> instance.
* @param graph The instance to validate.
* @param aRuleSet An optional rule set instance to validate against it. If not present, the default
* rule set for the the graph will be used.
* @param graphValidatorConsumer An optional consumer for the graph instance when is being validated.
* @param nodeValidatorConsumer An optional consumer each node instance when being validated.
* @param edgeValidatorConsumer An optional consumer each edge instance when being validated.
* @param resultConsumer The consumer for all the resulting validation violations produced during the
* validator for the graph, and all of its nodes and edges. It's being called once the
* validation has been completed.
*/
@SuppressWarnings("unchecked")
void validate(final Graph graph, final Optional<RuleSet> aRuleSet, final Optional<BiConsumer<Graph, Collection<RuleViolation>>> graphValidatorConsumer, final Optional<BiConsumer<Node, Collection<RuleViolation>>> nodeValidatorConsumer, final Optional<BiConsumer<Edge, Collection<RuleViolation>>> edgeValidatorConsumer, Consumer<Collection<RuleViolation>> resultConsumer) {
final RuleSet ruleSet = aRuleSet.orElse(getRuleSet(graph));
final ViolationsSet violations = new ViolationsSet();
treeWalkTraverseProcessor.traverse(graph, new AbstractTreeTraverseCallback<org.kie.workbench.common.stunner.core.graph.Graph, Node, Edge>() {
private final Stack<Node> currentParents = new Stack<Node>();
@Override
public void startGraphTraversal(final org.kie.workbench.common.stunner.core.graph.Graph graph) {
super.startGraphTraversal(graph);
currentParents.clear();
// Evaluate the graph's cardinality rules.
final Set<RuleViolation> graphCardinalityViolations = violations.addViolations(evaluateCardinality(ruleSet, graph));
graphValidatorConsumer.ifPresent(g -> g.accept(graph, graphCardinalityViolations));
}
@Override
public boolean startEdgeTraversal(final Edge edge) {
super.startEdgeTraversal(edge);
final Object content = edge.getContent();
final ViolationsSet edgeViolations = new ViolationsSet();
if (content instanceof Child) {
this.currentParents.push(edge.getSourceNode());
} else if (content instanceof View) {
final Optional<Node<? extends View<?>, ? extends Edge>> sourceOpt = Optional.ofNullable(edge.getSourceNode());
final Optional<Node<? extends View<?>, ? extends Edge>> targetOpt = Optional.ofNullable(edge.getTargetNode());
// Check not empty connections.
final Optional<RuleViolation> emptyConnectionViolation = evaluateNotEmptyConnections(graph, edge, sourceOpt, targetOpt);
emptyConnectionViolation.ifPresent(edgeViolations::add);
// Evaluate connection rules.
edgeViolations.addViolations(evaluateConnection(ruleSet, graph, edge, sourceOpt, targetOpt));
// Evaluate connector cardinality rules for this edge.
if (null != edge.getTargetNode()) {
edgeViolations.addViolations(evaluateIncomingEdgeCardinality(ruleSet, graph, edge));
}
if (null != edge.getSourceNode()) {
edgeViolations.addViolations(evaluateOutgoingEdgeCardinality(ruleSet, graph, edge));
}
} else if (content instanceof Dock) {
final Node parent = edge.getSourceNode();
final Node docked = edge.getTargetNode();
// Evaluate docking rules for the source & target nodes.
edgeViolations.addViolations(evaluateDocking(ruleSet, graph, parent, docked));
}
edgeValidatorConsumer.ifPresent(c -> c.accept(edge, edgeViolations));
violations.addAll(edgeViolations);
return true;
}
@Override
public void endEdgeTraversal(final Edge edge) {
super.endEdgeTraversal(edge);
if (edge.getContent() instanceof Child) {
this.currentParents.pop();
}
}
@Override
public boolean startNodeTraversal(final Node node) {
super.startNodeTraversal(node);
final Collection<RuleViolation> nodeViolations = evaluateNode(node, currentParents.isEmpty() ? null : currentParents.peek());
nodeValidatorConsumer.ifPresent(c -> c.accept(node, nodeViolations));
return true;
}
@Override
public void endGraphTraversal() {
super.endGraphTraversal();
// Finished - feed the consumer instance.
resultConsumer.accept(violations);
}
private Collection<RuleViolation> evaluateNode(final Node node, final Node parent) {
// Evaluate containment rules for this node.
return violations.addViolations(evaluateContainment(ruleSet, graph, null != parent ? parent : graph, node));
}
});
}
Aggregations