Examples with Timeout org.junit.rules.Timeout used on opensource projects

Example 11 with Timeout

use of org.junit.rules.Timeout in project pravega by pravega.

the class StreamSegmentContainerTests method testAttributeCleanup.

/**
 * Tests the ability to clean up Extended Attributes from Segment Metadatas that have not been used recently.
 * This test does the following:
 * 1. Sets up a custom SegmentContainer with a hook into the metadataCleanup task
 * 2. Creates a segment and appends something to it, each time updating attributes (and verifies they were updated correctly).
 * 3. Waits for the segment's attributes to be forgotten.
 * 4. Verifies that the forgotten attributes can be fetched from the Attribute Index and re-cached in memory.
 */
@Test
public void testAttributeCleanup() throws Exception {
    final String segmentName = "segment";
    final AttributeId[] attributes = new AttributeId[] { Attributes.EVENT_COUNT, AttributeId.uuid(0, 1), AttributeId.uuid(0, 2), AttributeId.uuid(0, 3) };
    Map<AttributeId, Long> allAttributes = new HashMap<>();
    final TestContainerConfig containerConfig = new TestContainerConfig();
    containerConfig.setSegmentMetadataExpiration(Duration.ofMillis(250));
    containerConfig.setMaxCachedExtendedAttributeCount(1);
    @Cleanup TestContext context = createContext(containerConfig);
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(FREQUENT_TRUNCATIONS_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    @Cleanup MetadataCleanupContainer localContainer = new MetadataCleanupContainer(CONTAINER_ID, containerConfig, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    localContainer.startAsync().awaitRunning();
    // Create segment with initial attributes and verify they were set correctly.
    localContainer.createStreamSegment(segmentName, getSegmentType(segmentName), null, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Add one append with some attribute changes and verify they were set correctly.
    val appendAttributes = createAttributeUpdates(attributes);
    applyAttributes(appendAttributes, allAttributes);
    for (val au : appendAttributes) {
        localContainer.updateAttributes(segmentName, AttributeUpdateCollection.from(au), TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    }
    SegmentProperties sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after initial updateAttributes() call.", allAttributes, sp, AUTO_ATTRIBUTES);
    // Wait until the attributes are forgotten
    localContainer.triggerAttributeCleanup(segmentName, containerConfig.getMaxCachedExtendedAttributeCount()).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Now get attributes again and verify them.
    sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // During attribute eviction, we expect all core attributes to be preserved, and only 1 extended attribute (as
    // defined in the config) to be preserved. This extended attribute should be the last one we updated.
    val expectedAttributes = new HashMap<>(Attributes.getCoreNonNullAttributes(allAttributes));
    val lastExtAttribute = appendAttributes.stream().filter(au -> !Attributes.isCoreAttribute(au.getAttributeId())).reduce((a, b) -> b).get();
    expectedAttributes.put(lastExtAttribute.getAttributeId(), lastExtAttribute.getValue());
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after eviction.", expectedAttributes, sp, AUTO_ATTRIBUTES);
    val fetchedAttributes = localContainer.getAttributes(segmentName, allAttributes.keySet(), true, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    AssertExtensions.assertMapEquals("Unexpected attributes after eviction & reload.", allAttributes, fetchedAttributes);
    sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after eviction & reload+getInfo.", allAttributes, sp, AUTO_ATTRIBUTES);
}

Example 12 with Timeout

use of org.junit.rules.Timeout in project pravega by pravega.

the class StreamSegmentContainerTests method tryActivate.

/**
 * Attempts to activate the targetSegment in the given Container. Since we do not have access to the internals of the
 * Container, we need to trigger this somehow, hence the need for this complex code. We need to trigger a truncation,
 * so we need an 'appendSegment' to which we continuously append so that the DurableDataLog is truncated. After truncation,
 * the Metadata should have enough leeway in making room for new activation.
 *
 * @return A Future that will complete either with an exception (failure) or SegmentProperties for the targetSegment.
 */
private CompletableFuture<SegmentProperties> tryActivate(MetadataCleanupContainer localContainer, String targetSegment, String appendSegment) {
    CompletableFuture<SegmentProperties> successfulMap = new CompletableFuture<>();
    // Append continuously to an existing segment in order to trigger truncations (these are necessary for forced evictions).
    val appendFuture = localContainer.appendRandomly(appendSegment, false, () -> !successfulMap.isDone());
    Futures.exceptionListener(appendFuture, successfulMap::completeExceptionally);
    // Repeatedly try to get info on 'segment1' (activate it), until we succeed or time out.
    TimeoutTimer remaining = new TimeoutTimer(TIMEOUT);
    Futures.loop(() -> !successfulMap.isDone(), () -> Futures.delayedFuture(Duration.ofMillis(EVICTION_SEGMENT_EXPIRATION_MILLIS_SHORT), executorService()).thenCompose(v -> localContainer.getStreamSegmentInfo(targetSegment, TIMEOUT)).thenAccept(successfulMap::complete).exceptionally(ex -> {
        if (!(Exceptions.unwrap(ex) instanceof TooManyActiveSegmentsException)) {
            // Some other error.
            successfulMap.completeExceptionally(ex);
        } else if (!remaining.hasRemaining()) {
            // Waited too long.
            successfulMap.completeExceptionally(new TimeoutException("No successful activation could be done in the allotted time."));
        }
        // Try again.
        return null;
    }), executorService());
    return successfulMap;
}

Example 13 with Timeout

use of org.junit.rules.Timeout in project pravega by pravega.

the class StreamSegmentContainerTests method testEventProcessorDurableQueueAndSwitchToConsumer.

/**
 * Test the EventProcessor in durable queue mode (no handler). Then, close it and recreate another one on the same
 * internal Segment (same name) that actually consumes the events stored previously.
 *
 * @throws Exception
 */
@Test(timeout = 10000)
public void testEventProcessorDurableQueueAndSwitchToConsumer() throws Exception {
    @Cleanup TestContext context = createContext();
    val container = (StreamSegmentContainer) context.container;
    container.startAsync().awaitRunning();
    int allEventsToProcess = 100;
    @Cleanup ContainerEventProcessorImpl containerEventProcessor = new ContainerEventProcessorImpl(container, container.metadataStore, TIMEOUT_EVENT_PROCESSOR_ITERATION, TIMEOUT_EVENT_PROCESSOR_ITERATION, this.executorService());
    ContainerEventProcessor.EventProcessor processor = containerEventProcessor.forDurableQueue("testDurableQueue").get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS);
    // At this point, we can only add events, but not consuming them as the EventProcessor works in durable queue mode.
    for (int i = 0; i < allEventsToProcess; i++) {
        BufferView event = new ByteArraySegment(ByteBuffer.allocate(Integer.BYTES).putInt(i).array());
        processor.add(event, TIMEOUT_FUTURE).join();
    }
    Assert.assertEquals("Event processor object not matching", processor, containerEventProcessor.forDurableQueue("testDurableQueue").get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS));
    // Close the processor and unregister it.
    processor.close();
    // Make sure that EventProcessor eventually terminates.
    ((ContainerEventProcessorImpl.EventProcessorImpl) processor).awaitTerminated();
    // Now, re-create the Event Processor with a handler to consume the events.
    ContainerEventProcessor.EventProcessorConfig eventProcessorConfig = new ContainerEventProcessor.EventProcessorConfig(EVENT_PROCESSOR_EVENTS_AT_ONCE, EVENT_PROCESSOR_MAX_OUTSTANDING_BYTES, EVENT_PROCESSOR_TRUNCATE_SIZE_BYTES);
    List<Integer> processorResults = new ArrayList<>();
    Function<List<BufferView>, CompletableFuture<Void>> handler = l -> {
        l.forEach(b -> {
            try {
                processorResults.add(ByteBuffer.wrap(b.getReader().readNBytes(Integer.BYTES)).getInt());
            } catch (IOException e) {
                throw new CompletionException(e);
            }
        });
        return CompletableFuture.completedFuture(null);
    };
    processor = containerEventProcessor.forConsumer("testDurableQueue", handler, eventProcessorConfig).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS);
    // Wait for all items to be processed.
    AssertExtensions.assertEventuallyEquals(true, () -> processorResults.size() == allEventsToProcess, 10000);
    Assert.assertArrayEquals(processorResults.toArray(), IntStream.iterate(0, v -> v + 1).limit(allEventsToProcess).boxed().toArray());
    // Just check failure callback.
    ((ContainerEventProcessorImpl.EventProcessorImpl) processor).failureCallback(new IntentionalException());
    // Close the processor and unregister it.
    processor.close();
    // Make sure that EventProcessor eventually terminates.
    ((ContainerEventProcessorImpl.EventProcessorImpl) processor).awaitTerminated();
}

Example 14 with Timeout

use of org.junit.rules.Timeout in project pravega by pravega.

the class StreamSegmentContainerTests method waitForOperationsInReadIndex.

/**
 * Blocks until all operations processed so far have been added to the ReadIndex and InMemoryOperationLog.
 * This is needed to simplify test verification due to the fact that the the OperationProcessor commits operations to
 * the ReadIndex and InMemoryOperationLog asynchronously, after those operations were ack-ed. This method makes use
 * of the fact that the OperationProcessor/MemoryStateUpdater will still commit such operations in sequence; it
 * creates a new segment, writes 1 byte to it and issues a read (actual/future) and waits until it's completed - when
 * it is, it is guaranteed that everything prior to that has been committed.
 */
private static void waitForOperationsInReadIndex(SegmentContainer container) throws Exception {
    TimeoutTimer timer = new TimeoutTimer(TIMEOUT);
    String segmentName = "test" + System.nanoTime();
    container.createStreamSegment(segmentName, BASIC_TYPE, null, timer.getRemaining()).thenCompose(v -> container.append(segmentName, new ByteArraySegment(new byte[1]), null, timer.getRemaining())).thenCompose(v -> container.read(segmentName, 0, 1, timer.getRemaining())).thenCompose(rr -> {
        ReadResultEntry rre = rr.next();
        rre.requestContent(TIMEOUT);
        return rre.getContent().thenRun(rr::close);
    }).thenCompose(v -> container.deleteStreamSegment(segmentName, timer.getRemaining())).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
}

Example 15 with Timeout

use of org.junit.rules.Timeout in project pravega by pravega.

the class StreamSegmentContainerTests method testBasicConditionalMergeScenarios.

/**
 * Test in detail the basic situations that a conditional segment merge can face.
 */
@Test
public void testBasicConditionalMergeScenarios() throws Exception {
    @Cleanup TestContext context = createContext();
    context.container.startAsync().awaitRunning();
    final String parentSegment = "parentSegment";
    // This will be the attribute update to execute against the parent segment.
    Function<String, AttributeUpdateCollection> attributeUpdateForTxn = txnName -> AttributeUpdateCollection.from(new AttributeUpdate(AttributeId.fromUUID(UUID.nameUUIDFromBytes(txnName.getBytes())), AttributeUpdateType.ReplaceIfEquals, txnName.hashCode() + 1, txnName.hashCode()));
    Function<String, Long> getAttributeValue = txnName -> {
        AttributeId attributeId = AttributeId.fromUUID(UUID.nameUUIDFromBytes(txnName.getBytes()));
        return context.container.getAttributes(parentSegment, Collections.singletonList(attributeId), true, TIMEOUT).join().get(attributeId);
    };
    // Create a parent Segment.
    context.container.createStreamSegment(parentSegment, getSegmentType(parentSegment), null, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    SegmentType segmentType = getSegmentType(parentSegment);
    // Case 1: Create and empty transaction that fails to merge conditionally due to bad attributes.
    String txnName = NameUtils.getTransactionNameFromId(parentSegment, UUID.randomUUID());
    AttributeId txnAttributeId = AttributeId.fromUUID(UUID.nameUUIDFromBytes(txnName.getBytes()));
    context.container.createStreamSegment(txnName, segmentType, null, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    AttributeUpdateCollection attributeUpdates = attributeUpdateForTxn.apply(txnName);
    AssertExtensions.assertFutureThrows("Transaction was expected to fail on attribute update", context.container.mergeStreamSegment(parentSegment, txnName, attributeUpdates, TIMEOUT), ex -> ex instanceof BadAttributeUpdateException);
    Assert.assertEquals(Attributes.NULL_ATTRIBUTE_VALUE, (long) getAttributeValue.apply(txnName));
    // Case 2: Now, we prepare the attributes in the parent segment so the merge of the empty transaction succeeds.
    context.container.updateAttributes(parentSegment, AttributeUpdateCollection.from(new AttributeUpdate(txnAttributeId, AttributeUpdateType.Replace, txnName.hashCode())), TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // As the source segment is empty, the amount of merged data should be 0.
    Assert.assertEquals(0L, context.container.mergeStreamSegment(parentSegment, txnName, attributeUpdates, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).getMergedDataLength());
    // But the attribute related to that transaction merge on the parent segment should have been updated.
    Assert.assertEquals(txnName.hashCode() + 1L, (long) context.container.getAttributes(parentSegment, Collections.singletonList(txnAttributeId), true, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).get(txnAttributeId));
    // Case 3: Create a non-empty transaction that should fail due to a conditional attribute update failure.
    txnName = NameUtils.getTransactionNameFromId(parentSegment, UUID.randomUUID());
    txnAttributeId = AttributeId.fromUUID(UUID.nameUUIDFromBytes(txnName.getBytes()));
    attributeUpdates = attributeUpdateForTxn.apply(txnName);
    context.container.createStreamSegment(txnName, segmentType, null, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Add some appends to the transaction.
    RefCountByteArraySegment appendData = getAppendData(txnName, 1);
    context.container.append(txnName, appendData, null, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Attempt the conditional merge.
    AssertExtensions.assertFutureThrows("Transaction was expected to fail on attribute update", context.container.mergeStreamSegment(parentSegment, txnName, attributeUpdates, TIMEOUT), ex -> ex instanceof BadAttributeUpdateException);
    Assert.assertEquals(Attributes.NULL_ATTRIBUTE_VALUE, (long) getAttributeValue.apply(txnName));
    // Case 4: Now, we prepare the attributes in the parent segment so the merge of the non-empty transaction succeeds.
    context.container.updateAttributes(parentSegment, AttributeUpdateCollection.from(new AttributeUpdate(txnAttributeId, AttributeUpdateType.Replace, txnName.hashCode())), TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // As the source segment is non-empty, the amount of merged data should be greater than 0.
    Assert.assertTrue(context.container.mergeStreamSegment(parentSegment, txnName, attributeUpdates, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).getMergedDataLength() > 0);
    // The attribute related to that transaction merge on the parent segment should have been updated as well.
    Assert.assertEquals(txnName.hashCode() + 1L, (long) context.container.getAttributes(parentSegment, Collections.singletonList(txnAttributeId), true, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).get(txnAttributeId));
    context.container.stopAsync().awaitTerminated();
}