Examples with Timeout org.junit.rules.Timeout used on opensource projects

Example 16 with Timeout

use of org.junit.rules.Timeout in project pravega by pravega.

the class StreamSegmentContainerTests method testForceFlush.

/**
 * Tests the {@link SegmentContainer#flushToStorage} method.
 */
@Test
public void testForceFlush() throws Exception {
    final AttributeId attributeReplace = AttributeId.randomUUID();
    final long expectedAttributeValue = APPENDS_PER_SEGMENT + ATTRIBUTE_UPDATES_PER_SEGMENT;
    final int entriesPerSegment = 10;
    @Cleanup TestContext context = new TestContext(DEFAULT_CONFIG, NO_TRUNCATIONS_DURABLE_LOG_CONFIG, INFREQUENT_FLUSH_WRITER_CONFIG, null);
    val durableLog = new AtomicReference<OperationLog>();
    val durableLogFactory = new WatchableOperationLogFactory(context.operationLogFactory, durableLog::set);
    @Cleanup val container = new StreamSegmentContainer(CONTAINER_ID, DEFAULT_CONFIG, durableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    container.startAsync().awaitRunning();
    Assert.assertNotNull(durableLog.get());
    val tableStore = container.getExtension(ContainerTableExtension.class);
    // 1. Create the StreamSegments and Table Segments.
    ArrayList<String> segmentNames = new ArrayList<>();
    ArrayList<String> tableSegmentNames = new ArrayList<>();
    ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
    for (int i = 0; i < SEGMENT_COUNT; i++) {
        String segmentName = getSegmentName(i);
        segmentNames.add(segmentName);
        opFutures.add(container.createStreamSegment(segmentName, getSegmentType(segmentName), null, TIMEOUT));
    }
    for (int i = 0; i < SEGMENT_COUNT; i++) {
        String segmentName = getSegmentName(i) + "_Table";
        tableSegmentNames.add(segmentName);
        val type = SegmentType.builder(getSegmentType(segmentName)).tableSegment().build();
        opFutures.add(tableStore.createSegment(segmentName, type, TIMEOUT));
    }
    // 1.1 Wait for all segments to be created prior to using them.
    Futures.allOf(opFutures).join();
    opFutures.clear();
    // 2. Add some appends and update some of the attributes.
    HashMap<String, Long> lengths = new HashMap<>();
    HashMap<String, ByteArrayOutputStream> segmentContents = new HashMap<>();
    for (String segmentName : segmentNames) {
        for (int i = 0; i < APPENDS_PER_SEGMENT; i++) {
            val attributeUpdates = AttributeUpdateCollection.from(new AttributeUpdate(attributeReplace, AttributeUpdateType.Replace, i + 1));
            val appendData = getAppendData(segmentName, i);
            long expectedLength = lengths.getOrDefault(segmentName, 0L) + appendData.getLength();
            val append = (i % 2 == 0) ? container.append(segmentName, appendData, attributeUpdates, TIMEOUT) : container.append(segmentName, lengths.get(segmentName), appendData, attributeUpdates, TIMEOUT);
            opFutures.add(Futures.toVoid(append));
            lengths.put(segmentName, expectedLength);
            recordAppend(segmentName, appendData, segmentContents, null);
        }
        for (int i = 0; i < ATTRIBUTE_UPDATES_PER_SEGMENT; i++) {
            val attributeUpdates = AttributeUpdateCollection.from(new AttributeUpdate(attributeReplace, AttributeUpdateType.Replace, APPENDS_PER_SEGMENT + i + 1));
            opFutures.add(container.updateAttributes(segmentName, attributeUpdates, TIMEOUT));
        }
    }
    // 2.2 Add some entries to the table segments.
    final BiFunction<String, Integer, TableEntry> createTableEntry = (segmentName, entryId) -> TableEntry.unversioned(new ByteArraySegment(String.format("Key_%s_%s", segmentName, entryId).getBytes()), new ByteArraySegment(String.format("Value_%s_%s", segmentName, entryId).getBytes()));
    for (String segmentName : tableSegmentNames) {
        for (int i = 0; i < entriesPerSegment; i++) {
            opFutures.add(Futures.toVoid(tableStore.put(segmentName, Collections.singletonList(createTableEntry.apply(segmentName, i)), TIMEOUT)));
        }
    }
    Futures.allOf(opFutures).join();
    // 3. Instead of waiting for the Writer to move data to Storage, we invoke the flushToStorage to verify that all
    // operations have been applied to Storage.
    val forceFlush = container.flushToStorage(TIMEOUT);
    forceFlush.get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkStorage(segmentContents, lengths, container, context.storage);
    // 4. Truncate all the data in the DurableLog and immediately shut down the container.
    val truncateSeqNo = container.metadata.getClosestValidTruncationPoint(container.metadata.getOperationSequenceNumber());
    durableLog.get().truncate(truncateSeqNo, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    container.close();
    // 5. Create a new container instance (from the nearly empty DurableLog) and with an empty cache.
    @Cleanup val container2 = new StreamSegmentContainer(CONTAINER_ID, DEFAULT_CONFIG, durableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    container2.startAsync().awaitRunning();
    // 5.1 Verify Segment Data.
    for (val sc : segmentContents.entrySet()) {
        // Contents.
        byte[] expectedData = sc.getValue().toByteArray();
        byte[] actualData = new byte[expectedData.length];
        container2.read(sc.getKey(), 0, actualData.length, TIMEOUT).join().readRemaining(actualData, TIMEOUT);
        Assert.assertArrayEquals("Unexpected contents for " + sc.getKey(), expectedData, actualData);
        // Length.
        val si = container2.getStreamSegmentInfo(sc.getKey(), TIMEOUT).join();
        Assert.assertEquals("Unexpected length for " + sc.getKey(), expectedData.length, si.getLength());
        // Attributes.
        val attributes = container2.getAttributes(sc.getKey(), Collections.singleton(attributeReplace), false, TIMEOUT).join();
        Assert.assertEquals("Unexpected attribute for " + sc.getKey(), expectedAttributeValue, (long) attributes.get(attributeReplace));
    }
    // 5.2 Verify table segment data.
    val tableStore2 = container2.getExtension(ContainerTableExtension.class);
    for (String segmentName : tableSegmentNames) {
        for (int i = 0; i < entriesPerSegment; i++) {
            val expected = createTableEntry.apply(segmentName, i);
            val actual = tableStore2.get(segmentName, Collections.singletonList(expected.getKey().getKey()), TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).get(0);
            Assert.assertTrue("Unexpected Table Entry for " + segmentName + " at position " + i, expected.getKey().getKey().equals(actual.getKey().getKey()) && expected.getValue().equals(actual.getValue()));
        }
    }
// Ending Note: if all the above tests passed, we have implicitly validated that the Container Metadata Segment has also
// been properly flushed.
}

Example 17 with Timeout

use of org.junit.rules.Timeout in project pravega by pravega.

the class StreamSegmentContainerTests method checkAttributeIterators.

private void checkAttributeIterators(DirectSegmentAccess segment, List<AttributeId> sortedAttributes, Map<AttributeId, Long> allExpectedValues) throws Exception {
    int skip = sortedAttributes.size() / 10;
    for (int i = 0; i < sortedAttributes.size() / 2; i += skip) {
        AttributeId fromId = sortedAttributes.get(i);
        AttributeId toId = sortedAttributes.get(sortedAttributes.size() - i - 1);
        val expectedValues = allExpectedValues.entrySet().stream().filter(e -> fromId.compareTo(e.getKey()) <= 0 && toId.compareTo(e.getKey()) >= 0).sorted(Comparator.comparing(Map.Entry::getKey)).collect(Collectors.toList());
        val actualValues = new ArrayList<Map.Entry<AttributeId, Long>>();
        val ids = new HashSet<AttributeId>();
        val iterator = segment.attributeIterator(fromId, toId, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        iterator.forEachRemaining(batch -> batch.forEach(attribute -> {
            Assert.assertTrue("Duplicate key found.", ids.add(attribute.getKey()));
            actualValues.add(attribute);
        }), executorService()).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        AssertExtensions.assertListEquals("Unexpected iterator result.", expectedValues, actualValues, (e1, e2) -> e1.getKey().equals(e2.getKey()) && e1.getValue().equals(e2.getValue()));
    }
}

Example 18 with Timeout

use of org.junit.rules.Timeout in project pravega by pravega.

the class StreamSegmentContainerTests method testExtendedAttributesConditionalUpdates.

/**
 * Test conditional updates for Extended Attributes when they are not loaded in memory (i.e., they will need to be
 * auto-fetched from the AttributeIndex so that the operation may succeed).
 */
@Test
public void testExtendedAttributesConditionalUpdates() throws Exception {
    final AttributeId ea1 = AttributeId.uuid(0, 1);
    final AttributeId ea2 = AttributeId.uuid(0, 2);
    final List<AttributeId> allAttributes = Stream.of(ea1, ea2).collect(Collectors.toList());
    // We set a longer Segment Expiration time, since this test executes more operations than the others.
    final TestContainerConfig containerConfig = new TestContainerConfig();
    containerConfig.setSegmentMetadataExpiration(Duration.ofMillis(EVICTION_SEGMENT_EXPIRATION_MILLIS_LONG));
    AtomicInteger expectedAttributeValue = new AtomicInteger(0);
    @Cleanup TestContext context = createContext();
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(FREQUENT_TRUNCATIONS_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    @Cleanup MetadataCleanupContainer localContainer = new MetadataCleanupContainer(CONTAINER_ID, containerConfig, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    localContainer.startAsync().awaitRunning();
    // 1. Create the StreamSegments and set the initial attribute values.
    ArrayList<String> segmentNames = createSegments(localContainer);
    ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
    // 2. Update some of the attributes.
    expectedAttributeValue.set(1);
    for (String segmentName : segmentNames) {
        AttributeUpdateCollection attributeUpdates = allAttributes.stream().map(attributeId -> new AttributeUpdate(attributeId, AttributeUpdateType.Accumulate, 1)).collect(Collectors.toCollection(AttributeUpdateCollection::new));
        opFutures.add(localContainer.updateAttributes(segmentName, attributeUpdates, TIMEOUT));
    }
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 3. Force these segments out of memory, so that we may verify conditional appends/updates on extended attributes.
    localContainer.triggerMetadataCleanup(segmentNames).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 4. Execute various conditional operations using these attributes as comparison.
    long compare = expectedAttributeValue.getAndIncrement();
    long set = expectedAttributeValue.get();
    boolean badUpdate = false;
    for (String segmentName : segmentNames) {
        if (badUpdate) {
            // For every other segment, try to do a bad update, then a good one. This helps us verify both direct
            // conditional operations, failed conditional operations and conditional operations with cached attributes.
            AssertExtensions.assertSuppliedFutureThrows("Conditional append succeeded with incorrect compare value.", () -> localContainer.append(segmentName, getAppendData(segmentName, 0), AttributeUpdateCollection.from(new AttributeUpdate(ea1, AttributeUpdateType.ReplaceIfEquals, set, compare - 1)), TIMEOUT), ex -> (ex instanceof BadAttributeUpdateException) && !((BadAttributeUpdateException) ex).isPreviousValueMissing());
            AssertExtensions.assertSuppliedFutureThrows("Conditional update-attributes succeeded with incorrect compare value.", () -> localContainer.updateAttributes(segmentName, AttributeUpdateCollection.from(new AttributeUpdate(ea2, AttributeUpdateType.ReplaceIfEquals, set, compare - 1)), TIMEOUT), ex -> (ex instanceof BadAttributeUpdateException) && !((BadAttributeUpdateException) ex).isPreviousValueMissing());
        }
        opFutures.add(Futures.toVoid(localContainer.append(segmentName, getAppendData(segmentName, 0), AttributeUpdateCollection.from(new AttributeUpdate(ea1, AttributeUpdateType.ReplaceIfEquals, set, compare)), TIMEOUT)));
        opFutures.add(localContainer.updateAttributes(segmentName, AttributeUpdateCollection.from(new AttributeUpdate(ea2, AttributeUpdateType.ReplaceIfEquals, set, compare)), TIMEOUT));
        badUpdate = !badUpdate;
    }
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 4. Evict the segment from memory, then verify results.
    localContainer.triggerMetadataCleanup(segmentNames).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    for (String segmentName : segmentNames) {
        // Verify all attribute values.
        val attributeValues = localContainer.getAttributes(segmentName, allAttributes, true, TIMEOUT).join();
        val sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).join();
        for (val attributeId : allAttributes) {
            Assert.assertEquals("Unexpected value for non-cached attribute " + attributeId + " for segment " + segmentName, expectedAttributeValue.get(), (long) attributeValues.getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
            Assert.assertEquals("Unexpected value for metadata attribute " + attributeId + " for segment " + segmentName, expectedAttributeValue.get(), (long) sp.getAttributes().getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
        }
    }
    localContainer.stopAsync().awaitTerminated();
}

Example 19 with Timeout

use of org.junit.rules.Timeout in project pravega by pravega.

the class StreamSegmentContainerTests method waitForSegmentInStorage.

private CompletableFuture<Void> waitForSegmentInStorage(SegmentProperties metadataProps, TestContext context) {
    if (metadataProps.getLength() == 0) {
        // Empty segments may or may not exist in Storage, so don't bother complicating ourselves with this.
        return CompletableFuture.completedFuture(null);
    }
    // Check if the Storage Segment is caught up. If sealed, we want to make sure that both the Segment and its
    // Attribute Segment are sealed (or the latter has been deleted - for transactions). For all other, we want to
    // ensure that the length and truncation offsets have  caught up.
    BiFunction<SegmentProperties, SegmentProperties, Boolean> meetsConditions = (segmentProps, attrProps) -> metadataProps.isSealed() == (segmentProps.isSealed() && (attrProps.isSealed() || attrProps.isDeleted())) && segmentProps.getLength() >= metadataProps.getLength() && context.storageFactory.truncationOffsets.getOrDefault(metadataProps.getName(), 0L) >= metadataProps.getStartOffset();
    String attributeSegmentName = NameUtils.getAttributeSegmentName(metadataProps.getName());
    AtomicBoolean canContinue = new AtomicBoolean(true);
    TimeoutTimer timer = new TimeoutTimer(TIMEOUT);
    return Futures.loop(canContinue::get, () -> {
        val segInfo = getStorageSegmentInfo(metadataProps.getName(), timer, context);
        val attrInfo = getStorageSegmentInfo(attributeSegmentName, timer, context);
        return CompletableFuture.allOf(segInfo, attrInfo).thenCompose(v -> {
            if (meetsConditions.apply(segInfo.join(), attrInfo.join())) {
                canContinue.set(false);
                return CompletableFuture.completedFuture(null);
            } else if (!timer.hasRemaining()) {
                return Futures.failedFuture(new TimeoutException());
            } else {
                return Futures.delayedFuture(Duration.ofMillis(10), executorService());
            }
        }).thenRun(Runnables.doNothing());
    }, executorService());
}

Example 20 with Timeout

use of org.junit.rules.Timeout in project pravega by pravega.

the class StreamSegmentContainerTests method testAttributeIterators.

/**
 * Tests the ability to run attribute iterators over all or a subset of attributes in a segment.
 */
@Test
public void testAttributeIterators() throws Exception {
    final List<AttributeId> sortedAttributes = IntStream.range(0, 100).mapToObj(i -> AttributeId.uuid(i, i)).sorted().collect(Collectors.toList());
    final Map<AttributeId, Long> expectedValues = sortedAttributes.stream().collect(Collectors.toMap(id -> id, id -> id.getBitGroup(0)));
    final TestContainerConfig containerConfig = new TestContainerConfig();
    containerConfig.setSegmentMetadataExpiration(Duration.ofMillis(EVICTION_SEGMENT_EXPIRATION_MILLIS_SHORT));
    @Cleanup TestContext context = createContext();
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(FREQUENT_TRUNCATIONS_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    @Cleanup MetadataCleanupContainer localContainer = new MetadataCleanupContainer(CONTAINER_ID, containerConfig, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    localContainer.startAsync().awaitRunning();
    // 1. Create the Segment.
    String segmentName = getSegmentName(0);
    localContainer.createStreamSegment(segmentName, getSegmentType(segmentName), null, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    val segment1 = localContainer.forSegment(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 2. Set some initial attribute values and verify in-memory iterator.
    AttributeUpdateCollection attributeUpdates = sortedAttributes.stream().map(attributeId -> new AttributeUpdate(attributeId, AttributeUpdateType.Replace, expectedValues.get(attributeId))).collect(Collectors.toCollection(AttributeUpdateCollection::new));
    segment1.updateAttributes(attributeUpdates, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkAttributeIterators(segment1, sortedAttributes, expectedValues);
    // 3. Force these segments out of memory and verify out-of-memory iterator.
    localContainer.triggerMetadataCleanup(Collections.singleton(segmentName)).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    val segment2 = localContainer.forSegment(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkAttributeIterators(segment2, sortedAttributes, expectedValues);
    // 4. Update some values, and verify mixed iterator.
    attributeUpdates.clear();
    for (int i = 0; i < sortedAttributes.size(); i += 3) {
        AttributeId attributeId = sortedAttributes.get(i);
        expectedValues.put(attributeId, expectedValues.get(attributeId) + 1);
        attributeUpdates.add(new AttributeUpdate(attributeId, AttributeUpdateType.Replace, expectedValues.get(attributeId)));
    }
    segment2.updateAttributes(attributeUpdates, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkAttributeIterators(segment2, sortedAttributes, expectedValues);
    localContainer.stopAsync().awaitTerminated();
}