Examples with TooManyActiveSegmentsException io.pravega.segmentstore.contracts.TooManyActiveSegmentsException used on opensource projects

Example 1 with TooManyActiveSegmentsException

use of io.pravega.segmentstore.contracts.TooManyActiveSegmentsException in project pravega by pravega.

the class StreamSegmentContainerTests method tryActivate.

/**
 * Attempts to activate the targetSegment in the given Container. Since we do not have access to the internals of the
 * Container, we need to trigger this somehow, hence the need for this complex code. We need to trigger a truncation,
 * so we need an 'appendSegment' to which we continuously append so that the DurableDataLog is truncated. After truncation,
 * the Metadata should have enough leeway in making room for new activation.
 *
 * @return A Future that will complete either with an exception (failure) or SegmentProperties for the targetSegment.
 */
private CompletableFuture<SegmentProperties> tryActivate(MetadataCleanupContainer localContainer, String targetSegment, String appendSegment) {
    CompletableFuture<SegmentProperties> successfulMap = new CompletableFuture<>();
    // Append continuously to an existing segment in order to trigger truncations (these are necessary for forced evictions).
    val appendFuture = localContainer.appendRandomly(appendSegment, false, () -> !successfulMap.isDone());
    Futures.exceptionListener(appendFuture, successfulMap::completeExceptionally);
    // Repeatedly try to get info on 'segment1' (activate it), until we succeed or time out.
    TimeoutTimer remaining = new TimeoutTimer(TIMEOUT);
    Futures.loop(() -> !successfulMap.isDone(), () -> Futures.delayedFuture(Duration.ofMillis(EVICTION_SEGMENT_EXPIRATION_MILLIS_SHORT), executorService()).thenCompose(v -> localContainer.getStreamSegmentInfo(targetSegment, TIMEOUT)).thenAccept(successfulMap::complete).exceptionally(ex -> {
        if (!(Exceptions.unwrap(ex) instanceof TooManyActiveSegmentsException)) {
            // Some other error.
            successfulMap.completeExceptionally(ex);
        } else if (!remaining.hasRemaining()) {
            // Waited too long.
            successfulMap.completeExceptionally(new TimeoutException("No successful activation could be done in the allotted time."));
        }
        // Try again.
        return null;
    }), executorService());
    return successfulMap;
}

Example 2 with TooManyActiveSegmentsException

use of io.pravega.segmentstore.contracts.TooManyActiveSegmentsException in project pravega by pravega.

the class MetadataStoreTestBase method testGetOrAssignStreamSegmentIdWithMetadataLimit.

/**
 * Tests the ability of getOrAssignSegmentId to handle the TooManyActiveSegmentsException.
 */
@Test
public void testGetOrAssignStreamSegmentIdWithMetadataLimit() {
    List<String> segmentNames = Arrays.asList("S1", "S2", "S3");
    AtomicBoolean cleanupInvoked = new AtomicBoolean();
    AtomicInteger exceptionCounter = new AtomicInteger();
    Supplier<CompletableFuture<Void>> noOpCleanup = () -> {
        if (!cleanupInvoked.compareAndSet(false, true)) {
            return Futures.failedFuture(new AssertionError("Cleanup invoked multiple times."));
        }
        return CompletableFuture.completedFuture(null);
    };
    try (TestContext context = createTestContext(noOpCleanup)) {
        for (val s : segmentNames) {
            context.getMetadataStore().createSegment(s, SEGMENT_TYPE, null, TIMEOUT).join();
        }
        // 1. Verify the behavior when even after the retry we still cannot map.
        // We use 'containerId' as a proxy for the exception id (to make sure we collect the right one).
        context.connector.setMapSegmentId((id, sp, pin, timeout) -> Futures.failedFuture(new TooManyActiveSegmentsException(exceptionCounter.incrementAndGet(), 0)));
        AssertExtensions.assertSuppliedFutureThrows("Unexpected outcome when trying to map a segment to a full metadata that cannot be cleaned.", () -> context.getMetadataStore().getOrAssignSegmentId(segmentNames.get(0), TIMEOUT), ex -> ex instanceof TooManyActiveSegmentsException && ((TooManyActiveSegmentsException) ex).getContainerId() == exceptionCounter.get());
        Assert.assertEquals("Unexpected number of attempts to map.", 2, exceptionCounter.get());
        Assert.assertTrue("Cleanup was not invoked.", cleanupInvoked.get());
        // Now with a Segment 3.
        exceptionCounter.set(0);
        cleanupInvoked.set(false);
        AssertExtensions.assertSuppliedFutureThrows("Unexpected outcome when trying to map a Segment to a full metadata that cannot be cleaned.", () -> context.getMetadataStore().getOrAssignSegmentId(segmentNames.get(2), TIMEOUT), ex -> ex instanceof TooManyActiveSegmentsException && ((TooManyActiveSegmentsException) ex).getContainerId() == exceptionCounter.get());
        Assert.assertEquals("Unexpected number of attempts to map.", 2, exceptionCounter.get());
        Assert.assertTrue("Cleanup was not invoked.", cleanupInvoked.get());
    }
    // 2. Verify the behavior when the first call fails, but the second one succeeds.
    exceptionCounter.set(0);
    cleanupInvoked.set(false);
    Supplier<CompletableFuture<Void>> workingCleanup = () -> {
        if (!cleanupInvoked.compareAndSet(false, true)) {
            return Futures.failedFuture(new AssertionError("Cleanup invoked multiple times."));
        }
        return CompletableFuture.completedFuture(null);
    };
    try (TestContext context = createTestContext(workingCleanup)) {
        for (val s : segmentNames) {
            context.getMetadataStore().createSegment(s, SEGMENT_TYPE, null, TIMEOUT).join();
        }
        // 1. Verify the behavior when even after the retry we still cannot map.
        // We use 'containerId' as a proxy for the exception id (to make sure we collect the right one).
        context.connector.setMapSegmentId((id, sp, pin, timeout) -> {
            context.connector.reset();
            return Futures.failedFuture(new TooManyActiveSegmentsException(exceptionCounter.incrementAndGet(), 0));
        });
        long id = context.getMetadataStore().getOrAssignSegmentId(segmentNames.get(0), TIMEOUT).join();
        Assert.assertEquals("Unexpected number of attempts to map.", 1, exceptionCounter.get());
        Assert.assertTrue("Cleanup was not invoked.", cleanupInvoked.get());
        Assert.assertNotEquals("No valid SegmentId assigned.", ContainerMetadata.NO_STREAM_SEGMENT_ID, id);
    }
}

Example 3 with TooManyActiveSegmentsException

use of io.pravega.segmentstore.contracts.TooManyActiveSegmentsException in project pravega by pravega.

the class StreamSegmentMapperTests method testGetOrAssignStreamSegmentIdWithMetadataLimit.

/**
 * Tests the ability of getOrAssignStreamSegmentId to handle the TooManyActiveSegmentsException.
 */
@Test
public void testGetOrAssignStreamSegmentIdWithMetadataLimit() throws Exception {
    // We use different "parent" segment names because it is possible that, if the test runs fast enough, and the
    // StreamSegmentMapper does not clean up its state quickly enough, subsequent mapping attempts will piggyback
    // on the first one, and thus not execute the test as desired.
    final String segmentName = "Segment";
    final String transactionParent = "SegmentWithTxn";
    final String transactionName = StreamSegmentNameUtils.getTransactionNameFromId(transactionParent, UUID.randomUUID());
    HashSet<String> storageSegments = new HashSet<>(Arrays.asList(segmentName, transactionParent, transactionName));
    @Cleanup TestContext context = new TestContext();
    setupStorageGetHandler(context, storageSegments, name -> StreamSegmentInformation.builder().name(name).build());
    // 1. Verify the behavior when even after the retry we still cannot map.
    AtomicInteger exceptionCounter = new AtomicInteger();
    AtomicBoolean cleanupInvoked = new AtomicBoolean();
    // We use 'containerId' as a proxy for the exception id (to make sure we collect the right one).
    context.operationLog.addHandler = op -> Futures.failedFuture(new TooManyActiveSegmentsException(exceptionCounter.incrementAndGet(), 0));
    Supplier<CompletableFuture<Void>> noOpCleanup = () -> {
        if (!cleanupInvoked.compareAndSet(false, true)) {
            return Futures.failedFuture(new AssertionError("Cleanup invoked multiple times."));
        }
        return CompletableFuture.completedFuture(null);
    };
    val mapper1 = new StreamSegmentMapper(context.metadata, context.operationLog, context.stateStore, noOpCleanup, context.storage, executorService());
    AssertExtensions.assertThrows("Unexpected outcome when trying to map a segment to a full metadata that cannot be cleaned.", () -> mapper1.getOrAssignStreamSegmentId(segmentName, TIMEOUT), ex -> ex instanceof TooManyActiveSegmentsException && ((TooManyActiveSegmentsException) ex).getContainerId() == exceptionCounter.get());
    Assert.assertEquals("Unexpected number of attempts to map.", 2, exceptionCounter.get());
    Assert.assertTrue("Cleanup was not invoked.", cleanupInvoked.get());
    // Now with a transaction.
    exceptionCounter.set(0);
    cleanupInvoked.set(false);
    AssertExtensions.assertThrows("Unexpected outcome when trying to map a transaction to a full metadata that cannot be cleaned.", () -> mapper1.getOrAssignStreamSegmentId(transactionName, TIMEOUT), ex -> ex instanceof TooManyActiveSegmentsException && ((TooManyActiveSegmentsException) ex).getContainerId() == exceptionCounter.get());
    Assert.assertEquals("Unexpected number of attempts to map.", 2, exceptionCounter.get());
    Assert.assertTrue("Cleanup was not invoked.", cleanupInvoked.get());
    // 2. Verify the behavior when the first call fails, but the second one succeeds.
    exceptionCounter.set(0);
    cleanupInvoked.set(false);
    Supplier<CompletableFuture<Void>> workingCleanup = () -> {
        if (!cleanupInvoked.compareAndSet(false, true)) {
            return Futures.failedFuture(new AssertionError("Cleanup invoked multiple times."));
        }
        // Setup the OperationLog to function correctly.
        setupOperationLog(context);
        return CompletableFuture.completedFuture(null);
    };
    val mapper2 = new StreamSegmentMapper(context.metadata, context.operationLog, context.stateStore, workingCleanup, context.storage, executorService());
    long id = mapper2.getOrAssignStreamSegmentId(segmentName, TIMEOUT).join();
    Assert.assertEquals("Unexpected number of attempts to map.", 1, exceptionCounter.get());
    Assert.assertTrue("Cleanup was not invoked.", cleanupInvoked.get());
    Assert.assertNotEquals("No valid SegmentId assigned.", ContainerMetadata.NO_STREAM_SEGMENT_ID, id);
}

Example 4 with TooManyActiveSegmentsException

use of io.pravega.segmentstore.contracts.TooManyActiveSegmentsException in project pravega by pravega.

the class StreamSegmentContainerTests method testForcedMetadataCleanup.

/**
 * Tests the case when the ContainerMetadata has filled up to capacity (with segments and we cannot map anymore segments).
 */
@Test
public void testForcedMetadataCleanup() throws Exception {
    final int maxSegmentCount = 3;
    final int createdSegmentCount = maxSegmentCount * 2;
    final ContainerConfig containerConfig = ContainerConfig.builder().with(ContainerConfig.SEGMENT_METADATA_EXPIRATION_SECONDS, (int) DEFAULT_CONFIG.getSegmentMetadataExpiration().getSeconds()).with(ContainerConfig.MAX_ACTIVE_SEGMENT_COUNT, maxSegmentCount + EXPECTED_PINNED_SEGMENT_COUNT).with(ContainerConfig.STORAGE_SNAPSHOT_TIMEOUT_SECONDS, (int) DEFAULT_CONFIG.getStorageSnapshotTimeout().getSeconds()).build();
    // We need a special DL config so that we can force truncations after every operation - this will speed up metadata
    // eviction eligibility.
    final DurableLogConfig durableLogConfig = DurableLogConfig.builder().with(DurableLogConfig.CHECKPOINT_MIN_COMMIT_COUNT, 1).with(DurableLogConfig.CHECKPOINT_COMMIT_COUNT, 10).with(DurableLogConfig.CHECKPOINT_TOTAL_COMMIT_LENGTH, 10L * 1024 * 1024).build();
    @Cleanup TestContext context = createContext(containerConfig);
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(durableLogConfig, context.dataLogFactory, executorService());
    @Cleanup MetadataCleanupContainer localContainer = new MetadataCleanupContainer(CONTAINER_ID, containerConfig, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    localContainer.startAsync().awaitRunning();
    // Create the segments.
    val segments = new ArrayList<String>();
    for (int i = 0; i < createdSegmentCount; i++) {
        String name = getSegmentName(i);
        segments.add(name);
        localContainer.createStreamSegment(name, getSegmentType(name), null, TIMEOUT).join();
    }
    // Activate three segments (this should fill up available capacity).
    activateSegment(segments.get(2), localContainer).join();
    activateSegment(segments.get(4), localContainer).join();
    activateSegment(segments.get(5), localContainer).join();
    // At this point, the active segments should be: 2, 4 and 5.
    // Verify we cannot activate any other segment.
    AssertExtensions.assertSuppliedFutureThrows("getSegmentId() allowed mapping more segments than the metadata can support.", () -> activateSegment(segments.get(1), localContainer), ex -> ex instanceof TooManyActiveSegmentsException);
    // Test the ability to forcefully evict items from the metadata when there is pressure and we need to register something new.
    // Case 1: following a Segment deletion.
    localContainer.deleteStreamSegment(segments.get(2), TIMEOUT).join();
    val segment1Activation = tryActivate(localContainer, segments.get(1), segments.get(4));
    val segment1Info = segment1Activation.get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    Assert.assertNotNull("Unable to properly activate dormant segment (1).", segment1Info);
    // Case 2: following a Merge.
    localContainer.sealStreamSegment(segments.get(5), TIMEOUT).join();
    localContainer.mergeStreamSegment(segments.get(4), segments.get(5), TIMEOUT).join();
    val segment0Activation = tryActivate(localContainer, segments.get(0), segments.get(3));
    val segment0Info = segment0Activation.get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    Assert.assertNotNull("Unable to properly activate dormant segment (0).", segment0Info);
    tryActivate(localContainer, segments.get(1), segments.get(3)).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // At this point the active segments should be: 0, 1 and 3.
    Assert.assertNotNull("Pre-activated segment did not stay in metadata (3).", localContainer.getStreamSegmentInfo(segments.get(3), TIMEOUT).join());
    Assert.assertNotNull("Pre-activated segment did not stay in metadata (1).", localContainer.getStreamSegmentInfo(segments.get(1), TIMEOUT).join());
    Assert.assertNotNull("Pre-activated segment did not stay in metadata (0).", localContainer.getStreamSegmentInfo(segments.get(0), TIMEOUT).join());
    context.container.stopAsync().awaitTerminated();
}

Example 5 with TooManyActiveSegmentsException

use of io.pravega.segmentstore.contracts.TooManyActiveSegmentsException in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testSegmentMapMax.

/**
 * Tests the ability to reject new StreamSegment/Transaction map operations that would exceed the max allowed counts.
 */
@Test
public void testSegmentMapMax() throws Exception {
    UpdateableContainerMetadata metadata = new MetadataBuilder(CONTAINER_ID).withMaxActiveSegmentCount(3).build();
    metadata.mapStreamSegmentId("a", SEGMENT_ID);
    metadata.mapStreamSegmentId("b", 123457);
    // Non-recovery mode.
    val txn1 = createUpdateTransaction(metadata);
    // Map one segment, which should fill up the quota.
    StreamSegmentMapOperation acceptedMap = createMap();
    txn1.preProcessOperation(acceptedMap);
    txn1.acceptOperation(acceptedMap);
    // Verify non-recovery mode.
    AssertExtensions.assertThrows("Unexpected behavior from preProcessOperation when attempting to map a StreamSegment that would exceed the active segment quota.", () -> txn1.preProcessOperation(createMap("foo")), ex -> ex instanceof TooManyActiveSegmentsException);
    // Verify recovery mode.
    metadata.enterRecoveryMode();
    val txn2 = createUpdateTransaction(metadata);
    // updater.setOperationSequenceNumber(10000);
    StreamSegmentMapOperation secondMap = createMap("c");
    secondMap.setStreamSegmentId(1234);
    txn2.preProcessOperation(secondMap);
    txn2.acceptOperation(secondMap);
    StreamSegmentMapOperation thirdMap = createTransactionMap("a_txn2");
    thirdMap.setStreamSegmentId(1235);
    txn2.preProcessOperation(thirdMap);
    txn2.acceptOperation(thirdMap);
    txn2.commit(metadata);
    metadata.exitRecoveryMode();
    Assert.assertNotNull("Updater did not create metadata for new segment in recovery mode even if quota is exceeded (1).", metadata.getStreamSegmentMetadata(secondMap.getStreamSegmentId()));
    Assert.assertNotNull("Updater did not create metadata for new segment in recovery mode even if quota is exceeded (2).", metadata.getStreamSegmentMetadata(thirdMap.getStreamSegmentId()));
}