Examples with Storage io.pravega.segmentstore.storage.Storage used on opensource projects

Example 6 with Storage

use of io.pravega.segmentstore.storage.Storage in project pravega by pravega.

the class DebugRecoveryProcessor method create.

/**
 * Creates a new instance of the DebugRecoveryProcessor class with the given arguments.
 *
 * @param containerId     The Id of the Container to recover.
 * @param durableDataLog  A DurableDataLog to recover from.
 * @param config          A ContainerConfig to use during recovery.
 * @param readIndexConfig A ReadIndexConfig to use during recovery.
 * @param executor        An Executor to use for background tasks.
 * @param callbacks       Callbacks to invoke during recovery.
 * @return A new instance of the DebugRecoveryProcessor.
 */
public static DebugRecoveryProcessor create(int containerId, DurableDataLog durableDataLog, ContainerConfig config, ReadIndexConfig readIndexConfig, ScheduledExecutorService executor, OperationCallbacks callbacks) {
    Preconditions.checkNotNull(durableDataLog, "durableDataLog");
    Preconditions.checkNotNull(config, "config");
    Preconditions.checkNotNull(readIndexConfig, "readIndexConfig");
    Preconditions.checkNotNull(executor, "executor");
    Preconditions.checkNotNull(callbacks, callbacks);
    StreamSegmentContainerMetadata metadata = new StreamSegmentContainerMetadata(containerId, config.getMaxActiveSegmentCount());
    ContainerReadIndexFactory rf = new ContainerReadIndexFactory(readIndexConfig, new NoOpCacheFactory(), executor);
    Storage s = new InMemoryStorageFactory(executor).createStorageAdapter();
    return new DebugRecoveryProcessor(metadata, durableDataLog, rf, s, callbacks);
}

Example 7 with Storage

use of io.pravega.segmentstore.storage.Storage in project pravega by pravega.

the class SegmentAggregator method reconcileAppendOperation.

/**
 * Attempts to reconcile the given Append Operation. Since Append Operations can be partially flushed, reconciliation
 * may be for the full operation or for a part of it.
 *
 * @param op          The Operation (StreamSegmentAppendOperation or CachedStreamSegmentAppendOperation) to reconcile.
 * @param storageInfo The current state of the Segment in Storage.
 * @param timer       Timer for the operation.
 * @return A CompletableFuture containing a FlushResult with the number of bytes reconciled, or failed with a ReconciliationFailureException,
 * if the operation cannot be reconciled, based on the in-memory metadata or the current state of the Segment in Storage.
 */
private CompletableFuture<FlushResult> reconcileAppendOperation(StorageOperation op, SegmentProperties storageInfo, TimeoutTimer timer) {
    Preconditions.checkArgument(op instanceof AggregatedAppendOperation, "Not given an append operation.");
    // Read data from Storage, and compare byte-by-byte.
    InputStream appendStream = this.dataSource.getAppendData(op.getStreamSegmentId(), op.getStreamSegmentOffset(), (int) op.getLength());
    if (appendStream == null) {
        return Futures.failedFuture(new ReconciliationFailureException(String.format("Unable to reconcile operation '%s' because no append data is associated with it.", op), this.metadata, storageInfo));
    }
    // Only read as much data as we need.
    long readLength = Math.min(op.getLastStreamSegmentOffset(), storageInfo.getLength()) - op.getStreamSegmentOffset();
    assert readLength > 0 : "Append Operation to be reconciled is beyond the Segment's StorageLength " + op;
    AtomicInteger bytesReadSoFar = new AtomicInteger();
    // Read all data from storage.
    byte[] storageData = new byte[(int) readLength];
    return Futures.loop(() -> bytesReadSoFar.get() < readLength, () -> this.storage.read(this.handle.get(), op.getStreamSegmentOffset() + bytesReadSoFar.get(), storageData, bytesReadSoFar.get(), (int) readLength - bytesReadSoFar.get(), timer.getRemaining()), bytesRead -> {
        assert bytesRead > 0 : String.format("Unable to make any read progress when reconciling operation '%s' after reading %s bytes.", op, bytesReadSoFar);
        bytesReadSoFar.addAndGet(bytesRead);
    }, this.executor).thenApplyAsync(v -> {
        // Compare, byte-by-byte, the contents of the append.
        verifySame(appendStream, storageData, op, storageInfo);
        if (readLength >= op.getLength() && op.getLastStreamSegmentOffset() <= storageInfo.getLength()) {
            // Operation has been completely validated; pop it off the list.
            StorageOperation removedOp = this.operations.removeFirst();
            assert op == removedOp : "Reconciled operation is not the same as removed operation";
        }
        return new FlushResult().withFlushedBytes(readLength);
    }, this.executor);
}

Example 8 with Storage

use of io.pravega.segmentstore.storage.Storage in project pravega by pravega.

the class SegmentStateMapperTests method testGetSegmentInfoFromStorage.

/**
 * Tests the ability to retrieve Segment Info from Storage (base info and its State).
 */
@Test
public void testGetSegmentInfoFromStorage() {
    final UUID attributeId = UUID.randomUUID();
    @Cleanup val s = createStorage();
    val stateStore = new InMemoryStateStore();
    val m = new SegmentStateMapper(stateStore, s);
    // Save some state.
    val info = s.create("s", TIMEOUT).thenCompose(si -> s.openWrite(si.getName())).thenCompose(handle -> s.write(handle, 0, new ByteArrayInputStream(new byte[10]), 10, TIMEOUT).thenCompose(v -> s.seal(handle, TIMEOUT)).thenCompose(v -> s.getStreamSegmentInfo(handle.getSegmentName(), TIMEOUT))).join();
    val expectedInfo = StreamSegmentInformation.from(info).startOffset(info.getLength() / 2).build();
    val attributes = Collections.singleton(new AttributeUpdate(attributeId, AttributeUpdateType.Replace, 100L));
    val expectedAttributes = Collections.singletonMap(attributeId, 100L);
    m.saveState(expectedInfo, attributes, TIMEOUT).join();
    // Retrieve the state and verify it.
    val actual = m.getSegmentInfoFromStorage(expectedInfo.getName(), TIMEOUT).join();
    Assert.assertEquals("Unexpected Name.", expectedInfo.getName(), actual.getName());
    Assert.assertEquals("Unexpected Length.", expectedInfo.getLength(), actual.getLength());
    Assert.assertEquals("Unexpected Sealed status.", expectedInfo.isSealed(), actual.isSealed());
    Assert.assertEquals("Unexpected Start Offset.", expectedInfo.getStartOffset(), actual.getStartOffset());
    AssertExtensions.assertMapEquals("Unexpected Attributes.", expectedAttributes, actual.getAttributes());
}

Example 9 with Storage

use of io.pravega.segmentstore.storage.Storage in project pravega by pravega.

the class SegmentStateStoreTests method testGetCorruptedState.

/**
 * Tests the get() method when there exists a state in Storage, however it is a corrupted file.
 */
@Test
public void testGetCorruptedState() throws Exception {
    final String segmentName = "foo";
    final String stateSegment = StreamSegmentNameUtils.getStateSegmentName(segmentName);
    val store = createStateStore();
    // Write some dummy contents in the file which is not how a SegmentState would be serialized.
    this.storage.create(stateSegment, TIMEOUT).thenCompose(si -> this.storage.openWrite(stateSegment)).thenCompose(handle -> this.storage.write(handle, 0, new ByteArrayInputStream(new byte[1]), 1, TIMEOUT)).join();
    AssertExtensions.assertThrows("Unexpected behavior when attempting to read a corrupted state file.", () -> store.get(segmentName, TIMEOUT), ex -> ex instanceof DataCorruptionException);
}

Example 10 with Storage

use of io.pravega.segmentstore.storage.Storage in project pravega by pravega.

the class StreamSegmentMapper method createNewTransactionStreamSegment.

/**
 * Creates a new Transaction StreamSegment for an existing Parent StreamSegment and persists the given attributes (in Storage).
 *
 * @param parentStreamSegmentName The case-sensitive StreamSegment Name of the Parent StreamSegment.
 * @param transactionId           A unique identifier for the transaction to be created.
 * @param attributes              The initial attributes for the Transaction, if any.
 * @param timeout                 Timeout for the operation.
 * @return A CompletableFuture that, when completed normally, will contain the name of the newly created Transaction StreamSegment.
 * If the operation failed, this will contain the exception that caused the failure.
 * @throws IllegalArgumentException If the given parent StreamSegment cannot have a Transaction (because it is deleted, sealed, inexistent).
 */
public CompletableFuture<String> createNewTransactionStreamSegment(String parentStreamSegmentName, UUID transactionId, Collection<AttributeUpdate> attributes, Duration timeout) {
    long traceId = LoggerHelpers.traceEnterWithContext(log, traceObjectId, "createNewTransactionStreamSegment", parentStreamSegmentName);
    // We cannot create a Transaction StreamSegment for a what looks like another Transaction.
    Exceptions.checkArgument(StreamSegmentNameUtils.getParentStreamSegmentName(parentStreamSegmentName) == null, "parentStreamSegmentName", "Cannot create a Transaction for a Transaction.");
    // Validate that Parent StreamSegment exists.
    TimeoutTimer timer = new TimeoutTimer(timeout);
    CompletableFuture<Void> parentCheck = null;
    long mappedParentId = this.containerMetadata.getStreamSegmentId(parentStreamSegmentName, true);
    if (isValidStreamSegmentId(mappedParentId)) {
        SegmentProperties parentInfo = this.containerMetadata.getStreamSegmentMetadata(mappedParentId);
        if (parentInfo != null) {
            parentCheck = validateParentSegmentEligibility(parentInfo);
        }
    }
    if (parentCheck == null) {
        // The parent is not registered in the metadata. Get required info from Storage and don't map it unnecessarily.
        parentCheck = this.storage.getStreamSegmentInfo(parentStreamSegmentName, timer.getRemaining()).thenCompose(this::validateParentSegmentEligibility);
    }
    String transactionName = StreamSegmentNameUtils.getTransactionNameFromId(parentStreamSegmentName, transactionId);
    return parentCheck.thenComposeAsync(parentId -> createSegmentInStorageWithRecovery(transactionName, attributes, timer), this.executor).thenApply(v -> {
        LoggerHelpers.traceLeave(log, traceObjectId, "createNewTransactionStreamSegment", traceId, parentStreamSegmentName, transactionName);
        return transactionName;
    });
}