Examples with StorageFactory io.pravega.segmentstore.storage.StorageFactory used on opensource projects

Example 1 with StorageFactory

use of io.pravega.segmentstore.storage.StorageFactory in project pravega by pravega.

the class DebugStreamSegmentContainerTests method testBackUpMetadataAndAttributeSegments.

/**
 * The test create a dummy metadata segment and its attribute segment using a storage instance. The method under the
 * test creates copies of the segments. After that, it is verified if the new segments exist.
 */
@Test
public void testBackUpMetadataAndAttributeSegments() {
    // Create a storage.
    StorageFactory storageFactory = new InMemoryStorageFactory(executorService());
    @Cleanup Storage s = storageFactory.createStorageAdapter();
    s.initialize(1);
    log.info("Created a storage instance");
    String metadataSegment = NameUtils.getMetadataSegmentName(CONTAINER_ID);
    String attributeSegment = NameUtils.getAttributeSegmentName(metadataSegment);
    String backUpMetadataSegment = "segment-" + RANDOM.nextInt();
    String backUpAttributeSegment = "segment-" + RANDOM.nextInt();
    s.create(metadataSegment, TIMEOUT).join();
    s.create(attributeSegment, TIMEOUT).join();
    ContainerRecoveryUtils.backUpMetadataAndAttributeSegments(s, CONTAINER_ID, backUpMetadataSegment, backUpAttributeSegment, executorService(), TIMEOUT).join();
    // back up metadata segment should exist
    Assert.assertTrue("Unexpected result for existing segment (no files).", s.exists(backUpMetadataSegment, TIMEOUT).join());
    // back up attribute segment should exist
    Assert.assertTrue("Unexpected result for existing segment (no files).", s.exists(backUpAttributeSegment, TIMEOUT).join());
}

Example 2 with StorageFactory

use of io.pravega.segmentstore.storage.StorageFactory in project pravega by pravega.

the class DebugStreamSegmentContainerTests method testCopySegment.

/**
 * The test creates a segment and then writes some data to it. The method under the test copies the contents of the
 * segment to a segment with a different name. At the end, it is verified that the new segment has the accurate
 * contents from the first one.
 */
@Test
public void testCopySegment() {
    int dataSize = 10 * 1024 * 1024;
    // Create a storage.
    StorageFactory storageFactory = new InMemoryStorageFactory(executorService());
    @Cleanup Storage s = storageFactory.createStorageAdapter();
    s.initialize(1);
    log.info("Created a storage instance");
    String sourceSegmentName = "segment-" + RANDOM.nextInt();
    String targetSegmentName = "segment-" + RANDOM.nextInt();
    // Create source segment
    s.create(sourceSegmentName, TIMEOUT).join();
    val handle = s.openWrite(sourceSegmentName).join();
    // do some writing
    byte[] writeData = populate(dataSize);
    val dataStream = new ByteArrayInputStream(writeData);
    s.write(handle, 0, dataStream, writeData.length, TIMEOUT).join();
    // copy segment
    ContainerRecoveryUtils.copySegment(s, sourceSegmentName, targetSegmentName, executorService(), TIMEOUT).join();
    // source segment should exist
    Assert.assertTrue("Unexpected result for existing segment (no files).", s.exists(sourceSegmentName, null).join());
    // target segment should exist
    Assert.assertTrue("Unexpected result for existing segment (no files).", s.exists(targetSegmentName, null).join());
    // Do reading on target segment to verify if the copy was successful or not
    val readHandle = s.openRead(targetSegmentName).join();
    int length = writeData.length;
    byte[] readBuffer = new byte[length];
    int bytesRead = s.read(readHandle, 0, readBuffer, 0, readBuffer.length, TIMEOUT).join();
    Assert.assertEquals(String.format("Unexpected number of bytes read."), length, bytesRead);
    AssertExtensions.assertArrayEquals(String.format("Unexpected read result."), writeData, 0, readBuffer, 0, bytesRead);
}

Example 3 with StorageFactory

use of io.pravega.segmentstore.storage.StorageFactory in project pravega by pravega.

the class DebugStreamSegmentContainerTests method testDataRecoveryContainerLevel.

/**
 * The test creates a container. Using it, some segments are created, data is written to them and their attributes are
 * updated. After Tier1 is flushed to the storage, the container is closed. From the storage, we recover the
 * segments and update their attributes. The data and attributes for each of the segment are verified to validate the
 * recovery process.
 */
@Test
public void testDataRecoveryContainerLevel() throws Exception {
    int attributesUpdatesPerSegment = 50;
    int segmentsCount = 10;
    final AttributeId attributeReplace = AttributeId.uuid(CORE_ATTRIBUTE_ID_PREFIX, RANDOM.nextInt(10));
    final long expectedAttributeValue = attributesUpdatesPerSegment;
    int containerId = 0;
    int containerCount = 1;
    // 1MB
    int maxDataSize = 1024 * 1024;
    StorageFactory storageFactory = new InMemoryStorageFactory(executorService());
    @Cleanup TestContext context = createContext(executorService());
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(DEFAULT_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    @Cleanup MetadataCleanupContainer container = new MetadataCleanupContainer(containerId, CONTAINER_CONFIG, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, storageFactory, context.getDefaultExtensions(), executorService());
    container.startAsync().awaitRunning();
    // 1. Create segments.
    ArrayList<String> segmentNames = new ArrayList<>();
    ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
    for (int i = 0; i < segmentsCount; i++) {
        String segmentName = getSegmentName(i);
        segmentNames.add(segmentName);
        opFutures.add(container.createStreamSegment(segmentName, getSegmentType(segmentName), null, TIMEOUT));
    }
    // 1.1 Wait for all segments to be created prior to using them.
    Futures.allOf(opFutures).join();
    opFutures.clear();
    // 2. Write data and update some of the attributes.
    HashMap<String, Long> lengths = new HashMap<>();
    HashMap<String, ByteArraySegment> segmentContents = new HashMap<>();
    for (String segmentName : segmentNames) {
        val dataSize = RANDOM.nextInt(maxDataSize);
        byte[] writeData = populate(dataSize);
        val appendData = new ByteArraySegment(writeData);
        val append = container.append(segmentName, appendData, null, TIMEOUT);
        opFutures.add(Futures.toVoid(append));
        lengths.put(segmentName, (long) dataSize);
        segmentContents.put(segmentName, appendData);
        for (int i = 0; i < attributesUpdatesPerSegment; i++) {
            AttributeUpdateCollection attributeUpdates = AttributeUpdateCollection.from(new AttributeUpdate(attributeReplace, AttributeUpdateType.Replace, i + 1));
            opFutures.add(container.updateAttributes(segmentName, attributeUpdates, TIMEOUT));
        }
    }
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 3. Instead of waiting for the Writer to move data to Storage, we invoke the flushToStorage to verify that all
    // operations have been applied to Storage.
    val forceFlush = container.flushToStorage(TIMEOUT);
    forceFlush.get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    container.close();
    // Get the storage instance
    @Cleanup Storage storage = storageFactory.createStorageAdapter();
    // 4. Move container metadata and its attribute segment to back up segments.
    Map<Integer, String> backUpMetadataSegments = ContainerRecoveryUtils.createBackUpMetadataSegments(storage, containerCount, executorService(), TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    OperationLogFactory localDurableLogFactory2 = new DurableLogFactory(NO_TRUNCATIONS_DURABLE_LOG_CONFIG, new InMemoryDurableDataLogFactory(MAX_DATA_LOG_APPEND_SIZE, executorService()), executorService());
    // Starts a DebugSegmentContainer with new Durable Log
    @Cleanup TestContext context1 = createContext(executorService());
    @Cleanup MetadataCleanupContainer container2 = new MetadataCleanupContainer(containerId, CONTAINER_CONFIG, localDurableLogFactory2, context1.readIndexFactory, context1.attributeIndexFactory, context1.writerFactory, storageFactory, context1.getDefaultExtensions(), executorService());
    container2.startAsync().awaitRunning();
    Map<Integer, DebugStreamSegmentContainer> debugStreamSegmentContainersMap = new HashMap<>();
    debugStreamSegmentContainersMap.put(containerId, container2);
    // 4. Recover all segments.
    recoverAllSegments(storage, debugStreamSegmentContainersMap, executorService(), TIMEOUT);
    // 5. Update core attributes using back up segments.
    updateCoreAttributes(backUpMetadataSegments, debugStreamSegmentContainersMap, executorService(), TIMEOUT);
    // 6. Verify Segment Data.
    for (val sc : segmentContents.entrySet()) {
        // Contents.
        byte[] expectedData = sc.getValue().array();
        byte[] actualData = new byte[expectedData.length];
        container2.read(sc.getKey(), 0, actualData.length, TIMEOUT).join().readRemaining(actualData, TIMEOUT);
        Assert.assertArrayEquals("Unexpected contents for " + sc.getKey(), expectedData, actualData);
        // Length.
        val si = container2.getStreamSegmentInfo(sc.getKey(), TIMEOUT).join();
        Assert.assertEquals("Unexpected length for " + sc.getKey(), expectedData.length, si.getLength());
        // Attributes.
        val attributes = container2.getAttributes(sc.getKey(), Collections.singleton(attributeReplace), false, TIMEOUT).join();
        Assert.assertEquals("Unexpected attribute for " + sc.getKey(), expectedAttributeValue, (long) attributes.get(attributeReplace));
    }
}

Example 4 with StorageFactory

use of io.pravega.segmentstore.storage.StorageFactory in project pravega by pravega.

the class StorageLoader method load.

public StorageFactory load(ConfigSetup setup, String storageImplementation, StorageLayoutType storageLayoutType, ScheduledExecutorService executor) {
    ServiceLoader<StorageFactoryCreator> loader = ServiceLoader.load(StorageFactoryCreator.class);
    StorageExtraConfig noOpConfig = setup.getConfig(StorageExtraConfig::builder);
    for (StorageFactoryCreator factoryCreator : loader) {
        val factories = factoryCreator.getStorageFactories();
        for (val factoryInfo : factories) {
            log.info("Loading {}, trying {}", storageImplementation, factoryInfo);
            if (factoryInfo.getName().equals(storageImplementation) && factoryInfo.getStorageLayoutType() == storageLayoutType) {
                StorageFactory factory = factoryCreator.createFactory(factoryInfo, setup, executor);
                if (!noOpConfig.isStorageNoOpMode()) {
                    return factory;
                } else {
                    // The specified storage implementation is in No-Op mode.
                    log.warn("{} IS IN NO-OP MODE: DATA LOSS WILL HAPPEN! MAKE SURE IT IS BY FULL INTENTION FOR TESTING PURPOSE!", storageImplementation);
                    return new NoOpStorageFactory(noOpConfig, executor, factory, null);
                }
            }
        }
    }
    return null;
}

Example 5 with StorageFactory

use of io.pravega.segmentstore.storage.StorageFactory in project pravega by pravega.

the class ServiceBuilder method createSegmentContainerFactory.

protected SegmentContainerFactory createSegmentContainerFactory() {
    ReadIndexFactory readIndexFactory = getSingleton(this.readIndexFactory, this::createReadIndexFactory);
    AttributeIndexFactory attributeIndexFactory = getSingleton(this.attributeIndexFactory, this::createAttributeIndexFactory);
    StorageFactory storageFactory = createStorageFactory();
    OperationLogFactory operationLogFactory = getSingleton(this.operationLogFactory, this::createOperationLogFactory);
    WriterFactory writerFactory = getSingleton(this.writerFactory, this::createWriterFactory);
    ContainerConfig containerConfig = this.serviceBuilderConfig.getConfig(ContainerConfig::builder);
    return new StreamSegmentContainerFactory(containerConfig, operationLogFactory, readIndexFactory, attributeIndexFactory, writerFactory, storageFactory, this::createContainerExtensions, this.coreExecutor);
}