Examples with SealedSegmentsMapShard io.pravega.controller.store.stream.records.SealedSegmentsMapShard used on opensource projects

Example 1 with SealedSegmentsMapShard

use of io.pravega.controller.store.stream.records.SealedSegmentsMapShard in project pravega by pravega.

the class PersistentStreamBase method updateSealedSegmentSizes.

private CompletableFuture<Void> updateSealedSegmentSizes(Map<Long, Long> sealedSegmentSizes, OperationContext context) {
    Map<Integer, List<Long>> shards = sealedSegmentSizes.keySet().stream().collect(Collectors.groupingBy(this::getShardNumber));
    return Futures.allOf(shards.entrySet().stream().map(x -> {
        int shard = x.getKey();
        List<Long> segments = x.getValue();
        return Futures.exceptionallyComposeExpecting(getSealedSegmentSizesMapShardData(shard, context), DATA_NOT_FOUND_PREDICATE, () -> createSealedSegmentSizeMapShardIfAbsent(shard, context).thenCompose(v -> getSealedSegmentSizesMapShardData(shard, context))).thenCompose(mapShardData -> {
            SealedSegmentsMapShard mapShard = mapShardData.getObject();
            segments.forEach(z -> mapShard.addSealedSegmentSize(z, sealedSegmentSizes.get(z)));
            return updateSealedSegmentSizesMapShardData(shard, new VersionedMetadata<>(mapShard, mapShardData.getVersion()), context);
        });
    }).collect(Collectors.toList()));
}

Example 2 with SealedSegmentsMapShard

use of io.pravega.controller.store.stream.records.SealedSegmentsMapShard in project pravega by pravega.

the class StreamMetadataStoreTest method testSealedSegmentSizeMapShard.

@Test(timeout = 30000)
public void testSealedSegmentSizeMapShard() throws Exception {
    String scope = "sealedMap";
    String stream = "sealedMap";
    createAndScaleStream(store, scope, stream, 2);
    SealedSegmentsMapShard shard = store.getSealedSegmentSizeMapShard(scope, stream, 0, null, executor).join();
    assertEquals(shard.getSize(NameUtils.computeSegmentId(0, 0)).longValue(), 0L);
    assertEquals(shard.getSize(NameUtils.computeSegmentId(1, 1)).longValue(), 1L);
    assertNull(shard.getSize(NameUtils.computeSegmentId(2, 2)));
}

Example 3 with SealedSegmentsMapShard

use of io.pravega.controller.store.stream.records.SealedSegmentsMapShard in project pravega by pravega.

the class StreamTestBase method testSealedSegmentSizesMapShards.

/**
 * Stream history.
 * epoch0 = 0, 1, 2, 3, 4
 * epoch1 = 5, 1, 2, 3, 4
 * epoch2 = 5, 6, 2, 3, 4
 * epoch3 = 5, 6, 7, 3, 4
 * epoch4 = 5, 6, 7, 8, 4
 * epoch5 = 5, 6, 7, 8, 9
 * epoch6 = 0`, 1`, 2`, 3`, 4`
 * epoch7 = 5`, 6`, 7`, 8`, 9`
 * epoch8 = 10, 6`, 7`, 8`, 9`
 * epoch9 = 10, 11, 7`, 8`, 9`
 * epoch10 = 10, 11, 12, 8`, 9`
 * epoch11 = 10, 11, 12, 13, 9`
 * epoch12 = 10, 11, 12, 13, 14
 */
@Test(timeout = 30000L)
public void testSealedSegmentSizesMapShards() {
    String scope = "sealedSizeTest";
    String name = "sealedSizeTest";
    int startingSegmentNumber = new Random().nextInt(2000);
    PersistentStreamBase stream = createScaleAndRollStreamForMultiChunkTests(name, scope, startingSegmentNumber, System::currentTimeMillis);
    OperationContext context = getContext();
    SealedSegmentsMapShard shard0 = stream.getSealedSegmentSizeMapShard(0, context).join();
    // 5 segments created in epoch 0 and 1 segment in epoch 1
    assertTrue(shard0.getSealedSegmentsSizeMap().keySet().stream().allMatch(x -> getEpoch(x) == 0 || getEpoch(x) == 1));
    assertEquals(5, shard0.getSealedSegmentsSizeMap().keySet().stream().filter(x -> getEpoch(x) == 0).collect(Collectors.toList()).size());
    assertEquals(1, shard0.getSealedSegmentsSizeMap().keySet().stream().filter(x -> getEpoch(x) == 1).collect(Collectors.toList()).size());
    // 1 segment created in epoch 2 and 1 segment in epoch 3
    SealedSegmentsMapShard shard1 = stream.getSealedSegmentSizeMapShard(1, context).join();
    assertTrue(shard1.getSealedSegmentsSizeMap().keySet().stream().allMatch(x -> getEpoch(x) == 2 || getEpoch(x) == 3));
    assertEquals(1, shard1.getSealedSegmentsSizeMap().keySet().stream().filter(x -> getEpoch(x) == 2).collect(Collectors.toList()).size());
    assertEquals(1, shard1.getSealedSegmentsSizeMap().keySet().stream().filter(x -> getEpoch(x) == 3).collect(Collectors.toList()).size());
    // 1 segment created in epoch 3 and 1 segment in epoch 4
    SealedSegmentsMapShard shard2 = stream.getSealedSegmentSizeMapShard(2, context).join();
    assertTrue(shard2.getSealedSegmentsSizeMap().keySet().stream().allMatch(x -> getEpoch(x) == 4 || getEpoch(x) == 5));
    assertEquals(1, shard2.getSealedSegmentsSizeMap().keySet().stream().filter(x -> getEpoch(x) == 4).count());
    assertEquals(1, shard2.getSealedSegmentsSizeMap().keySet().stream().filter(x -> getEpoch(x) == 5).count());
    // rolling transaction, 10 segments created across two epochs mapped to the shard
    SealedSegmentsMapShard shard3 = stream.getSealedSegmentSizeMapShard(3, context).join();
    assertTrue(shard3.getSealedSegmentsSizeMap().keySet().stream().allMatch(x -> getEpoch(x) == 6 || getEpoch(x) == 7));
    assertEquals(5, shard3.getSealedSegmentsSizeMap().keySet().stream().filter(x -> getEpoch(x) == 6).count());
    assertEquals(5, shard3.getSealedSegmentsSizeMap().keySet().stream().filter(x -> getEpoch(x) == 7).count());
    // 1 segment created in epoch 8 and 1 segment in epoch 9 but they are not sealed yet
    SealedSegmentsMapShard shard4 = stream.getSealedSegmentSizeMapShard(4, context).join();
    assertTrue(shard4.getSealedSegmentsSizeMap().isEmpty());
    // 1 segment created in epoch 10 and 1 segment in epoch 11 but they are not sealed yet
    SealedSegmentsMapShard shard5 = stream.getSealedSegmentSizeMapShard(5, context).join();
    assertTrue(shard5.getSealedSegmentsSizeMap().isEmpty());
    // 1 segment created in epoch 12 but nothing is sealed yet
    SealedSegmentsMapShard shard6 = stream.getSealedSegmentSizeMapShard(6, context).join();
    assertTrue(shard6.getSealedSegmentsSizeMap().isEmpty());
    // now seal all of them again
    EpochRecord activeEpoch = stream.getActiveEpoch(true, context).join();
    List<Map.Entry<Double, Double>> newRanges = new LinkedList<>();
    newRanges.add(new AbstractMap.SimpleEntry<>(0.0, 1.0));
    Map<Long, Long> sealedSizeMap = new HashMap<>();
    activeEpoch.getSegments().forEach(x -> sealedSizeMap.put(x.segmentId(), 100L));
    scaleStream(stream, System.currentTimeMillis(), Lists.newArrayList(activeEpoch.getSegmentIds()), newRanges, sealedSizeMap);
    // 1 segment created in epoch 8 and 1 segment in epoch 9
    shard4 = stream.getSealedSegmentSizeMapShard(4, context).join();
    assertTrue(shard4.getSealedSegmentsSizeMap().keySet().stream().allMatch(x -> getEpoch(x) == 8 || getEpoch(x) == 9));
    assertEquals(1, shard4.getSealedSegmentsSizeMap().keySet().stream().filter(x -> getEpoch(x) == 8).collect(Collectors.toList()).size());
    assertEquals(1, shard4.getSealedSegmentsSizeMap().keySet().stream().filter(x -> getEpoch(x) == 9).collect(Collectors.toList()).size());
    // 1 segment created in epoch 10 and 1 segment in epoch 11
    shard5 = stream.getSealedSegmentSizeMapShard(5, context).join();
    assertTrue(shard5.getSealedSegmentsSizeMap().keySet().stream().allMatch(x -> getEpoch(x) == 10 || getEpoch(x) == 11));
    assertEquals(1, shard5.getSealedSegmentsSizeMap().keySet().stream().filter(x -> getEpoch(x) == 10).collect(Collectors.toList()).size());
    assertEquals(1, shard5.getSealedSegmentsSizeMap().keySet().stream().filter(x -> getEpoch(x) == 11).collect(Collectors.toList()).size());
    // 1 segment created in epoch 12
    shard6 = stream.getSealedSegmentSizeMapShard(6, context).join();
    assertTrue(shard6.getSealedSegmentsSizeMap().keySet().stream().allMatch(x -> getEpoch(x) == 12));
    assertEquals(1, shard6.getSealedSegmentsSizeMap().keySet().stream().filter(x -> getEpoch(x) == 12).collect(Collectors.toList()).size());
}

Example 4 with SealedSegmentsMapShard

use of io.pravega.controller.store.stream.records.SealedSegmentsMapShard in project pravega by pravega.

the class ControllerMetadataJsonSerializerTest method testSealedSegmentsMapShard.

@Test
public void testSealedSegmentsMapShard() {
    Map<Long, Long> map = new HashMap<>();
    map.put(0L, 0L);
    map.put(1L, 0L);
    map.put(2L, 0L);
    SealedSegmentsMapShard record = SealedSegmentsMapShard.builder().sealedSegmentsSizeMap(map).build();
    testRecordSerialization(record, SealedSegmentsMapShard.class);
}