Examples with Region com.alipay.sofa.jraft.rhea.metadata.Region used on opensource projects

Example 1 with Region

use of com.alipay.sofa.jraft.rhea.metadata.Region in project sofa-jraft by sofastack.

the class RocksKVStoreTest method multiGroupSnapshotTest.

public void multiGroupSnapshotTest() throws Exception {
    final File backupDir = new File("multi-backup");
    if (backupDir.exists()) {
        FileUtils.deleteDirectory(backupDir);
    }
    final List<Region> regions = Lists.newArrayList();
    regions.add(new Region(1, makeKey("0"), makeKey("1"), null, null));
    regions.add(new Region(2, makeKey("1"), makeKey("2"), null, null));
    regions.add(new Region(3, makeKey("2"), makeKey("3"), null, null));
    regions.add(new Region(4, makeKey("3"), makeKey("4"), null, null));
    regions.add(new Region(5, makeKey("4"), makeKey("5"), null, null));
    for (int i = 0; i < 5; i++) {
        final String v = String.valueOf(i);
        this.kvStore.put(makeKey(v), makeValue(v), null);
    }
    for (int i = 0; i < 5; i++) {
        this.kvStore.getSequence(makeKey(i + "_seq_test"), 10, null);
    }
    KVStoreSnapshotFile kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
    final ExecutorService snapshotPool = StoreEngineHelper.createSnapshotExecutor(1, 5);
    final List<TestSnapshotWriter> writers = Lists.newArrayList();
    for (int i = 0; i < 4; i++) {
        final Path p = Paths.get(backupDir.getAbsolutePath(), String.valueOf(i));
        final TestSnapshotWriter snapshotWriter = new TestSnapshotWriter(p.toString());
        writers.add(snapshotWriter);
        final CountDownLatch latch = new CountDownLatch(1);
        final Closure done = status -> {
            assertTrue(status.isOk());
            latch.countDown();
        };
        kvStoreSnapshotFile.save(snapshotWriter, regions.get(i), done, snapshotPool);
        latch.await();
        final LocalFileMeta meta = (LocalFileMeta) snapshotWriter.getFileMeta(SNAPSHOT_ARCHIVE);
        assertNotNull(meta);
    }
    this.kvStore.shutdown();
    FileUtils.deleteDirectory(new File(this.tempPath));
    FileUtils.forceMkdir(new File(this.tempPath));
    this.kvStore = new RocksRawKVStore();
    this.kvStore.init(this.dbOptions);
    kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
    for (int i = 0; i < 4; i++) {
        final Path p = Paths.get(backupDir.getAbsolutePath(), String.valueOf(i));
        final TestSnapshotReader snapshotReader = new TestSnapshotReader(writers.get(i).metaTable, p.toString());
        final boolean ret = kvStoreSnapshotFile.load(snapshotReader, regions.get(i));
        assertTrue(ret);
    }
    for (int i = 0; i < 4; i++) {
        final String v = String.valueOf(i);
        final byte[] seqKey = makeKey(i + "_seq_test");
        assertArrayEquals(makeValue(v), get(makeKey(v)));
        final Sequence sequence = new SyncKVStore<Sequence>() {

            @Override
            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
                kvStore.getSequence(seqKey, 10, closure);
            }
        }.apply(this.kvStore);
        assertEquals(10L, sequence.getStartValue());
        assertEquals(20L, sequence.getEndValue());
    }
    assertNull(get(makeKey("5")));
    final Sequence sequence = new SyncKVStore<Sequence>() {

        @Override
        public void execute(RawKVStore kvStore, KVStoreClosure closure) {
            kvStore.getSequence(makeKey("4_seq_test"), 10, closure);
        }
    }.apply(this.kvStore);
    assertEquals(0L, sequence.getStartValue());
    FileUtils.deleteDirectory(backupDir);
    ExecutorServiceHelper.shutdownAndAwaitTermination(snapshotPool);
}

Example 2 with Region

use of com.alipay.sofa.jraft.rhea.metadata.Region in project sofa-jraft by sofastack.

the class MemoryKVStoreTest method snapshotTest.

@Test
public void snapshotTest() throws Exception {
    final File backupDir = new File("backup");
    if (backupDir.exists()) {
        FileUtils.deleteDirectory(backupDir);
    }
    FileUtils.forceMkdir(backupDir);
    for (int i = 0; i < 100000; i++) {
        final String v = String.valueOf(i);
        this.kvStore.put(makeKey(v), makeValue(v), null);
    }
    for (int i = 0; i < 10000; i++) {
        this.kvStore.getSequence(makeKey((i % 100) + "seq_test"), 10, null);
    }
    final Region region = new Region();
    KVStoreSnapshotFile kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
    final ExecutorService snapshotPool = StoreEngineHelper.createSnapshotExecutor(1, 2);
    final TestSnapshotWriter snapshotWriter = new TestSnapshotWriter(backupDir.getAbsolutePath());
    final CountDownLatch latch = new CountDownLatch(1);
    final Closure done = status -> {
        assertTrue(status.isOk());
        latch.countDown();
    };
    kvStoreSnapshotFile.save(snapshotWriter, region, done, snapshotPool);
    latch.await();
    final LocalFileMeta meta = (LocalFileMeta) snapshotWriter.getFileMeta(SNAPSHOT_ARCHIVE);
    assertNotNull(meta);
    assertNotNull(get(makeKey("1")));
    this.kvStore.put(makeKey("100001"), makeValue("100001"), null);
    assertNotNull(get(makeKey("100001")));
    this.kvStore.shutdown();
    this.kvStore = new MemoryRawKVStore();
    final MemoryDBOptions dbOpts = new MemoryDBOptions();
    this.kvStore.init(dbOpts);
    assertNull(get(makeKey("1")));
    final TestSnapshotReader snapshotReader = new TestSnapshotReader(snapshotWriter.metaTable, backupDir.getAbsolutePath());
    kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
    final boolean ret = kvStoreSnapshotFile.load(snapshotReader, region);
    assertTrue(ret);
    for (int i = 0; i < 100000; i++) {
        final String v = String.valueOf(i);
        assertArrayEquals(makeValue(v), get(makeKey(v)));
    }
    // Key[100001] is put after the snapshot, so key[100001] should not exist
    assertNull(get(makeKey("100001")));
    FileUtils.deleteDirectory(backupDir);
    ExecutorServiceHelper.shutdownAndAwaitTermination(snapshotPool);
}

Example 3 with Region

use of com.alipay.sofa.jraft.rhea.metadata.Region in project sofa-jraft by sofastack.

the class MemoryKVStoreTest method multiGroupSnapshotTest.

@Test
public void multiGroupSnapshotTest() throws Exception {
    final File backupDir = new File("multi-backup");
    if (backupDir.exists()) {
        FileUtils.deleteDirectory(backupDir);
    }
    final List<Region> regions = Lists.newArrayList();
    regions.add(new Region(1, makeKey("0"), makeKey("1"), null, null));
    regions.add(new Region(2, makeKey("1"), makeKey("2"), null, null));
    regions.add(new Region(3, makeKey("2"), makeKey("3"), null, null));
    regions.add(new Region(4, makeKey("3"), makeKey("4"), null, null));
    regions.add(new Region(5, makeKey("4"), makeKey("5"), null, null));
    for (int i = 0; i < 5; i++) {
        final String v = String.valueOf(i);
        this.kvStore.put(makeKey(v), makeValue(v), null);
    }
    for (int i = 0; i < 5; i++) {
        this.kvStore.getSequence(makeKey(i + "_seq_test"), 10, null);
    }
    KVStoreSnapshotFile kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
    final ExecutorService snapshotPool = StoreEngineHelper.createSnapshotExecutor(1, 2);
    final List<TestSnapshotWriter> writers = Lists.newArrayList();
    for (int i = 0; i < 4; i++) {
        final Path p = Paths.get(backupDir.getAbsolutePath(), String.valueOf(i));
        final TestSnapshotWriter snapshotWriter = new TestSnapshotWriter(p.toString());
        writers.add(snapshotWriter);
        final CountDownLatch latch = new CountDownLatch(1);
        final Closure done = status -> {
            assertTrue(status.isOk());
            latch.countDown();
        };
        kvStoreSnapshotFile.save(snapshotWriter, regions.get(i), done, snapshotPool);
        latch.await();
        final LocalFileMeta meta = (LocalFileMeta) snapshotWriter.getFileMeta(SNAPSHOT_ARCHIVE);
        assertNotNull(meta);
    }
    this.kvStore.shutdown();
    this.kvStore = new MemoryRawKVStore();
    final MemoryDBOptions dbOpts = new MemoryDBOptions();
    this.kvStore.init(dbOpts);
    kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
    for (int i = 0; i < 4; i++) {
        final Path p = Paths.get(backupDir.getAbsolutePath(), String.valueOf(i));
        final TestSnapshotReader snapshotReader = new TestSnapshotReader(writers.get(i).metaTable, p.toString());
        final boolean ret = kvStoreSnapshotFile.load(snapshotReader, regions.get(i));
        assertTrue(ret);
    }
    for (int i = 0; i < 4; i++) {
        final String v = String.valueOf(i);
        final byte[] seqKey = makeKey(i + "_seq_test");
        assertArrayEquals(makeValue(v), get(makeKey(v)));
        final Sequence sequence = new SyncKVStore<Sequence>() {

            @Override
            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
                kvStore.getSequence(seqKey, 10, closure);
            }
        }.apply(this.kvStore);
        assertEquals(10L, sequence.getStartValue());
        assertEquals(20L, sequence.getEndValue());
    }
    assertNull(get(makeKey("5")));
    final Sequence sequence = new SyncKVStore<Sequence>() {

        @Override
        public void execute(RawKVStore kvStore, KVStoreClosure closure) {
            kvStore.getSequence(makeKey("4_seq_test"), 10, closure);
        }
    }.apply(this.kvStore);
    assertEquals(0L, sequence.getStartValue());
    FileUtils.deleteDirectory(backupDir);
    ExecutorServiceHelper.shutdownAndAwaitTermination(snapshotPool);
}

Example 4 with Region

use of com.alipay.sofa.jraft.rhea.metadata.Region in project sofa-jraft by sofastack.

the class RegionStatsValidator method readMessage.

@Override
public void readMessage(final HandlerContext ctx, final RegionPingEvent event) throws Exception {
    final MetadataStore metadataStore = event.getMetadataStore();
    final RegionHeartbeatRequest request = event.getMessage();
    final List<Pair<Region, RegionStats>> regionStatsList = request.getRegionStatsList();
    if (regionStatsList == null || regionStatsList.isEmpty()) {
        LOG.error("Empty [RegionStatsList] by event: {}.", event);
        throw Errors.INVALID_REGION_STATS.exception();
    }
    for (final Pair<Region, RegionStats> pair : regionStatsList) {
        final Region region = pair.getKey();
        if (region == null) {
            LOG.error("Empty [Region] by event: {}.", event);
            throw Errors.INVALID_REGION_STATS.exception();
        }
        final RegionEpoch regionEpoch = region.getRegionEpoch();
        if (regionEpoch == null) {
            LOG.error("Empty [RegionEpoch] by event: {}.", event);
            throw Errors.INVALID_REGION_STATS.exception();
        }
        final RegionStats regionStats = pair.getValue();
        if (regionStats == null) {
            LOG.error("Empty [RegionStats] by event: {}.", event);
            throw Errors.INVALID_REGION_STATS.exception();
        }
        final Pair<Region, RegionStats> currentRegionInfo = metadataStore.getRegionStats(request.getClusterId(), region);
        if (currentRegionInfo == null) {
            // new data
            return;
        }
        final Region currentRegion = currentRegionInfo.getKey();
        if (regionEpoch.compareTo(currentRegion.getRegionEpoch()) < 0) {
            LOG.error("The region epoch is out of date: {}.", event);
            throw Errors.REGION_HEARTBEAT_OUT_OF_DATE.exception();
        }
        final TimeInterval interval = regionStats.getInterval();
        if (interval == null) {
            LOG.error("Empty [TimeInterval] by event: {}.", event);
            throw Errors.INVALID_REGION_STATS.exception();
        }
        final TimeInterval currentInterval = currentRegionInfo.getValue().getInterval();
        if (interval.getEndTimestamp() < currentInterval.getEndTimestamp()) {
            LOG.error("The [TimeInterval] is out of date: {}.", event);
            throw Errors.REGION_HEARTBEAT_OUT_OF_DATE.exception();
        }
    }
}

Example 5 with Region

use of com.alipay.sofa.jraft.rhea.metadata.Region in project sofa-jraft by sofastack.

the class SplittingJudgeByApproximateKeysHandler method readMessage.

@Override
public void readMessage(final HandlerContext ctx, final RegionPingEvent event) throws Exception {
    if (event.isReady()) {
        return;
    }
    final MetadataStore metadataStore = event.getMetadataStore();
    final RegionHeartbeatRequest request = event.getMessage();
    final long clusterId = request.getClusterId();
    final ClusterStatsManager clusterStatsManager = ClusterStatsManager.getInstance(clusterId);
    clusterStatsManager.addOrUpdateRegionStats(request.getRegionStatsList());
    final Set<Long> stores = metadataStore.unsafeGetStoreIds(clusterId);
    if (stores == null || stores.isEmpty()) {
        return;
    }
    if (clusterStatsManager.regionSize() >= stores.size()) {
        // one store one region is perfect
        return;
    }
    final Pair<Region, RegionStats> modelWorker = clusterStatsManager.findModelWorkerRegion();
    if (!isSplitNeeded(request, modelWorker)) {
        return;
    }
    LOG.info("[Cluster: {}] model worker region is: {}.", clusterId, modelWorker);
    final Long newRegionId = metadataStore.createRegionId(clusterId);
    final Instruction.RangeSplit rangeSplit = new Instruction.RangeSplit();
    rangeSplit.setNewRegionId(newRegionId);
    final Instruction instruction = new Instruction();
    instruction.setRegion(modelWorker.getKey().copy());
    instruction.setRangeSplit(rangeSplit);
    event.addInstruction(instruction);
}