Search in sources :

Example 1 with TestSnapshotWriter

use of com.alipay.sofa.jraft.rhea.storage.TestSnapshotWriter in project sofa-jraft by sofastack.

the class RocksKVStoreTest method multiGroupSnapshotTest.

public void multiGroupSnapshotTest() throws Exception {
    final File backupDir = new File("multi-backup");
    if (backupDir.exists()) {
        FileUtils.deleteDirectory(backupDir);
    }
    final List<Region> regions = Lists.newArrayList();
    regions.add(new Region(1, makeKey("0"), makeKey("1"), null, null));
    regions.add(new Region(2, makeKey("1"), makeKey("2"), null, null));
    regions.add(new Region(3, makeKey("2"), makeKey("3"), null, null));
    regions.add(new Region(4, makeKey("3"), makeKey("4"), null, null));
    regions.add(new Region(5, makeKey("4"), makeKey("5"), null, null));
    for (int i = 0; i < 5; i++) {
        final String v = String.valueOf(i);
        this.kvStore.put(makeKey(v), makeValue(v), null);
    }
    for (int i = 0; i < 5; i++) {
        this.kvStore.getSequence(makeKey(i + "_seq_test"), 10, null);
    }
    KVStoreSnapshotFile kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
    final ExecutorService snapshotPool = StoreEngineHelper.createSnapshotExecutor(1, 5);
    final List<TestSnapshotWriter> writers = Lists.newArrayList();
    for (int i = 0; i < 4; i++) {
        final Path p = Paths.get(backupDir.getAbsolutePath(), String.valueOf(i));
        final TestSnapshotWriter snapshotWriter = new TestSnapshotWriter(p.toString());
        writers.add(snapshotWriter);
        final CountDownLatch latch = new CountDownLatch(1);
        final Closure done = status -> {
            assertTrue(status.isOk());
            latch.countDown();
        };
        kvStoreSnapshotFile.save(snapshotWriter, regions.get(i), done, snapshotPool);
        latch.await();
        final LocalFileMeta meta = (LocalFileMeta) snapshotWriter.getFileMeta(SNAPSHOT_ARCHIVE);
        assertNotNull(meta);
    }
    this.kvStore.shutdown();
    FileUtils.deleteDirectory(new File(this.tempPath));
    FileUtils.forceMkdir(new File(this.tempPath));
    this.kvStore = new RocksRawKVStore();
    this.kvStore.init(this.dbOptions);
    kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
    for (int i = 0; i < 4; i++) {
        final Path p = Paths.get(backupDir.getAbsolutePath(), String.valueOf(i));
        final TestSnapshotReader snapshotReader = new TestSnapshotReader(writers.get(i).metaTable, p.toString());
        final boolean ret = kvStoreSnapshotFile.load(snapshotReader, regions.get(i));
        assertTrue(ret);
    }
    for (int i = 0; i < 4; i++) {
        final String v = String.valueOf(i);
        final byte[] seqKey = makeKey(i + "_seq_test");
        assertArrayEquals(makeValue(v), get(makeKey(v)));
        final Sequence sequence = new SyncKVStore<Sequence>() {

            @Override
            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
                kvStore.getSequence(seqKey, 10, closure);
            }
        }.apply(this.kvStore);
        assertEquals(10L, sequence.getStartValue());
        assertEquals(20L, sequence.getEndValue());
    }
    assertNull(get(makeKey("5")));
    final Sequence sequence = new SyncKVStore<Sequence>() {

        @Override
        public void execute(RawKVStore kvStore, KVStoreClosure closure) {
            kvStore.getSequence(makeKey("4_seq_test"), 10, closure);
        }
    }.apply(this.kvStore);
    assertEquals(0L, sequence.getStartValue());
    FileUtils.deleteDirectory(backupDir);
    ExecutorServiceHelper.shutdownAndAwaitTermination(snapshotPool);
}
Also used : Path(java.nio.file.Path) StoreEngineHelper(com.alipay.sofa.jraft.rhea.StoreEngineHelper) Region(com.alipay.sofa.jraft.rhea.metadata.Region) TestSnapshotReader(com.alipay.sofa.jraft.rhea.storage.TestSnapshotReader) SyncKVStore(com.alipay.sofa.jraft.rhea.storage.SyncKVStore) Lists(com.alipay.sofa.jraft.rhea.util.Lists) SstColumnFamily(com.alipay.sofa.jraft.rhea.storage.SstColumnFamily) Map(java.util.Map) After(org.junit.After) KVStoreSnapshotFileFactory(com.alipay.sofa.jraft.rhea.storage.KVStoreSnapshotFileFactory) Method(java.lang.reflect.Method) Path(java.nio.file.Path) DistributedLock(com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock) EnumMap(java.util.EnumMap) KVStoreClosure(com.alipay.sofa.jraft.rhea.storage.KVStoreClosure) RawKVStore(com.alipay.sofa.jraft.rhea.storage.RawKVStore) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) KVOperation(com.alipay.sofa.jraft.rhea.storage.KVOperation) Assert.assertFalse(org.junit.Assert.assertFalse) RocksDBOptions(com.alipay.sofa.jraft.rhea.options.RocksDBOptions) TestClosure(com.alipay.sofa.jraft.rhea.storage.TestClosure) KVState(com.alipay.sofa.jraft.rhea.storage.KVState) BaseKVStoreClosure(com.alipay.sofa.jraft.rhea.storage.BaseKVStoreClosure) KVStoreSnapshotFile(com.alipay.sofa.jraft.rhea.storage.KVStoreSnapshotFile) KeyValueTool.makeKey(com.alipay.sofa.jraft.rhea.KeyValueTool.makeKey) Closure(com.alipay.sofa.jraft.Closure) KVIterator(com.alipay.sofa.jraft.rhea.storage.KVIterator) Assert.assertArrayEquals(org.junit.Assert.assertArrayEquals) ExecutorServiceHelper(com.alipay.sofa.jraft.util.ExecutorServiceHelper) LocalLock(com.alipay.sofa.jraft.rhea.storage.LocalLock) ExecutorService(java.util.concurrent.ExecutorService) Before(org.junit.Before) KVStoreAccessHelper(com.alipay.sofa.jraft.rhea.storage.KVStoreAccessHelper) TestSnapshotWriter(com.alipay.sofa.jraft.rhea.storage.TestSnapshotWriter) RocksRawKVStore(com.alipay.sofa.jraft.rhea.storage.RocksRawKVStore) Assert.assertNotNull(org.junit.Assert.assertNotNull) Assert.assertTrue(org.junit.Assert.assertTrue) IOException(java.io.IOException) FileUtils(org.apache.commons.io.FileUtils) Test(org.junit.Test) Status(com.alipay.sofa.jraft.Status) ByteArray(com.alipay.sofa.jraft.rhea.util.ByteArray) File(java.io.File) TimeUnit(java.util.concurrent.TimeUnit) Assert.assertNull(org.junit.Assert.assertNull) Paths(java.nio.file.Paths) LocalFileMeta(com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta) Sequence(com.alipay.sofa.jraft.rhea.storage.Sequence) KVEntry(com.alipay.sofa.jraft.rhea.storage.KVEntry) KVStateOutputList(com.alipay.sofa.jraft.rhea.storage.KVStateOutputList) KeyValueTool.makeValue(com.alipay.sofa.jraft.rhea.KeyValueTool.makeValue) Assert.assertEquals(org.junit.Assert.assertEquals) BytesUtil(com.alipay.sofa.jraft.util.BytesUtil) KVStoreClosure(com.alipay.sofa.jraft.rhea.storage.KVStoreClosure) TestClosure(com.alipay.sofa.jraft.rhea.storage.TestClosure) BaseKVStoreClosure(com.alipay.sofa.jraft.rhea.storage.BaseKVStoreClosure) Closure(com.alipay.sofa.jraft.Closure) RocksRawKVStore(com.alipay.sofa.jraft.rhea.storage.RocksRawKVStore) TestSnapshotReader(com.alipay.sofa.jraft.rhea.storage.TestSnapshotReader) Sequence(com.alipay.sofa.jraft.rhea.storage.Sequence) CountDownLatch(java.util.concurrent.CountDownLatch) RawKVStore(com.alipay.sofa.jraft.rhea.storage.RawKVStore) RocksRawKVStore(com.alipay.sofa.jraft.rhea.storage.RocksRawKVStore) ExecutorService(java.util.concurrent.ExecutorService) Region(com.alipay.sofa.jraft.rhea.metadata.Region) KVStoreSnapshotFile(com.alipay.sofa.jraft.rhea.storage.KVStoreSnapshotFile) KVStoreClosure(com.alipay.sofa.jraft.rhea.storage.KVStoreClosure) BaseKVStoreClosure(com.alipay.sofa.jraft.rhea.storage.BaseKVStoreClosure) LocalFileMeta(com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta) KVStoreSnapshotFile(com.alipay.sofa.jraft.rhea.storage.KVStoreSnapshotFile) File(java.io.File) TestSnapshotWriter(com.alipay.sofa.jraft.rhea.storage.TestSnapshotWriter)

Example 2 with TestSnapshotWriter

use of com.alipay.sofa.jraft.rhea.storage.TestSnapshotWriter in project sofa-jraft by sofastack.

the class MemoryKVStoreTest method snapshotTest.

@Test
public void snapshotTest() throws Exception {
    final File backupDir = new File("backup");
    if (backupDir.exists()) {
        FileUtils.deleteDirectory(backupDir);
    }
    FileUtils.forceMkdir(backupDir);
    for (int i = 0; i < 100000; i++) {
        final String v = String.valueOf(i);
        this.kvStore.put(makeKey(v), makeValue(v), null);
    }
    for (int i = 0; i < 10000; i++) {
        this.kvStore.getSequence(makeKey((i % 100) + "seq_test"), 10, null);
    }
    final Region region = new Region();
    KVStoreSnapshotFile kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
    final ExecutorService snapshotPool = StoreEngineHelper.createSnapshotExecutor(1, 2);
    final TestSnapshotWriter snapshotWriter = new TestSnapshotWriter(backupDir.getAbsolutePath());
    final CountDownLatch latch = new CountDownLatch(1);
    final Closure done = status -> {
        assertTrue(status.isOk());
        latch.countDown();
    };
    kvStoreSnapshotFile.save(snapshotWriter, region, done, snapshotPool);
    latch.await();
    final LocalFileMeta meta = (LocalFileMeta) snapshotWriter.getFileMeta(SNAPSHOT_ARCHIVE);
    assertNotNull(meta);
    assertNotNull(get(makeKey("1")));
    this.kvStore.put(makeKey("100001"), makeValue("100001"), null);
    assertNotNull(get(makeKey("100001")));
    this.kvStore.shutdown();
    this.kvStore = new MemoryRawKVStore();
    final MemoryDBOptions dbOpts = new MemoryDBOptions();
    this.kvStore.init(dbOpts);
    assertNull(get(makeKey("1")));
    final TestSnapshotReader snapshotReader = new TestSnapshotReader(snapshotWriter.metaTable, backupDir.getAbsolutePath());
    kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
    final boolean ret = kvStoreSnapshotFile.load(snapshotReader, region);
    assertTrue(ret);
    for (int i = 0; i < 100000; i++) {
        final String v = String.valueOf(i);
        assertArrayEquals(makeValue(v), get(makeKey(v)));
    }
    // Key[100001] is put after the snapshot, so key[100001] should not exist
    assertNull(get(makeKey("100001")));
    FileUtils.deleteDirectory(backupDir);
    ExecutorServiceHelper.shutdownAndAwaitTermination(snapshotPool);
}
Also used : StoreEngineHelper(com.alipay.sofa.jraft.rhea.StoreEngineHelper) Region(com.alipay.sofa.jraft.rhea.metadata.Region) TestSnapshotReader(com.alipay.sofa.jraft.rhea.storage.TestSnapshotReader) SyncKVStore(com.alipay.sofa.jraft.rhea.storage.SyncKVStore) TestClosure(com.alipay.sofa.jraft.rhea.storage.TestClosure) Lists(com.alipay.sofa.jraft.rhea.util.Lists) BaseKVStoreClosure(com.alipay.sofa.jraft.rhea.storage.BaseKVStoreClosure) KVStoreSnapshotFile(com.alipay.sofa.jraft.rhea.storage.KVStoreSnapshotFile) KeyValueTool.makeKey(com.alipay.sofa.jraft.rhea.KeyValueTool.makeKey) Closure(com.alipay.sofa.jraft.Closure) Map(java.util.Map) After(org.junit.After) KVIterator(com.alipay.sofa.jraft.rhea.storage.KVIterator) Assert.assertArrayEquals(org.junit.Assert.assertArrayEquals) ExecutorServiceHelper(com.alipay.sofa.jraft.util.ExecutorServiceHelper) KVStoreSnapshotFileFactory(com.alipay.sofa.jraft.rhea.storage.KVStoreSnapshotFileFactory) LocalLock(com.alipay.sofa.jraft.rhea.storage.LocalLock) Method(java.lang.reflect.Method) Path(java.nio.file.Path) ExecutorService(java.util.concurrent.ExecutorService) Before(org.junit.Before) DistributedLock(com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock) MemoryDBOptions(com.alipay.sofa.jraft.rhea.options.MemoryDBOptions) TestSnapshotWriter(com.alipay.sofa.jraft.rhea.storage.TestSnapshotWriter) Assert.assertNotNull(org.junit.Assert.assertNotNull) KVStoreClosure(com.alipay.sofa.jraft.rhea.storage.KVStoreClosure) Assert.assertTrue(org.junit.Assert.assertTrue) FileUtils(org.apache.commons.io.FileUtils) Test(org.junit.Test) Status(com.alipay.sofa.jraft.Status) ByteArray(com.alipay.sofa.jraft.rhea.util.ByteArray) RawKVStore(com.alipay.sofa.jraft.rhea.storage.RawKVStore) File(java.io.File) TimeUnit(java.util.concurrent.TimeUnit) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) Assert.assertNull(org.junit.Assert.assertNull) Paths(java.nio.file.Paths) LocalFileMeta(com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta) Sequence(com.alipay.sofa.jraft.rhea.storage.Sequence) Assert.assertFalse(org.junit.Assert.assertFalse) KVEntry(com.alipay.sofa.jraft.rhea.storage.KVEntry) MemoryRawKVStore(com.alipay.sofa.jraft.rhea.storage.MemoryRawKVStore) KeyValueTool.makeValue(com.alipay.sofa.jraft.rhea.KeyValueTool.makeValue) Assert.assertEquals(org.junit.Assert.assertEquals) BytesUtil(com.alipay.sofa.jraft.util.BytesUtil) TestClosure(com.alipay.sofa.jraft.rhea.storage.TestClosure) BaseKVStoreClosure(com.alipay.sofa.jraft.rhea.storage.BaseKVStoreClosure) Closure(com.alipay.sofa.jraft.Closure) KVStoreClosure(com.alipay.sofa.jraft.rhea.storage.KVStoreClosure) MemoryDBOptions(com.alipay.sofa.jraft.rhea.options.MemoryDBOptions) TestSnapshotReader(com.alipay.sofa.jraft.rhea.storage.TestSnapshotReader) CountDownLatch(java.util.concurrent.CountDownLatch) MemoryRawKVStore(com.alipay.sofa.jraft.rhea.storage.MemoryRawKVStore) ExecutorService(java.util.concurrent.ExecutorService) Region(com.alipay.sofa.jraft.rhea.metadata.Region) KVStoreSnapshotFile(com.alipay.sofa.jraft.rhea.storage.KVStoreSnapshotFile) LocalFileMeta(com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta) KVStoreSnapshotFile(com.alipay.sofa.jraft.rhea.storage.KVStoreSnapshotFile) File(java.io.File) TestSnapshotWriter(com.alipay.sofa.jraft.rhea.storage.TestSnapshotWriter) Test(org.junit.Test)

Example 3 with TestSnapshotWriter

use of com.alipay.sofa.jraft.rhea.storage.TestSnapshotWriter in project sofa-jraft by sofastack.

the class MemoryKVStoreTest method multiGroupSnapshotTest.

@Test
public void multiGroupSnapshotTest() throws Exception {
    final File backupDir = new File("multi-backup");
    if (backupDir.exists()) {
        FileUtils.deleteDirectory(backupDir);
    }
    final List<Region> regions = Lists.newArrayList();
    regions.add(new Region(1, makeKey("0"), makeKey("1"), null, null));
    regions.add(new Region(2, makeKey("1"), makeKey("2"), null, null));
    regions.add(new Region(3, makeKey("2"), makeKey("3"), null, null));
    regions.add(new Region(4, makeKey("3"), makeKey("4"), null, null));
    regions.add(new Region(5, makeKey("4"), makeKey("5"), null, null));
    for (int i = 0; i < 5; i++) {
        final String v = String.valueOf(i);
        this.kvStore.put(makeKey(v), makeValue(v), null);
    }
    for (int i = 0; i < 5; i++) {
        this.kvStore.getSequence(makeKey(i + "_seq_test"), 10, null);
    }
    KVStoreSnapshotFile kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
    final ExecutorService snapshotPool = StoreEngineHelper.createSnapshotExecutor(1, 2);
    final List<TestSnapshotWriter> writers = Lists.newArrayList();
    for (int i = 0; i < 4; i++) {
        final Path p = Paths.get(backupDir.getAbsolutePath(), String.valueOf(i));
        final TestSnapshotWriter snapshotWriter = new TestSnapshotWriter(p.toString());
        writers.add(snapshotWriter);
        final CountDownLatch latch = new CountDownLatch(1);
        final Closure done = status -> {
            assertTrue(status.isOk());
            latch.countDown();
        };
        kvStoreSnapshotFile.save(snapshotWriter, regions.get(i), done, snapshotPool);
        latch.await();
        final LocalFileMeta meta = (LocalFileMeta) snapshotWriter.getFileMeta(SNAPSHOT_ARCHIVE);
        assertNotNull(meta);
    }
    this.kvStore.shutdown();
    this.kvStore = new MemoryRawKVStore();
    final MemoryDBOptions dbOpts = new MemoryDBOptions();
    this.kvStore.init(dbOpts);
    kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
    for (int i = 0; i < 4; i++) {
        final Path p = Paths.get(backupDir.getAbsolutePath(), String.valueOf(i));
        final TestSnapshotReader snapshotReader = new TestSnapshotReader(writers.get(i).metaTable, p.toString());
        final boolean ret = kvStoreSnapshotFile.load(snapshotReader, regions.get(i));
        assertTrue(ret);
    }
    for (int i = 0; i < 4; i++) {
        final String v = String.valueOf(i);
        final byte[] seqKey = makeKey(i + "_seq_test");
        assertArrayEquals(makeValue(v), get(makeKey(v)));
        final Sequence sequence = new SyncKVStore<Sequence>() {

            @Override
            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
                kvStore.getSequence(seqKey, 10, closure);
            }
        }.apply(this.kvStore);
        assertEquals(10L, sequence.getStartValue());
        assertEquals(20L, sequence.getEndValue());
    }
    assertNull(get(makeKey("5")));
    final Sequence sequence = new SyncKVStore<Sequence>() {

        @Override
        public void execute(RawKVStore kvStore, KVStoreClosure closure) {
            kvStore.getSequence(makeKey("4_seq_test"), 10, closure);
        }
    }.apply(this.kvStore);
    assertEquals(0L, sequence.getStartValue());
    FileUtils.deleteDirectory(backupDir);
    ExecutorServiceHelper.shutdownAndAwaitTermination(snapshotPool);
}
Also used : Path(java.nio.file.Path) StoreEngineHelper(com.alipay.sofa.jraft.rhea.StoreEngineHelper) Region(com.alipay.sofa.jraft.rhea.metadata.Region) TestSnapshotReader(com.alipay.sofa.jraft.rhea.storage.TestSnapshotReader) SyncKVStore(com.alipay.sofa.jraft.rhea.storage.SyncKVStore) TestClosure(com.alipay.sofa.jraft.rhea.storage.TestClosure) Lists(com.alipay.sofa.jraft.rhea.util.Lists) BaseKVStoreClosure(com.alipay.sofa.jraft.rhea.storage.BaseKVStoreClosure) KVStoreSnapshotFile(com.alipay.sofa.jraft.rhea.storage.KVStoreSnapshotFile) KeyValueTool.makeKey(com.alipay.sofa.jraft.rhea.KeyValueTool.makeKey) Closure(com.alipay.sofa.jraft.Closure) Map(java.util.Map) After(org.junit.After) KVIterator(com.alipay.sofa.jraft.rhea.storage.KVIterator) Assert.assertArrayEquals(org.junit.Assert.assertArrayEquals) ExecutorServiceHelper(com.alipay.sofa.jraft.util.ExecutorServiceHelper) KVStoreSnapshotFileFactory(com.alipay.sofa.jraft.rhea.storage.KVStoreSnapshotFileFactory) LocalLock(com.alipay.sofa.jraft.rhea.storage.LocalLock) Method(java.lang.reflect.Method) Path(java.nio.file.Path) ExecutorService(java.util.concurrent.ExecutorService) Before(org.junit.Before) DistributedLock(com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock) MemoryDBOptions(com.alipay.sofa.jraft.rhea.options.MemoryDBOptions) TestSnapshotWriter(com.alipay.sofa.jraft.rhea.storage.TestSnapshotWriter) Assert.assertNotNull(org.junit.Assert.assertNotNull) KVStoreClosure(com.alipay.sofa.jraft.rhea.storage.KVStoreClosure) Assert.assertTrue(org.junit.Assert.assertTrue) FileUtils(org.apache.commons.io.FileUtils) Test(org.junit.Test) Status(com.alipay.sofa.jraft.Status) ByteArray(com.alipay.sofa.jraft.rhea.util.ByteArray) RawKVStore(com.alipay.sofa.jraft.rhea.storage.RawKVStore) File(java.io.File) TimeUnit(java.util.concurrent.TimeUnit) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) Assert.assertNull(org.junit.Assert.assertNull) Paths(java.nio.file.Paths) LocalFileMeta(com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta) Sequence(com.alipay.sofa.jraft.rhea.storage.Sequence) Assert.assertFalse(org.junit.Assert.assertFalse) KVEntry(com.alipay.sofa.jraft.rhea.storage.KVEntry) MemoryRawKVStore(com.alipay.sofa.jraft.rhea.storage.MemoryRawKVStore) KeyValueTool.makeValue(com.alipay.sofa.jraft.rhea.KeyValueTool.makeValue) Assert.assertEquals(org.junit.Assert.assertEquals) BytesUtil(com.alipay.sofa.jraft.util.BytesUtil) TestClosure(com.alipay.sofa.jraft.rhea.storage.TestClosure) BaseKVStoreClosure(com.alipay.sofa.jraft.rhea.storage.BaseKVStoreClosure) Closure(com.alipay.sofa.jraft.Closure) KVStoreClosure(com.alipay.sofa.jraft.rhea.storage.KVStoreClosure) MemoryDBOptions(com.alipay.sofa.jraft.rhea.options.MemoryDBOptions) TestSnapshotReader(com.alipay.sofa.jraft.rhea.storage.TestSnapshotReader) Sequence(com.alipay.sofa.jraft.rhea.storage.Sequence) CountDownLatch(java.util.concurrent.CountDownLatch) MemoryRawKVStore(com.alipay.sofa.jraft.rhea.storage.MemoryRawKVStore) RawKVStore(com.alipay.sofa.jraft.rhea.storage.RawKVStore) MemoryRawKVStore(com.alipay.sofa.jraft.rhea.storage.MemoryRawKVStore) ExecutorService(java.util.concurrent.ExecutorService) Region(com.alipay.sofa.jraft.rhea.metadata.Region) KVStoreSnapshotFile(com.alipay.sofa.jraft.rhea.storage.KVStoreSnapshotFile) BaseKVStoreClosure(com.alipay.sofa.jraft.rhea.storage.BaseKVStoreClosure) KVStoreClosure(com.alipay.sofa.jraft.rhea.storage.KVStoreClosure) LocalFileMeta(com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta) KVStoreSnapshotFile(com.alipay.sofa.jraft.rhea.storage.KVStoreSnapshotFile) File(java.io.File) TestSnapshotWriter(com.alipay.sofa.jraft.rhea.storage.TestSnapshotWriter) Test(org.junit.Test)

Example 4 with TestSnapshotWriter

use of com.alipay.sofa.jraft.rhea.storage.TestSnapshotWriter in project sofa-jraft by sofastack.

the class RocksKVStoreTest method snapshotTest.

public void snapshotTest() throws Exception {
    final File backupDir = new File("backup");
    if (backupDir.exists()) {
        FileUtils.deleteDirectory(backupDir);
    }
    FileUtils.forceMkdir(backupDir);
    for (int i = 0; i < 100000; i++) {
        final String v = String.valueOf(i);
        this.kvStore.put(makeKey(v), makeValue(v), null);
    }
    final Region region = new Region();
    KVStoreSnapshotFile kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
    final ExecutorService snapshotPool = StoreEngineHelper.createSnapshotExecutor(1, 2);
    final TestSnapshotWriter snapshotWriter = new TestSnapshotWriter(backupDir.getAbsolutePath());
    final CountDownLatch latch = new CountDownLatch(1);
    final Closure done = status -> {
        assertTrue(status.isOk());
        latch.countDown();
    };
    kvStoreSnapshotFile.save(snapshotWriter, region, done, snapshotPool);
    latch.await();
    final LocalFileMeta meta = (LocalFileMeta) snapshotWriter.getFileMeta(SNAPSHOT_ARCHIVE);
    assertNotNull(meta);
    assertNotNull(get(makeKey("1")));
    this.kvStore.put(makeKey("100001"), makeValue("100001"), null);
    assertNotNull(get(makeKey("100001")));
    this.kvStore.shutdown();
    FileUtils.deleteDirectory(new File(this.tempPath));
    FileUtils.forceMkdir(new File(this.tempPath));
    this.kvStore = new RocksRawKVStore();
    this.kvStore.init(this.dbOptions);
    assertNull(get(makeKey("1")));
    final TestSnapshotReader snapshotReader = new TestSnapshotReader(snapshotWriter.metaTable, backupDir.getAbsolutePath());
    kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
    final boolean ret = kvStoreSnapshotFile.load(snapshotReader, region);
    assertTrue(ret);
    for (int i = 0; i < 100000; i++) {
        final String v = String.valueOf(i);
        assertArrayEquals(makeValue(v), get(makeKey(v)));
    }
    // key[100001] is put after the snapshot, so key[100001] should not exist.
    assertNull(get(makeKey("100001")));
    FileUtils.deleteDirectory(backupDir);
    ExecutorServiceHelper.shutdownAndAwaitTermination(snapshotPool);
}
Also used : StoreEngineHelper(com.alipay.sofa.jraft.rhea.StoreEngineHelper) Region(com.alipay.sofa.jraft.rhea.metadata.Region) TestSnapshotReader(com.alipay.sofa.jraft.rhea.storage.TestSnapshotReader) SyncKVStore(com.alipay.sofa.jraft.rhea.storage.SyncKVStore) Lists(com.alipay.sofa.jraft.rhea.util.Lists) SstColumnFamily(com.alipay.sofa.jraft.rhea.storage.SstColumnFamily) Map(java.util.Map) After(org.junit.After) KVStoreSnapshotFileFactory(com.alipay.sofa.jraft.rhea.storage.KVStoreSnapshotFileFactory) Method(java.lang.reflect.Method) Path(java.nio.file.Path) DistributedLock(com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock) EnumMap(java.util.EnumMap) KVStoreClosure(com.alipay.sofa.jraft.rhea.storage.KVStoreClosure) RawKVStore(com.alipay.sofa.jraft.rhea.storage.RawKVStore) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) KVOperation(com.alipay.sofa.jraft.rhea.storage.KVOperation) Assert.assertFalse(org.junit.Assert.assertFalse) RocksDBOptions(com.alipay.sofa.jraft.rhea.options.RocksDBOptions) TestClosure(com.alipay.sofa.jraft.rhea.storage.TestClosure) KVState(com.alipay.sofa.jraft.rhea.storage.KVState) BaseKVStoreClosure(com.alipay.sofa.jraft.rhea.storage.BaseKVStoreClosure) KVStoreSnapshotFile(com.alipay.sofa.jraft.rhea.storage.KVStoreSnapshotFile) KeyValueTool.makeKey(com.alipay.sofa.jraft.rhea.KeyValueTool.makeKey) Closure(com.alipay.sofa.jraft.Closure) KVIterator(com.alipay.sofa.jraft.rhea.storage.KVIterator) Assert.assertArrayEquals(org.junit.Assert.assertArrayEquals) ExecutorServiceHelper(com.alipay.sofa.jraft.util.ExecutorServiceHelper) LocalLock(com.alipay.sofa.jraft.rhea.storage.LocalLock) ExecutorService(java.util.concurrent.ExecutorService) Before(org.junit.Before) KVStoreAccessHelper(com.alipay.sofa.jraft.rhea.storage.KVStoreAccessHelper) TestSnapshotWriter(com.alipay.sofa.jraft.rhea.storage.TestSnapshotWriter) RocksRawKVStore(com.alipay.sofa.jraft.rhea.storage.RocksRawKVStore) Assert.assertNotNull(org.junit.Assert.assertNotNull) Assert.assertTrue(org.junit.Assert.assertTrue) IOException(java.io.IOException) FileUtils(org.apache.commons.io.FileUtils) Test(org.junit.Test) Status(com.alipay.sofa.jraft.Status) ByteArray(com.alipay.sofa.jraft.rhea.util.ByteArray) File(java.io.File) TimeUnit(java.util.concurrent.TimeUnit) Assert.assertNull(org.junit.Assert.assertNull) Paths(java.nio.file.Paths) LocalFileMeta(com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta) Sequence(com.alipay.sofa.jraft.rhea.storage.Sequence) KVEntry(com.alipay.sofa.jraft.rhea.storage.KVEntry) KVStateOutputList(com.alipay.sofa.jraft.rhea.storage.KVStateOutputList) KeyValueTool.makeValue(com.alipay.sofa.jraft.rhea.KeyValueTool.makeValue) Assert.assertEquals(org.junit.Assert.assertEquals) BytesUtil(com.alipay.sofa.jraft.util.BytesUtil) KVStoreClosure(com.alipay.sofa.jraft.rhea.storage.KVStoreClosure) TestClosure(com.alipay.sofa.jraft.rhea.storage.TestClosure) BaseKVStoreClosure(com.alipay.sofa.jraft.rhea.storage.BaseKVStoreClosure) Closure(com.alipay.sofa.jraft.Closure) RocksRawKVStore(com.alipay.sofa.jraft.rhea.storage.RocksRawKVStore) TestSnapshotReader(com.alipay.sofa.jraft.rhea.storage.TestSnapshotReader) CountDownLatch(java.util.concurrent.CountDownLatch) ExecutorService(java.util.concurrent.ExecutorService) Region(com.alipay.sofa.jraft.rhea.metadata.Region) KVStoreSnapshotFile(com.alipay.sofa.jraft.rhea.storage.KVStoreSnapshotFile) LocalFileMeta(com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta) KVStoreSnapshotFile(com.alipay.sofa.jraft.rhea.storage.KVStoreSnapshotFile) File(java.io.File) TestSnapshotWriter(com.alipay.sofa.jraft.rhea.storage.TestSnapshotWriter)

Aggregations

Closure (com.alipay.sofa.jraft.Closure)4 Status (com.alipay.sofa.jraft.Status)4 LocalFileMeta (com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta)4 KeyValueTool.makeKey (com.alipay.sofa.jraft.rhea.KeyValueTool.makeKey)4 KeyValueTool.makeValue (com.alipay.sofa.jraft.rhea.KeyValueTool.makeValue)4 StoreEngineHelper (com.alipay.sofa.jraft.rhea.StoreEngineHelper)4 Region (com.alipay.sofa.jraft.rhea.metadata.Region)4 BaseKVStoreClosure (com.alipay.sofa.jraft.rhea.storage.BaseKVStoreClosure)4 KVEntry (com.alipay.sofa.jraft.rhea.storage.KVEntry)4 KVIterator (com.alipay.sofa.jraft.rhea.storage.KVIterator)4 KVStoreClosure (com.alipay.sofa.jraft.rhea.storage.KVStoreClosure)4 KVStoreSnapshotFile (com.alipay.sofa.jraft.rhea.storage.KVStoreSnapshotFile)4 KVStoreSnapshotFileFactory (com.alipay.sofa.jraft.rhea.storage.KVStoreSnapshotFileFactory)4 LocalLock (com.alipay.sofa.jraft.rhea.storage.LocalLock)4 RawKVStore (com.alipay.sofa.jraft.rhea.storage.RawKVStore)4 Sequence (com.alipay.sofa.jraft.rhea.storage.Sequence)4 SyncKVStore (com.alipay.sofa.jraft.rhea.storage.SyncKVStore)4 TestClosure (com.alipay.sofa.jraft.rhea.storage.TestClosure)4 TestSnapshotReader (com.alipay.sofa.jraft.rhea.storage.TestSnapshotReader)4 TestSnapshotWriter (com.alipay.sofa.jraft.rhea.storage.TestSnapshotWriter)4