use of com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta in project sofa-jraft by sofastack.
the class RocksKVStoreTest method multiGroupSnapshotTest.
public void multiGroupSnapshotTest() throws Exception {
final File backupDir = new File("multi-backup");
if (backupDir.exists()) {
FileUtils.deleteDirectory(backupDir);
}
final List<Region> regions = Lists.newArrayList();
regions.add(new Region(1, makeKey("0"), makeKey("1"), null, null));
regions.add(new Region(2, makeKey("1"), makeKey("2"), null, null));
regions.add(new Region(3, makeKey("2"), makeKey("3"), null, null));
regions.add(new Region(4, makeKey("3"), makeKey("4"), null, null));
regions.add(new Region(5, makeKey("4"), makeKey("5"), null, null));
for (int i = 0; i < 5; i++) {
final String v = String.valueOf(i);
this.kvStore.put(makeKey(v), makeValue(v), null);
}
for (int i = 0; i < 5; i++) {
this.kvStore.getSequence(makeKey(i + "_seq_test"), 10, null);
}
KVStoreSnapshotFile kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
final ExecutorService snapshotPool = StoreEngineHelper.createSnapshotExecutor(1, 5);
final List<TestSnapshotWriter> writers = Lists.newArrayList();
for (int i = 0; i < 4; i++) {
final Path p = Paths.get(backupDir.getAbsolutePath(), String.valueOf(i));
final TestSnapshotWriter snapshotWriter = new TestSnapshotWriter(p.toString());
writers.add(snapshotWriter);
final CountDownLatch latch = new CountDownLatch(1);
final Closure done = status -> {
assertTrue(status.isOk());
latch.countDown();
};
kvStoreSnapshotFile.save(snapshotWriter, regions.get(i), done, snapshotPool);
latch.await();
final LocalFileMeta meta = (LocalFileMeta) snapshotWriter.getFileMeta(SNAPSHOT_ARCHIVE);
assertNotNull(meta);
}
this.kvStore.shutdown();
FileUtils.deleteDirectory(new File(this.tempPath));
FileUtils.forceMkdir(new File(this.tempPath));
this.kvStore = new RocksRawKVStore();
this.kvStore.init(this.dbOptions);
kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
for (int i = 0; i < 4; i++) {
final Path p = Paths.get(backupDir.getAbsolutePath(), String.valueOf(i));
final TestSnapshotReader snapshotReader = new TestSnapshotReader(writers.get(i).metaTable, p.toString());
final boolean ret = kvStoreSnapshotFile.load(snapshotReader, regions.get(i));
assertTrue(ret);
}
for (int i = 0; i < 4; i++) {
final String v = String.valueOf(i);
final byte[] seqKey = makeKey(i + "_seq_test");
assertArrayEquals(makeValue(v), get(makeKey(v)));
final Sequence sequence = new SyncKVStore<Sequence>() {
@Override
public void execute(RawKVStore kvStore, KVStoreClosure closure) {
kvStore.getSequence(seqKey, 10, closure);
}
}.apply(this.kvStore);
assertEquals(10L, sequence.getStartValue());
assertEquals(20L, sequence.getEndValue());
}
assertNull(get(makeKey("5")));
final Sequence sequence = new SyncKVStore<Sequence>() {
@Override
public void execute(RawKVStore kvStore, KVStoreClosure closure) {
kvStore.getSequence(makeKey("4_seq_test"), 10, closure);
}
}.apply(this.kvStore);
assertEquals(0L, sequence.getStartValue());
FileUtils.deleteDirectory(backupDir);
ExecutorServiceHelper.shutdownAndAwaitTermination(snapshotPool);
}
use of com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta in project sofa-jraft by sofastack.
the class MemoryKVStoreTest method snapshotTest.
@Test
public void snapshotTest() throws Exception {
final File backupDir = new File("backup");
if (backupDir.exists()) {
FileUtils.deleteDirectory(backupDir);
}
FileUtils.forceMkdir(backupDir);
for (int i = 0; i < 100000; i++) {
final String v = String.valueOf(i);
this.kvStore.put(makeKey(v), makeValue(v), null);
}
for (int i = 0; i < 10000; i++) {
this.kvStore.getSequence(makeKey((i % 100) + "seq_test"), 10, null);
}
final Region region = new Region();
KVStoreSnapshotFile kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
final ExecutorService snapshotPool = StoreEngineHelper.createSnapshotExecutor(1, 2);
final TestSnapshotWriter snapshotWriter = new TestSnapshotWriter(backupDir.getAbsolutePath());
final CountDownLatch latch = new CountDownLatch(1);
final Closure done = status -> {
assertTrue(status.isOk());
latch.countDown();
};
kvStoreSnapshotFile.save(snapshotWriter, region, done, snapshotPool);
latch.await();
final LocalFileMeta meta = (LocalFileMeta) snapshotWriter.getFileMeta(SNAPSHOT_ARCHIVE);
assertNotNull(meta);
assertNotNull(get(makeKey("1")));
this.kvStore.put(makeKey("100001"), makeValue("100001"), null);
assertNotNull(get(makeKey("100001")));
this.kvStore.shutdown();
this.kvStore = new MemoryRawKVStore();
final MemoryDBOptions dbOpts = new MemoryDBOptions();
this.kvStore.init(dbOpts);
assertNull(get(makeKey("1")));
final TestSnapshotReader snapshotReader = new TestSnapshotReader(snapshotWriter.metaTable, backupDir.getAbsolutePath());
kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
final boolean ret = kvStoreSnapshotFile.load(snapshotReader, region);
assertTrue(ret);
for (int i = 0; i < 100000; i++) {
final String v = String.valueOf(i);
assertArrayEquals(makeValue(v), get(makeKey(v)));
}
// Key[100001] is put after the snapshot, so key[100001] should not exist
assertNull(get(makeKey("100001")));
FileUtils.deleteDirectory(backupDir);
ExecutorServiceHelper.shutdownAndAwaitTermination(snapshotPool);
}
use of com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta in project sofa-jraft by sofastack.
the class MemoryKVStoreTest method multiGroupSnapshotTest.
@Test
public void multiGroupSnapshotTest() throws Exception {
final File backupDir = new File("multi-backup");
if (backupDir.exists()) {
FileUtils.deleteDirectory(backupDir);
}
final List<Region> regions = Lists.newArrayList();
regions.add(new Region(1, makeKey("0"), makeKey("1"), null, null));
regions.add(new Region(2, makeKey("1"), makeKey("2"), null, null));
regions.add(new Region(3, makeKey("2"), makeKey("3"), null, null));
regions.add(new Region(4, makeKey("3"), makeKey("4"), null, null));
regions.add(new Region(5, makeKey("4"), makeKey("5"), null, null));
for (int i = 0; i < 5; i++) {
final String v = String.valueOf(i);
this.kvStore.put(makeKey(v), makeValue(v), null);
}
for (int i = 0; i < 5; i++) {
this.kvStore.getSequence(makeKey(i + "_seq_test"), 10, null);
}
KVStoreSnapshotFile kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
final ExecutorService snapshotPool = StoreEngineHelper.createSnapshotExecutor(1, 2);
final List<TestSnapshotWriter> writers = Lists.newArrayList();
for (int i = 0; i < 4; i++) {
final Path p = Paths.get(backupDir.getAbsolutePath(), String.valueOf(i));
final TestSnapshotWriter snapshotWriter = new TestSnapshotWriter(p.toString());
writers.add(snapshotWriter);
final CountDownLatch latch = new CountDownLatch(1);
final Closure done = status -> {
assertTrue(status.isOk());
latch.countDown();
};
kvStoreSnapshotFile.save(snapshotWriter, regions.get(i), done, snapshotPool);
latch.await();
final LocalFileMeta meta = (LocalFileMeta) snapshotWriter.getFileMeta(SNAPSHOT_ARCHIVE);
assertNotNull(meta);
}
this.kvStore.shutdown();
this.kvStore = new MemoryRawKVStore();
final MemoryDBOptions dbOpts = new MemoryDBOptions();
this.kvStore.init(dbOpts);
kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
for (int i = 0; i < 4; i++) {
final Path p = Paths.get(backupDir.getAbsolutePath(), String.valueOf(i));
final TestSnapshotReader snapshotReader = new TestSnapshotReader(writers.get(i).metaTable, p.toString());
final boolean ret = kvStoreSnapshotFile.load(snapshotReader, regions.get(i));
assertTrue(ret);
}
for (int i = 0; i < 4; i++) {
final String v = String.valueOf(i);
final byte[] seqKey = makeKey(i + "_seq_test");
assertArrayEquals(makeValue(v), get(makeKey(v)));
final Sequence sequence = new SyncKVStore<Sequence>() {
@Override
public void execute(RawKVStore kvStore, KVStoreClosure closure) {
kvStore.getSequence(seqKey, 10, closure);
}
}.apply(this.kvStore);
assertEquals(10L, sequence.getStartValue());
assertEquals(20L, sequence.getEndValue());
}
assertNull(get(makeKey("5")));
final Sequence sequence = new SyncKVStore<Sequence>() {
@Override
public void execute(RawKVStore kvStore, KVStoreClosure closure) {
kvStore.getSequence(makeKey("4_seq_test"), 10, closure);
}
}.apply(this.kvStore);
assertEquals(0L, sequence.getStartValue());
FileUtils.deleteDirectory(backupDir);
ExecutorServiceHelper.shutdownAndAwaitTermination(snapshotPool);
}
use of com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta in project sofa-jraft by sofastack.
the class LocalSnapshotCopier method copyFile.
void copyFile(final String fileName) throws IOException, InterruptedException {
if (this.writer.getFileMeta(fileName) != null) {
LOG.info("Skipped downloading {}", fileName);
return;
}
if (!checkFile(fileName)) {
return;
}
final String filePath = this.writer.getPath() + File.separator + fileName;
final Path subPath = Paths.get(filePath);
if (!subPath.equals(subPath.getParent()) && !subPath.getParent().getFileName().toString().equals(".")) {
final File parentDir = subPath.getParent().toFile();
if (!parentDir.exists() && !parentDir.mkdirs()) {
LOG.error("Fail to create directory for {}", filePath);
setError(RaftError.EIO, "Fail to create directory");
return;
}
}
final LocalFileMeta meta = (LocalFileMeta) this.remoteSnapshot.getFileMeta(fileName);
Session session = null;
try {
this.lock.lock();
try {
if (this.cancelled) {
if (isOk()) {
setError(RaftError.ECANCELED, "ECANCELED");
}
return;
}
session = this.copier.startCopyToFile(fileName, filePath, null);
if (session == null) {
LOG.error("Fail to copy {}", fileName);
setError(-1, "Fail to copy %s", fileName);
return;
}
this.curSession = session;
} finally {
this.lock.unlock();
}
// join out of lock
session.join();
this.lock.lock();
try {
this.curSession = null;
} finally {
this.lock.unlock();
}
if (!session.status().isOk() && isOk()) {
setError(session.status().getCode(), session.status().getErrorMsg());
return;
}
if (!this.writer.addFile(fileName, meta)) {
setError(RaftError.EIO, "Fail to add file to writer");
return;
}
if (!this.writer.sync()) {
setError(RaftError.EIO, "Fail to sync writer");
}
} finally {
if (session != null) {
Utils.closeQuietly(session);
}
}
}
use of com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta in project sofa-jraft by sofastack.
the class LocalSnapshotWriter method addFile.
@Override
public boolean addFile(final String fileName, final Message fileMeta) {
final Builder metaBuilder = LocalFileMeta.newBuilder();
if (fileMeta != null) {
metaBuilder.mergeFrom(fileMeta);
}
final LocalFileMeta meta = metaBuilder.build();
return this.metaTable.addFile(fileName, meta);
}
Aggregations