Search in sources :

Example 1 with SimpleStateMachineStorage

use of org.apache.ratis.statemachine.impl.SimpleStateMachineStorage in project alluxio by Alluxio.

the class RaftJournalDumper method readRatisSnapshotFromDir.

private void readRatisSnapshotFromDir() throws IOException {
    try (RaftStorage storage = new RaftStorageImpl(getJournalDir(), RaftServerConfigKeys.Log.CorruptionPolicy.getDefault())) {
        SimpleStateMachineStorage stateMachineStorage = new SimpleStateMachineStorage();
        stateMachineStorage.init(storage);
        SingleFileSnapshotInfo currentSnapshot = stateMachineStorage.getLatestSnapshot();
        if (currentSnapshot == null) {
            LOG.debug("No snapshot found");
            return;
        }
        final File snapshotFile = currentSnapshot.getFile().getPath().toFile();
        String checkpointPath = String.format("%s-%s-%s", mCheckpointsDir, currentSnapshot.getIndex(), snapshotFile.lastModified());
        try (DataInputStream inputStream = new DataInputStream(new FileInputStream(snapshotFile))) {
            LOG.debug("Reading snapshot-Id: {}", inputStream.readLong());
            try (CheckpointInputStream checkpointStream = new CheckpointInputStream(inputStream)) {
                readCheckpoint(checkpointStream, Paths.get(checkpointPath));
            } catch (Exception e) {
                LOG.error("Failed to read snapshot from journal.", e);
            }
        } catch (Exception e) {
            LOG.error("Failed to load snapshot {}", snapshotFile, e);
            throw e;
        }
    }
}
Also used : SingleFileSnapshotInfo(org.apache.ratis.statemachine.impl.SingleFileSnapshotInfo) RaftStorage(org.apache.ratis.server.storage.RaftStorage) SimpleStateMachineStorage(org.apache.ratis.statemachine.impl.SimpleStateMachineStorage) RaftStorageImpl(org.apache.ratis.server.storage.RaftStorageImpl) DataInputStream(java.io.DataInputStream) CheckpointInputStream(alluxio.master.journal.checkpoint.CheckpointInputStream) File(java.io.File) FileInputStream(java.io.FileInputStream) IOException(java.io.IOException) FileNotFoundException(java.io.FileNotFoundException)

Example 2 with SimpleStateMachineStorage

use of org.apache.ratis.statemachine.impl.SimpleStateMachineStorage in project alluxio by Alluxio.

the class SnapshotReplicationManagerTest method getSimpleStateMachineStorage.

private SimpleStateMachineStorage getSimpleStateMachineStorage() throws IOException {
    RaftStorage rs = new RaftStorageImpl(mFolder.newFolder(CommonUtils.randomAlphaNumString(6)), RaftServerConfigKeys.Log.CorruptionPolicy.getDefault());
    SimpleStateMachineStorage snapshotStore = new SimpleStateMachineStorage();
    snapshotStore.init(rs);
    return snapshotStore;
}
Also used : RaftStorage(org.apache.ratis.server.storage.RaftStorage) SimpleStateMachineStorage(org.apache.ratis.statemachine.impl.SimpleStateMachineStorage) RaftStorageImpl(org.apache.ratis.server.storage.RaftStorageImpl)

Example 3 with SimpleStateMachineStorage

use of org.apache.ratis.statemachine.impl.SimpleStateMachineStorage in project alluxio by Alluxio.

the class JournalToolTest method getCurrentRatisSnapshotIndex.

private long getCurrentRatisSnapshotIndex(String journalFolder) throws Throwable {
    try (RaftStorage storage = new RaftStorageImpl(new File(RaftJournalUtils.getRaftJournalDir(new File(journalFolder)), RaftJournalSystem.RAFT_GROUP_UUID.toString()), RaftServerConfigKeys.Log.CorruptionPolicy.getDefault())) {
        SimpleStateMachineStorage stateMachineStorage = new SimpleStateMachineStorage();
        stateMachineStorage.init(storage);
        SingleFileSnapshotInfo snapshot = stateMachineStorage.getLatestSnapshot();
        if (snapshot == null) {
            throw new IOException("Failed to find a valid snapshot");
        }
        return snapshot.getIndex();
    }
}
Also used : SingleFileSnapshotInfo(org.apache.ratis.statemachine.impl.SingleFileSnapshotInfo) RaftStorage(org.apache.ratis.server.storage.RaftStorage) SimpleStateMachineStorage(org.apache.ratis.statemachine.impl.SimpleStateMachineStorage) RaftStorageImpl(org.apache.ratis.server.storage.RaftStorageImpl) IOException(java.io.IOException) File(java.io.File)

Example 4 with SimpleStateMachineStorage

use of org.apache.ratis.statemachine.impl.SimpleStateMachineStorage in project alluxio by Alluxio.

the class EmbeddedJournalIntegrationTestFaultTolerance method copySnapshotToFollower.

@Test
public void copySnapshotToFollower() throws Exception {
    mCluster = MultiProcessCluster.newBuilder(PortCoordination.EMBEDDED_JOURNAL_SNAPSHOT_FOLLOWER).setClusterName("EmbeddedJournalFaultTolerance_copySnapshotToFollower").setNumMasters(NUM_MASTERS).setNumWorkers(NUM_WORKERS).addProperty(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.EMBEDDED.toString()).addProperty(PropertyKey.MASTER_JOURNAL_FLUSH_TIMEOUT_MS, "5min").addProperty(PropertyKey.MASTER_JOURNAL_CHECKPOINT_PERIOD_ENTRIES, "1000").addProperty(PropertyKey.MASTER_JOURNAL_LOG_SIZE_BYTES_MAX, "50KB").addProperty(PropertyKey.MASTER_EMBEDDED_JOURNAL_MIN_ELECTION_TIMEOUT, "3s").addProperty(PropertyKey.MASTER_EMBEDDED_JOURNAL_MAX_ELECTION_TIMEOUT, "6s").addProperty(PropertyKey.MASTER_STANDBY_HEARTBEAT_INTERVAL, "5s").build();
    mCluster.start();
    int catchUpMasterIndex = (mCluster.getPrimaryMasterIndex(MASTER_INDEX_WAIT_TIME) + 1) % NUM_MASTERS;
    AlluxioURI testDir = new AlluxioURI("/dir");
    FileSystem fs = mCluster.getFileSystemClient();
    fs.createDirectory(testDir);
    for (int i = 0; i < 2000; i++) {
        fs.createDirectory(testDir.join("file" + i));
    }
    mCluster.getMetaMasterClient().checkpoint();
    mCluster.stopMaster(catchUpMasterIndex);
    File catchupJournalDir = new File(mCluster.getJournalDir(catchUpMasterIndex));
    FileUtils.deleteDirectory(catchupJournalDir);
    assertTrue(catchupJournalDir.mkdirs());
    mCluster.startMaster(catchUpMasterIndex);
    File raftDir = new File(RaftJournalUtils.getRaftJournalDir(catchupJournalDir), RaftJournalSystem.RAFT_GROUP_UUID.toString());
    waitForSnapshot(raftDir);
    mCluster.stopMaster(catchUpMasterIndex);
    SimpleStateMachineStorage storage = new SimpleStateMachineStorage();
    storage.init(new RaftStorageImpl(raftDir, RaftServerConfigKeys.Log.CorruptionPolicy.getDefault()));
    SingleFileSnapshotInfo snapshot = storage.findLatestSnapshot();
    assertNotNull(snapshot);
    mCluster.notifySuccess();
}
Also used : SingleFileSnapshotInfo(org.apache.ratis.statemachine.impl.SingleFileSnapshotInfo) FileSystem(alluxio.client.file.FileSystem) SimpleStateMachineStorage(org.apache.ratis.statemachine.impl.SimpleStateMachineStorage) RaftStorageImpl(org.apache.ratis.server.storage.RaftStorageImpl) File(java.io.File) AlluxioURI(alluxio.AlluxioURI) Test(org.junit.Test)

Example 5 with SimpleStateMachineStorage

use of org.apache.ratis.statemachine.impl.SimpleStateMachineStorage in project alluxio by Alluxio.

the class EmbeddedJournalIntegrationTestFaultTolerance method copySnapshotToMaster.

@Test
public void copySnapshotToMaster() throws Exception {
    mCluster = MultiProcessCluster.newBuilder(PortCoordination.EMBEDDED_JOURNAL_SNAPSHOT_MASTER).setClusterName("EmbeddedJournalFaultTolerance_copySnapshotToMaster").setNumMasters(NUM_MASTERS).setNumWorkers(NUM_WORKERS).addProperty(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.EMBEDDED.toString()).addProperty(PropertyKey.MASTER_JOURNAL_FLUSH_TIMEOUT_MS, "5min").addProperty(PropertyKey.MASTER_JOURNAL_CHECKPOINT_PERIOD_ENTRIES, "1000").addProperty(PropertyKey.MASTER_JOURNAL_LOG_SIZE_BYTES_MAX, "50KB").addProperty(PropertyKey.MASTER_EMBEDDED_JOURNAL_MIN_ELECTION_TIMEOUT, "3s").addProperty(PropertyKey.MASTER_EMBEDDED_JOURNAL_MAX_ELECTION_TIMEOUT, "6s").addProperty(PropertyKey.MASTER_STANDBY_HEARTBEAT_INTERVAL, "5s").build();
    mCluster.start();
    AlluxioURI testDir = new AlluxioURI("/dir");
    FileSystem fs = mCluster.getFileSystemClient();
    fs.createDirectory(testDir);
    for (int i = 0; i < 2000; i++) {
        fs.createDirectory(testDir.join("file" + i));
    }
    int primaryMasterIndex = mCluster.getPrimaryMasterIndex(MASTER_INDEX_WAIT_TIME);
    String leaderJournalPath = mCluster.getJournalDir(primaryMasterIndex);
    File raftDir = new File(RaftJournalUtils.getRaftJournalDir(new File(leaderJournalPath)), RaftJournalSystem.RAFT_GROUP_UUID.toString());
    waitForSnapshot(raftDir);
    mCluster.stopMasters();
    SimpleStateMachineStorage storage = new SimpleStateMachineStorage();
    storage.init(new RaftStorageImpl(raftDir, RaftServerConfigKeys.Log.CorruptionPolicy.getDefault()));
    SingleFileSnapshotInfo snapshot = storage.findLatestSnapshot();
    assertNotNull(snapshot);
    mCluster.notifySuccess();
}
Also used : SingleFileSnapshotInfo(org.apache.ratis.statemachine.impl.SingleFileSnapshotInfo) FileSystem(alluxio.client.file.FileSystem) SimpleStateMachineStorage(org.apache.ratis.statemachine.impl.SimpleStateMachineStorage) RaftStorageImpl(org.apache.ratis.server.storage.RaftStorageImpl) File(java.io.File) AlluxioURI(alluxio.AlluxioURI) Test(org.junit.Test)

Aggregations

RaftStorageImpl (org.apache.ratis.server.storage.RaftStorageImpl)5 SimpleStateMachineStorage (org.apache.ratis.statemachine.impl.SimpleStateMachineStorage)5 File (java.io.File)4 SingleFileSnapshotInfo (org.apache.ratis.statemachine.impl.SingleFileSnapshotInfo)4 RaftStorage (org.apache.ratis.server.storage.RaftStorage)3 AlluxioURI (alluxio.AlluxioURI)2 FileSystem (alluxio.client.file.FileSystem)2 IOException (java.io.IOException)2 Test (org.junit.Test)2 CheckpointInputStream (alluxio.master.journal.checkpoint.CheckpointInputStream)1 DataInputStream (java.io.DataInputStream)1 FileInputStream (java.io.FileInputStream)1 FileNotFoundException (java.io.FileNotFoundException)1