Examples with Journal alluxio.proto.journal.Journal used on opensource projects

Example 11 with Journal

use of alluxio.proto.journal.Journal in project alluxio by Alluxio.

the class UfsJournalDumper method dumpJournal.

@Override
public void dumpJournal() throws Throwable {
    try (UfsJournal journal = new UfsJournalSystem(getJournalLocation(mInputDir), 0).createJournal(new NoopMaster(mMaster));
        PrintStream out = new PrintStream(new BufferedOutputStream(new FileOutputStream(mJournalEntryFile)));
        JournalReader reader = new UfsJournalReader(journal, mStart, true)) {
        boolean done = false;
        while (!done && reader.getNextSequenceNumber() < mEnd) {
            JournalReader.State state = reader.advance();
            switch(state) {
                case CHECKPOINT:
                    try (CheckpointInputStream checkpoint = reader.getCheckpoint()) {
                        Path dir = Paths.get(mCheckpointsDir + "-" + reader.getNextSequenceNumber());
                        Files.createDirectories(dir);
                        readCheckpoint(checkpoint, dir);
                    }
                    break;
                case LOG:
                    Journal.JournalEntry entry = reader.getEntry();
                    out.println(ENTRY_SEPARATOR);
                    out.print(entry);
                    break;
                case DONE:
                    done = true;
                    break;
                default:
                    throw new RuntimeException("Unknown state: " + state);
            }
        }
    }
}

Example 12 with Journal

use of alluxio.proto.journal.Journal in project alluxio by Alluxio.

the class RaftJournalDumper method readRatisLogFromDir.

private void readRatisLogFromDir() {
    try (PrintStream out = new PrintStream(new BufferedOutputStream(new FileOutputStream(mJournalEntryFile)));
        RaftStorage storage = new RaftStorageImpl(getJournalDir(), RaftServerConfigKeys.Log.CorruptionPolicy.getDefault())) {
        List<LogSegmentPath> paths = LogSegmentPath.getLogSegmentPaths(storage);
        for (LogSegmentPath path : paths) {
            final int entryCount = LogSegment.readSegmentFile(path.getPath().toFile(), path.getStartEnd(), RaftServerConfigKeys.Log.CorruptionPolicy.EXCEPTION, null, (proto) -> {
                if (proto.hasStateMachineLogEntry()) {
                    try {
                        Journal.JournalEntry entry = Journal.JournalEntry.parseFrom(proto.getStateMachineLogEntry().getLogData().asReadOnlyByteBuffer());
                        writeSelected(out, entry);
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                }
            });
            LOG.info("Read {} entries from log {}.", entryCount, path.getPath());
        }
    } catch (Exception e) {
        LOG.error("Failed to read logs from journal.", e);
    }
}

Example 13 with Journal

use of alluxio.proto.journal.Journal in project alluxio by Alluxio.

the class AccessTimeUpdaterTest method updateAccessTimePrecision.

@Test
public void updateAccessTimePrecision() throws Exception {
    mAccessTimeUpdater = new AccessTimeUpdater(mFileSystemMaster, mInodeTree, mContext.getJournalSystem(), 0, Constants.HOUR_MS, 0);
    mAccessTimeUpdater.start();
    String path = "/foo";
    createInode(path, CreateFileContext.defaults());
    JournalContext journalContext = mock(JournalContext.class);
    when(journalContext.get()).thenReturn(journalContext);
    when(mFileSystemMaster.createJournalContext()).thenReturn(journalContext);
    long accessTime = CommonUtils.getCurrentMs() + 100L;
    long inodeId;
    try (LockedInodePath lockedInodes = mInodeTree.lockFullInodePath(new AlluxioURI(path), InodeTree.LockPattern.READ)) {
        mAccessTimeUpdater.updateAccessTime(journalContext, lockedInodes.getInode(), accessTime);
        inodeId = lockedInodes.getInode().getId();
    }
    // verify inode attribute is not updated
    assertNotEquals(accessTime, mInodeStore.get(inodeId).get().getLastAccessTimeMs());
    // verify journal entry is not logged yet
    verify(journalContext, never()).append(any(Journal.JournalEntry.class));
    long newAccessTime = CommonUtils.getCurrentMs() + 2 * Constants.HOUR_MS;
    // update access time with a much later timestamp
    try (LockedInodePath lockedInodes = mInodeTree.lockFullInodePath(new AlluxioURI(path), InodeTree.LockPattern.READ)) {
        mAccessTimeUpdater.updateAccessTime(journalContext, lockedInodes.getInode(), newAccessTime);
        inodeId = lockedInodes.getInode().getId();
    }
    // verify inode attribute is updated
    assertEquals(newAccessTime, mInodeStore.get(inodeId).get().getLastAccessTimeMs());
    // / verify journal entry is logged
    ArgumentCaptor<Journal.JournalEntry> captor = ArgumentCaptor.forClass(Journal.JournalEntry.class);
    verify(journalContext).append(captor.capture());
    assertTrue(captor.getValue().hasUpdateInode());
    assertEquals(inodeId, captor.getValue().getUpdateInode().getId());
    assertEquals(newAccessTime, captor.getValue().getUpdateInode().getLastAccessTimeMs());
}

Example 14 with Journal

use of alluxio.proto.journal.Journal in project alluxio by Alluxio.

the class InodeSyncStream method mergeCreateComplete.

/**
 * Merge inode entry with subsequent update inode and update inode file entries.
 *
 * @param entries list of journal entries
 * @return a list of compacted journal entries
 */
public static List<Journal.JournalEntry> mergeCreateComplete(List<alluxio.proto.journal.Journal.JournalEntry> entries) {
    List<alluxio.proto.journal.Journal.JournalEntry> newEntries = new ArrayList<>();
    // file id : index in the newEntries, InodeFileEntry
    Map<Long, Pair<Integer, MutableInodeFile>> fileEntryMap = new HashMap<>();
    for (alluxio.proto.journal.Journal.JournalEntry oldEntry : entries) {
        if (oldEntry.hasInodeFile()) {
            // Use the old entry as a placeholder, to be replaced later
            newEntries.add(oldEntry);
            fileEntryMap.put(oldEntry.getInodeFile().getId(), new Pair<>(newEntries.size() - 1, MutableInodeFile.fromJournalEntry(oldEntry.getInodeFile())));
        } else if (oldEntry.hasUpdateInode()) {
            File.UpdateInodeEntry entry = oldEntry.getUpdateInode();
            if (fileEntryMap.get(entry.getId()) == null) {
                newEntries.add(oldEntry);
                continue;
            }
            MutableInodeFile inode = fileEntryMap.get(entry.getId()).getSecond();
            inode.updateFromEntry(entry);
        } else if (oldEntry.hasUpdateInodeFile()) {
            File.UpdateInodeFileEntry entry = oldEntry.getUpdateInodeFile();
            if (fileEntryMap.get(entry.getId()) == null) {
                newEntries.add(oldEntry);
                continue;
            }
            MutableInodeFile inode = fileEntryMap.get(entry.getId()).getSecond();
            inode.updateFromEntry(entry);
        } else {
            newEntries.add(oldEntry);
        }
    }
    for (Pair<Integer, MutableInodeFile> pair : fileEntryMap.values()) {
        // Replace the old entry placeholder with the new entry,
        // to create the file in the same place in the journal
        newEntries.set(pair.getFirst(), pair.getSecond().toJournalEntry());
    }
    return newEntries;
}

Example 15 with Journal

use of alluxio.proto.journal.Journal in project SSM by Intel-bigdata.

the class AlluxioJournalUtil method getCurrentSeqNum.

/**
 * @param conf smart configuration
 * @return the current entry sequence number
 */
public static Long getCurrentSeqNum(SmartConf conf) {
    UfsJournal journal = new UfsJournalSystem(getJournalLocation(conf), 0).createJournal(new NoopMaster(sMaster));
    UfsJournalFile currentLog;
    try {
        currentLog = UfsJournalSnapshot.getCurrentLog(journal);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    long sn = -1L;
    if (currentLog != null) {
        try (JournalReader reader = new UfsJournalReader(journal, currentLog.getStart(), true)) {
            Journal.JournalEntry entry;
            while ((entry = reader.read()) != null) {
                sn = entry.getSequenceNumber();
                if (sn >= Long.MAX_VALUE) {
                    break;
                }
            }
        } catch (Exception e) {
            LOG.error("Failed to read next journal entry.", e);
        }
    }
    return sn;
}