Examples with JournalContext alluxio.master.journal.JournalContext used on opensource projects

Example 16 with JournalContext

use of alluxio.master.journal.JournalContext in project alluxio by Alluxio.

the class DefaultBlockMaster method commitBlock.

// TODO(binfan): check the logic is correct or not when commitBlock is a retry
@Override
public void commitBlock(long workerId, long usedBytesOnTier, String tierAlias, String mediumType, long blockId, long length) throws NotFoundException, UnavailableException {
    LOG.debug("Commit block from workerId: {}, usedBytesOnTier: {}, blockId: {}, length: {}", workerId, usedBytesOnTier, blockId, length);
    MasterWorkerInfo worker = mWorkers.getFirstByField(ID_INDEX, workerId);
    // TODO(peis): Check lost workers as well.
    if (worker == null) {
        throw new NotFoundException(ExceptionMessage.NO_WORKER_FOUND.getMessage(workerId));
    }
    try (JournalContext journalContext = createJournalContext()) {
        // The worker metadata must be locked before the blocks
        try (LockResource lr = worker.lockWorkerMeta(EnumSet.of(WorkerMetaLockSection.USAGE, WorkerMetaLockSection.BLOCKS), false)) {
            try (LockResource r = lockBlock(blockId)) {
                Optional<BlockMeta> block = mBlockStore.getBlock(blockId);
                if (!block.isPresent() || block.get().getLength() != length) {
                    if (block.isPresent() && block.get().getLength() != Constants.UNKNOWN_SIZE) {
                        LOG.warn("Rejecting attempt to change block length from {} to {}", block.get().getLength(), length);
                    } else {
                        mBlockStore.putBlock(blockId, BlockMeta.newBuilder().setLength(length).build());
                        BlockInfoEntry blockInfo = BlockInfoEntry.newBuilder().setBlockId(blockId).setLength(length).build();
                        journalContext.append(JournalEntry.newBuilder().setBlockInfo(blockInfo).build());
                    }
                }
                // Update the block metadata with the new worker location.
                mBlockStore.addLocation(blockId, BlockLocation.newBuilder().setWorkerId(workerId).setTier(tierAlias).setMediumType(mediumType).build());
                // This worker has this block, so it is no longer lost.
                mLostBlocks.remove(blockId);
                // Update the worker information for this new block.
                // TODO(binfan): when retry commitBlock on master is expected, make sure metrics are not
                // double counted.
                worker.addBlock(blockId);
                worker.updateUsedBytes(tierAlias, usedBytesOnTier);
            }
        }
        worker.updateLastUpdatedTimeMs();
    }
}

Example 17 with JournalContext

use of alluxio.master.journal.JournalContext in project alluxio by Alluxio.

the class DefaultBlockMaster method removeBlocks.

@Override
public void removeBlocks(Collection<Long> blockIds, boolean delete) throws UnavailableException {
    try (JournalContext journalContext = createJournalContext()) {
        for (long blockId : blockIds) {
            Set<Long> workerIds;
            try (LockResource r = lockBlock(blockId)) {
                Optional<BlockMeta> block = mBlockStore.getBlock(blockId);
                if (!block.isPresent()) {
                    continue;
                }
                List<BlockLocation> locations = mBlockStore.getLocations(blockId);
                workerIds = new HashSet<>(locations.size());
                for (BlockLocation loc : locations) {
                    workerIds.add(loc.getWorkerId());
                }
                // processWorkerRemovedBlocks
                if (delete) {
                    // Make sure blockId is removed from mLostBlocks when the block metadata is deleted.
                    // Otherwise blockId in mLostBlock can be dangling index if the metadata is gone.
                    mLostBlocks.remove(blockId);
                    mBlockStore.removeBlock(blockId);
                    JournalEntry entry = JournalEntry.newBuilder().setDeleteBlock(DeleteBlockEntry.newBuilder().setBlockId(blockId)).build();
                    journalContext.append(entry);
                }
            }
            // workerRegister should be changed to address this race condition.
            for (long workerId : workerIds) {
                MasterWorkerInfo worker = mWorkers.getFirstByField(ID_INDEX, workerId);
                if (worker != null) {
                    try (LockResource r = worker.lockWorkerMeta(EnumSet.of(WorkerMetaLockSection.BLOCKS), false)) {
                        worker.updateToRemovedBlock(true, blockId);
                    }
                }
            }
        }
    }
}

Example 18 with JournalContext

use of alluxio.master.journal.JournalContext in project alluxio by Alluxio.

the class RaftJournalTest method catchUpInSteps.

// Raft journal receives leader knowledge in chunks.
// So advancing should take into account seeing partial knowledge.
@Test
public void catchUpInSteps() throws Exception {
    // Create a counting master implementation that counts how many journal entries it processed.
    CountingDummyFileSystemMaster countingMaster = new CountingDummyFileSystemMaster();
    mFollowerJournalSystem.createJournal(countingMaster);
    // Suspend follower journal system.
    mFollowerJournalSystem.suspend(null);
    final int entryBatchCount = 5;
    // Create batch of entries on the leader journal context.
    try (JournalContext journalContext = mLeaderJournalSystem.createJournal(new NoopMaster()).createJournalContext()) {
        for (int i = 0; i < entryBatchCount; i++) {
            journalContext.append(alluxio.proto.journal.Journal.JournalEntry.newBuilder().setInodeLastModificationTime(File.InodeLastModificationTimeEntry.newBuilder().setId(i).build()).build());
        }
    }
    // Catch up follower journal system to target-index:(fileCount * 2) - 1.
    Map<String, Long> backupSequences = new HashMap<>();
    backupSequences.put("FileSystemMaster", (long) (entryBatchCount * 2) - 1);
    CatchupFuture catchupFuture = mFollowerJournalSystem.catchup(backupSequences);
    // Create next batch of entries on the leader journal context.
    try (JournalContext journalContext = mLeaderJournalSystem.createJournal(new NoopMaster()).createJournalContext()) {
        for (int i = 0; i < entryBatchCount; i++) {
            journalContext.append(alluxio.proto.journal.Journal.JournalEntry.newBuilder().setInodeLastModificationTime(File.InodeLastModificationTimeEntry.newBuilder().setId(i).build()).build());
        }
    }
    // Wait for sequence to be caught up.
    catchupFuture.waitTermination();
    Assert.assertEquals(entryBatchCount * 2, countingMaster.getApplyCount());
    // Catchup on the already met sequence.
    mFollowerJournalSystem.catchup(backupSequences);
    Assert.assertEquals(entryBatchCount * 2, countingMaster.getApplyCount());
}

Example 19 with JournalContext

use of alluxio.master.journal.JournalContext in project alluxio by Alluxio.

the class RaftJournalTest method suspendCatchupResume.

@Test
public void suspendCatchupResume() throws Exception {
    // Create a counting master implementation that counts how many journal entries it processed.
    CountingDummyFileSystemMaster countingMaster = new CountingDummyFileSystemMaster();
    mFollowerJournalSystem.createJournal(countingMaster);
    // Suspend follower journal system.
    mFollowerJournalSystem.suspend(null);
    try {
        mFollowerJournalSystem.suspend(null);
        Assert.fail("Suspend succeeded for already suspended journal.");
    } catch (Exception e) {
    // Expected to fail when suspending a suspended journal.
    }
    // Catch up follower journal system to target-index:5.
    final long catchupIndex = 5;
    Map<String, Long> backupSequences = new HashMap<>();
    backupSequences.put("FileSystemMaster", catchupIndex);
    CatchupFuture catchupFuture = mFollowerJournalSystem.catchup(backupSequences);
    // Create entries on the leader journal context.
    // These will be replicated to follower journal context.
    final int entryCount = 10;
    try (JournalContext journalContext = mLeaderJournalSystem.createJournal(new NoopMaster()).createJournalContext()) {
        for (int i = 0; i < entryCount; i++) {
            journalContext.append(alluxio.proto.journal.Journal.JournalEntry.newBuilder().setInodeLastModificationTime(File.InodeLastModificationTimeEntry.newBuilder().setId(i).build()).build());
        }
    }
    // Wait for sequences to be caught up.
    catchupFuture.waitTermination();
    Assert.assertEquals(catchupIndex + 1, countingMaster.getApplyCount());
    // Wait for election timeout and verify follower master state hasn't changed.
    Thread.sleep(ServerConfiguration.getMs(PropertyKey.MASTER_EMBEDDED_JOURNAL_MAX_ELECTION_TIMEOUT));
    Assert.assertEquals(catchupIndex + 1, countingMaster.getApplyCount());
    // Exit backup mode and wait until follower master acquires the current knowledge.
    mFollowerJournalSystem.resume();
    CommonUtils.waitFor("full state acquired", () -> countingMaster.getApplyCount() == entryCount, mWaitOptions);
    // Write more entries and validate they are replicated to follower.
    try (JournalContext journalContext = mLeaderJournalSystem.createJournal(new NoopMaster()).createJournalContext()) {
        journalContext.append(alluxio.proto.journal.Journal.JournalEntry.newBuilder().setInodeLastModificationTime(File.InodeLastModificationTimeEntry.newBuilder().setId(entryCount).build()).build());
    }
    CommonUtils.waitFor("full state acquired after resume", () -> countingMaster.getApplyCount() == entryCount + 1, mWaitOptions);
}

Example 20 with JournalContext

use of alluxio.master.journal.JournalContext in project alluxio by Alluxio.

the class RaftJournalTest method joinCluster.

@Test
public void joinCluster() throws Exception {
    // Create entries on the leader journal context.
    // These will be replicated to follower journal context.
    final int entryCount = 10;
    try (JournalContext journalContext = mLeaderJournalSystem.createJournal(new NoopMaster()).createJournalContext()) {
        for (int i = 0; i < entryCount; i++) {
            journalContext.append(alluxio.proto.journal.Journal.JournalEntry.newBuilder().setInodeLastModificationTime(File.InodeLastModificationTimeEntry.newBuilder().setId(i).build()).build());
        }
    }
    RaftJournalSystem newJs = createNewJournalSystem(mLeaderJournalSystem);
    // Create a counting master implementation that counts how many journal entries it processed.
    CountingDummyFileSystemMaster countingMaster = new CountingDummyFileSystemMaster();
    newJs.createJournal(countingMaster);
    newJs.start();
    // Write more entries and validate they are replicated to follower.
    try (JournalContext journalContext = mLeaderJournalSystem.createJournal(new NoopMaster()).createJournalContext()) {
        journalContext.append(alluxio.proto.journal.Journal.JournalEntry.newBuilder().setInodeLastModificationTime(File.InodeLastModificationTimeEntry.newBuilder().setId(entryCount).build()).build());
    }
    CommonUtils.waitFor("follower catches up on all changes", () -> countingMaster.getApplyCount() == entryCount + 1, mWaitOptions);
}