Examples with JournalEntry alluxio.proto.journal.Journal.JournalEntry used on opensource projects

Example 31 with JournalEntry

use of alluxio.proto.journal.Journal.JournalEntry in project alluxio by Alluxio.

the class DefaultBlockMaster method removeBlocks.

@Override
public void removeBlocks(Collection<Long> blockIds, boolean delete) throws UnavailableException {
    try (JournalContext journalContext = createJournalContext()) {
        for (long blockId : blockIds) {
            Set<Long> workerIds;
            try (LockResource r = lockBlock(blockId)) {
                Optional<BlockMeta> block = mBlockStore.getBlock(blockId);
                if (!block.isPresent()) {
                    continue;
                }
                List<BlockLocation> locations = mBlockStore.getLocations(blockId);
                workerIds = new HashSet<>(locations.size());
                for (BlockLocation loc : locations) {
                    workerIds.add(loc.getWorkerId());
                }
                // processWorkerRemovedBlocks
                if (delete) {
                    // Make sure blockId is removed from mLostBlocks when the block metadata is deleted.
                    // Otherwise blockId in mLostBlock can be dangling index if the metadata is gone.
                    mLostBlocks.remove(blockId);
                    mBlockStore.removeBlock(blockId);
                    JournalEntry entry = JournalEntry.newBuilder().setDeleteBlock(DeleteBlockEntry.newBuilder().setBlockId(blockId)).build();
                    journalContext.append(entry);
                }
            }
            // workerRegister should be changed to address this race condition.
            for (long workerId : workerIds) {
                MasterWorkerInfo worker = mWorkers.getFirstByField(ID_INDEX, workerId);
                if (worker != null) {
                    try (LockResource r = worker.lockWorkerMeta(EnumSet.of(WorkerMetaLockSection.BLOCKS), false)) {
                        worker.updateToRemovedBlock(true, blockId);
                    }
                }
            }
        }
    }
}

Example 32 with JournalEntry

use of alluxio.proto.journal.Journal.JournalEntry in project alluxio by Alluxio.

the class AsyncJournalWriterTest method failedWriteInternal.

/**
 * Simulates that exception occurs when writing {@link JournalEntry}.
 *
 * @param batchingEnabled whether to use a batch flush or not
 */
public void failedWriteInternal(boolean batchingEnabled) throws Exception {
    setupAsyncJournalWriter(batchingEnabled);
    // Start failing journal writes.
    // PS: Need to stop the writer before mocking because mMockJournalWriter could be
    // acceessed concurrently by the internal flush thread.
    mAsyncJournalWriter.stop();
    doThrow(new IOException("entry write failed")).when(mMockJournalWriter).write(any(JournalEntry.class));
    mAsyncJournalWriter.start();
    int entries = 5;
    for (int i = 0; i < entries; i++) {
        long flushCounter = mAsyncJournalWriter.appendEntry(JournalEntry.getDefaultInstance());
        // Assuming the flush counter starts from 0
        assertEquals(i + 1, flushCounter);
    }
    // Flushes should fail.
    for (int i = 1; i <= entries; i++) {
        try {
            mAsyncJournalWriter.flush(1);
            fail("journal flush should not succeed if journal write fails.");
        } catch (IOException e) {
        // This is expected.
        }
    }
    // Allow journal writes to succeed.
    // PS: Need to stop the writer before mocking because mMockJournalWriter could be
    // acceessed concurrently by the internal flush thread.
    mAsyncJournalWriter.stop();
    doNothing().when(mMockJournalWriter).write(any(JournalEntry.class));
    mAsyncJournalWriter.start();
    // Flushes should succeed.
    for (int i = 1; i <= entries; i++) {
        mAsyncJournalWriter.flush(i);
    }
    verify(mMockJournalWriter, atLeastOnce()).flush();
}

Example 33 with JournalEntry

use of alluxio.proto.journal.Journal.JournalEntry in project SSM by Intel-bigdata.

the class AlluxioEntryFetcher method canReadFromLastSeqNum.

public static boolean canReadFromLastSeqNum(Long lastSn) {
    try {
        if (AlluxioJournalUtil.getCurrentSeqNum(conf) == lastSn) {
            return true;
        }
        JournalReader reader = AlluxioJournalUtil.getJournalReaderFromSn(conf, lastSn);
        JournalEntry entry = reader.read();
        return entry != null;
    } catch (Exception e) {
        return false;
    }
}

Example 34 with JournalEntry

use of alluxio.proto.journal.Journal.JournalEntry in project SSM by Intel-bigdata.

the class TestAlluxioEntryApplier method testSetAttributeApplier.

@Test
public void testSetAttributeApplier() throws Exception {
    FileSystem fs = Mockito.mock(FileSystem.class);
    AlluxioEntryApplier entryApplier = new AlluxioEntryApplier(metaStore, fs);
    FileInfo fooFile = FileInfo.newBuilder().setFileId(33554431).setIsdir(false).setPath("/foo/foobar").build();
    metaStore.insertFile(fooFile);
    BackUpInfo backUpInfo = new BackUpInfo(1L, "/foo/foobar", "remote/dest/", 10);
    metaStore.insertBackUpInfo(backUpInfo);
    alluxio.wire.FileInfo info1 = new alluxio.wire.FileInfo().setFileId(33554431).setPath("/foo/foobar").setLength(100L).setFolder(false).setBlockSizeBytes(210000).setLastModificationTimeMs(1515665470681L).setCreationTimeMs(1515665470681L).setMode(493).setOwner("user1").setGroup("group1");
    URIStatus status1 = new URIStatus(info1);
    Mockito.when(fs.getStatus(new AlluxioURI("/foo/foobar"))).thenReturn(status1);
    SetAttributeEntry setAttributeEntry = SetAttributeEntry.newBuilder().setId(33554431).setOpTimeMs(1515667208590658L).setPermission(511).build();
    JournalEntry setAttributeJEntry = JournalEntry.newBuilder().setSetAttribute(setAttributeEntry).build();
    entryApplier.apply(setAttributeJEntry);
    List<FileDiff> fileDiffs = metaStore.getFileDiffsByFileName("/foo/foobar");
    Assert.assertTrue(fileDiffs.size() > 0);
    for (FileDiff fileDiff : fileDiffs) {
        if (fileDiff.getDiffType().equals(FileDiffType.METADATA)) {
            Assert.assertEquals("511", fileDiff.getParameters().get("-permission"));
        }
    }
}