Examples with TieredBlockStore alluxio.worker.block.TieredBlockStore used on opensource projects

Example 1 with TieredBlockStore

use of alluxio.worker.block.TieredBlockStore in project alluxio by Alluxio.

the class BaseTierManagementTaskTest method init.

/**
 * Sets up all dependencies before a test runs.
 */
protected void init() throws Exception {
    // Disable reviewer to make sure the allocator behavior stays deterministic
    ServerConfiguration.set(PropertyKey.WORKER_REVIEWER_CLASS, "alluxio.worker.block.reviewer.AcceptingReviewer");
    // Use LRU for stronger overlap guarantee.
    ServerConfiguration.set(PropertyKey.WORKER_BLOCK_ANNOTATOR_CLASS, LRUAnnotator.class.getName());
    ServerConfiguration.set(PropertyKey.USER_BLOCK_SIZE_BYTES_DEFAULT, BLOCK_SIZE);
    // Set timeout for faster task execution.
    ServerConfiguration.set(PropertyKey.WORKER_MANAGEMENT_LOAD_DETECTION_COOL_DOWN_TIME, "100ms");
    File tempFolder = mTestFolder.newFolder();
    TieredBlockStoreTestUtils.setupDefaultConf(tempFolder.getAbsolutePath());
    mBlockStore = new TieredBlockStore();
    Field field = mBlockStore.getClass().getDeclaredField("mMetaManager");
    field.setAccessible(true);
    mMetaManager = (BlockMetadataManager) field.get(mBlockStore);
    mBlockIterator = mMetaManager.getBlockIterator();
    mTestDir1 = mMetaManager.getTier(FIRST_TIER_ALIAS).getDir(0);
    mTestDir2 = mMetaManager.getTier(FIRST_TIER_ALIAS).getDir(1);
    mTestDir3 = mMetaManager.getTier(SECOND_TIER_ALIAS).getDir(1);
    mTestDir4 = mMetaManager.getTier(SECOND_TIER_ALIAS).getDir(2);
}

Example 2 with TieredBlockStore

use of alluxio.worker.block.TieredBlockStore in project alluxio by Alluxio.

the class BlockWorkerDataReaderTest method before.

@Before
public void before() throws Exception {
    BlockMasterClient blockMasterClient = mock(BlockMasterClient.class);
    BlockMasterClientPool blockMasterClientPool = spy(new BlockMasterClientPool());
    when(blockMasterClientPool.createNewResource()).thenReturn(blockMasterClient);
    TieredBlockStore blockStore = new TieredBlockStore();
    FileSystemMasterClient fileSystemMasterClient = mock(FileSystemMasterClient.class);
    Sessions sessions = mock(Sessions.class);
    // Connect to the real UFS for UFS read testing
    UfsManager ufsManager = mock(UfsManager.class);
    mRootUfs = ServerConfiguration.getString(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS);
    UfsManager.UfsClient ufsClient = new UfsManager.UfsClient(() -> UnderFileSystem.Factory.create(mRootUfs, UnderFileSystemConfiguration.defaults(ServerConfiguration.global())), new AlluxioURI(mRootUfs));
    when(ufsManager.get(anyLong())).thenReturn(ufsClient);
    mBlockWorker = new DefaultBlockWorker(blockMasterClientPool, fileSystemMasterClient, sessions, blockStore, ufsManager);
    URIStatus dummyStatus = new URIStatus(new FileInfo().setBlockIds(Collections.singletonList(BLOCK_ID)));
    InStreamOptions options = new InStreamOptions(dummyStatus, FileSystemOptions.openFileDefaults(mConf), mConf);
    mDataReaderFactory = new BlockWorkerDataReader.Factory(mBlockWorker, BLOCK_ID, CHUNK_SIZE, options);
}

Example 3 with TieredBlockStore

use of alluxio.worker.block.TieredBlockStore in project alluxio by Alluxio.

the class BlockWorkerRegisterStreamIntegrationTest method initBlockWorker.

public void initBlockWorker() throws Exception {
    // Prepare a block worker
    mBlockMasterClientPool = spy(new BlockMasterClientPool());
    when(mBlockMasterClientPool.createNewResource()).thenReturn(mBlockMasterClient);
    when(mBlockMasterClientPool.acquire()).thenReturn(mBlockMasterClient);
    TieredBlockStore mBlockStore = spy(new TieredBlockStore());
    FileSystemMasterClient mFileSystemMasterClient = mock(FileSystemMasterClient.class);
    Sessions mSessions = mock(Sessions.class);
    UfsManager mUfsManager = mock(UfsManager.class);
    mBlockWorker = new DefaultBlockWorker(mBlockMasterClientPool, mFileSystemMasterClient, mSessions, mBlockStore, mUfsManager);
}

Example 4 with TieredBlockStore

use of alluxio.worker.block.TieredBlockStore in project alluxio by Alluxio.

the class UfsFallbackBlockWriteHandlerTest method before.

@Before
public void before() throws Exception {
    mFile = mTestFolder.newFile();
    mOutputStream = new FileOutputStream(mFile);
    mBlockStore = new TieredBlockStore();
    mBlockWorker = new MockBlockWorker();
    UnderFileSystem mockUfs = Mockito.mock(UnderFileSystem.class);
    UfsManager ufsManager = Mockito.mock(UfsManager.class);
    UfsManager.UfsClient ufsClient = new UfsManager.UfsClient(() -> mockUfs, AlluxioURI.EMPTY_URI);
    Mockito.when(ufsManager.get(Mockito.anyLong())).thenReturn(ufsClient);
    Mockito.when(mockUfs.createNonexistingFile(Mockito.anyString(), Mockito.any(CreateOptions.class))).thenReturn(mOutputStream).thenReturn(new FileOutputStream(mFile, true));
    mResponseObserver = Mockito.mock(StreamObserver.class);
    mWriteHandler = new UfsFallbackBlockWriteHandler(mBlockWorker, ufsManager, mResponseObserver, mUserInfo, false);
    setupResponseTrigger();
    // create a partial block in block store first
    mBlockStore.createBlock(TEST_SESSION_ID, TEST_BLOCK_ID, AllocateOptions.forCreate(CHUNK_SIZE, BlockStoreLocation.anyDirInTier(Constants.MEDIUM_MEM)));
    BlockWriter writer = mBlockStore.getBlockWriter(TEST_SESSION_ID, TEST_BLOCK_ID);
    DataBuffer buffer = newDataBuffer(PARTIAL_WRITTEN);
    mPartialChecksum = getChecksum(buffer);
    writer.append((ByteBuf) buffer.getNettyOutput());
    writer.close();
}