Examples with BlockWorker alluxio.worker.block.BlockWorker used on opensource projects

Example 6 with BlockWorker

use of alluxio.worker.block.BlockWorker in project alluxio by Alluxio.

the class BlockMasterIntegrityIntegrationTest method deleteInvalidBlocks.

@Test
@LocalAlluxioClusterResource.Config(confParams = { PropertyKey.Name.MASTER_STARTUP_BLOCK_INTEGRITY_CHECK_ENABLED, "true" })
public void deleteInvalidBlocks() throws Exception {
    AlluxioURI uri = new AlluxioURI("/test");
    int len = 10;
    FileSystem fs = mCluster.getClient();
    BlockWorker worker = mCluster.getWorkerProcess().getWorker(BlockWorker.class);
    FileSystemTestUtils.createByteFile(fs, uri, WritePType.MUST_CACHE, len);
    Assert.assertEquals(1, worker.getStoreMetaFull().getNumberOfBlocks());
    removeFileMetadata(uri);
    mCluster.stopWorkers();
    mCluster.restartMasters();
    // creates a new worker, so need to get the new BlockWorker
    mCluster.startWorkers();
    BlockWorker newWorker = mCluster.getWorkerProcess().getWorker(BlockWorker.class);
    CommonUtils.waitFor("invalid blocks to be deleted", () -> newWorker.getStoreMetaFull().getNumberOfBlocks() == 0, WaitForOptions.defaults().setTimeoutMs(2000));
}

Example 7 with BlockWorker

use of alluxio.worker.block.BlockWorker in project alluxio by Alluxio.

the class BlockWorkerDataWriter method create.

/**
 * Creates an instance of {@link BlockWorkerDataWriter}.
 *
 * @param context the file system context
 * @param blockId the block ID
 * @param blockSize the block size in bytes
 * @param options the output stream options
 * @return the {@link BlockWorkerDataWriter} created
 */
public static BlockWorkerDataWriter create(final FileSystemContext context, long blockId, long blockSize, OutStreamOptions options) throws IOException {
    AlluxioConfiguration conf = context.getClusterConf();
    int chunkSize = (int) conf.getBytes(PropertyKey.USER_LOCAL_WRITER_CHUNK_SIZE_BYTES);
    long reservedBytes = Math.min(blockSize, conf.getBytes(PropertyKey.USER_FILE_RESERVED_BYTES));
    BlockWorker blockWorker = context.getProcessLocalWorker();
    Preconditions.checkNotNull(blockWorker, "blockWorker");
    long sessionId = SessionIdUtils.createSessionId();
    try {
        blockWorker.createBlock(sessionId, blockId, options.getWriteTier(), options.getMediumType(), reservedBytes);
        BlockWriter blockWriter = blockWorker.createBlockWriter(sessionId, blockId);
        return new BlockWorkerDataWriter(sessionId, blockId, options, blockWriter, blockWorker, chunkSize, reservedBytes, conf);
    } catch (BlockAlreadyExistsException | WorkerOutOfSpaceException | BlockDoesNotExistException | InvalidWorkerStateException e) {
        throw new IOException(e);
    }
}

Example 8 with BlockWorker

use of alluxio.worker.block.BlockWorker in project alluxio by Alluxio.

the class LocalFirstPolicyIntegrationTest method test.

@Test
public void test() throws Exception {
    AlluxioMasterProcess master = AlluxioMasterProcess.Factory.create();
    WorkerProcess worker1 = AlluxioWorkerProcess.Factory.create(TieredIdentityFactory.fromString("node=node1,rack=rack1", ServerConfiguration.global()));
    WorkerProcess worker2 = AlluxioWorkerProcess.Factory.create(TieredIdentityFactory.fromString("node=node2,rack=rack2", ServerConfiguration.global()));
    runProcess(mExecutor, master);
    runProcess(mExecutor, worker1);
    runProcess(mExecutor, worker2);
    TestUtils.waitForReady(master);
    TestUtils.waitForReady(worker1);
    TestUtils.waitForReady(worker2);
    FileSystem fs = FileSystem.Factory.create(ServerConfiguration.global());
    // Write to the worker in node1
    {
        Whitebox.setInternalState(TieredIdentityFactory.class, "sInstance", TieredIdentityFactory.fromString("node=node1,rack=rack1", ServerConfiguration.global()));
        try {
            FileSystemTestUtils.createByteFile(fs, "/file1", WritePType.MUST_CACHE, 100);
        } finally {
            Whitebox.setInternalState(TieredIdentityFactory.class, "sInstance", (Object) null);
        }
        BlockWorker blockWorker1 = worker1.getWorker(BlockWorker.class);
        BlockWorker blockWorker2 = worker2.getWorker(BlockWorker.class);
        assertEquals(100, blockWorker1.getStoreMeta().getUsedBytes());
        assertEquals(0, blockWorker2.getStoreMeta().getUsedBytes());
    }
    // Write to the worker in rack2
    {
        Whitebox.setInternalState(TieredIdentityFactory.class, "sInstance", TieredIdentityFactory.fromString("node=node3,rack=rack2", ServerConfiguration.global()));
        try {
            FileSystemTestUtils.createByteFile(fs, "/file2", WritePType.MUST_CACHE, 10);
        } finally {
            Whitebox.setInternalState(TieredIdentityFactory.class, "sInstance", (Object) null);
        }
        BlockWorker blockWorker1 = worker1.getWorker(BlockWorker.class);
        BlockWorker blockWorker2 = worker2.getWorker(BlockWorker.class);
        assertEquals(100, blockWorker1.getStoreMeta().getUsedBytes());
        assertEquals(10, blockWorker2.getStoreMeta().getUsedBytes());
    }
}

Example 9 with BlockWorker

use of alluxio.worker.block.BlockWorker in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getInStreamUfsProcessLocal.

@Test
public void getInStreamUfsProcessLocal() throws Exception {
    WorkerNetAddress remote = new WorkerNetAddress().setHost("remote");
    WorkerNetAddress local = new WorkerNetAddress().setHost(WORKER_HOSTNAME_LOCAL);
    BlockInfo info = new BlockInfo().setBlockId(0);
    URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L)).setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info))));
    OpenFilePOptions readOptions = OpenFilePOptions.newBuilder().build();
    InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, sConf);
    when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo());
    when(mContext.getCachedWorkers()).thenReturn(Lists.newArrayList(new BlockWorkerInfo(remote, 100, 0), new BlockWorkerInfo(local, 100, 0)));
    when(mContext.getNodeLocalWorker()).thenReturn(local);
    when(mContext.hasProcessLocalWorker()).thenReturn(true);
    BlockWorker blockWorker = Mockito.mock(BlockWorker.class);
    when(mContext.getProcessLocalWorker()).thenReturn(blockWorker);
    BlockInStream stream = mBlockStore.getInStream(BLOCK_ID, options);
    assertEquals(local, stream.getAddress());
    assertEquals(BlockWorkerDataReader.Factory.class.getName(), stream.getDataReaderFactory().getClass().getName());
}

Example 10 with BlockWorker

use of alluxio.worker.block.BlockWorker in project alluxio by Alluxio.

the class BlockInStreamTest method createProcessLocal.

@Test
public void createProcessLocal() throws Exception {
    WorkerNetAddress dataSource = new WorkerNetAddress();
    when(mMockContext.getNodeLocalWorker()).thenReturn(dataSource);
    when(mMockContext.getClientContext()).thenReturn(ClientContext.create(mConf));
    BlockWorker blockWorker = Mockito.mock(BlockWorker.class);
    when(mMockContext.getProcessLocalWorker()).thenReturn(blockWorker);
    BlockInStream.BlockInStreamSource dataSourceType = BlockInStream.BlockInStreamSource.PROCESS_LOCAL;
    BlockInStream stream = BlockInStream.create(mMockContext, mInfo, dataSource, dataSourceType, mOptions);
    assertEquals(BlockWorkerDataReader.Factory.class.getName(), stream.getDataReaderFactory().getClass().getName());
}