Examples with MasterClientContext alluxio.master.MasterClientContext used on opensource projects

Example 1 with MasterClientContext

use of alluxio.master.MasterClientContext in project presto by prestodb.

the class AlluxioMetastoreModule method provideCatalogMasterClient.

@Provides
@Inject
TableMasterClient provideCatalogMasterClient(AlluxioHiveMetastoreConfig config) {
    InstancedConfiguration conf = new InstancedConfiguration(ConfigurationUtils.defaults());
    if (config.isZookeeperEnabled()) {
        conf.set(PropertyKey.ZOOKEEPER_ENABLED, true);
        conf.set(PropertyKey.ZOOKEEPER_ADDRESS, config.getZookeeperAddress());
    } else {
        String address = config.getMasterAddress();
        String[] parts = address.split(":", 2);
        conf.set(PropertyKey.MASTER_HOSTNAME, parts[0]);
        if (parts.length > 1) {
            conf.set(PropertyKey.MASTER_RPC_PORT, parts[1]);
        }
    }
    MasterClientContext context = MasterClientContext.newBuilder(ClientContext.create(conf)).build();
    return new RetryHandlingTableMasterClient(context);
}

Example 2 with MasterClientContext

use of alluxio.master.MasterClientContext in project alluxio by Alluxio.

the class BlockWorkerRegisterStreamIntegrationTest method deleteDuringRegisterStream.

/**
 * Tests below cover the race conditions during concurrent executions.
 *
 * If a worker is new to the cluster, no clients should know this worker.
 * Therefore there is no concurrent client-incurred write operations on this worker.
 * The races happen typically when the worker re-registers with the master,
 * where some clients already know this worker and can direct invoke writes on the worker.
 *
 * Tests here verify the integrity of the worker-side metadata.
 * In other words, even a delete/move happens on the worker during the register stream,
 * the change should be successful and the update should be recorded correctly.
 * The update should later be reported to the master.
 */
@Test
public void deleteDuringRegisterStream() throws Exception {
    // Generate a request stream of blocks
    List<RegisterWorkerPRequest> requestChunks = RegisterStreamTestUtils.generateRegisterStreamForWorker(WORKER_ID);
    // Select a block to remove concurrent with the stream
    long blockToRemove = findFirstBlock(requestChunks);
    // Manually create a stream and control the request/response observers
    BlockMasterWorkerServiceStub asyncClient = PowerMockito.mock(BlockMasterWorkerServiceStub.class);
    when(asyncClient.withDeadlineAfter(anyLong(), any())).thenReturn(asyncClient);
    CountDownLatch signalLatch = new CountDownLatch(1);
    ManualRegisterStreamObserver requestObserver = new ManualRegisterStreamObserver(MasterMode.SIGNAL_IN_STREAM, signalLatch);
    when(asyncClient.registerWorkerStream(any())).thenReturn(requestObserver);
    RegisterStreamer registerStreamer = new RegisterStreamer(asyncClient, WORKER_ID, mTierAliases, mCapacityMap, mUsedMap, mBlockMap, LOST_STORAGE, EMPTY_CONFIG);
    StreamObserver<RegisterWorkerPResponse> responseObserver = getResponseObserver(registerStreamer);
    requestObserver.setResponseObserver(responseObserver);
    // Prepare the block worker to use the overriden stream
    MasterClientContext context = MasterClientContext.newBuilder(ClientContext.create(ServerConfiguration.global())).build();
    // On heartbeat, the expected values will be checked against
    List<Long> expectedLostBlocks = ImmutableList.of(blockToRemove);
    Map<BlockStoreLocation, List<Long>> expectedAddedBlocks = ImmutableMap.of();
    mBlockMasterClient = new TestBlockMasterClient(context, registerStreamer, expectedLostBlocks, expectedAddedBlocks);
    initBlockWorker();
    // Prepare the blocks in the BlockWorker storage that match the register stream
    prepareBlocksOnWorker(TIER_CONFIG);
    // Let a delete job wait on the signal
    AtomicReference<Throwable> error = new AtomicReference<>();
    Future f = mExecutorService.submit(() -> {
        try {
            signalLatch.await();
            mBlockWorker.removeBlock(1L, blockToRemove);
        } catch (Exception e) {
            error.set(e);
        }
    });
    // Kick off the register stream
    AtomicReference<Long> workerId = new AtomicReference<>(WORKER_ID);
    BlockMasterSync sync = new BlockMasterSync(mBlockWorker, workerId, NET_ADDRESS_1, mBlockMasterClientPool);
    // Check the next heartbeat to be sent to the master
    f.get();
    assertNull(error.get());
    // Validation will happen on the heartbeat
    sync.heartbeat();
}

Example 3 with MasterClientContext

use of alluxio.master.MasterClientContext in project alluxio by Alluxio.

the class MultiProcessCluster method getClients.

/**
 * @return clients for communicating with the cluster
 */
public synchronized Clients getClients() {
    Preconditions.checkState(mState == State.STARTED, "must be in the started state to create a meta master client, but state was %s", mState);
    MasterClientContext config = MasterClientContext.newBuilder(ClientContext.create(ServerConfiguration.global())).setMasterInquireClient(getMasterInquireClient()).build();
    return new Clients(getFileSystemClient(), new RetryHandlingFileSystemMasterClient(config), new RetryHandlingMetaMasterClient(config), new RetryHandlingBlockMasterClient(config));
}

Example 4 with MasterClientContext

use of alluxio.master.MasterClientContext in project alluxio by Alluxio.

the class FileSystemMasterClientPoolTest method create.

@Test
public void create() throws Exception {
    AlluxioConfiguration conf = ConfigurationTestUtils.defaults();
    FileSystemMasterClient expectedClient = Mockito.mock(FileSystemMasterClient.class);
    PowerMockito.mockStatic(FileSystemMasterClient.Factory.class);
    Mockito.when(FileSystemMasterClient.Factory.create(Mockito.any(MasterClientContext.class))).thenReturn(expectedClient);
    FileSystemMasterClient client;
    ClientContext clientContext = ClientContext.create(conf);
    MasterInquireClient masterInquireClient = MasterInquireClient.Factory.create(conf, clientContext.getUserState());
    MasterClientContext masterClientContext = MasterClientContext.newBuilder(clientContext).setMasterInquireClient(masterInquireClient).build();
    try (FileSystemMasterClientPool pool = new FileSystemMasterClientPool(masterClientContext)) {
        client = pool.acquire();
        assertEquals(expectedClient, client);
        pool.release(client);
    }
    Mockito.verify(client).close();
}

Example 5 with MasterClientContext

use of alluxio.master.MasterClientContext in project alluxio by Alluxio.

the class BlockMasterClientPoolTest method create.

@Test
public void create() throws Exception {
    BlockMasterClient expectedClient = mock(BlockMasterClient.class);
    PowerMockito.mockStatic(BlockMasterClient.Factory.class);
    when(BlockMasterClient.Factory.create(any(MasterClientContext.class))).thenReturn(expectedClient);
    BlockMasterClient client;
    ClientContext clientContext = ClientContext.create(mConf);
    MasterInquireClient masterInquireClient = MasterInquireClient.Factory.create(mConf, clientContext.getUserState());
    MasterClientContext masterClientContext = MasterClientContext.newBuilder(clientContext).setMasterInquireClient(masterInquireClient).build();
    try (BlockMasterClientPool pool = new BlockMasterClientPool(masterClientContext)) {
        client = pool.acquire();
        assertEquals(expectedClient, client);
        pool.release(client);
    }
    verify(client).close();
}