Examples with WorkerNetAddress alluxio.wire.WorkerNetAddress used on opensource projects

Example 16 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class ServiceSocketBindIntegrationTest method connectServices.

private void connectServices() throws IOException, ConnectionFailedException {
    // connect Master RPC service
    mBlockMasterClient = BlockMasterClient.Factory.create(new InetSocketAddress(mLocalAlluxioCluster.getHostname(), mLocalAlluxioCluster.getMasterRpcPort()));
    mBlockMasterClient.connect();
    // connect Worker RPC service
    WorkerNetAddress workerAddress = mLocalAlluxioCluster.getWorkerAddress();
    mBlockWorkerClient = FileSystemContext.INSTANCE.createBlockWorkerClient(workerAddress);
    // connect Worker data service
    mWorkerDataService = SocketChannel.open(new InetSocketAddress(workerAddress.getHost(), workerAddress.getDataPort()));
    // connect Master Web service
    InetSocketAddress masterWebAddr = NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_WEB);
    mMasterWebService = (HttpURLConnection) new URL("http://" + masterWebAddr.getAddress().getHostAddress() + ":" + masterWebAddr.getPort() + "/css/custom.min.css").openConnection();
    mMasterWebService.connect();
    // connect Worker Web service
    InetSocketAddress workerWebAddr = new InetSocketAddress(workerAddress.getHost(), workerAddress.getWebPort());
    mWorkerWebService = (HttpURLConnection) new URL("http://" + workerWebAddr.getAddress().getHostAddress() + ":" + workerWebAddr.getPort() + "/css/custom.min.css").openConnection();
    mWorkerWebService.connect();
}

Example 17 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class FileInStream method updateCacheStream.

/**
   * Updates {@link #mCurrentCacheStream}. When {@code mShouldCache} is true, {@code FileInStream}
   * will create an {@code BlockOutStream} to cache the data read only if
   * <ol>
   *   <li>the file is read from under storage, or</li>
   *   <li>the file is read from a remote worker and we have an available local worker.</li>
   * </ol>
   * The following preconditions are checked inside:
   * <ol>
   *   <li>{@link #mCurrentCacheStream} is either done or null.</li>
   *   <li>EOF is reached or {@link #mCurrentBlockInStream} must be valid.</li>
   * </ol>
   * After this call, {@link #mCurrentCacheStream} is either null or freshly created.
   * {@link #mCurrentCacheStream} is created only if the block is not cached in a chosen machine
   * and mPos is at the beginning of a block.
   * This function is only called by {@link #updateStreams()}.
   *
   * @param blockId cached result of {@link #getCurrentBlockId()}
   * @throws IOException if the next cache stream cannot be created
   */
private void updateCacheStream(long blockId) throws IOException {
    // We should really only close a cache stream here. This check is to verify this.
    Preconditions.checkState(mCurrentCacheStream == null || cacheStreamRemaining() == 0);
    closeOrCancelCacheStream();
    Preconditions.checkState(mCurrentCacheStream == null);
    if (blockId < 0) {
        // End of file.
        return;
    }
    Preconditions.checkNotNull(mCurrentBlockInStream);
    if (!mShouldCache || isReadingFromLocalBlockWorker()) {
        return;
    }
    // If this block is read from a remote worker but we don't have a local worker, don't cache
    if (isReadingFromRemoteBlockWorker() && !mContext.hasLocalWorker()) {
        return;
    }
    // middle of a block.
    if (mPos % mBlockSize != 0) {
        return;
    }
    try {
        WorkerNetAddress address = mCacheLocationPolicy.getWorkerForNextBlock(mBlockStore.getWorkerInfoList(), getBlockSizeAllocation(mPos));
        // If we reach here, we need to cache.
        mCurrentCacheStream = mBlockStore.getOutStream(blockId, getBlockSize(mPos), address, mOutStreamOptions);
    } catch (IOException e) {
        handleCacheStreamIOException(e);
    } catch (AlluxioException e) {
        LOG.warn("The block with ID {} could not be cached into Alluxio storage.", blockId, e);
    }
}

Example 18 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class FileSystemContext method getWorkerAddresses.

/**
   * @return if there are any local workers, the returned list will ONLY contain the local workers,
   *         otherwise a list of all remote workers will be returned
   * @throws IOException if an error occurs communicating with the master
   */
private List<WorkerNetAddress> getWorkerAddresses() throws IOException {
    List<WorkerInfo> infos;
    BlockMasterClient blockMasterClient = mBlockMasterClientPool.acquire();
    try {
        infos = blockMasterClient.getWorkerInfoList();
    } catch (AlluxioException e) {
        throw new IOException(e);
    } finally {
        mBlockMasterClientPool.release(blockMasterClient);
    }
    if (infos.isEmpty()) {
        throw new IOException(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage());
    }
    // Convert the worker infos into net addresses, if there are local addresses, only keep those
    List<WorkerNetAddress> workerNetAddresses = new ArrayList<>();
    List<WorkerNetAddress> localWorkerNetAddresses = new ArrayList<>();
    String localHostname = NetworkAddressUtils.getClientHostName();
    for (WorkerInfo info : infos) {
        WorkerNetAddress netAddress = info.getAddress();
        if (netAddress.getHost().equals(localHostname)) {
            localWorkerNetAddresses.add(netAddress);
        }
        workerNetAddresses.add(netAddress);
    }
    return localWorkerNetAddresses.isEmpty() ? workerNetAddresses : localWorkerNetAddresses;
}

Example 19 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class FileSystemContext method createFileSystemWorkerClient.

/**
   * Creates a new file system worker client, prioritizing local workers if available. This method
   * initializes the list of worker addresses if it has not been initialized.
   *
   * @return a file system worker client to a worker in the system
   * @throws IOException if an error occurs getting the list of workers in the system
   */
public FileSystemWorkerClient createFileSystemWorkerClient() throws IOException {
    WorkerNetAddress address;
    synchronized (this) {
        if (mWorkerAddresses == null) {
            mWorkerAddresses = getWorkerAddresses();
        }
        address = mWorkerAddresses.get(ThreadLocalRandom.current().nextInt(mWorkerAddresses.size()));
    }
    InetSocketAddress rpcAddress = NetworkAddressUtils.getRpcPortSocketAddress(address);
    if (!mFileSystemWorkerClientPools.containsKey(rpcAddress)) {
        FileSystemWorkerThriftClientPool pool = new FileSystemWorkerThriftClientPool(mParentSubject, rpcAddress, Configuration.getInt(PropertyKey.USER_FILE_WORKER_CLIENT_POOL_SIZE_MAX), Configuration.getLong(PropertyKey.USER_FILE_WORKER_CLIENT_POOL_GC_THRESHOLD_MS));
        if (mFileSystemWorkerClientPools.putIfAbsent(rpcAddress, pool) != null) {
            pool.close();
        }
    }
    if (!mFileSystemWorkerClientHeartbeatPools.containsKey(rpcAddress)) {
        FileSystemWorkerThriftClientPool pool = new FileSystemWorkerThriftClientPool(mParentSubject, rpcAddress, Configuration.getInt(PropertyKey.USER_FILE_WORKER_CLIENT_POOL_SIZE_MAX), Configuration.getLong(PropertyKey.USER_FILE_WORKER_CLIENT_POOL_GC_THRESHOLD_MS));
        if (mFileSystemWorkerClientHeartbeatPools.putIfAbsent(rpcAddress, pool) != null) {
            pool.close();
        }
    }
    long sessionId = IdUtils.getRandomNonNegativeLong();
    return FileSystemWorkerClient.Factory.create(mFileSystemWorkerClientPools.get(rpcAddress), mFileSystemWorkerClientHeartbeatPools.get(rpcAddress), address, sessionId);
}

Example 20 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class DeterministicHashPolicy method getWorker.

@Override
public WorkerNetAddress getWorker(GetWorkerOptions options) {
    List<BlockWorkerInfo> workerInfos = Lists.newArrayList(options.getBlockWorkerInfos());
    Collections.sort(workerInfos, new Comparator<BlockWorkerInfo>() {

        @Override
        public int compare(BlockWorkerInfo o1, BlockWorkerInfo o2) {
            return o1.getNetAddress().toString().compareToIgnoreCase(o2.getNetAddress().toString());
        }
    });
    HashMap<WorkerNetAddress, BlockWorkerInfo> blockWorkerInfoMap = new HashMap<>();
    for (BlockWorkerInfo workerInfo : options.getBlockWorkerInfos()) {
        blockWorkerInfoMap.put(workerInfo.getNetAddress(), workerInfo);
    }
    List<WorkerNetAddress> workers = new ArrayList<>();
    // Try the next one if the worker mapped from the blockId doesn't work until all the workers
    // are examined.
    int index = (int) (options.getBlockId() % (long) workerInfos.size());
    for (BlockWorkerInfo blockWorkerInfoUnused : workerInfos) {
        WorkerNetAddress candidate = workerInfos.get(index).getNetAddress();
        BlockWorkerInfo workerInfo = blockWorkerInfoMap.get(candidate);
        if (workerInfo != null && workerInfo.getCapacityBytes() >= options.getBlockSize()) {
            workers.add(candidate);
            if (workers.size() >= mShards) {
                break;
            }
        }
        index = (index + 1) % workerInfos.size();
    }
    return workers.isEmpty() ? null : workers.get(mRandom.nextInt(workers.size()));
}