Examples with WorkerNetAddress alluxio.wire.WorkerNetAddress used on opensource projects

Example 56 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class MultiWorkerIntegrationTest method replicateFileBlocks.

private void replicateFileBlocks(AlluxioURI filePath) throws Exception {
    FileSystemContext fsContext = FileSystemContext.create(ServerConfiguration.global());
    AlluxioBlockStore store = AlluxioBlockStore.create(fsContext);
    URIStatus status = mResource.get().getClient().getStatus(filePath);
    List<FileBlockInfo> blocks = status.getFileBlockInfos();
    List<BlockWorkerInfo> workers = fsContext.getCachedWorkers();
    for (FileBlockInfo block : blocks) {
        BlockInfo blockInfo = block.getBlockInfo();
        WorkerNetAddress src = blockInfo.getLocations().get(0).getWorkerAddress();
        WorkerNetAddress dest = workers.stream().filter(candidate -> !candidate.getNetAddress().equals(src)).findFirst().get().getNetAddress();
        try (OutputStream outStream = store.getOutStream(blockInfo.getBlockId(), blockInfo.getLength(), dest, OutStreamOptions.defaults(fsContext.getClientContext()).setBlockSizeBytes(8 * Constants.MB).setWriteType(WriteType.MUST_CACHE))) {
            try (InputStream inStream = store.getInStream(blockInfo.getBlockId(), new InStreamOptions(status, ServerConfiguration.global()))) {
                ByteStreams.copy(inStream, outStream);
            }
        }
    }
}

Example 57 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class ServiceSocketBindIntegrationTest method connectServices.

private void connectServices() throws IOException, ConnectionFailedException {
    // connect Master RPC service
    mBlockMasterClient = BlockMasterClient.Factory.create(new InetSocketAddress(mLocalAlluxioCluster.getHostname(), mLocalAlluxioCluster.getMasterRpcPort()));
    mBlockMasterClient.connect();
    // connect Worker RPC service
    WorkerNetAddress workerAddress = mLocalAlluxioCluster.getWorkerAddress();
    mBlockWorkerClient = FileSystemContext.INSTANCE.createBlockWorkerClient(workerAddress);
    // connect Worker data service
    mWorkerDataService = SocketChannel.open(new InetSocketAddress(workerAddress.getHost(), workerAddress.getDataPort()));
    // connect Master Web service
    InetSocketAddress masterWebAddr = NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_WEB);
    mMasterWebService = (HttpURLConnection) new URL("http://" + masterWebAddr.getAddress().getHostAddress() + ":" + masterWebAddr.getPort() + "/css/custom.min.css").openConnection();
    mMasterWebService.connect();
    // connect Worker Web service
    InetSocketAddress workerWebAddr = new InetSocketAddress(workerAddress.getHost(), workerAddress.getWebPort());
    mWorkerWebService = (HttpURLConnection) new URL("http://" + workerWebAddr.getAddress().getHostAddress() + ":" + workerWebAddr.getPort() + "/css/custom.min.css").openConnection();
    mWorkerWebService.connect();
}

Example 58 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class FileInStream method updateCacheStream.

/**
   * Updates {@link #mCurrentCacheStream}. When {@code mShouldCache} is true, {@code FileInStream}
   * will create an {@code BlockOutStream} to cache the data read only if
   * <ol>
   *   <li>the file is read from under storage, or</li>
   *   <li>the file is read from a remote worker and we have an available local worker.</li>
   * </ol>
   * The following preconditions are checked inside:
   * <ol>
   *   <li>{@link #mCurrentCacheStream} is either done or null.</li>
   *   <li>EOF is reached or {@link #mCurrentBlockInStream} must be valid.</li>
   * </ol>
   * After this call, {@link #mCurrentCacheStream} is either null or freshly created.
   * {@link #mCurrentCacheStream} is created only if the block is not cached in a chosen machine
   * and mPos is at the beginning of a block.
   * This function is only called by {@link #updateStreams()}.
   *
   * @param blockId cached result of {@link #getCurrentBlockId()}
   * @throws IOException if the next cache stream cannot be created
   */
private void updateCacheStream(long blockId) throws IOException {
    // We should really only close a cache stream here. This check is to verify this.
    Preconditions.checkState(mCurrentCacheStream == null || cacheStreamRemaining() == 0);
    closeOrCancelCacheStream();
    Preconditions.checkState(mCurrentCacheStream == null);
    if (blockId < 0) {
        // End of file.
        return;
    }
    Preconditions.checkNotNull(mCurrentBlockInStream);
    if (!mShouldCache || isReadingFromLocalBlockWorker()) {
        return;
    }
    // If this block is read from a remote worker but we don't have a local worker, don't cache
    if (isReadingFromRemoteBlockWorker() && !mContext.hasLocalWorker()) {
        return;
    }
    // middle of a block.
    if (mPos % mBlockSize != 0) {
        return;
    }
    try {
        WorkerNetAddress address = mCacheLocationPolicy.getWorkerForNextBlock(mBlockStore.getWorkerInfoList(), getBlockSizeAllocation(mPos));
        // If we reach here, we need to cache.
        mCurrentCacheStream = mBlockStore.getOutStream(blockId, getBlockSize(mPos), address, mOutStreamOptions);
    } catch (IOException e) {
        handleCacheStreamIOException(e);
    } catch (AlluxioException e) {
        LOG.warn("The block with ID {} could not be cached into Alluxio storage.", blockId, e);
    }
}

Example 59 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class FileSystemContext method createFileSystemWorkerClient.

/**
   * Creates a new file system worker client, prioritizing local workers if available. This method
   * initializes the list of worker addresses if it has not been initialized.
   *
   * @return a file system worker client to a worker in the system
   * @throws IOException if an error occurs getting the list of workers in the system
   */
public FileSystemWorkerClient createFileSystemWorkerClient() throws IOException {
    WorkerNetAddress address;
    synchronized (this) {
        if (mWorkerAddresses == null) {
            mWorkerAddresses = getWorkerAddresses();
        }
        address = mWorkerAddresses.get(ThreadLocalRandom.current().nextInt(mWorkerAddresses.size()));
    }
    InetSocketAddress rpcAddress = NetworkAddressUtils.getRpcPortSocketAddress(address);
    if (!mFileSystemWorkerClientPools.containsKey(rpcAddress)) {
        FileSystemWorkerThriftClientPool pool = new FileSystemWorkerThriftClientPool(mParentSubject, rpcAddress, Configuration.getInt(PropertyKey.USER_FILE_WORKER_CLIENT_POOL_SIZE_MAX), Configuration.getLong(PropertyKey.USER_FILE_WORKER_CLIENT_POOL_GC_THRESHOLD_MS));
        if (mFileSystemWorkerClientPools.putIfAbsent(rpcAddress, pool) != null) {
            pool.close();
        }
    }
    if (!mFileSystemWorkerClientHeartbeatPools.containsKey(rpcAddress)) {
        FileSystemWorkerThriftClientPool pool = new FileSystemWorkerThriftClientPool(mParentSubject, rpcAddress, Configuration.getInt(PropertyKey.USER_FILE_WORKER_CLIENT_POOL_SIZE_MAX), Configuration.getLong(PropertyKey.USER_FILE_WORKER_CLIENT_POOL_GC_THRESHOLD_MS));
        if (mFileSystemWorkerClientHeartbeatPools.putIfAbsent(rpcAddress, pool) != null) {
            pool.close();
        }
    }
    long sessionId = IdUtils.getRandomNonNegativeLong();
    return FileSystemWorkerClient.Factory.create(mFileSystemWorkerClientPools.get(rpcAddress), mFileSystemWorkerClientHeartbeatPools.get(rpcAddress), address, sessionId);
}

Example 60 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class RoundRobinPolicy method getWorkerForNextBlock.

/**
   * The policy uses the first fetch of worker info list as the base, and visits each of them in a
   * round-robin manner in the subsequent calls. The policy doesn't assume the list of worker info
   * in the subsequent calls has the same order from the first, and it will skip the workers that
   * are no longer active.
   *
   * @param workerInfoList the info of the active workers
   * @param blockSizeBytes the size of the block in bytes
   * @return the address of the worker to write to
   */
@Override
public WorkerNetAddress getWorkerForNextBlock(Iterable<BlockWorkerInfo> workerInfoList, long blockSizeBytes) {
    if (!mInitialized) {
        mWorkerInfoList = Lists.newArrayList(workerInfoList);
        Collections.shuffle(mWorkerInfoList);
        mIndex = 0;
        mInitialized = true;
    }
    // at most try all the workers
    for (int i = 0; i < mWorkerInfoList.size(); i++) {
        WorkerNetAddress candidate = mWorkerInfoList.get(mIndex).getNetAddress();
        BlockWorkerInfo workerInfo = findBlockWorkerInfo(workerInfoList, candidate);
        mIndex = (mIndex + 1) % mWorkerInfoList.size();
        if (workerInfo != null && workerInfo.getCapacityBytes() >= blockSizeBytes) {
            return candidate;
        }
    }
    return null;
}