Examples with WorkerNetAddress alluxio.wire.WorkerNetAddress used on opensource projects

Example 36 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class AlluxioBlockStore method getOutStream.

/**
 * Gets a stream to write data to a block based on the options. The stream can only be backed by
 * Alluxio storage.
 *
 * @param blockId the block to write
 * @param blockSize the standard block size to write
 * @param options the output stream option
 * @return a {@link BlockOutStream} which can be used to write data to the block in a streaming
 *         fashion
 */
public BlockOutStream getOutStream(long blockId, long blockSize, OutStreamOptions options) throws IOException {
    WorkerNetAddress address;
    BlockLocationPolicy locationPolicy = Preconditions.checkNotNull(options.getLocationPolicy(), PreconditionMessage.BLOCK_WRITE_LOCATION_POLICY_UNSPECIFIED);
    GetWorkerOptions workerOptions = GetWorkerOptions.defaults().setBlockInfo(new BlockInfo().setBlockId(blockId).setLength(blockSize)).setBlockWorkerInfos(new ArrayList<>(mContext.getCachedWorkers()));
    // The number of initial copies depends on the write type: if ASYNC_THROUGH, it is the property
    // "alluxio.user.file.replication.durable" before data has been persisted; otherwise
    // "alluxio.user.file.replication.min"
    int initialReplicas = (options.getWriteType() == WriteType.ASYNC_THROUGH && options.getReplicationDurable() > options.getReplicationMin()) ? options.getReplicationDurable() : options.getReplicationMin();
    if (initialReplicas <= 1) {
        address = locationPolicy.getWorker(workerOptions);
        if (address == null) {
            if (mContext.getCachedWorkers().isEmpty()) {
                throw new UnavailableException(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage());
            }
            throw new UnavailableException(ExceptionMessage.NO_SPACE_FOR_BLOCK_ON_WORKER.getMessage(blockSize));
        }
        // TODO(ggezer): Retry on another worker if this has no storage.
        return getOutStream(blockId, blockSize, address, options);
    }
    // Group different block workers by their hostnames
    Map<String, Set<BlockWorkerInfo>> blockWorkersByHost = new HashMap<>();
    for (BlockWorkerInfo blockWorker : workerOptions.getBlockWorkerInfos()) {
        String hostName = blockWorker.getNetAddress().getHost();
        if (blockWorkersByHost.containsKey(hostName)) {
            blockWorkersByHost.get(hostName).add(blockWorker);
        } else {
            blockWorkersByHost.put(hostName, com.google.common.collect.Sets.newHashSet(blockWorker));
        }
    }
    // Select N workers on different hosts where N is the value of initialReplicas for this block
    List<WorkerNetAddress> workerAddressList = new ArrayList<>();
    List<BlockWorkerInfo> updatedInfos = Lists.newArrayList(workerOptions.getBlockWorkerInfos());
    for (int i = 0; i < initialReplicas; i++) {
        address = locationPolicy.getWorker(workerOptions);
        if (address == null) {
            break;
        }
        workerAddressList.add(address);
        updatedInfos.removeAll(blockWorkersByHost.get(address.getHost()));
        workerOptions.setBlockWorkerInfos(updatedInfos);
    }
    if (workerAddressList.size() < initialReplicas) {
        throw new alluxio.exception.status.ResourceExhaustedException(String.format("Not enough workers for replications, %d workers selected but %d required", workerAddressList.size(), initialReplicas));
    }
    return BlockOutStream.createReplicatedBlockOutStream(mContext, blockId, blockSize, workerAddressList, options);
}

Example 37 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class LocalFirstAvoidEvictionPolicy method getWorker.

@Override
public WorkerNetAddress getWorker(GetWorkerOptions options) {
    List<BlockWorkerInfo> allWorkers = Lists.newArrayList(options.getBlockWorkerInfos());
    // Prefer workers with enough availability.
    List<BlockWorkerInfo> workers = allWorkers.stream().filter(worker -> getAvailableBytes(worker) >= options.getBlockInfo().getLength()).collect(Collectors.toList());
    if (workers.isEmpty()) {
        workers = allWorkers;
    }
    GetWorkerOptions filteredWorkers = GetWorkerOptions.defaults().setBlockInfo(options.getBlockInfo()).setBlockWorkerInfos(workers);
    return mPolicy.getWorker(filteredWorkers);
}

Example 38 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class MostAvailableFirstPolicy method getWorker.

/**
 * The policy returns null if no worker is qualified.
 */
@Override
public WorkerNetAddress getWorker(GetWorkerOptions options) {
    long mostAvailableBytes = -1;
    WorkerNetAddress result = null;
    for (BlockWorkerInfo workerInfo : options.getBlockWorkerInfos()) {
        if (workerInfo.getCapacityBytes() - workerInfo.getUsedBytes() > mostAvailableBytes) {
            mostAvailableBytes = workerInfo.getCapacityBytes() - workerInfo.getUsedBytes();
            result = workerInfo.getNetAddress();
        }
    }
    return result;
}

Example 39 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class FileOutStreamTest method before.

/**
 * Sets up the different contexts and clients before a test runs.
 */
@Before
public void before() throws Exception {
    GroupMappingServiceTestUtils.resetCache();
    ClientTestUtils.setSmallBufferSizes(sConf);
    mClientContext = ClientContext.create(sConf);
    // PowerMock enums and final classes
    mFileSystemContext = PowerMockito.mock(FileSystemContext.class);
    when(mFileSystemContext.getClientContext()).thenReturn(mClientContext);
    when(mFileSystemContext.getClusterConf()).thenReturn(sConf);
    when(mFileSystemContext.getPathConf(any(AlluxioURI.class))).thenReturn(sConf);
    mBlockStore = PowerMockito.mock(AlluxioBlockStore.class);
    mFileSystemMasterClient = PowerMockito.mock(FileSystemMasterClient.class);
    PowerMockito.mockStatic(AlluxioBlockStore.class);
    PowerMockito.when(AlluxioBlockStore.create(mFileSystemContext)).thenReturn(mBlockStore);
    when(mFileSystemContext.acquireMasterClientResource()).thenReturn(new DummyCloseableResource<>(mFileSystemMasterClient));
    when(mFileSystemMasterClient.getStatus(any(AlluxioURI.class), any(GetStatusPOptions.class))).thenReturn(new URIStatus(new FileInfo()));
    // Return sequentially increasing numbers for new block ids
    when(mFileSystemMasterClient.getNewBlockIdForFile(FILE_NAME)).thenAnswer(new Answer<Long>() {

        private long mCount = 0;

        @Override
        public Long answer(InvocationOnMock invocation) throws Throwable {
            return mCount++;
        }
    });
    // Set up out streams. When they are created, add them to outStreamMap
    final Map<Long, TestBlockOutStream> outStreamMap = new HashMap<>();
    when(mBlockStore.getOutStream(anyLong(), eq(BLOCK_LENGTH), any(OutStreamOptions.class))).thenAnswer(new Answer<TestBlockOutStream>() {

        @Override
        public TestBlockOutStream answer(InvocationOnMock invocation) throws Throwable {
            Long blockId = invocation.getArgument(0, Long.class);
            if (!outStreamMap.containsKey(blockId)) {
                TestBlockOutStream newStream = new TestBlockOutStream(ByteBuffer.allocate(1000), BLOCK_LENGTH);
                outStreamMap.put(blockId, newStream);
            }
            return outStreamMap.get(blockId);
        }
    });
    BlockWorkerInfo workerInfo = new BlockWorkerInfo(new WorkerNetAddress().setHost("localhost").setTieredIdentity(TieredIdentityFactory.fromString("node=localhost", sConf)).setRpcPort(1).setDataPort(2).setWebPort(3), Constants.GB, 0);
    when(mFileSystemContext.getCachedWorkers()).thenReturn(Lists.newArrayList(workerInfo));
    mAlluxioOutStreamMap = outStreamMap;
    // Create an under storage stream so that we can check whether it has been flushed
    final AtomicBoolean underStorageFlushed = new AtomicBoolean(false);
    mUnderStorageOutputStream = new TestUnderFileSystemFileOutStream(ByteBuffer.allocate(5000)) {

        @Override
        public void flush() throws IOException {
            super.flush();
            underStorageFlushed.set(true);
        }
    };
    mUnderStorageFlushed = underStorageFlushed;
    PowerMockito.mockStatic(UnderFileSystemFileOutStream.class);
    PowerMockito.when(UnderFileSystemFileOutStream.create(any(FileSystemContext.class), any(WorkerNetAddress.class), any(OutStreamOptions.class))).thenReturn(mUnderStorageOutputStream);
    OutStreamOptions options = OutStreamOptions.defaults(mClientContext).setBlockSizeBytes(BLOCK_LENGTH).setWriteType(WriteType.CACHE_THROUGH).setUfsPath(FILE_NAME.getPath());
    mTestStream = createTestStream(FILE_NAME, options);
}

Example 40 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class BlockLocationUtilsTest method chooseLocalAccessibleDomainSocket.

@Test
public void chooseLocalAccessibleDomainSocket() throws Exception {
    List<BlockWorkerInfo> workers = new ArrayList<>();
    workers.add(worker(Constants.GB, "node2", "rack2"));
    // create worker info with domain socket path
    BlockWorkerInfo workerWithDomainSocket = worker(Constants.GB, "node3", "rack3");
    String domainSocketPath = "/tmp/domain/uuid-node3";
    workerWithDomainSocket.getNetAddress().setDomainSocketPath(domainSocketPath);
    workers.add(workerWithDomainSocket);
    // mock NettyUtils
    PowerMockito.mockStatic(NettyUtils.class);
    when(NettyUtils.isDomainSocketAccessible(eq(workerWithDomainSocket.getNetAddress()), any(AlluxioConfiguration.class))).thenReturn(true);
    // choose worker with domain socket accessible ignoring rack
    InstancedConfiguration conf = ConfigurationTestUtils.defaults();
    conf.set(PropertyKey.WORKER_DATA_SERVER_DOMAIN_SOCKET_AS_UUID, true);
    List<WorkerNetAddress> addresses = workers.stream().map(worker -> worker.getNetAddress()).filter(Objects::nonNull).collect(Collectors.toList());
    Optional<Pair<WorkerNetAddress, Boolean>> chosen = BlockLocationUtils.nearest(TieredIdentityFactory.fromString("node=node1,rack=rack2", conf), addresses, conf);
    assertTrue(chosen.isPresent());
    assertTrue(chosen.get().getSecond());
    assertEquals(domainSocketPath, chosen.get().getFirst().getDomainSocketPath());
    assertEquals("node3", chosen.get().getFirst().getHost());
}