Examples with GetWorkerOptions alluxio.client.block.policy.options.GetWorkerOptions used on opensource projects

Example 1 with GetWorkerOptions

use of alluxio.client.block.policy.options.GetWorkerOptions in project alluxio by Alluxio.

the class LocalFirstPolicyTest method getOthersWhenNotEnoughCapacityOnLocal.

/**
 * Tests that another worker is picked in case the local host does not have enough capacity.
 */
@Test
public void getOthersWhenNotEnoughCapacityOnLocal() {
    String localhostName = NetworkAddressUtils.getLocalHostName(sResolutionTimeout);
    LocalFirstPolicy policy = new LocalFirstPolicy(sConf);
    List<BlockWorkerInfo> workers = new ArrayList<>();
    workers.add(worker(Constants.GB, "worker1", ""));
    workers.add(worker(Constants.MB, localhostName, ""));
    GetWorkerOptions options = GetWorkerOptions.defaults().setBlockWorkerInfos(workers).setBlockInfo(new BlockInfo().setLength(Constants.GB));
    assertEquals("worker1", policy.getWorker(options).getHost());
}

Example 2 with GetWorkerOptions

use of alluxio.client.block.policy.options.GetWorkerOptions in project alluxio by Alluxio.

the class LocalFirstPolicyTest method getOthersRandomly.

/**
 * Tests that non-local workers are randomly selected.
 */
@Test
public void getOthersRandomly() {
    LocalFirstPolicy policy = new LocalFirstPolicy(sConf);
    List<BlockWorkerInfo> workers = new ArrayList<>();
    workers.add(worker(Constants.GB, "worker1", ""));
    workers.add(worker(Constants.GB, "worker2", ""));
    boolean success = false;
    GetWorkerOptions options = GetWorkerOptions.defaults().setBlockWorkerInfos(workers).setBlockInfo(new BlockInfo().setLength(Constants.MB));
    for (int i = 0; i < 100; i++) {
        String host = policy.getWorker(options).getHost();
        if (!host.equals(policy.getWorker(options).getHost())) {
            success = true;
            break;
        }
    }
    assertTrue(success);
}

Example 3 with GetWorkerOptions

use of alluxio.client.block.policy.options.GetWorkerOptions in project alluxio by Alluxio.

the class LocalFirstAvoidEvictionPolicyTest method getLocalWhenNoneHasAvailability.

/**
 * Tests that local host is picked if none of the workers has enough availability.
 */
@Test
public void getLocalWhenNoneHasAvailability() {
    String localhostName = NetworkAddressUtils.getLocalHostName(1000);
    BlockLocationPolicy policy = new LocalFirstAvoidEvictionPolicy(mConf);
    List<BlockWorkerInfo> workers = new ArrayList<>();
    workers.add(worker(Constants.GB, Constants.MB, "worker1", ""));
    workers.add(worker(Constants.GB, Constants.MB, localhostName, ""));
    GetWorkerOptions options = GetWorkerOptions.defaults().setBlockWorkerInfos(workers).setBlockInfo(new BlockInfo().setLength(Constants.GB));
    assertEquals(localhostName, policy.getWorker(options).getHost());
}

Example 4 with GetWorkerOptions

use of alluxio.client.block.policy.options.GetWorkerOptions in project alluxio by Alluxio.

the class LocalFirstAvoidEvictionPolicyTest method chooseClosestTierAvoidEviction.

@Test
public void chooseClosestTierAvoidEviction() throws Exception {
    List<BlockWorkerInfo> workers = new ArrayList<>();
    workers.add(worker(Constants.GB, Constants.MB, "node2", "rack3"));
    workers.add(worker(Constants.GB, 0, "node3", "rack2"));
    workers.add(worker(Constants.GB, 0, "node4", "rack3"));
    BlockLocationPolicy policy;
    WorkerNetAddress chosen;
    // local rack with enough availability
    policy = new LocalFirstAvoidEvictionPolicy(mConf.getBytes(PropertyKey.USER_BLOCK_AVOID_EVICTION_POLICY_RESERVED_BYTES), TieredIdentityFactory.fromString("node=node2,rack=rack3", mConf), mConf);
    GetWorkerOptions options = GetWorkerOptions.defaults().setBlockWorkerInfos(workers).setBlockInfo(new BlockInfo().setLength(Constants.GB));
    chosen = policy.getWorker(options);
    assertEquals("node4", chosen.getTieredIdentity().getTier(0).getValue());
}

Example 5 with GetWorkerOptions

use of alluxio.client.block.policy.options.GetWorkerOptions in project alluxio by Alluxio.

the class AlluxioBlockStore method getOutStream.

/**
 * Gets a stream to write data to a block based on the options. The stream can only be backed by
 * Alluxio storage.
 *
 * @param blockId the block to write
 * @param blockSize the standard block size to write
 * @param options the output stream option
 * @return a {@link BlockOutStream} which can be used to write data to the block in a streaming
 *         fashion
 */
public BlockOutStream getOutStream(long blockId, long blockSize, OutStreamOptions options) throws IOException {
    WorkerNetAddress address;
    BlockLocationPolicy locationPolicy = Preconditions.checkNotNull(options.getLocationPolicy(), PreconditionMessage.BLOCK_WRITE_LOCATION_POLICY_UNSPECIFIED);
    GetWorkerOptions workerOptions = GetWorkerOptions.defaults().setBlockInfo(new BlockInfo().setBlockId(blockId).setLength(blockSize)).setBlockWorkerInfos(new ArrayList<>(mContext.getCachedWorkers()));
    // The number of initial copies depends on the write type: if ASYNC_THROUGH, it is the property
    // "alluxio.user.file.replication.durable" before data has been persisted; otherwise
    // "alluxio.user.file.replication.min"
    int initialReplicas = (options.getWriteType() == WriteType.ASYNC_THROUGH && options.getReplicationDurable() > options.getReplicationMin()) ? options.getReplicationDurable() : options.getReplicationMin();
    if (initialReplicas <= 1) {
        address = locationPolicy.getWorker(workerOptions);
        if (address == null) {
            if (mContext.getCachedWorkers().isEmpty()) {
                throw new UnavailableException(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage());
            }
            throw new UnavailableException(ExceptionMessage.NO_SPACE_FOR_BLOCK_ON_WORKER.getMessage(blockSize));
        }
        // TODO(ggezer): Retry on another worker if this has no storage.
        return getOutStream(blockId, blockSize, address, options);
    }
    // Group different block workers by their hostnames
    Map<String, Set<BlockWorkerInfo>> blockWorkersByHost = new HashMap<>();
    for (BlockWorkerInfo blockWorker : workerOptions.getBlockWorkerInfos()) {
        String hostName = blockWorker.getNetAddress().getHost();
        if (blockWorkersByHost.containsKey(hostName)) {
            blockWorkersByHost.get(hostName).add(blockWorker);
        } else {
            blockWorkersByHost.put(hostName, com.google.common.collect.Sets.newHashSet(blockWorker));
        }
    }
    // Select N workers on different hosts where N is the value of initialReplicas for this block
    List<WorkerNetAddress> workerAddressList = new ArrayList<>();
    List<BlockWorkerInfo> updatedInfos = Lists.newArrayList(workerOptions.getBlockWorkerInfos());
    for (int i = 0; i < initialReplicas; i++) {
        address = locationPolicy.getWorker(workerOptions);
        if (address == null) {
            break;
        }
        workerAddressList.add(address);
        updatedInfos.removeAll(blockWorkersByHost.get(address.getHost()));
        workerOptions.setBlockWorkerInfos(updatedInfos);
    }
    if (workerAddressList.size() < initialReplicas) {
        throw new alluxio.exception.status.ResourceExhaustedException(String.format("Not enough workers for replications, %d workers selected but %d required", workerAddressList.size(), initialReplicas));
    }
    return BlockOutStream.createReplicatedBlockOutStream(mContext, blockId, blockSize, workerAddressList, options);
}