Examples with AlluxioConfiguration alluxio.conf.AlluxioConfiguration used on opensource projects

Example 11 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class BlockInStream method create.

/**
 * Creates a {@link BlockInStream}.
 *
 * One of several read behaviors:
 *
 * 1. Domain socket - if the data source is the local worker and the local worker has a domain
 * socket server
 * 2. Short-Circuit - if the data source is the local worker
 * 3. Local Loopback Read - if the data source is the local worker and short circuit is disabled
 * 4. Read from remote worker - if the data source is a remote worker
 * 5. UFS Read from worker - if the data source is UFS, read from the UFS policy's designated
 * worker (ufs -> local or remote worker -> client)
 *
 * @param context the file system context
 * @param info the block info
 * @param dataSource the Alluxio worker which should read the data
 * @param dataSourceType the source location of the block
 * @param options the instream options
 * @return the {@link BlockInStream} object
 */
public static BlockInStream create(FileSystemContext context, BlockInfo info, WorkerNetAddress dataSource, BlockInStreamSource dataSourceType, InStreamOptions options) throws IOException {
    long blockId = info.getBlockId();
    long blockSize = info.getLength();
    if (dataSourceType == BlockInStreamSource.PROCESS_LOCAL) {
        // Interaction between the current client and the worker it embedded to should
        // go through worker internal communication directly without RPC involves
        LOG.debug("Creating worker process local input stream for block {} @ {}", blockId, dataSource);
        return createProcessLocalBlockInStream(context, dataSource, blockId, blockSize, options);
    }
    AlluxioConfiguration alluxioConf = context.getClusterConf();
    boolean shortCircuit = alluxioConf.getBoolean(PropertyKey.USER_SHORT_CIRCUIT_ENABLED);
    boolean shortCircuitPreferred = alluxioConf.getBoolean(PropertyKey.USER_SHORT_CIRCUIT_PREFERRED);
    boolean sourceSupportsDomainSocket = NettyUtils.isDomainSocketSupported(dataSource);
    boolean sourceIsLocal = dataSourceType == BlockInStreamSource.NODE_LOCAL;
    // OR alluxio.user.short.circuit.preferred is true
    if (sourceIsLocal && shortCircuit && (shortCircuitPreferred || !sourceSupportsDomainSocket)) {
        LOG.debug("Creating short circuit input stream for block {} @ {}", blockId, dataSource);
        try {
            return createLocalBlockInStream(context, dataSource, blockId, blockSize, options);
        } catch (NotFoundException e) {
            // Failed to do short circuit read because the block is not available in Alluxio.
            // We will try to read via gRPC. So this exception is ignored.
            LOG.warn("Failed to create short circuit input stream for block {} @ {}. Falling back to " + "network transfer", blockId, dataSource);
        }
    }
    // gRPC
    LOG.debug("Creating gRPC input stream for block {} @ {} from client {} reading through {} (" + "data locates in the local worker {}, shortCircuitEnabled {}, " + "shortCircuitPreferred {}, sourceSupportDomainSocket {})", blockId, dataSource, NetworkAddressUtils.getClientHostName(alluxioConf), dataSource, sourceIsLocal, shortCircuit, shortCircuitPreferred, sourceSupportsDomainSocket);
    return createGrpcBlockInStream(context, dataSource, dataSourceType, blockId, blockSize, options);
}

Example 12 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class BlockInStream method createGrpcBlockInStream.

/**
 * Creates a {@link BlockInStream} to read from a gRPC data server.
 *
 * @param context the file system context
 * @param address the address of the gRPC data server
 * @param blockSource the source location of the block
 * @param blockSize the block size
 * @param blockId the block id
 * @return the {@link BlockInStream} created
 */
private static BlockInStream createGrpcBlockInStream(FileSystemContext context, WorkerNetAddress address, BlockInStreamSource blockSource, long blockId, long blockSize, InStreamOptions options) {
    AlluxioConfiguration conf = context.getClusterConf();
    long chunkSize = conf.getBytes(PropertyKey.USER_STREAMING_READER_CHUNK_SIZE_BYTES);
    // Construct the partial read request
    ReadRequest.Builder builder = ReadRequest.newBuilder().setBlockId(blockId).setPromote(ReadType.fromProto(options.getOptions().getReadType()).isPromote()).setOpenUfsBlockOptions(// Add UFS fallback options
    options.getOpenUfsBlockOptions(blockId)).setPositionShort(options.getPositionShort()).setChunkSize(chunkSize);
    DataReader.Factory factory;
    if (context.getClusterConf().getBoolean(PropertyKey.FUSE_SHARED_CACHING_READER_ENABLED) && blockSize > chunkSize * 4) {
        // Heuristic to resolve issues/12146, guarded by alluxio.fuse.shared.caching.reader.enabled
        // GrpcDataReader instances are shared across FileInStreams to mitigate seek cost
        factory = new SharedGrpcDataReader.Factory(context, address, builder, blockSize);
    } else {
        factory = new GrpcDataReader.Factory(context, address, builder);
    }
    return new BlockInStream(factory, conf, address, blockSource, blockId, blockSize);
}

Example 13 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class ConfigurationUtils method getPathConf.

/**
 * Loads the path level configuration from the get configuration response.
 *
 * Only client scope properties will be loaded.
 *
 * @param response the get configuration RPC response
 * @param clusterConf cluster level configuration
 * @return the loaded path level configuration
 */
public static PathConfiguration getPathConf(GetConfigurationPResponse response, AlluxioConfiguration clusterConf) {
    String clientVersion = clusterConf.getString(PropertyKey.VERSION);
    LOG.debug("Alluxio client (version {}) is trying to load path level configurations", clientVersion);
    Map<String, AlluxioConfiguration> pathConfs = new HashMap<>();
    response.getPathConfigsMap().forEach((path, conf) -> {
        Properties props = filterAndLoadProperties(conf.getPropertiesList(), Scope.CLIENT, (key, value) -> String.format("Loading property: %s (%s) -> %s for path %s", key, key.getScope(), value, path));
        AlluxioProperties properties = new AlluxioProperties();
        properties.merge(props, Source.PATH_DEFAULT);
        pathConfs.put(path, new InstancedConfiguration(properties, true));
    });
    LOG.debug("Alluxio client has loaded path level configurations");
    return PathConfiguration.create(pathConfs);
}

Example 14 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class ConfigurationUtils method getConfiguration.

/**
 * Gets all configuration properties filtered by the specified scope.
 *
 * @param conf the configuration to use
 * @param scope the scope to filter by
 * @return the properties
 */
public static List<ConfigProperty> getConfiguration(AlluxioConfiguration conf, Scope scope) {
    ConfigurationValueOptions useRawDisplayValue = ConfigurationValueOptions.defaults().useDisplayValue(true);
    List<ConfigProperty> configs = new ArrayList<>();
    List<PropertyKey> selectedKeys = conf.keySet().stream().filter(key -> GrpcUtils.contains(key.getScope(), scope)).filter(key -> key.isValid(key.getName())).collect(toList());
    for (PropertyKey key : selectedKeys) {
        ConfigProperty.Builder configProp = ConfigProperty.newBuilder().setName(key.getName()).setSource(conf.getSource(key).toString());
        if (conf.isSet(key)) {
            configProp.setValue(String.valueOf(conf.get(key, useRawDisplayValue)));
        }
        configs.add(configProp.build());
    }
    return configs;
}

Example 15 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class MetricsSystem method constructSourceName.

/**
 * Constructs the source name of metrics in this {@link MetricsSystem}.
 */
private static String constructSourceName() {
    PropertyKey sourceKey = null;
    switch(CommonUtils.PROCESS_TYPE.get()) {
        case MASTER:
            sourceKey = PropertyKey.MASTER_HOSTNAME;
            break;
        case WORKER:
            sourceKey = PropertyKey.WORKER_HOSTNAME;
            break;
        case CLIENT:
            sourceKey = PropertyKey.USER_APP_ID;
            break;
        case JOB_MASTER:
            sourceKey = PropertyKey.JOB_MASTER_HOSTNAME;
            break;
        case JOB_WORKER:
            sourceKey = PropertyKey.JOB_WORKER_HOSTNAME;
            break;
        default:
            break;
    }
    AlluxioConfiguration conf = new InstancedConfiguration(ConfigurationUtils.defaults());
    if (sourceKey != null && conf.isSet(sourceKey)) {
        return conf.getString(sourceKey);
    }
    String hostName;
    // is not resolved on metrics reporting
    try {
        hostName = NetworkAddressUtils.getLocalHostMetricName(sResolveTimeout);
    } catch (RuntimeException e) {
        hostName = "unknown";
        LOG.error("Can't find local host name", e);
    }
    return hostName;
}