Examples with PropertyKey alluxio.conf.PropertyKey used on opensource projects

Example 1 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class RetryHandlingMetaMasterConfigClient method removePathConfiguration.

@Override
public void removePathConfiguration(AlluxioURI path, Set<PropertyKey> keys) throws IOException {
    Set<String> keySet = new HashSet<>();
    for (PropertyKey key : keys) {
        keySet.add(key.getName());
    }
    retryRPC(() -> mClient.removePathConfiguration(RemovePathConfigurationPRequest.newBuilder().setPath(path.getPath()).addAllKeys(keySet).build()), RPC_LOG, "removePathConfiguration", "path=%s,keys=%s", path, keys);
}

Example 2 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class AbstractFileSystemTest method hadoopShouldLoadFileSystemWhenConfigured.

@Test
public void hadoopShouldLoadFileSystemWhenConfigured() throws Exception {
    org.apache.hadoop.conf.Configuration conf = getConf();
    URI uri = URI.create(Constants.HEADER + "localhost:19998/tmp/path.txt");
    Map<PropertyKey, Object> properties = new HashMap<>();
    properties.put(PropertyKey.MASTER_HOSTNAME, uri.getHost());
    properties.put(PropertyKey.MASTER_RPC_PORT, Integer.toString(uri.getPort()));
    properties.put(PropertyKey.ZOOKEEPER_ENABLED, false);
    properties.put(PropertyKey.ZOOKEEPER_ADDRESS, null);
    try (Closeable c = new ConfigurationRule(properties, mConfiguration).toResource()) {
        final org.apache.hadoop.fs.FileSystem fs = org.apache.hadoop.fs.FileSystem.get(uri, conf);
        assertTrue(fs instanceof FileSystem);
    }
}

Example 3 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class Format method format.

/**
 * Formats the Alluxio file system.
 *
 * @param mode either {@code MASTER} or {@code WORKER}
 * @param alluxioConf Alluxio configuration
 */
public static void format(Mode mode, AlluxioConfiguration alluxioConf) throws IOException {
    NoopUfsManager noopUfsManager = new NoopUfsManager();
    switch(mode) {
        case MASTER:
            URI journalLocation = JournalUtils.getJournalLocation();
            LOG.info("Formatting master journal: {}", journalLocation);
            JournalSystem journalSystem = new JournalSystem.Builder().setLocation(journalLocation).build(CommonUtils.ProcessType.MASTER);
            for (String masterServiceName : ServiceUtils.getMasterServiceNames()) {
                journalSystem.createJournal(new NoopMaster(masterServiceName, noopUfsManager));
            }
            journalSystem.format();
            break;
        case WORKER:
            String workerDataFolder = ServerConfiguration.getString(PropertyKey.WORKER_DATA_FOLDER);
            LOG.info("Formatting worker data folder: {}", workerDataFolder);
            int storageLevels = ServerConfiguration.getInt(PropertyKey.WORKER_TIERED_STORE_LEVELS);
            for (int level = 0; level < storageLevels; level++) {
                PropertyKey tierLevelDirPath = PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_PATH.format(level);
                String[] dirPaths = ServerConfiguration.getString(tierLevelDirPath).split(",");
                String name = "Data path for tier " + level;
                for (String dirPath : dirPaths) {
                    String dirWorkerDataFolder = CommonUtils.getWorkerDataDirectory(dirPath, alluxioConf);
                    LOG.info("Formatting {}:{}", name, dirWorkerDataFolder);
                    formatWorkerDataFolder(dirWorkerDataFolder);
                }
            }
            break;
        default:
            throw new RuntimeException(String.format("Unrecognized format mode: %s", mode));
    }
}

Example 4 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class ConfigurationUtils method getConfiguration.

/**
 * Gets all configuration properties filtered by the specified scope.
 *
 * @param conf the configuration to use
 * @param scope the scope to filter by
 * @return the properties
 */
public static List<ConfigProperty> getConfiguration(AlluxioConfiguration conf, Scope scope) {
    ConfigurationValueOptions useRawDisplayValue = ConfigurationValueOptions.defaults().useDisplayValue(true);
    List<ConfigProperty> configs = new ArrayList<>();
    List<PropertyKey> selectedKeys = conf.keySet().stream().filter(key -> GrpcUtils.contains(key.getScope(), scope)).filter(key -> key.isValid(key.getName())).collect(toList());
    for (PropertyKey key : selectedKeys) {
        ConfigProperty.Builder configProp = ConfigProperty.newBuilder().setName(key.getName()).setSource(conf.getSource(key).toString());
        if (conf.isSet(key)) {
            configProp.setValue(String.valueOf(conf.get(key, useRawDisplayValue)));
        }
        configs.add(configProp.build());
    }
    return configs;
}

Example 5 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class ConfigurationTestUtils method testConfigurationDefaults.

/**
 * Returns reasonable default configuration values for running integration tests.
 *
 * These defaults are mostly aimed at getting tests to run faster. Individual tests may override
 * them to test specific functionality.
 *
 * @param hostname the master hostname
 * @param workDirectory the work directory in which to configure the journal and tiered storage
 * @return the configuration
 */
public static Map<PropertyKey, Object> testConfigurationDefaults(AlluxioConfiguration alluxioConf, String hostname, String workDirectory) {
    Map<PropertyKey, Object> conf = new HashMap<>();
    conf.put(PropertyKey.MASTER_HOSTNAME, hostname);
    conf.put(PropertyKey.WORKER_BIND_HOST, hostname);
    conf.put(PropertyKey.WORKER_WEB_BIND_HOST, hostname);
    conf.put(PropertyKey.MASTER_BIND_HOST, hostname);
    conf.put(PropertyKey.MASTER_WEB_BIND_HOST, hostname);
    conf.put(PropertyKey.WORK_DIR, workDirectory);
    // Sets up the tiered store
    conf.put(PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_ALIAS.format(0), Constants.MEDIUM_MEM);
    String ramdiskPath = PathUtils.concatPath(workDirectory, "ramdisk");
    conf.put(PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_PATH.format(0), ramdiskPath);
    int numLevel = alluxioConf.getInt(PropertyKey.WORKER_TIERED_STORE_LEVELS);
    for (int level = 1; level < numLevel; level++) {
        PropertyKey tierLevelDirPath = PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_PATH.format(level);
        String[] dirPaths = alluxioConf.getString(tierLevelDirPath).split(",");
        List<String> newPaths = new ArrayList<>();
        for (String dirPath : dirPaths) {
            String newPath = workDirectory + dirPath;
            newPaths.add(newPath);
        }
        conf.put(PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_PATH.format(level), Joiner.on(',').join(newPaths));
    }
    // Sets up the block allocation and review policy
    conf.put(PropertyKey.WORKER_REVIEWER_CLASS, "alluxio.worker.block.reviewer.AcceptingReviewer");
    // Sets up the journal folder
    conf.put(PropertyKey.MASTER_JOURNAL_TYPE, "UFS");
    conf.put(PropertyKey.MASTER_JOURNAL_FOLDER, PathUtils.concatPath(workDirectory, "journal"));
    conf.put(PropertyKey.MASTER_METASTORE_DIR, PathUtils.concatPath(workDirectory, "metastore"));
    conf.put(PropertyKey.USER_BLOCK_SIZE_BYTES_DEFAULT, "1KB");
    conf.put(PropertyKey.USER_BLOCK_REMOTE_READ_BUFFER_SIZE_BYTES, "64");
    conf.put(PropertyKey.USER_STREAMING_READER_CHUNK_SIZE_BYTES, "64");
    conf.put(PropertyKey.MASTER_TTL_CHECKER_INTERVAL_MS, "1sec");
    conf.put(PropertyKey.MASTER_JOURNAL_FLUSH_TIMEOUT_MS, "1sec");
    // This cannot be too short, since sometimes there are grpc channel startup delays, which
    // affect authentication
    conf.put(PropertyKey.NETWORK_CONNECTION_AUTH_TIMEOUT, "5sec");
    conf.put(PropertyKey.NETWORK_CONNECTION_SHUTDOWN_GRACEFUL_TIMEOUT, "3sec");
    conf.put(PropertyKey.NETWORK_CONNECTION_SERVER_SHUTDOWN_TIMEOUT, "10sec");
    // Shutdown journal tailer quickly. Graceful shutdown is unnecessarily slow.
    conf.put(PropertyKey.MASTER_JOURNAL_TAILER_SHUTDOWN_QUIET_WAIT_TIME_MS, "50ms");
    conf.put(PropertyKey.MASTER_JOURNAL_TAILER_SLEEP_TIME_MS, "10ms");
    // To keep tests fast, we should do more retries with a lower max wait time.
    conf.put(PropertyKey.USER_RPC_RETRY_MAX_DURATION, "2s");
    conf.put(PropertyKey.USER_RPC_RETRY_MAX_SLEEP_MS, "200");
    conf.put(PropertyKey.USER_RPC_RETRY_BASE_SLEEP_MS, "20");
    // Do not engage safe mode by default since the worker is connected when test starts.
    conf.put(PropertyKey.MASTER_WORKER_CONNECT_WAIT_TIME, "0sec");
    // Since tests are always running on a single host keep the resolution timeout low as otherwise
    // people running with strange network configurations will see very slow tests
    conf.put(PropertyKey.NETWORK_HOST_RESOLUTION_TIMEOUT_MS, "250ms");
    // default write type becomes MUST_CACHE, set this value to CACHE_THROUGH for tests.
    // default Alluxio storage is STORE, and under storage is SYNC_PERSIST for tests.
    // TODO(binfan): eliminate this setting after updating integration tests
    // conf.put(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, "CACHE_THROUGH");
    conf.put(PropertyKey.WEB_THREADS, "1");
    conf.put(PropertyKey.WEB_RESOURCES, PathUtils.concatPath(System.getProperty("user.dir"), "../webui"));
    conf.put(PropertyKey.WORKER_RAMDISK_SIZE, "100MB");
    conf.put(PropertyKey.MASTER_LOST_WORKER_FILE_DETECTION_INTERVAL, "15ms");
    conf.put(PropertyKey.MASTER_LOST_WORKER_DETECTION_INTERVAL, "15ms");
    conf.put(PropertyKey.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS, "15ms");
    conf.put(PropertyKey.WORKER_NETWORK_NETTY_WORKER_THREADS, "2");
    // Shutdown data server quickly. Graceful shutdown is unnecessarily slow.
    conf.put(PropertyKey.WORKER_NETWORK_NETTY_SHUTDOWN_QUIET_PERIOD, "0ms");
    conf.put(PropertyKey.WORKER_NETWORK_SHUTDOWN_TIMEOUT, "0ms");
    conf.put(PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_ALIAS.format(0), Constants.MEDIUM_MEM);
    conf.put(PropertyKey.MASTER_EMBEDDED_JOURNAL_MIN_ELECTION_TIMEOUT, "260ms");
    conf.put(PropertyKey.MASTER_EMBEDDED_JOURNAL_MAX_ELECTION_TIMEOUT, "520ms");
    // Reset the value to avoid raft journal system complaining about log size < 65
    conf.put(PropertyKey.MASTER_JOURNAL_LOG_SIZE_BYTES_MAX, PropertyKey.MASTER_JOURNAL_LOG_SIZE_BYTES_MAX.getDefaultValue());
    conf.put(PropertyKey.USER_WORKER_LIST_REFRESH_INTERVAL, "1s");
    // faster persists
    conf.put(PropertyKey.JOB_MASTER_WORKER_HEARTBEAT_INTERVAL, "20ms");
    conf.put(PropertyKey.MASTER_PERSISTENCE_CHECKER_INTERVAL_MS, "20ms");
    conf.put(PropertyKey.MASTER_PERSISTENCE_INITIAL_INTERVAL_MS, "20ms");
    conf.put(PropertyKey.MASTER_PERSISTENCE_SCHEDULER_INTERVAL_MS, "20ms");
    // faster refresh
    conf.put(PropertyKey.MASTER_WORKER_INFO_CACHE_REFRESH_TIME, "20ms");
    // faster I/O retries.
    conf.put(PropertyKey.USER_BLOCK_READ_RETRY_SLEEP_MIN, "1ms");
    conf.put(PropertyKey.USER_BLOCK_READ_RETRY_SLEEP_MIN, "5ms");
    conf.put(PropertyKey.USER_BLOCK_READ_RETRY_MAX_DURATION, "10ms");
    conf.put(PropertyKey.TEST_MODE, true);
    return conf;
}