Examples with PropertyKey alluxio.conf.PropertyKey used on opensource projects

Example 26 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class DefaultStorageTierTest method tolerantFailureInStorageDir.

@Test
public void tolerantFailureInStorageDir() throws Exception {
    PropertyKey tierDirPathConf = PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_PATH.format(0);
    ServerConfiguration.set(tierDirPathConf, "/dev/null/invalid," + mTestDirPath1);
    mTier = DefaultStorageTier.newStorageTier(Constants.MEDIUM_MEM, false);
    List<StorageDir> dirs = mTier.getStorageDirs();
    Assert.assertEquals(1, dirs.size());
    Assert.assertEquals(mTestBlockDirPath1, dirs.get(0).getDirPath());
}

Example 27 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class MultiProcessCluster method createMaster.

/**
 * Creates the specified master without starting it.
 *
 * @param i the index of the master to create
 */
private synchronized Master createMaster(int i) throws IOException {
    Preconditions.checkState(mState == State.STARTED, "Must be in a started state to create masters");
    MasterNetAddress address = mMasterAddresses.get(i);
    String extension = "-" + mMasterIds.get(i);
    File confDir = new File(mWorkDir, "conf-master" + extension);
    File metastoreDir = new File(mWorkDir, "metastore-master" + extension);
    File logsDir = new File(mWorkDir, "logs-master" + extension);
    logsDir.mkdirs();
    Map<PropertyKey, Object> conf = new HashMap<>();
    conf.put(PropertyKey.LOGGER_TYPE, "MASTER_LOGGER");
    conf.put(PropertyKey.CONF_DIR, confDir.getAbsolutePath());
    conf.put(PropertyKey.MASTER_METASTORE_DIR, metastoreDir.getAbsolutePath());
    conf.put(PropertyKey.LOGS_DIR, logsDir.getAbsolutePath());
    conf.put(PropertyKey.MASTER_HOSTNAME, address.getHostname());
    conf.put(PropertyKey.MASTER_RPC_PORT, Integer.toString(address.getRpcPort()));
    conf.put(PropertyKey.MASTER_WEB_PORT, Integer.toString(address.getWebPort()));
    conf.put(PropertyKey.MASTER_EMBEDDED_JOURNAL_PORT, Integer.toString(address.getEmbeddedJournalPort()));
    if (mDeployMode.equals(DeployMode.EMBEDDED)) {
        File journalDir = new File(mWorkDir, "journal" + extension);
        journalDir.mkdirs();
        conf.put(PropertyKey.MASTER_JOURNAL_FOLDER, journalDir.getAbsolutePath());
    }
    Master master = mCloser.register(new Master(logsDir, conf));
    mMasters.add(master);
    return master;
}

Example 28 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class MultiProcessCluster method writeConfToFile.

/**
 * Creates the conf directory and file.
 * Writes the properties to the generated file.
 *
 * @param dir the conf directory to create
 * @param properties the specific properties of the current node
 */
private void writeConfToFile(File dir, Map<PropertyKey, String> properties) throws IOException {
    // Generates the full set of properties to write
    Map<PropertyKey, Object> map = new HashMap<>(mProperties);
    for (Map.Entry<PropertyKey, String> entry : properties.entrySet()) {
        map.put(entry.getKey(), entry.getValue());
    }
    StringBuilder sb = new StringBuilder();
    for (Entry<PropertyKey, Object> entry : map.entrySet()) {
        sb.append(String.format("%s=%s%n", entry.getKey(), entry.getValue()));
    }
    dir.mkdirs();
    try (FileOutputStream fos = new FileOutputStream(new File(dir, "alluxio-site.properties"))) {
        fos.write(sb.toString().getBytes(Charsets.UTF_8));
    }
}

Example 29 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class GetConf method getConfImpl.

/**
 * Implements get configuration.
 *
 * @param clientSupplier a functor to return a config client of meta master
 * @param alluxioConf Alluxio configuration
 * @param args list of arguments
 * @return 0 on success, 1 on failures
 */
@VisibleForTesting
public static int getConfImpl(Supplier<RetryHandlingMetaMasterConfigClient> clientSupplier, AlluxioConfiguration alluxioConf, String... args) {
    CommandLineParser parser = new DefaultParser();
    CommandLine cmd;
    try {
        cmd = parser.parse(OPTIONS, args, true);
    } catch (ParseException e) {
        printHelp("Unable to parse input args: " + e.getMessage());
        return 1;
    }
    args = cmd.getArgs();
    Map<String, ConfigProperty> confMap = new HashMap<>();
    if (cmd.hasOption(MASTER_OPTION_NAME)) {
        // load cluster-wide configuration
        try (RetryHandlingMetaMasterConfigClient client = clientSupplier.get()) {
            client.getConfiguration(GetConfigurationPOptions.newBuilder().setIgnorePathConf(true).build()).getClusterConf().forEach(prop -> confMap.put(prop.getName(), prop.toProto()));
        } catch (IOException e) {
            System.out.println("Unable to get master-side configuration: " + e.getMessage());
            return -1;
        }
    } else {
        // load local configuration
        for (PropertyKey key : alluxioConf.keySet()) {
            if (key.isBuiltIn()) {
                ConfigProperty.Builder config = ConfigProperty.newBuilder().setName(key.getName()).setSource(alluxioConf.getSource(key).toString());
                Object val = alluxioConf.getOrDefault(key, null, ConfigurationValueOptions.defaults().useDisplayValue(true));
                if (val != null) {
                    config.setValue(String.valueOf(val));
                }
                confMap.put(key.getName(), config.build());
            }
        }
    }
    StringBuilder output = new StringBuilder();
    switch(args.length) {
        case 0:
            List<ConfigProperty> properties = new ArrayList<>(confMap.values());
            properties.sort(Comparator.comparing(ConfigProperty::getName));
            for (ConfigProperty property : properties) {
                String value = ConfigurationUtils.valueAsString(property.getValue());
                output.append(String.format("%s=%s", property.getName(), value));
                if (cmd.hasOption(SOURCE_OPTION_NAME)) {
                    output.append(String.format(" (%s)", property.getSource()));
                }
                output.append("\n");
            }
            System.out.print(output.toString());
            break;
        case 1:
            if (!PropertyKey.isValid(args[0])) {
                printHelp(String.format("%s is not a valid configuration key", args[0]));
                return 1;
            }
            // args[0] can be the alias
            String key = PropertyKey.fromString(args[0]).getName();
            ConfigProperty property = confMap.get(key);
            if (property == null) {
                printHelp(String.format("%s is not found", key));
                return 1;
            }
            if (property.getValue() == null) {
                // value not set
                System.out.println("");
            } else {
                if (cmd.hasOption(SOURCE_OPTION_NAME)) {
                    System.out.println(property.getSource());
                } else if (cmd.hasOption(UNIT_OPTION_NAME)) {
                    String arg = cmd.getOptionValue(UNIT_OPTION_NAME).toUpperCase();
                    try {
                        ByteUnit byteUnit;
                        byteUnit = ByteUnit.valueOf(arg);
                        System.out.println(FormatUtils.parseSpaceSize(property.getValue()) / byteUnit.getValue());
                        break;
                    } catch (Exception e) {
                    // try next unit parse
                    }
                    try {
                        TimeUnit timeUnit;
                        timeUnit = TimeUnit.valueOf(arg);
                        System.out.println(FormatUtils.parseTimeSize(property.getValue()) / timeUnit.getValue());
                        break;
                    } catch (IllegalArgumentException ex) {
                    // try next unit parse
                    }
                    printHelp(String.format("%s is not a valid unit", arg));
                    return 1;
                } else {
                    System.out.println(property.getValue());
                }
            }
            break;
        default:
            printHelp(String.format("More arguments than expected. Args: %s", Arrays.toString(args)));
            return 1;
    }
    return 0;
}

Example 30 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class ConfigurationDocGenerator method writeCSVFile.

/**
 * Writes property key to csv files.
 *
 * @param defaultKeys Collection which is from PropertyKey DEFAULT_KEYS_MAP.values()
 * @param filePath    path for csv files
 */
@VisibleForTesting
public static void writeCSVFile(Collection<? extends PropertyKey> defaultKeys, String filePath) throws IOException {
    if (defaultKeys.size() == 0) {
        return;
    }
    FileWriter fileWriter;
    Closer closer = Closer.create();
    String[] fileNames = { "user-configuration.csv", "master-configuration.csv", "worker-configuration.csv", "security-configuration.csv", "common-configuration.csv", "cluster-management-configuration.csv" };
    try {
        // HashMap for FileWriter per each category
        Map<String, FileWriter> fileWriterMap = new HashMap<>();
        for (String fileName : fileNames) {
            fileWriter = new FileWriter(PathUtils.concatPath(filePath, fileName));
            // Write the CSV file header and line separator after the header
            fileWriter.append(CSV_FILE_HEADER + "\n");
            // put fileWriter
            String key = fileName.substring(0, fileName.indexOf("configuration") - 1);
            fileWriterMap.put(key, fileWriter);
            // register file writer
            closer.register(fileWriter);
        }
        // Sort defaultKeys
        List<PropertyKey> dfkeys = new ArrayList<>(defaultKeys);
        Collections.sort(dfkeys);
        for (PropertyKey propertyKey : dfkeys) {
            String pKey = propertyKey.toString();
            String defaultDescription;
            if (propertyKey.getDefaultSupplier().get() == null) {
                defaultDescription = "";
            } else {
                defaultDescription = propertyKey.getDefaultSupplier().getDescription();
            }
            // Quote the whole description to escape characters such as commas.
            defaultDescription = String.format("\"%s\"", defaultDescription);
            // Write property key and default value to CSV
            String keyValueStr = pKey + "," + defaultDescription + "\n";
            if (pKey.startsWith("alluxio.user.")) {
                fileWriter = fileWriterMap.get("user");
            } else if (pKey.startsWith("alluxio.master.")) {
                fileWriter = fileWriterMap.get("master");
            } else if (pKey.startsWith("alluxio.worker.")) {
                fileWriter = fileWriterMap.get("worker");
            } else if (pKey.startsWith("alluxio.security.")) {
                fileWriter = fileWriterMap.get("security");
            } else if (pKey.startsWith("alluxio.integration")) {
                fileWriter = fileWriterMap.get("cluster-management");
            } else {
                fileWriter = fileWriterMap.get("common");
            }
            fileWriter.append(keyValueStr);
        }
        LOG.info("Property Key CSV files were created successfully.");
    } catch (Exception e) {
        throw closer.rethrow(e);
    } finally {
        try {
            closer.close();
        } catch (IOException e) {
            LOG.error("Error while flushing/closing Property Key CSV FileWriter", e);
        }
    }
}