Examples with PropertyKey alluxio.conf.PropertyKey used on opensource projects

Example 16 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class ServerConfigurationStoreTest method before.

@Before
public void before() {
    PropertyKey keyEnforce = PropertyKey.ZOOKEEPER_ELECTION_PATH;
    PropertyKey keyWarn = PropertyKey.WORKER_FREE_SPACE_TIMEOUT;
    mConfigListOne = Arrays.asList(ConfigProperty.newBuilder().setName(keyEnforce.getName()).setSource("Test").setValue("Value").build(), ConfigProperty.newBuilder().setName(keyWarn.getName()).setSource("Test").setValue("Value2").build());
    mConfigListTwo = Arrays.asList(ConfigProperty.newBuilder().setName(keyEnforce.getName()).setSource("Test").setValue("Value3").build(), ConfigProperty.newBuilder().setName(keyWarn.getName()).setSource("Test").setValue("Value4").build());
    Random random = new Random();
    mAddressOne = new Address(RandomStringUtils.randomAlphanumeric(10), random.nextInt());
    mAddressTwo = new Address(RandomStringUtils.randomAlphanumeric(10), random.nextInt());
}

Example 17 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class ServerConfigurationCheckerTest method checkConf.

@Test
public void checkConf() {
    // Prepare data
    PropertyKey keyMasterEnforce = new PropertyKey.Builder("TestKey1").setConsistencyCheckLevel(PropertyKey.ConsistencyCheckLevel.ENFORCE).setScope(Scope.MASTER).build();
    ConfigProperty masterEnforceProp = ConfigProperty.newBuilder().setName(keyMasterEnforce.getName()).setSource("Test").setValue("Value").build();
    PropertyKey keyWorkerWarn = new PropertyKey.Builder("TestKey2").setConsistencyCheckLevel(PropertyKey.ConsistencyCheckLevel.WARN).setScope(Scope.WORKER).build();
    ConfigProperty workerWarnProp = ConfigProperty.newBuilder().setName(keyWorkerWarn.getName()).setSource("Test").setValue("Value").build();
    PropertyKey keyServerEnforce = new PropertyKey.Builder("TestKey3").setConsistencyCheckLevel(PropertyKey.ConsistencyCheckLevel.ENFORCE).setScope(Scope.SERVER).build();
    ConfigProperty serverEnforceProp = ConfigProperty.newBuilder().setName(keyServerEnforce.getName()).setSource("Test").setValue("Value").build();
    Random random = new Random();
    Address addressOne = new Address(RandomStringUtils.randomAlphanumeric(10), random.nextInt());
    Address addressTwo = new Address(RandomStringUtils.randomAlphanumeric(10), random.nextInt());
    // When records have nothing different, no errors or warns will be found
    mRecordOne.registerNewConf(addressOne, Arrays.asList(masterEnforceProp, workerWarnProp));
    mRecordTwo.registerNewConf(addressTwo, Arrays.asList(masterEnforceProp, workerWarnProp));
    checkResults(0, 0, ConfigStatus.PASSED);
    // When records have a wrong warn property, checker should be able to find config warns
    ConfigProperty wrongWorkerWarnProp = ConfigProperty.newBuilder().setName(workerWarnProp.getName()).setSource(workerWarnProp.getSource()).setValue("WrongValue").build();
    mRecordOne.registerNewConf(addressOne, Arrays.asList(masterEnforceProp, wrongWorkerWarnProp));
    checkResults(0, 1, ConfigStatus.WARN);
    // When records have a wrong enforce property, checker should be able to find config errors
    ConfigProperty wrongMasterEnforceProp = ConfigProperty.newBuilder().setName(masterEnforceProp.getName()).setSource(masterEnforceProp.getSource()).setValue("WrongValue").build();
    mRecordTwo.registerNewConf(addressTwo, Arrays.asList(wrongMasterEnforceProp, workerWarnProp));
    checkResults(1, 1, ConfigStatus.FAILED);
    ConfigProperty wrongServerEnforceProp = ConfigProperty.newBuilder().setName(serverEnforceProp.getName()).setSource(serverEnforceProp.getSource()).setValue("WrongValue").build();
    mRecordOne.registerNewConf(addressOne, Arrays.asList(masterEnforceProp, workerWarnProp, serverEnforceProp));
    mRecordTwo.registerNewConf(addressTwo, Arrays.asList(masterEnforceProp, workerWarnProp, wrongServerEnforceProp));
    checkResults(1, 0, ConfigStatus.FAILED);
}

Example 18 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class MultiProcessCluster method startNewMasters.

/**
 * Start a number of new master nodes.
 *
 * @param count the count of the masters to start
 * @param format whether to format the master first
 * @throws Exception if any error occurs
 */
public synchronized void startNewMasters(int count, boolean format) throws Exception {
    int startIndex = 0;
    if (mMasterAddresses != null) {
        startIndex = mMasterAddresses.size();
    } else {
        mMasterAddresses = new ArrayList<>();
    }
    List<MasterNetAddress> masterAddresses = generateMasterAddresses(count);
    for (MasterNetAddress newMasterAddress : masterAddresses) {
        String id = newMasterAddress.getEmbeddedJournalPort() + "-" + RandomString.make(RandomString.DEFAULT_LENGTH);
        mMasterIds.add(id);
    }
    mMasterAddresses.addAll(masterAddresses);
    mNumMasters = mMasterAddresses.size();
    LOG.info("Master addresses: {}", mMasterAddresses);
    switch(mDeployMode) {
        case UFS_NON_HA:
            MasterNetAddress masterAddress = mMasterAddresses.get(0);
            mProperties.put(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.UFS.toString());
            mProperties.put(PropertyKey.MASTER_HOSTNAME, masterAddress.getHostname());
            mProperties.put(PropertyKey.MASTER_RPC_PORT, Integer.toString(masterAddress.getRpcPort()));
            mProperties.put(PropertyKey.MASTER_WEB_PORT, Integer.toString(masterAddress.getWebPort()));
            break;
        case EMBEDDED:
            List<String> journalAddresses = new ArrayList<>();
            List<String> rpcAddresses = new ArrayList<>();
            for (MasterNetAddress address : mMasterAddresses) {
                journalAddresses.add(String.format("%s:%d", address.getHostname(), address.getEmbeddedJournalPort()));
                rpcAddresses.add(String.format("%s:%d", address.getHostname(), address.getRpcPort()));
            }
            mProperties.put(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.EMBEDDED.toString());
            mProperties.put(PropertyKey.MASTER_EMBEDDED_JOURNAL_ADDRESSES, com.google.common.base.Joiner.on(",").join(journalAddresses));
            mProperties.put(PropertyKey.MASTER_RPC_ADDRESSES, com.google.common.base.Joiner.on(",").join(rpcAddresses));
            break;
        case ZOOKEEPER_HA:
            mCuratorServer = mCloser.register(new TestingServer(-1, AlluxioTestDirectory.createTemporaryDirectory("zk")));
            mProperties.put(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.UFS.toString());
            mProperties.put(PropertyKey.ZOOKEEPER_ENABLED, "true");
            mProperties.put(PropertyKey.ZOOKEEPER_ADDRESS, mCuratorServer.getConnectString());
            break;
        default:
            throw new IllegalStateException("Unknown deploy mode: " + mDeployMode.toString());
    }
    for (Entry<PropertyKey, Object> entry : ConfigurationTestUtils.testConfigurationDefaults(ServerConfiguration.global(), NetworkAddressUtils.getLocalHostName((int) ServerConfiguration.getMs(PropertyKey.NETWORK_HOST_RESOLUTION_TIMEOUT_MS)), mWorkDir.getAbsolutePath()).entrySet()) {
        // Don't overwrite explicitly set properties.
        if (mProperties.containsKey(entry.getKey())) {
            continue;
        }
        // Keep the default RPC timeout.
        if (entry.getKey().equals(PropertyKey.USER_RPC_RETRY_MAX_DURATION)) {
            continue;
        }
        mProperties.put(entry.getKey(), entry.getValue());
    }
    mProperties.put(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS, PathUtils.concatPath(mWorkDir, "underFSStorage"));
    new File((String) mProperties.get(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS)).mkdirs();
    if (format) {
        formatJournal();
    }
    writeConf();
    ServerConfiguration.merge(mProperties, Source.RUNTIME);
    // in ms
    final int MASTER_START_DELAY_MS = 500;
    for (int i = 0; i < count; i++) {
        createMaster(startIndex + i).start();
        wait(MASTER_START_DELAY_MS);
    }
    mFilesystemContext = null;
}

Example 19 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class MultiProcessCluster method createWorker.

/**
 * Creates the specified worker without starting it.
 *
 * @param i the index of the worker to create
 */
private synchronized Worker createWorker(int i) throws IOException {
    Preconditions.checkState(mState == State.STARTED, "Must be in a started state to create workers");
    File confDir = new File(mWorkDir, "conf-worker" + i);
    File logsDir = new File(mWorkDir, "logs-worker" + i);
    File ramdisk = new File(mWorkDir, "ramdisk" + i);
    logsDir.mkdirs();
    ramdisk.mkdirs();
    int rpcPort = getNewPort();
    int dataPort = getNewPort();
    int webPort = getNewPort();
    Map<PropertyKey, Object> conf = new HashMap<>();
    conf.put(PropertyKey.LOGGER_TYPE, "WORKER_LOGGER");
    conf.put(PropertyKey.CONF_DIR, confDir.getAbsolutePath());
    conf.put(PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_PATH.format(0), ramdisk.getAbsolutePath());
    conf.put(PropertyKey.LOGS_DIR, logsDir.getAbsolutePath());
    conf.put(PropertyKey.WORKER_RPC_PORT, Integer.toString(rpcPort));
    conf.put(PropertyKey.WORKER_WEB_PORT, Integer.toString(webPort));
    Worker worker = mCloser.register(new Worker(logsDir, conf));
    mWorkers.add(worker);
    LOG.info("Created worker with (rpc, data, web) ports ({}, {}, {})", rpcPort, dataPort, webPort);
    return worker;
}

Example 20 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class ConfigurationDocGenerator method writeYMLFile.

/**
 * Writes description of property key to yml files.
 *
 * @param defaultKeys Collection which is from PropertyKey DEFAULT_KEYS_MAP.values()
 * @param filePath path for csv files
 */
@VisibleForTesting
public static void writeYMLFile(Collection<? extends PropertyKey> defaultKeys, String filePath) throws IOException {
    if (defaultKeys.size() == 0) {
        return;
    }
    FileWriter fileWriter;
    Closer closer = Closer.create();
    String[] fileNames = { "user-configuration.yml", "master-configuration.yml", "worker-configuration.yml", "security-configuration.yml", "common-configuration.yml", "cluster-management-configuration.yml" };
    try {
        // HashMap for FileWriter per each category
        Map<String, FileWriter> fileWriterMap = new HashMap<>();
        for (String fileName : fileNames) {
            fileWriter = new FileWriter(PathUtils.concatPath(filePath, fileName));
            // put fileWriter
            String key = fileName.substring(0, fileName.indexOf("configuration") - 1);
            fileWriterMap.put(key, fileWriter);
            // register file writer
            closer.register(fileWriter);
        }
        // Sort defaultKeys
        List<PropertyKey> dfkeys = new ArrayList<>(defaultKeys);
        Collections.sort(dfkeys);
        for (PropertyKey iteratorPK : dfkeys) {
            String pKey = iteratorPK.toString();
            // Puts descriptions in single quotes to avoid having to escaping reserved characters.
            // Still needs to escape single quotes with double single quotes.
            String description = iteratorPK.getDescription().replace("'", "''");
            // Write property key and default value to yml files
            if (iteratorPK.isIgnoredSiteProperty()) {
                description += " Note: overwriting this property will only work when it is passed as a " + "JVM system property (e.g., appending \"-D" + iteratorPK + "\"=<NEW_VALUE>\" to " + "$ALLUXIO_JAVA_OPTS). Setting it in alluxio-site.properties will not work.";
            }
            String keyValueStr = pKey + ":\n  '" + description + "'\n";
            if (pKey.startsWith("alluxio.user.")) {
                fileWriter = fileWriterMap.get("user");
            } else if (pKey.startsWith("alluxio.master.")) {
                fileWriter = fileWriterMap.get("master");
            } else if (pKey.startsWith("alluxio.worker.")) {
                fileWriter = fileWriterMap.get("worker");
            } else if (pKey.startsWith("alluxio.security.")) {
                fileWriter = fileWriterMap.get("security");
            } else if (pKey.startsWith("alluxio.integration.")) {
                fileWriter = fileWriterMap.get("cluster-management");
            } else {
                fileWriter = fileWriterMap.get("common");
            }
            fileWriter.append(StringEscapeUtils.escapeHtml4(keyValueStr));
        }
        LOG.info("YML files for description of Property Keys were created successfully.");
    } catch (Exception e) {
        throw closer.rethrow(e);
    } finally {
        try {
            closer.close();
        } catch (IOException e) {
            LOG.error("Error while flushing/closing YML files for description of Property Keys " + "FileWriter", e);
        }
    }
}