Examples with PropertyKey alluxio.conf.PropertyKey used on opensource projects

Example 11 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class EmulatingBlockIterator method initEvictorConfiguration.

/**
 * Used to calculate configuration that evictors need to run.
 */
private void initEvictorConfiguration() {
    StorageTierAssoc storageTierAssoc = new WorkerStorageTierAssoc();
    // Calculate tier capacities.
    Map<String, Long> tierCapacities = new HashMap<>();
    for (StorageTier tier : mMetadataManager.getTiers()) {
        tierCapacities.put(tier.getTierAlias(), tier.getCapacityBytes());
    }
    long lastTierReservedBytes = 0;
    for (int ordinal = 0; ordinal < storageTierAssoc.size(); ordinal++) {
        String tierAlias = storageTierAssoc.getAlias(ordinal);
        long tierCapacity = tierCapacities.get(tierAlias);
        // High watermark defines when to start the space reserving process.
        // It's only validated in this emulator, it doesn't trigger any background task.
        PropertyKey tierHighWatermarkProp = PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_HIGH_WATERMARK_RATIO.format(ordinal);
        double tierHighWatermarkConf = ServerConfiguration.getDouble(tierHighWatermarkProp);
        Preconditions.checkArgument(tierHighWatermarkConf > 0, "The high watermark of tier %s should be positive, but is %s", Integer.toString(ordinal), tierHighWatermarkConf);
        Preconditions.checkArgument(tierHighWatermarkConf < 1, "The high watermark of tier %s should be less than 1.0, but is %s", Integer.toString(ordinal), tierHighWatermarkConf);
        // Low watermark defines when to stop the space reserving process if started
        PropertyKey tierLowWatermarkProp = PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_LOW_WATERMARK_RATIO.format(ordinal);
        double tierLowWatermarkConf = ServerConfiguration.getDouble(tierLowWatermarkProp);
        Preconditions.checkArgument(tierLowWatermarkConf >= 0, "The low watermark of tier %s should not be negative, but is %s", Integer.toString(ordinal), tierLowWatermarkConf);
        Preconditions.checkArgument(tierLowWatermarkConf < tierHighWatermarkConf, "The low watermark (%s) of tier %d should not be smaller than the high watermark (%s)", tierLowWatermarkConf, ordinal, tierHighWatermarkConf);
        long reservedSpace = (long) (tierCapacity - tierCapacity * tierLowWatermarkConf);
        lastTierReservedBytes += reservedSpace;
        // On each tier, we reserve no more than its capacity
        lastTierReservedBytes = (lastTierReservedBytes <= tierCapacity) ? lastTierReservedBytes : tierCapacity;
        mReservedSpaces.put(tierAlias, lastTierReservedBytes);
        // Update special ANY_TIER to have total reserved capacity.
        if (mReservedSpaces.containsKey(BlockStoreLocation.ANY_TIER)) {
            mReservedSpaces.put(BlockStoreLocation.ANY_TIER, mReservedSpaces.get(BlockStoreLocation.ANY_TIER) + lastTierReservedBytes);
        } else {
            mReservedSpaces.put(BlockStoreLocation.ANY_TIER, lastTierReservedBytes);
        }
    }
}

Example 12 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class AbstractLocalAlluxioCluster method setupTest.

/**
 * Sets up corresponding directories for tests.
 */
protected void setupTest() throws IOException {
    UnderFileSystem ufs = UnderFileSystem.Factory.createForRoot(ServerConfiguration.global());
    String underfsAddress = ServerConfiguration.getString(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS);
    // Deletes the ufs dir for this test from to avoid permission problems
    UnderFileSystemUtils.deleteDirIfExists(ufs, underfsAddress);
    // Creates ufs dir. This must be called before starting UFS with UnderFileSystemCluster.create()
    UnderFileSystemUtils.mkdirIfNotExists(ufs, underfsAddress);
    // Creates storage dirs for worker
    int numLevel = ServerConfiguration.getInt(PropertyKey.WORKER_TIERED_STORE_LEVELS);
    for (int level = 0; level < numLevel; level++) {
        PropertyKey tierLevelDirPath = PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_PATH.format(level);
        String[] dirPaths = ServerConfiguration.getString(tierLevelDirPath).split(",");
        for (String dirPath : dirPaths) {
            FileUtils.createDir(dirPath);
        }
    }
    // Formats the journal
    Format.format(Format.Mode.MASTER, ServerConfiguration.global());
}

Example 13 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class ConfigCheckerIntegrationTest method multiMasters.

@Test
public void multiMasters() throws Exception {
    PropertyKey key = PropertyKey.MASTER_JOURNAL_FLUSH_TIMEOUT_MS;
    Map<Integer, Map<PropertyKey, String>> masterProperties = generatePropertyWithDifferentValues(TEST_NUM_MASTERS, key);
    mCluster = MultiProcessCluster.newBuilder(PortCoordination.CONFIG_CHECKER_MULTI_MASTERS).setClusterName("ConfigCheckerMultiMastersTest").setNumMasters(TEST_NUM_MASTERS).setNumWorkers(0).setMasterProperties(masterProperties).build();
    mCluster.start();
    ConfigCheckReport report = getReport();
    // When using embedded journal, the journal paths are different
    assertEquals(mCluster.getDeployMode().equals(DeployMode.ZOOKEEPER_HA) ? ConfigStatus.WARN : ConfigStatus.FAILED, report.getConfigStatus());
    assertThat(report.getConfigWarns().toString(), CoreMatchers.containsString(key.getName()));
    mCluster.notifySuccess();
}

Example 14 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class ConfigCheckerIntegrationTest method multiNodes.

@Test
public void multiNodes() throws Exception {
    PropertyKey key = PropertyKey.UNDERFS_LISTING_LENGTH;
    // Prepare properties
    Map<Integer, Map<PropertyKey, String>> properties = generatePropertyWithDifferentValues(TEST_NUM_MASTERS + TEST_NUM_WORKERS, key);
    Map<Integer, Map<PropertyKey, String>> masterProperties = properties.entrySet().stream().filter(entry -> (entry.getKey() < TEST_NUM_MASTERS)).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
    Map<Integer, Map<PropertyKey, String>> workerProperties = properties.entrySet().stream().filter(entry -> (entry.getKey() >= TEST_NUM_MASTERS)).collect(Collectors.toMap(entry -> entry.getKey() - TEST_NUM_MASTERS, Map.Entry::getValue));
    mCluster = MultiProcessCluster.newBuilder(PortCoordination.CONFIG_CHECKER_MULTI_NODES).setClusterName("ConfigCheckerMultiNodesTest").setNumMasters(TEST_NUM_MASTERS).setNumWorkers(TEST_NUM_WORKERS).setMasterProperties(masterProperties).setWorkerProperties(workerProperties).build();
    mCluster.start();
    ConfigCheckReport report = getReport();
    assertEquals(ConfigStatus.FAILED, report.getConfigStatus());
    assertThat(report.getConfigErrors().toString(), CoreMatchers.containsString(key.getName()));
    mCluster.notifySuccess();
}

Example 15 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class ConfigCheckerIntegrationTest method multiWorkers.

@Test
public void multiWorkers() throws Exception {
    PropertyKey key = PropertyKey.WORKER_FREE_SPACE_TIMEOUT;
    Map<Integer, Map<PropertyKey, String>> workerProperties = generatePropertyWithDifferentValues(TEST_NUM_WORKERS, key);
    mCluster = MultiProcessCluster.newBuilder(PortCoordination.CONFIG_CHECKER_MULTI_WORKERS).setClusterName("ConfigCheckerMultiWorkersTest").setNumMasters(1).setNumWorkers(TEST_NUM_WORKERS).setWorkerProperties(workerProperties).build();
    mCluster.start();
    ConfigCheckReport report = getReport();
    // The workers values of many directory related properties are different
    assertEquals(ConfigStatus.WARN, report.getConfigStatus());
    assertThat(report.getConfigWarns().toString(), CoreMatchers.containsString(key.getName()));
    mCluster.notifySuccess();
}