Examples with PropertyKey alluxio.PropertyKey used on opensource projects

Example 1 with PropertyKey

use of alluxio.PropertyKey in project alluxio by Alluxio.

the class StorageTier method initStorageTier.

private void initStorageTier() throws BlockAlreadyExistsException, IOException, WorkerOutOfSpaceException {
    String workerDataFolder = Configuration.get(PropertyKey.WORKER_DATA_FOLDER);
    String tmpDir = Configuration.get(PropertyKey.WORKER_DATA_TMP_FOLDER);
    PropertyKey tierDirPathConf = PropertyKeyFormat.WORKER_TIERED_STORE_LEVEL_DIRS_PATH_FORMAT.format(mTierOrdinal);
    String[] dirPaths = Configuration.get(tierDirPathConf).split(",");
    // /mnt/ramdisk/alluxioworker
    for (int i = 0; i < dirPaths.length; i++) {
        dirPaths[i] = PathUtils.concatPath(dirPaths[i].trim(), workerDataFolder);
    }
    PropertyKey tierDirCapacityConf = PropertyKeyFormat.WORKER_TIERED_STORE_LEVEL_DIRS_QUOTA_FORMAT.format(mTierOrdinal);
    String rawDirQuota = Configuration.get(tierDirCapacityConf);
    Preconditions.checkState(rawDirQuota.length() > 0, PreconditionMessage.ERR_TIER_QUOTA_BLANK);
    String[] dirQuotas = rawDirQuota.split(",");
    mDirs = new ArrayList<>(dirPaths.length);
    long totalCapacity = 0;
    for (int i = 0; i < dirPaths.length; i++) {
        int index = i >= dirQuotas.length ? dirQuotas.length - 1 : i;
        long capacity = FormatUtils.parseSpaceSize(dirQuotas[index]);
        totalCapacity += capacity;
        mDirs.add(StorageDir.newStorageDir(this, i, capacity, dirPaths[i]));
        // Delete tmp directory.
        String tmpDirPath = PathUtils.concatPath(dirPaths[i], tmpDir);
        try {
            FileUtils.deletePathRecursively(tmpDirPath);
        } catch (IOException e) {
            if (FileUtils.exists(tmpDirPath)) {
                LOG.error("Failed to clean up temporary directory: {}.", tmpDirPath);
            }
        }
    }
    mCapacityBytes = totalCapacity;
}

Example 2 with PropertyKey

use of alluxio.PropertyKey in project alluxio by Alluxio.

the class SpaceReserverTest method reserveCorrectAmountsOfSpace.

@Test
public void reserveCorrectAmountsOfSpace() throws Exception {
    BlockWorker blockWorker = PowerMockito.mock(BlockWorker.class);
    BlockStoreMeta storeMeta = PowerMockito.mock(BlockStoreMeta.class);
    Mockito.when(blockWorker.getStoreMeta()).thenReturn(storeMeta);
    Map<String, Long> capacityBytesOnTiers = ImmutableMap.of("MEM", 400L, "HDD", 1000L);
    Mockito.when(storeMeta.getCapacityBytesOnTiers()).thenReturn(capacityBytesOnTiers);
    // Create two tiers named "MEM" and "HDD" with aliases 0 and 1.
    TieredBlockStoreTestUtils.setupConfWithMultiTier("/", new int[] { 0, 1 }, new String[] { "MEM", "HDD" }, new String[][] { new String[] { "/a" }, new String[] { "/b" } }, new long[][] { new long[] { 0 }, new long[] { 0 } }, "/");
    PropertyKey reserveRatioProp = PropertyKeyFormat.WORKER_TIERED_STORE_LEVEL_RESERVED_RATIO_FORMAT.format(0);
    Configuration.set(reserveRatioProp, "0.2");
    reserveRatioProp = PropertyKeyFormat.WORKER_TIERED_STORE_LEVEL_RESERVED_RATIO_FORMAT.format(1);
    Configuration.set(reserveRatioProp, "0.3");
    SpaceReserver spaceReserver = new SpaceReserver(blockWorker);
    mExecutorService.submit(new HeartbeatThread(HeartbeatContext.WORKER_SPACE_RESERVER, spaceReserver, 0));
    // Run the space reserver once.
    HeartbeatScheduler.execute(HeartbeatContext.WORKER_SPACE_RESERVER);
    // 400 * 0.2 = 80
    Mockito.verify(blockWorker).freeSpace(Sessions.MIGRATE_DATA_SESSION_ID, 80L, "MEM");
    // 400 * 0.2 + 1000 * 0.3 = 380
    Mockito.verify(blockWorker).freeSpace(Sessions.MIGRATE_DATA_SESSION_ID, 380L, "HDD");
}

Example 3 with PropertyKey

use of alluxio.PropertyKey in project alluxio by Alluxio.

the class StorageTierTest method blankStorageTier.

@Test
public void blankStorageTier() throws Exception {
    PropertyKey tierDirCapacityConf = PropertyKeyFormat.WORKER_TIERED_STORE_LEVEL_DIRS_QUOTA_FORMAT.format(0);
    Configuration.set(tierDirCapacityConf, "");
    mThrown.expect(IllegalStateException.class);
    mThrown.expectMessage(PreconditionMessage.ERR_TIER_QUOTA_BLANK.toString());
    mTier = StorageTier.newStorageTier("MEM");
}

Example 4 with PropertyKey

use of alluxio.PropertyKey in project alluxio by Alluxio.

the class HdfsUnderFileSystemUtilsTest method addKeyFromSystemProperty.

/**
   * Tests {@link HdfsUnderFileSystemUtils#addKey} method when a property is set in system property.
   */
@Test
public void addKeyFromSystemProperty() {
    PropertyKey key = PropertyKey.HOME;
    org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration();
    System.setProperty(key.toString(), "systemKey");
    ConfigurationTestUtils.resetConfiguration();
    HdfsUnderFileSystemUtils.addKey(conf, key);
    Assert.assertEquals("systemKey", conf.get(key.toString()));
    System.clearProperty(key.toString());
    ConfigurationTestUtils.resetConfiguration();
}

Example 5 with PropertyKey

use of alluxio.PropertyKey in project alluxio by Alluxio.

the class AbstractLocalAlluxioCluster method setupTest.

/**
   * Sets up corresponding directories for tests.
   *
   * @throws IOException when creating or deleting dirs failed
   */
protected void setupTest() throws IOException {
    String underfsAddress = Configuration.get(PropertyKey.UNDERFS_ADDRESS);
    // Deletes the ufs dir for this test from to avoid permission problems
    UnderFileSystemUtils.deleteDirIfExists(underfsAddress);
    // Creates ufs dir. This must be called before starting UFS with UnderFileSystemCluster.get().
    UnderFileSystemUtils.mkdirIfNotExists(underfsAddress);
    // Creates storage dirs for worker
    int numLevel = Configuration.getInt(PropertyKey.WORKER_TIERED_STORE_LEVELS);
    for (int level = 0; level < numLevel; level++) {
        PropertyKey tierLevelDirPath = PropertyKeyFormat.WORKER_TIERED_STORE_LEVEL_DIRS_PATH_FORMAT.format(level);
        String[] dirPaths = Configuration.get(tierLevelDirPath).split(",");
        for (String dirPath : dirPaths) {
            UnderFileSystemUtils.mkdirIfNotExists(dirPath);
        }
    }
    // Starts the UFS for integration tests. If this is for HDFS profiles, it starts miniDFSCluster
    // (see also {@link alluxio.LocalMiniDFSCluster} and sets up the folder like
    // "hdfs://xxx:xxx/alluxio*".
    mUfsCluster = UnderFileSystemCluster.get(mWorkDirectory);
    // Sets the journal folder
    String journalFolder = mUfsCluster.getUnderFilesystemAddress() + "/journal" + RANDOM_GENERATOR.nextLong();
    Configuration.set(PropertyKey.MASTER_JOURNAL_FOLDER, journalFolder);
    // Formats the journal
    UnderFileSystemUtils.mkdirIfNotExists(journalFolder);
    for (String masterServiceName : ServerUtils.getMasterServiceNames()) {
        UnderFileSystemUtils.mkdirIfNotExists(PathUtils.concatPath(journalFolder, masterServiceName));
    }
    UnderFileSystemUtils.touch(PathUtils.concatPath(journalFolder, "_format_" + System.currentTimeMillis()));
    // This must happen after UFS is started with UnderFileSystemCluster.get().
    if (!mUfsCluster.getClass().getName().equals(LocalFileSystemCluster.class.getName())) {
        String ufsAddress = mUfsCluster.getUnderFilesystemAddress() + mWorkDirectory;
        UnderFileSystemUtils.mkdirIfNotExists(ufsAddress);
        Configuration.set(PropertyKey.UNDERFS_ADDRESS, ufsAddress);
    }
}