Examples with PropertyKey alluxio.PropertyKey used on opensource projects

Example 6 with PropertyKey

use of alluxio.PropertyKey in project alluxio by Alluxio.

the class GetConf method getConf.

/**
   * Implements get configuration.
   *
   * @param args list of arguments
   * @return 0 on success, 1 on failures
   */
public static int getConf(String... args) {
    CommandLineParser parser = new DefaultParser();
    CommandLine cmd;
    try {
        cmd = parser.parse(OPTIONS, args, true);
    } catch (ParseException e) {
        printHelp("Unable to parse input args: " + e.getMessage());
        return 1;
    }
    Preconditions.checkNotNull(cmd, "Unable to parse input args");
    args = cmd.getArgs();
    switch(args.length) {
        case 0:
            for (Entry<String, String> entry : Configuration.toMap().entrySet()) {
                String key = entry.getKey();
                String value = entry.getValue();
                System.out.println(String.format("%s=%s", key, value));
            }
            break;
        case 1:
            if (!PropertyKey.isValid(args[0])) {
                printHelp(String.format("%s is not a valid configuration key", args[0]));
                return 1;
            }
            PropertyKey key = PropertyKey.fromString(args[0]);
            if (!Configuration.containsKey(key)) {
                System.out.println("");
            } else {
                if (cmd.hasOption(UNIT_OPTION_NAME)) {
                    String arg = cmd.getOptionValue(UNIT_OPTION_NAME).toUpperCase();
                    Unit unit;
                    try {
                        unit = Unit.valueOf(arg);
                        System.out.println(Configuration.getBytes(key) / unit.getValue());
                    } catch (IllegalArgumentException e) {
                        printHelp(String.format("%s is not a valid unit", arg));
                        return 1;
                    }
                } else {
                    System.out.println(Configuration.get(key));
                }
            }
            break;
        default:
            printHelp("More arguments than expected");
            return 1;
    }
    return 0;
}

Example 7 with PropertyKey

use of alluxio.PropertyKey in project alluxio by Alluxio.

the class SwiftUnderFileSystemFactory method addAndCheckSwiftCredentials.

/**
   * Adds Swift credentials from system properties to the Alluxio configuration if they are not
   * already present.
   *
   * @return true if simulation mode or if all required authentication credentials are present
   */
private boolean addAndCheckSwiftCredentials() {
    PropertyKey[] propertiesToRead = { PropertyKey.SWIFT_API_KEY, PropertyKey.SWIFT_TENANT_KEY, PropertyKey.SWIFT_USER_KEY, PropertyKey.SWIFT_AUTH_URL_KEY, PropertyKey.SWIFT_AUTH_METHOD_KEY, PropertyKey.SWIFT_PASSWORD_KEY, PropertyKey.SWIFT_SIMULATION, PropertyKey.SWIFT_REGION_KEY };
    for (PropertyKey property : propertiesToRead) {
        if (System.getProperty(property.toString()) != null && (!Configuration.containsKey(property) || Configuration.get(property) == null)) {
            Configuration.set(property, System.getProperty(property.toString()));
        }
    }
    // We do not need authentication credentials in simulation mode
    if (Configuration.containsKey(PropertyKey.SWIFT_SIMULATION) && Configuration.getBoolean(PropertyKey.SWIFT_SIMULATION)) {
        return true;
    }
    // API or Password Key is required
    PropertyKey apiOrPasswordKey = Configuration.containsKey(PropertyKey.SWIFT_API_KEY) ? PropertyKey.SWIFT_API_KEY : PropertyKey.SWIFT_PASSWORD_KEY;
    // Check if required credentials exist
    PropertyKey[] requiredProperties = { apiOrPasswordKey, PropertyKey.SWIFT_TENANT_KEY, PropertyKey.SWIFT_AUTH_URL_KEY, PropertyKey.SWIFT_USER_KEY };
    for (PropertyKey propertyName : requiredProperties) {
        if (Configuration.get(propertyName) == null) {
            return false;
        }
    }
    return true;
}

Example 8 with PropertyKey

use of alluxio.PropertyKey in project alluxio by Alluxio.

the class HdfsUnderFileSystemUtilsTest method addKeyFromAlluxioConf.

/**
   * Tests {@link HdfsUnderFileSystemUtils#addKey} method when a property is set in Alluxio.
   */
@Test
public void addKeyFromAlluxioConf() {
    PropertyKey key = PropertyKey.HOME;
    org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration();
    Configuration.set(key, "alluxioKey");
    HdfsUnderFileSystemUtils.addKey(conf, key);
    Assert.assertEquals("alluxioKey", conf.get(key.toString()));
    ConfigurationTestUtils.resetConfiguration();
}

Example 9 with PropertyKey

use of alluxio.PropertyKey in project alluxio by Alluxio.

the class AbstractLocalAlluxioCluster method initConfiguration.

/**
   * Creates a default {@link Configuration} for testing.
   *
   * @throws IOException when the operation fails
   */
public void initConfiguration() throws IOException {
    setAlluxioWorkDirectory();
    setHostname();
    Configuration.set(PropertyKey.TEST_MODE, "true");
    Configuration.set(PropertyKey.WORK_DIR, mWorkDirectory);
    Configuration.set(PropertyKey.USER_BLOCK_SIZE_BYTES_DEFAULT, Integer.toString(DEFAULT_BLOCK_SIZE_BYTES));
    Configuration.set(PropertyKey.USER_BLOCK_REMOTE_READ_BUFFER_SIZE_BYTES, Integer.toString(64));
    Configuration.set(PropertyKey.MASTER_HOSTNAME, mHostname);
    Configuration.set(PropertyKey.MASTER_RPC_PORT, Integer.toString(0));
    Configuration.set(PropertyKey.MASTER_WEB_PORT, Integer.toString(0));
    Configuration.set(PropertyKey.MASTER_TTL_CHECKER_INTERVAL_MS, Integer.toString(1000));
    Configuration.set(PropertyKey.MASTER_WORKER_THREADS_MIN, "1");
    Configuration.set(PropertyKey.MASTER_WORKER_THREADS_MAX, "100");
    Configuration.set(PropertyKey.MASTER_STARTUP_CONSISTENCY_CHECK_ENABLED, false);
    Configuration.set(PropertyKey.MASTER_JOURNAL_FLUSH_TIMEOUT_MS, 1000);
    Configuration.set(PropertyKey.MASTER_BIND_HOST, mHostname);
    Configuration.set(PropertyKey.MASTER_WEB_BIND_HOST, mHostname);
    // If tests fail to connect they should fail early rather than using the default ridiculously
    // high retries
    Configuration.set(PropertyKey.USER_RPC_RETRY_MAX_NUM_RETRY, "3");
    // Since tests are always running on a single host keep the resolution timeout low as otherwise
    // people running with strange network configurations will see very slow tests
    Configuration.set(PropertyKey.NETWORK_HOST_RESOLUTION_TIMEOUT_MS, "250");
    Configuration.set(PropertyKey.PROXY_WEB_PORT, Integer.toString(0));
    // default write type becomes MUST_CACHE, set this value to CACHE_THROUGH for tests.
    // default Alluxio storage is STORE, and under storage is SYNC_PERSIST for tests.
    // TODO(binfan): eliminate this setting after updating integration tests
    Configuration.set(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, "CACHE_THROUGH");
    Configuration.set(PropertyKey.WEB_THREADS, "1");
    Configuration.set(PropertyKey.WEB_RESOURCES, PathUtils.concatPath(System.getProperty("user.dir"), "../core/server/common/src/main/webapp"));
    Configuration.set(PropertyKey.WORKER_RPC_PORT, Integer.toString(0));
    Configuration.set(PropertyKey.WORKER_DATA_PORT, Integer.toString(0));
    Configuration.set(PropertyKey.WORKER_WEB_PORT, Integer.toString(0));
    Configuration.set(PropertyKey.WORKER_DATA_FOLDER, "/datastore");
    Configuration.set(PropertyKey.WORKER_MEMORY_SIZE, Long.toString(DEFAULT_WORKER_MEMORY_BYTES));
    Configuration.set(PropertyKey.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS, Integer.toString(15));
    Configuration.set(PropertyKey.WORKER_BLOCK_THREADS_MIN, Integer.toString(1));
    Configuration.set(PropertyKey.WORKER_BLOCK_THREADS_MAX, Integer.toString(2048));
    Configuration.set(PropertyKey.WORKER_NETWORK_NETTY_WORKER_THREADS, Integer.toString(2));
    Configuration.set(PropertyKey.WORKER_BIND_HOST, mHostname);
    Configuration.set(PropertyKey.WORKER_DATA_BIND_HOST, mHostname);
    Configuration.set(PropertyKey.WORKER_WEB_BIND_HOST, mHostname);
    // Performs an immediate shutdown of data server. Graceful shutdown is unnecessary and slow
    Configuration.set(PropertyKey.WORKER_NETWORK_NETTY_SHUTDOWN_QUIET_PERIOD, Integer.toString(0));
    Configuration.set(PropertyKey.WORKER_NETWORK_NETTY_SHUTDOWN_TIMEOUT, Integer.toString(0));
    // Sets up the tiered store
    String ramdiskPath = PathUtils.concatPath(mWorkDirectory, "ramdisk");
    Configuration.set(PropertyKeyFormat.WORKER_TIERED_STORE_LEVEL_ALIAS_FORMAT.format(0), "MEM");
    Configuration.set(PropertyKeyFormat.WORKER_TIERED_STORE_LEVEL_DIRS_PATH_FORMAT.format(0), ramdiskPath);
    int numLevel = Configuration.getInt(PropertyKey.WORKER_TIERED_STORE_LEVELS);
    for (int level = 1; level < numLevel; level++) {
        PropertyKey tierLevelDirPath = PropertyKeyFormat.WORKER_TIERED_STORE_LEVEL_DIRS_PATH_FORMAT.format(level);
        String[] dirPaths = Configuration.get(tierLevelDirPath).split(",");
        List<String> newPaths = new ArrayList<>();
        for (String dirPath : dirPaths) {
            String newPath = mWorkDirectory + dirPath;
            newPaths.add(newPath);
        }
        Configuration.set(PropertyKeyFormat.WORKER_TIERED_STORE_LEVEL_DIRS_PATH_FORMAT.format(level), Joiner.on(',').join(newPaths));
    }
    // For some test profiles, default properties get overwritten by system properties (e.g., s3
    // credentials for s3Test).
    // TODO(binfan): have one dedicated property (e.g., alluxio.test.properties) to carry on all the
    // properties we want to overwrite in tests, rather than simply merging all system properties.
    Configuration.merge(System.getProperties());
}

Example 10 with PropertyKey

use of alluxio.PropertyKey in project alluxio by Alluxio.

the class AlluxioFrameworkIntegrationTest method testMesosDeploy.

private void testMesosDeploy(Map<PropertyKey, String> properties) throws Exception {
    StringBuilder alluxioJavaOpts = new StringBuilder(System.getProperty("ALLUXIO_JAVA_OPTS", ""));
    for (Entry<PropertyKey, String> entry : properties.entrySet()) {
        alluxioJavaOpts.append(String.format(" -D%s=%s", entry.getKey().toString(), entry.getValue()));
    }
    Map<String, String> env = ImmutableMap.of("ALLUXIO_JAVA_OPTS", alluxioJavaOpts.toString());
    try {
        startAlluxioFramework(env);
        LOG.info("Launched Alluxio cluster, waiting for worker to register with master");
        String masterHostName = NetworkAddressUtils.getLocalHostName();
        int masterPort = Configuration.getInt(PropertyKey.MASTER_RPC_PORT);
        InetSocketAddress masterAddress = new InetSocketAddress(masterHostName, masterPort);
        try (final BlockMasterClient client = BlockMasterClient.Factory.create(masterAddress)) {
            CommonUtils.waitFor("Alluxio worker to register with master", new Function<Void, Boolean>() {

                @Override
                public Boolean apply(Void input) {
                    try {
                        try {
                            return !client.getWorkerInfoList().isEmpty();
                        } catch (ConnectionFailedException e) {
                            // block master isn't up yet, keep waiting
                            return false;
                        }
                    } catch (Exception e) {
                        throw Throwables.propagate(e);
                    }
                }
            }, WaitForOptions.defaults().setTimeout(15 * Constants.MINUTE_MS));
        }
        LOG.info("Worker registered");
        basicAlluxioTests();
    } finally {
        stopAlluxioFramework();
    }
}