Examples with InstancedConfiguration alluxio.conf.InstancedConfiguration used on opensource projects

Example 11 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class AbstractFileSystem method initialize.

/**
 * Initialize the {@link alluxio.hadoop.FileSystem}.
 * @param uri file system Uri
 * @param conf hadoop configuration
 * @param alluxioConfiguration [optional] alluxio configuration
 * @throws IOException
 */
@VisibleForTesting
public synchronized void initialize(URI uri, org.apache.hadoop.conf.Configuration conf, @Nullable AlluxioConfiguration alluxioConfiguration) throws IOException {
    // Validates scheme and authority of FS Uri.
    validateFsUri(uri);
    super.initialize(uri, conf);
    LOG.debug("initialize({}, {}). Connecting to Alluxio", uri, conf);
    HadoopUtils.addSwiftCredentials(conf);
    setConf(conf);
    // HDFS doesn't allow the authority to be empty; it must be "/" instead.
    String authority = uri.getAuthority() == null ? "/" : uri.getAuthority();
    mAlluxioHeader = getFsScheme(uri) + "://" + authority;
    // Set the statistics member. Use mStatistics instead of the parent class's variable.
    mStatistics = statistics;
    mUri = URI.create(mAlluxioHeader);
    // take the URI properties, hadoop configuration, and given Alluxio configuration and merge
    // all three into a single object.
    Map<String, Object> uriConfProperties = getConfigurationFromUri(uri, conf);
    Map<String, Object> hadoopConfProperties = HadoopConfigurationUtils.getConfigurationFromHadoop(conf);
    LOG.info("Creating Alluxio configuration from Hadoop configuration {}, uri configuration {}", hadoopConfProperties, uriConfProperties);
    AlluxioProperties alluxioProps = (alluxioConfiguration != null) ? alluxioConfiguration.copyProperties() : ConfigurationUtils.defaults();
    // Merge relevant Hadoop configuration into Alluxio's configuration.
    alluxioProps.merge(hadoopConfProperties, Source.RUNTIME);
    // Merge relevant connection details in the URI with the highest priority
    alluxioProps.merge(uriConfProperties, Source.RUNTIME);
    // Creating a new instanced configuration from an AlluxioProperties object isn't expensive.
    mAlluxioConf = new InstancedConfiguration(alluxioProps);
    mAlluxioConf.validate();
    if (mFileSystem != null) {
        return;
    }
    Subject subject = getHadoopSubject();
    LOG.debug("Using Hadoop subject: {}", subject);
    LOG.info("Initializing filesystem with connect details {}", Factory.getConnectDetails(mAlluxioConf));
    // Create FileSystem for accessing Alluxio.
    // Disable URI validation for non-Alluxio schemes.
    boolean enableUriValidation = (uri.getScheme() == null) || uri.getScheme().equals(Constants.SCHEME);
    mFileSystem = FileSystem.Factory.create(ClientContext.create(subject, mAlluxioConf).setUriValidationEnabled(enableUriValidation));
}

Example 12 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class MetricsHeartbeatContextTest method testCancelFuture.

@Test
public void testCancelFuture() {
    Map<MasterInquireClient.ConnectDetails, MetricsHeartbeatContext> map = getContextMap();
    assertTrue(map.isEmpty());
    ScheduledFuture<?> future = Mockito.mock(ScheduledFuture.class);
    when(future.cancel(any(Boolean.class))).thenReturn(true);
    InstancedConfiguration conf = ConfigurationTestUtils.defaults();
    conf.set(PropertyKey.USER_RPC_RETRY_MAX_DURATION, "1s");
    ClientContext ctx = ClientContext.create(conf);
    MasterInquireClient client = MasterInquireClient.Factory.create(ctx.getClusterConf(), ctx.getUserState());
    MetricsHeartbeatContext.addHeartbeat(ctx, client);
    assertFalse(map.isEmpty());
    map.forEach((addr, heartbeat) -> {
        ScheduledFuture<?> realFuture = Whitebox.getInternalState(heartbeat, "mMetricsMasterHeartbeatTask");
        // Should be created and scheduled
        assertNotNull(realFuture);
        // Scheduled indefinitely
        assertFalse(realFuture.isDone());
        Whitebox.setInternalState(heartbeat, "mMetricsMasterHeartbeatTask", future);
        // Cancel the real one once replaced with a mock.
        realFuture.cancel(false);
    });
    MetricsHeartbeatContext.removeHeartbeat(ctx);
    // Make sure the future is canceled afterwards
    verify(future).cancel(false);
}

Example 13 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class MetricsHeartbeatContextTest method testContextCounter.

@Test
public void testContextCounter() {
    Map<MasterInquireClient.ConnectDetails, MetricsHeartbeatContext> map = getContextMap();
    assertTrue(map.isEmpty());
    InstancedConfiguration conf = ConfigurationTestUtils.defaults();
    conf.set(PropertyKey.USER_RPC_RETRY_MAX_DURATION, "1s");
    ClientContext ctx = ClientContext.create(conf);
    MasterInquireClient client = MasterInquireClient.Factory.create(ctx.getClusterConf(), ctx.getUserState());
    MetricsHeartbeatContext.addHeartbeat(ctx, client);
    assertFalse(map.isEmpty());
    map.forEach((details, context) -> assertEquals(1, (int) Whitebox.getInternalState(context, "mCtxCount")));
    MetricsHeartbeatContext.addHeartbeat(ctx, client);
    map.forEach((details, context) -> assertEquals(2, (int) Whitebox.getInternalState(context, "mCtxCount")));
    conf = ConfigurationTestUtils.defaults();
    conf.set(PropertyKey.USER_RPC_RETRY_MAX_DURATION, "1s");
    conf.set(PropertyKey.MASTER_RPC_ADDRESSES, "master1:19998,master2:19998,master3:19998");
    ClientContext haCtx = ClientContext.create(conf);
    MetricsHeartbeatContext.addHeartbeat(haCtx, MasterInquireClient.Factory.create(conf, haCtx.getUserState()));
    assertEquals(2, map.size());
    MetricsHeartbeatContext.removeHeartbeat(ctx);
    assertEquals(2, map.size());
    map.forEach((details, context) -> assertEquals(1, (int) Whitebox.getInternalState(context, "mCtxCount")));
    assertNotNull(getInternalExecutor());
    MetricsHeartbeatContext.removeHeartbeat(ctx);
    MetricsHeartbeatContext.removeHeartbeat(haCtx);
    assertNull(getInternalExecutor());
    assertTrue(map.isEmpty());
}

Example 14 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class NondeterministicLRUCacheEvictorTest method before.

/**
 * Sets up the instances.
 */
@Before
public void before() {
    InstancedConfiguration conf = ConfigurationTestUtils.defaults();
    mEvictor = new NondeterministicLRUCacheEvictor(conf);
    mEvictor.setNumOfCandidate(2);
}

Example 15 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class FileSystemCacheTest method createTestFSKey.

private Key createTestFSKey(String username) {
    User user = new User(username);
    Set<Principal> principals = new HashSet<>();
    principals.add(user);
    return new FileSystemCache.Key(new Subject(false, principals, new HashSet<>(), new HashSet<>()), new InstancedConfiguration(ConfigurationUtils.defaults()));
}