Examples with AzureFileSystemInstrumentation org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation used on opensource projects

Example 1 with AzureFileSystemInstrumentation

use of org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation in project hadoop by apache.

the class AzureBlobStorageTestAccount method createOutOfBandStore.

public static AzureBlobStorageTestAccount createOutOfBandStore(int uploadBlockSize, int downloadBlockSize) throws Exception {
    saveMetricsConfigFile();
    CloudBlobContainer container = null;
    Configuration conf = createTestConfiguration();
    CloudStorageAccount account = createTestAccount(conf);
    if (null == account) {
        return null;
    }
    String containerName = String.format("wasbtests-%s-%tQ", System.getProperty("user.name"), new Date());
    // Create the container.
    container = account.createCloudBlobClient().getContainerReference(containerName);
    container.create();
    String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
    // Ensure that custom throttling is disabled and tolerate concurrent
    // out-of-band appends.
    conf.setBoolean(KEY_DISABLE_THROTTLING, true);
    conf.setBoolean(KEY_READ_TOLERATE_CONCURRENT_APPEND, true);
    configureSecureModeTestSettings(conf);
    // Set account URI and initialize Azure file system.
    URI accountUri = createAccountUri(accountName, containerName);
    // Set up instrumentation.
    //
    AzureFileSystemMetricsSystem.fileSystemStarted();
    String sourceName = NativeAzureFileSystem.newMetricsSourceName();
    String sourceDesc = "Azure Storage Volume File System metrics";
    AzureFileSystemInstrumentation instrumentation = new AzureFileSystemInstrumentation(conf);
    AzureFileSystemMetricsSystem.registerSource(sourceName, sourceDesc, instrumentation);
    // Create a new AzureNativeFileSystemStore object.
    AzureNativeFileSystemStore testStorage = new AzureNativeFileSystemStore();
    // Initialize the store with the throttling feedback interfaces.
    testStorage.initialize(accountUri, conf, instrumentation);
    // Create test account initializing the appropriate member variables.
    //
    AzureBlobStorageTestAccount testAcct = new AzureBlobStorageTestAccount(testStorage, account, container);
    return testAcct;
}

Example 2 with AzureFileSystemInstrumentation

use of org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation in project hadoop by apache.

the class TestBlobTypeSpeedDifference method writeTestFile.

/**
   * Writes data to the given file of the given size, flushing every
   * x bytes. Measure performance of that and return it.
   */
private static TestResult writeTestFile(NativeAzureFileSystem fs, Path path, long size, long flushInterval) throws IOException {
    AzureFileSystemInstrumentation instrumentation = fs.getInstrumentation();
    long initialRequests = instrumentation.getCurrentWebResponses();
    Date start = new Date();
    OutputStream output = fs.create(path);
    writeTestFile(output, size, flushInterval);
    output.close();
    long finalRequests = instrumentation.getCurrentWebResponses();
    return new TestResult(new Date().getTime() - start.getTime(), finalRequests - initialRequests);
}

Example 3 with AzureFileSystemInstrumentation

use of org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation in project hadoop by apache.

the class NativeAzureFileSystem method initialize.

@Override
public void initialize(URI uri, Configuration conf) throws IOException, IllegalArgumentException {
    // Check authority for the URI to guarantee that it is non-null.
    uri = reconstructAuthorityIfNeeded(uri, conf);
    if (null == uri.getAuthority()) {
        final String errMsg = String.format("Cannot initialize WASB file system, URI authority not recognized.");
        throw new IllegalArgumentException(errMsg);
    }
    super.initialize(uri, conf);
    if (store == null) {
        store = createDefaultStore(conf);
    }
    instrumentation = new AzureFileSystemInstrumentation(conf);
    if (!conf.getBoolean(SKIP_AZURE_METRICS_PROPERTY_NAME, false)) {
        // Make sure the metrics system is available before interacting with Azure
        AzureFileSystemMetricsSystem.fileSystemStarted();
        metricsSourceName = newMetricsSourceName();
        String sourceDesc = "Azure Storage Volume File System metrics";
        AzureFileSystemMetricsSystem.registerSource(metricsSourceName, sourceDesc, instrumentation);
    }
    store.initialize(uri, conf, instrumentation);
    setConf(conf);
    this.ugi = UserGroupInformation.getCurrentUser();
    this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
    this.workingDir = new Path("/user", UserGroupInformation.getCurrentUser().getShortUserName()).makeQualified(getUri(), getWorkingDirectory());
    this.blockSize = conf.getLong(AZURE_BLOCK_SIZE_PROPERTY_NAME, MAX_AZURE_BLOCK_SIZE);
    this.appendSupportEnabled = conf.getBoolean(APPEND_SUPPORT_ENABLE_PROPERTY_NAME, false);
    LOG.debug("NativeAzureFileSystem. Initializing.");
    LOG.debug("  blockSize  = {}", conf.getLong(AZURE_BLOCK_SIZE_PROPERTY_NAME, MAX_AZURE_BLOCK_SIZE));
    // Initialize thread counts from user configuration
    deleteThreadCount = conf.getInt(AZURE_DELETE_THREADS, DEFAULT_AZURE_DELETE_THREADS);
    renameThreadCount = conf.getInt(AZURE_RENAME_THREADS, DEFAULT_AZURE_RENAME_THREADS);
    boolean useSecureMode = conf.getBoolean(AzureNativeFileSystemStore.KEY_USE_SECURE_MODE, AzureNativeFileSystemStore.DEFAULT_USE_SECURE_MODE);
    this.azureAuthorization = useSecureMode && conf.getBoolean(KEY_AZURE_AUTHORIZATION, DEFAULT_AZURE_AUTHORIZATION);
    if (this.azureAuthorization) {
        this.authorizer = new RemoteWasbAuthorizerImpl();
        authorizer.init(conf);
    }
}