Examples with ServiceCallMetric org.apache.beam.runners.core.metrics.ServiceCallMetric used on opensource projects

Example 1 with ServiceCallMetric

use of org.apache.beam.runners.core.metrics.ServiceCallMetric in project beam by apache.

the class GcsUtil method open.

/**
 * Opens an object in GCS.
 *
 * <p>Returns a SeekableByteChannel that provides access to data in the bucket.
 *
 * @param path the GCS filename to read from
 * @param readOptions Fine-grained options for behaviors of retries, buffering, etc.
 * @return a SeekableByteChannel that can read the object data
 */
@VisibleForTesting
SeekableByteChannel open(GcsPath path, GoogleCloudStorageReadOptions readOptions) throws IOException {
    HashMap<String, String> baseLabels = new HashMap<>();
    baseLabels.put(MonitoringInfoConstants.Labels.PTRANSFORM, "");
    baseLabels.put(MonitoringInfoConstants.Labels.SERVICE, "Storage");
    baseLabels.put(MonitoringInfoConstants.Labels.METHOD, "GcsGet");
    baseLabels.put(MonitoringInfoConstants.Labels.RESOURCE, GcpResourceIdentifiers.cloudStorageBucket(path.getBucket()));
    baseLabels.put(MonitoringInfoConstants.Labels.GCS_PROJECT_ID, googleCloudStorageOptions.getProjectId());
    baseLabels.put(MonitoringInfoConstants.Labels.GCS_BUCKET, path.getBucket());
    ServiceCallMetric serviceCallMetric = new ServiceCallMetric(MonitoringInfoConstants.Urns.API_REQUEST_COUNT, baseLabels);
    try {
        SeekableByteChannel channel = googleCloudStorage.open(new StorageResourceId(path.getBucket(), path.getObject()), readOptions);
        serviceCallMetric.call("ok");
        return channel;
    } catch (IOException e) {
        if (e.getCause() instanceof GoogleJsonResponseException) {
            serviceCallMetric.call(((GoogleJsonResponseException) e.getCause()).getDetails().getCode());
        }
        throw e;
    }
}

Example 2 with ServiceCallMetric

use of org.apache.beam.runners.core.metrics.ServiceCallMetric in project beam by apache.

the class GcsUtil method create.

/**
 * Creates an object in GCS and prepares for uploading its contents.
 *
 * @param path the GCS file to write to
 * @param options to be used for creating and configuring file upload
 * @return a WritableByteChannel that can be used to write data to the object.
 */
public WritableByteChannel create(GcsPath path, CreateOptions options) throws IOException {
    AsyncWriteChannelOptions wcOptions = googleCloudStorageOptions.getWriteChannelOptions();
    @Nullable Integer uploadBufferSizeBytes = options.getUploadBufferSizeBytes() != null ? options.getUploadBufferSizeBytes() : getUploadBufferSizeBytes();
    if (uploadBufferSizeBytes != null) {
        wcOptions = wcOptions.toBuilder().setUploadChunkSize(uploadBufferSizeBytes).build();
    }
    GoogleCloudStorageOptions newGoogleCloudStorageOptions = googleCloudStorageOptions.toBuilder().setWriteChannelOptions(wcOptions).build();
    GoogleCloudStorage gcpStorage = new GoogleCloudStorageImpl(newGoogleCloudStorageOptions, this.storageClient, this.credentials);
    StorageResourceId resourceId = new StorageResourceId(path.getBucket(), path.getObject(), // See {@link GoogleCloudStorage#create(StorageResourceId, GoogleCloudStorageOptions)}
    options.getExpectFileToNotExist() ? 0L : StorageResourceId.UNKNOWN_GENERATION_ID);
    CreateObjectOptions.Builder createBuilder = CreateObjectOptions.builder().setOverwriteExisting(true);
    if (options.getContentType() != null) {
        createBuilder = createBuilder.setContentType(options.getContentType());
    }
    HashMap<String, String> baseLabels = new HashMap<>();
    baseLabels.put(MonitoringInfoConstants.Labels.PTRANSFORM, "");
    baseLabels.put(MonitoringInfoConstants.Labels.SERVICE, "Storage");
    baseLabels.put(MonitoringInfoConstants.Labels.METHOD, "GcsInsert");
    baseLabels.put(MonitoringInfoConstants.Labels.RESOURCE, GcpResourceIdentifiers.cloudStorageBucket(path.getBucket()));
    baseLabels.put(MonitoringInfoConstants.Labels.GCS_PROJECT_ID, googleCloudStorageOptions.getProjectId());
    baseLabels.put(MonitoringInfoConstants.Labels.GCS_BUCKET, path.getBucket());
    ServiceCallMetric serviceCallMetric = new ServiceCallMetric(MonitoringInfoConstants.Urns.API_REQUEST_COUNT, baseLabels);
    try {
        WritableByteChannel channel = gcpStorage.create(resourceId, createBuilder.build());
        serviceCallMetric.call("ok");
        return channel;
    } catch (IOException e) {
        if (e.getCause() instanceof GoogleJsonResponseException) {
            serviceCallMetric.call(((GoogleJsonResponseException) e.getCause()).getDetails().getCode());
        }
        throw e;
    }
}