use of com.google.cloud.hadoop.gcsio.GoogleCloudStorageOptions in project beam by apache.
the class GcsUtil method create.
/**
* Creates an object in GCS and prepares for uploading its contents.
*
* @param path the GCS file to write to
* @param options to be used for creating and configuring file upload
* @return a WritableByteChannel that can be used to write data to the object.
*/
public WritableByteChannel create(GcsPath path, CreateOptions options) throws IOException {
AsyncWriteChannelOptions wcOptions = googleCloudStorageOptions.getWriteChannelOptions();
@Nullable Integer uploadBufferSizeBytes = options.getUploadBufferSizeBytes() != null ? options.getUploadBufferSizeBytes() : getUploadBufferSizeBytes();
if (uploadBufferSizeBytes != null) {
wcOptions = wcOptions.toBuilder().setUploadChunkSize(uploadBufferSizeBytes).build();
}
GoogleCloudStorageOptions newGoogleCloudStorageOptions = googleCloudStorageOptions.toBuilder().setWriteChannelOptions(wcOptions).build();
GoogleCloudStorage gcpStorage = new GoogleCloudStorageImpl(newGoogleCloudStorageOptions, this.storageClient, this.credentials);
StorageResourceId resourceId = new StorageResourceId(path.getBucket(), path.getObject(), // See {@link GoogleCloudStorage#create(StorageResourceId, GoogleCloudStorageOptions)}
options.getExpectFileToNotExist() ? 0L : StorageResourceId.UNKNOWN_GENERATION_ID);
CreateObjectOptions.Builder createBuilder = CreateObjectOptions.builder().setOverwriteExisting(true);
if (options.getContentType() != null) {
createBuilder = createBuilder.setContentType(options.getContentType());
}
HashMap<String, String> baseLabels = new HashMap<>();
baseLabels.put(MonitoringInfoConstants.Labels.PTRANSFORM, "");
baseLabels.put(MonitoringInfoConstants.Labels.SERVICE, "Storage");
baseLabels.put(MonitoringInfoConstants.Labels.METHOD, "GcsInsert");
baseLabels.put(MonitoringInfoConstants.Labels.RESOURCE, GcpResourceIdentifiers.cloudStorageBucket(path.getBucket()));
baseLabels.put(MonitoringInfoConstants.Labels.GCS_PROJECT_ID, googleCloudStorageOptions.getProjectId());
baseLabels.put(MonitoringInfoConstants.Labels.GCS_BUCKET, path.getBucket());
ServiceCallMetric serviceCallMetric = new ServiceCallMetric(MonitoringInfoConstants.Urns.API_REQUEST_COUNT, baseLabels);
try {
WritableByteChannel channel = gcpStorage.create(resourceId, createBuilder.build());
serviceCallMetric.call("ok");
return channel;
} catch (IOException e) {
if (e.getCause() instanceof GoogleJsonResponseException) {
serviceCallMetric.call(((GoogleJsonResponseException) e.getCause()).getDetails().getCode());
}
throw e;
}
}
Aggregations