Examples with GoogleCloudStorage com.google.cloud.hadoop.gcsio.GoogleCloudStorage used on opensource projects

Example 1 with GoogleCloudStorage

use of com.google.cloud.hadoop.gcsio.GoogleCloudStorage in project beam by apache.

the class GcsUtilTest method testGCSWriteMetricsIsSet.

@Test
public void testGCSWriteMetricsIsSet() throws IOException {
    GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
    GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
    GoogleCloudStorage mockStorage = Mockito.mock(GoogleCloudStorage.class);
    gcsUtil.setCloudStorageImpl(GoogleCloudStorageOptions.builder().setAppName("Beam").setGrpcEnabled(true).setProjectId("my_project").build());
    when(mockStorage.create(new StorageResourceId("testbucket", "testobject"), CreateObjectOptions.builder().setOverwriteExisting(true).setContentType("type").build())).thenThrow(IOException.class);
    GcsPath gcsPath = GcsPath.fromComponents("testbucket", "testobject");
    assertThrows(IOException.class, () -> gcsUtil.create(gcsPath, ""));
    verifyMetricWasSet("my_project", "testbucket", "GcsInsert", "permission_denied", 1);
}

Example 2 with GoogleCloudStorage

use of com.google.cloud.hadoop.gcsio.GoogleCloudStorage in project beam by apache.

the class GcsUtil method create.

/**
 * Creates an object in GCS and prepares for uploading its contents.
 *
 * @param path the GCS file to write to
 * @param options to be used for creating and configuring file upload
 * @return a WritableByteChannel that can be used to write data to the object.
 */
public WritableByteChannel create(GcsPath path, CreateOptions options) throws IOException {
    AsyncWriteChannelOptions wcOptions = googleCloudStorageOptions.getWriteChannelOptions();
    @Nullable Integer uploadBufferSizeBytes = options.getUploadBufferSizeBytes() != null ? options.getUploadBufferSizeBytes() : getUploadBufferSizeBytes();
    if (uploadBufferSizeBytes != null) {
        wcOptions = wcOptions.toBuilder().setUploadChunkSize(uploadBufferSizeBytes).build();
    }
    GoogleCloudStorageOptions newGoogleCloudStorageOptions = googleCloudStorageOptions.toBuilder().setWriteChannelOptions(wcOptions).build();
    GoogleCloudStorage gcpStorage = new GoogleCloudStorageImpl(newGoogleCloudStorageOptions, this.storageClient, this.credentials);
    StorageResourceId resourceId = new StorageResourceId(path.getBucket(), path.getObject(), // See {@link GoogleCloudStorage#create(StorageResourceId, GoogleCloudStorageOptions)}
    options.getExpectFileToNotExist() ? 0L : StorageResourceId.UNKNOWN_GENERATION_ID);
    CreateObjectOptions.Builder createBuilder = CreateObjectOptions.builder().setOverwriteExisting(true);
    if (options.getContentType() != null) {
        createBuilder = createBuilder.setContentType(options.getContentType());
    }
    HashMap<String, String> baseLabels = new HashMap<>();
    baseLabels.put(MonitoringInfoConstants.Labels.PTRANSFORM, "");
    baseLabels.put(MonitoringInfoConstants.Labels.SERVICE, "Storage");
    baseLabels.put(MonitoringInfoConstants.Labels.METHOD, "GcsInsert");
    baseLabels.put(MonitoringInfoConstants.Labels.RESOURCE, GcpResourceIdentifiers.cloudStorageBucket(path.getBucket()));
    baseLabels.put(MonitoringInfoConstants.Labels.GCS_PROJECT_ID, googleCloudStorageOptions.getProjectId());
    baseLabels.put(MonitoringInfoConstants.Labels.GCS_BUCKET, path.getBucket());
    ServiceCallMetric serviceCallMetric = new ServiceCallMetric(MonitoringInfoConstants.Urns.API_REQUEST_COUNT, baseLabels);
    try {
        WritableByteChannel channel = gcpStorage.create(resourceId, createBuilder.build());
        serviceCallMetric.call("ok");
        return channel;
    } catch (IOException e) {
        if (e.getCause() instanceof GoogleJsonResponseException) {
            serviceCallMetric.call(((GoogleJsonResponseException) e.getCause()).getDetails().getCode());
        }
        throw e;
    }
}