Search in sources :

Example 21 with StorageObject

use of com.google.api.services.storage.model.StorageObject in project google-cloud-java by GoogleCloudPlatform.

the class HttpStorageRpc method compose.

@Override
public StorageObject compose(Iterable<StorageObject> sources, StorageObject target, Map<Option, ?> targetOptions) {
    ComposeRequest request = new ComposeRequest();
    request.setDestination(target);
    List<ComposeRequest.SourceObjects> sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
        ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
        sourceObject.setName(source.getName());
        Long generation = source.getGeneration();
        if (generation != null) {
            sourceObject.setGeneration(generation);
            sourceObject.setObjectPreconditions(new ObjectPreconditions().setIfGenerationMatch(generation));
        }
        sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
        return storage.objects().compose(target.getBucket(), target.getName(), request).setIfMetagenerationMatch(Option.IF_METAGENERATION_MATCH.getLong(targetOptions)).setIfGenerationMatch(Option.IF_GENERATION_MATCH.getLong(targetOptions)).execute();
    } catch (IOException ex) {
        throw translate(ex);
    }
}
Also used : StorageObject(com.google.api.services.storage.model.StorageObject) ArrayList(java.util.ArrayList) ObjectPreconditions(com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions) IOException(java.io.IOException) ComposeRequest(com.google.api.services.storage.model.ComposeRequest)

Example 22 with StorageObject

use of com.google.api.services.storage.model.StorageObject in project google-cloud-java by GoogleCloudPlatform.

the class StorageImplTest method testUpdateAllIterable.

@Test
public void testUpdateAllIterable() {
    RpcBatch batchMock = EasyMock.createMock(RpcBatch.class);
    Capture<RpcBatch.Callback<StorageObject>> callback1 = Capture.newInstance();
    Capture<RpcBatch.Callback<StorageObject>> callback2 = Capture.newInstance();
    batchMock.addPatch(EasyMock.eq(BLOB_INFO1.toPb()), EasyMock.capture(callback1), EasyMock.eq(ImmutableMap.<StorageRpc.Option, Object>of()));
    batchMock.addPatch(EasyMock.eq(BLOB_INFO2.toPb()), EasyMock.capture(callback2), EasyMock.eq(ImmutableMap.<StorageRpc.Option, Object>of()));
    EasyMock.expect(storageRpcMock.createBatch()).andReturn(batchMock);
    batchMock.submit();
    EasyMock.replay(storageRpcMock, batchMock);
    initializeService();
    List<Blob> resultBlobs = storage.update(ImmutableList.of(BLOB_INFO1, BLOB_INFO2));
    callback1.getValue().onSuccess(BLOB_INFO1.toPb());
    callback2.getValue().onFailure(new GoogleJsonError());
    assertEquals(2, resultBlobs.size());
    assertEquals(new Blob(storage, new BlobInfo.BuilderImpl(BLOB_INFO1)), resultBlobs.get(0));
    assertNull(resultBlobs.get(1));
    EasyMock.verify(batchMock);
}
Also used : BlobWriteOption(com.google.cloud.storage.Storage.BlobWriteOption) BucketSourceOption(com.google.cloud.storage.Storage.BucketSourceOption) BlobSourceOption(com.google.cloud.storage.Storage.BlobSourceOption) BlobTargetOption(com.google.cloud.storage.Storage.BlobTargetOption) StorageObject(com.google.api.services.storage.model.StorageObject) GoogleJsonError(com.google.api.client.googleapis.json.GoogleJsonError) RpcBatch(com.google.cloud.storage.spi.v1.RpcBatch) Test(org.junit.Test)

Example 23 with StorageObject

use of com.google.api.services.storage.model.StorageObject in project google-cloud-java by GoogleCloudPlatform.

the class BlobInfoTest method testToPbAndFromPb.

@Test
public void testToPbAndFromPb() {
    compareCustomerEncryptions(CUSTOMER_ENCRYPTION, CustomerEncryption.fromPb(CUSTOMER_ENCRYPTION.toPb()));
    compareBlobs(BLOB_INFO, BlobInfo.fromPb(BLOB_INFO.toPb()));
    BlobInfo blobInfo = BlobInfo.newBuilder(BlobId.of("b", "n")).build();
    compareBlobs(blobInfo, BlobInfo.fromPb(blobInfo.toPb()));
    StorageObject object = new StorageObject().setName("n/").setBucket("b").setSize(BigInteger.ZERO).set("isDirectory", true);
    blobInfo = BlobInfo.fromPb(object);
    assertEquals("b", blobInfo.getBucket());
    assertEquals("n/", blobInfo.getName());
    assertNull(blobInfo.getAcl());
    assertNull(blobInfo.getComponentCount());
    assertNull(blobInfo.getContentType());
    assertNull(blobInfo.getCacheControl());
    assertNull(blobInfo.getContentDisposition());
    assertNull(blobInfo.getContentEncoding());
    assertNull(blobInfo.getContentLanguage());
    assertNull(blobInfo.getCustomerEncryption());
    assertNull(blobInfo.getCrc32c());
    assertNull(blobInfo.getCreateTime());
    assertNull(blobInfo.getDeleteTime());
    assertNull(blobInfo.getEtag());
    assertNull(blobInfo.getGeneration());
    assertNull(blobInfo.getGeneratedId());
    assertNull(blobInfo.getMd5());
    assertNull(blobInfo.getMediaLink());
    assertNull(blobInfo.getMetadata());
    assertNull(blobInfo.getMetageneration());
    assertNull(blobInfo.getOwner());
    assertNull(blobInfo.getSelfLink());
    assertEquals(0L, (long) blobInfo.getSize());
    assertNull(blobInfo.getUpdateTime());
    assertNull(blobInfo.getStorageClass());
    assertTrue(blobInfo.isDirectory());
}
Also used : StorageObject(com.google.api.services.storage.model.StorageObject) Test(org.junit.Test)

Example 24 with StorageObject

use of com.google.api.services.storage.model.StorageObject in project google-cloud-java by GoogleCloudPlatform.

the class FakeStorageRpc method list.

@Override
public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?> options) throws StorageException {
    String delimiter = null;
    String preprefix = "";
    for (Map.Entry<Option, ?> e : options.entrySet()) {
        switch(e.getKey()) {
            case PREFIX:
                preprefix = (String) e.getValue();
                if (preprefix.startsWith("/")) {
                    preprefix = preprefix.substring(1);
                }
                break;
            case DELIMITER:
                delimiter = (String) e.getValue();
                break;
            case FIELDS:
                // ignore and return all the fields
                break;
            default:
                throw new UnsupportedOperationException("Unknown option: " + e.getKey());
        }
    }
    final String prefix = preprefix;
    List<StorageObject> values = new ArrayList<>();
    Map<String, StorageObject> folders = new HashMap<>();
    for (StorageObject so : metadata.values()) {
        if (!so.getName().startsWith(prefix)) {
            continue;
        }
        if (processedAsFolder(so, delimiter, prefix, folders)) {
            continue;
        }
        values.add(so);
    }
    values.addAll(folders.values());
    // The type cast seems to be necessary to help Java's typesystem remember that collections are iterable.
    return Tuple.of(null, (Iterable<StorageObject>) values);
}
Also used : StorageObject(com.google.api.services.storage.model.StorageObject) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) HashMap(java.util.HashMap) Map(java.util.Map)

Example 25 with StorageObject

use of com.google.api.services.storage.model.StorageObject in project beam by apache.

the class GcsFileSystemTest method testMatch.

@Test
public void testMatch() throws Exception {
    Objects modelObjects = new Objects();
    List<StorageObject> items = new ArrayList<>();
    // A directory
    items.add(new StorageObject().setBucket("testbucket").setName("testdirectory/"));
    // Files within the directory
    items.add(createStorageObject("gs://testbucket/testdirectory/file1name", 1L));
    items.add(createStorageObject("gs://testbucket/testdirectory/file2name", 2L));
    items.add(createStorageObject("gs://testbucket/testdirectory/file3name", 3L));
    items.add(createStorageObject("gs://testbucket/testdirectory/file4name", 4L));
    items.add(createStorageObject("gs://testbucket/testdirectory/otherfile", 5L));
    items.add(createStorageObject("gs://testbucket/testdirectory/anotherfile", 6L));
    modelObjects.setItems(items);
    when(mockGcsUtil.listObjects(eq("testbucket"), anyString(), isNull(String.class))).thenReturn(modelObjects);
    List<GcsPath> gcsPaths = ImmutableList.of(GcsPath.fromUri("gs://testbucket/testdirectory/non-exist-file"), GcsPath.fromUri("gs://testbucket/testdirectory/otherfile"));
    when(mockGcsUtil.getObjects(eq(gcsPaths))).thenReturn(ImmutableList.of(StorageObjectOrIOException.create(new FileNotFoundException()), StorageObjectOrIOException.create(createStorageObject("gs://testbucket/testdirectory/otherfile", 4L))));
    List<String> specs = ImmutableList.of("gs://testbucket/testdirectory/file[1-3]*", "gs://testbucket/testdirectory/non-exist-file", "gs://testbucket/testdirectory/otherfile");
    List<MatchResult> matchResults = gcsFileSystem.match(specs);
    assertEquals(3, matchResults.size());
    assertEquals(Status.OK, matchResults.get(0).status());
    assertThat(ImmutableList.of("gs://testbucket/testdirectory/file1name", "gs://testbucket/testdirectory/file2name", "gs://testbucket/testdirectory/file3name"), contains(toFilenames(matchResults.get(0)).toArray()));
    assertEquals(Status.NOT_FOUND, matchResults.get(1).status());
    assertEquals(Status.OK, matchResults.get(2).status());
    assertThat(ImmutableList.of("gs://testbucket/testdirectory/otherfile"), contains(toFilenames(matchResults.get(2)).toArray()));
}
Also used : StorageObject(com.google.api.services.storage.model.StorageObject) Objects(com.google.api.services.storage.model.Objects) ArrayList(java.util.ArrayList) GcsPath(org.apache.beam.sdk.util.gcsfs.GcsPath) FileNotFoundException(java.io.FileNotFoundException) Matchers.anyString(org.mockito.Matchers.anyString) MatchResult(org.apache.beam.sdk.io.fs.MatchResult) Test(org.junit.Test)

Aggregations

StorageObject (com.google.api.services.storage.model.StorageObject)35 Test (org.junit.Test)13 Objects (com.google.api.services.storage.model.Objects)10 RetryHelperException (com.google.cloud.RetryHelper.RetryHelperException)7 ArrayList (java.util.ArrayList)7 GoogleJsonError (com.google.api.client.googleapis.json.GoogleJsonError)6 IOException (java.io.IOException)6 GcsPath (org.apache.beam.sdk.util.gcsfs.GcsPath)6 RpcBatch (com.google.cloud.storage.spi.v1.RpcBatch)5 Storage (com.google.api.services.storage.Storage)4 BlobSourceOption (com.google.cloud.storage.Storage.BlobSourceOption)4 BlobTargetOption (com.google.cloud.storage.Storage.BlobTargetOption)4 BlobWriteOption (com.google.cloud.storage.Storage.BlobWriteOption)4 BucketSourceOption (com.google.cloud.storage.Storage.BucketSourceOption)4 GcsOptions (org.apache.beam.sdk.extensions.gcp.options.GcsOptions)4 ImmutableList (com.google.common.collect.ImmutableList)3 List (java.util.List)3 Map (java.util.Map)3 Callable (java.util.concurrent.Callable)3 GoogleJsonResponseException (com.google.api.client.googleapis.json.GoogleJsonResponseException)2