use of com.google.api.services.storage.model.StorageObject in project druid by druid-io.
the class GoogleTimestampVersionedDataFinderTest method getLatestVersionTrailingSlashKeyPrefix.
@Test
public void getLatestVersionTrailingSlashKeyPrefix() {
String bucket = "bucket";
String keyPrefix = "prefix/dir/0/";
// object for directory prefix/dir/0/
final StorageObject storageObject1 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/", 0);
storageObject1.setUpdated(new DateTime(System.currentTimeMillis()));
final StorageObject storageObject2 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "v1", 1);
storageObject2.setUpdated(new DateTime(System.currentTimeMillis()));
final StorageObject storageObject3 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "v2", 1);
storageObject3.setUpdated(new DateTime(System.currentTimeMillis() + 100));
final StorageObject storageObject4 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "other", 4);
storageObject4.setUpdated(new DateTime(System.currentTimeMillis() + 100));
final GoogleStorage storage = ObjectStorageIteratorTest.makeMockClient(ImmutableList.of(storageObject1, storageObject2, storageObject3, storageObject4));
final GoogleTimestampVersionedDataFinder finder = new GoogleTimestampVersionedDataFinder(storage);
Pattern pattern = Pattern.compile("v.*");
URI latest = finder.getLatestVersion(URI.create(StringUtils.format("gs://%s/%s", bucket, keyPrefix)), pattern);
URI expected = URI.create(StringUtils.format("gs://%s/%s", bucket, storageObject3.getName()));
Assert.assertEquals(expected, latest);
}
use of com.google.api.services.storage.model.StorageObject in project druid by druid-io.
the class ObjectStorageIteratorTest method makeStorageObject.
static StorageObject makeStorageObject(final String bucket, final String key, final long size) {
final StorageObject summary = new StorageObject();
summary.setBucket(bucket);
summary.setName(key);
summary.setSize(BigInteger.valueOf(size));
return summary;
}
use of com.google.api.services.storage.model.StorageObject in project beam by apache.
the class GcsUtil method expand.
/**
* Expands a pattern into matched paths. The pattern path may contain globs, which are expanded in
* the result. For patterns that only match a single object, we ensure that the object exists.
*/
public List<GcsPath> expand(GcsPath gcsPattern) throws IOException {
Pattern p = null;
String prefix = null;
if (isWildcard(gcsPattern)) {
// Part before the first wildcard character.
prefix = getNonWildcardPrefix(gcsPattern.getObject());
p = Pattern.compile(wildcardToRegexp(gcsPattern.getObject()));
} else {
// Not a wildcard.
try {
// Use a get request to fetch the metadata of the object, and ignore the return value.
// The request has strong global consistency.
getObject(gcsPattern);
return ImmutableList.of(gcsPattern);
} catch (FileNotFoundException e) {
// If the path was not found, return an empty list.
return ImmutableList.of();
}
}
LOG.debug("matching files in bucket {}, prefix {} against pattern {}", gcsPattern.getBucket(), prefix, p.toString());
String pageToken = null;
List<GcsPath> results = new ArrayList<>();
do {
Objects objects = listObjects(gcsPattern.getBucket(), prefix, pageToken);
if (objects.getItems() == null) {
break;
}
// Filter objects based on the regex.
for (StorageObject o : objects.getItems()) {
String name = o.getName();
// Skip directories, which end with a slash.
if (p.matcher(name).matches() && !name.endsWith("/")) {
LOG.debug("Matched object: {}", name);
results.add(GcsPath.fromObject(o));
}
}
pageToken = objects.getNextPageToken();
} while (pageToken != null);
return results;
}
use of com.google.api.services.storage.model.StorageObject in project beam by apache.
the class GcsFileSystemTest method createStorageObject.
private StorageObject createStorageObject(String gcsFilename, long fileSize) {
GcsPath gcsPath = GcsPath.fromUri(gcsFilename);
// Google APIs will use null for empty files.
@Nullable BigInteger size = (fileSize == 0) ? null : BigInteger.valueOf(fileSize);
return new StorageObject().setBucket(gcsPath.getBucket()).setName(gcsPath.getObject()).setSize(size);
}
use of com.google.api.services.storage.model.StorageObject in project beam by apache.
the class GcsFileSystemTest method testMatch.
@Test
public void testMatch() throws Exception {
Objects modelObjects = new Objects();
List<StorageObject> items = new ArrayList<>();
// A directory
items.add(new StorageObject().setBucket("testbucket").setName("testdirectory/"));
// Files within the directory
items.add(createStorageObject("gs://testbucket/testdirectory/file1name", 1L));
items.add(createStorageObject("gs://testbucket/testdirectory/file2name", 2L));
items.add(createStorageObject("gs://testbucket/testdirectory/file3name", 3L));
items.add(createStorageObject("gs://testbucket/testdirectory/file4name", 4L));
items.add(createStorageObject("gs://testbucket/testdirectory/otherfile", 5L));
items.add(createStorageObject("gs://testbucket/testdirectory/anotherfile", 6L));
modelObjects.setItems(items);
when(mockGcsUtil.listObjects(eq("testbucket"), anyString(), isNull(String.class))).thenReturn(modelObjects);
List<GcsPath> gcsPaths = ImmutableList.of(GcsPath.fromUri("gs://testbucket/testdirectory/non-exist-file"), GcsPath.fromUri("gs://testbucket/testdirectory/otherfile"));
when(mockGcsUtil.getObjects(eq(gcsPaths))).thenReturn(ImmutableList.of(StorageObjectOrIOException.create(new FileNotFoundException()), StorageObjectOrIOException.create(createStorageObject("gs://testbucket/testdirectory/otherfile", 4L))));
List<String> specs = ImmutableList.of("gs://testbucket/testdirectory/file[1-3]*", "gs://testbucket/testdirectory/non-exist-file", "gs://testbucket/testdirectory/otherfile");
List<MatchResult> matchResults = gcsFileSystem.match(specs);
assertEquals(3, matchResults.size());
assertEquals(Status.OK, matchResults.get(0).status());
assertThat(ImmutableList.of("gs://testbucket/testdirectory/file1name", "gs://testbucket/testdirectory/file2name", "gs://testbucket/testdirectory/file3name"), contains(toFilenames(matchResults.get(0)).toArray()));
assertEquals(Status.NOT_FOUND, matchResults.get(1).status());
assertEquals(Status.OK, matchResults.get(2).status());
assertThat(ImmutableList.of("gs://testbucket/testdirectory/otherfile"), contains(toFilenames(matchResults.get(2)).toArray()));
}
Aggregations