use of com.google.cloud.datacatalog.v1.DataCatalogClient in project DataflowTemplates by GoogleCloudPlatform.
the class DataCatalogSchemaUtils method getSchemasForEntryGroup.
/**
* Retrieve all of the {@link Schema}s associated to {@link Entry}s in an {@link EntryGroup}.
*/
public static Map<String, Schema> getSchemasForEntryGroup(String gcpProject, String entryGroupId) {
DataCatalogClient client = null;
try {
client = DataCatalogClient.create();
} catch (IOException e) {
throw new RuntimeException("Unable to create a DataCatalogClient", e);
}
if (client == null) {
return null;
}
String formattedParent = DataCatalogClient.formatEntryGroupName(gcpProject, DEFAULT_LOCATION, entryGroupId);
List<Entry> entries = new ArrayList<>();
ListEntriesRequest request = ListEntriesRequest.newBuilder().setParent(formattedParent).build();
while (true) {
ListEntriesResponse response = client.listEntriesCallable().call(request);
entries.addAll(response.getEntriesList());
String nextPageToken = response.getNextPageToken();
if (!Strings.isNullOrEmpty(nextPageToken)) {
request = request.toBuilder().setPageToken(nextPageToken).build();
} else {
break;
}
}
LOG.debug("Fetched entries: {}", entries);
return entries.stream().collect(Collectors.toMap(Entry::getDescription, e -> SchemaUtils.toBeamSchema(e.getSchema())));
}
use of com.google.cloud.datacatalog.v1.DataCatalogClient in project DataflowTemplates by GoogleCloudPlatform.
the class DataCatalogSchemaUtils method getSchemaFromPubSubTopic.
/**
* Retrieve the {@link Schema} associated to a Pub/Sub topics in an.
*
* <p>This method is to be used in multi-topic mode, where a single {@link Schema} is associated
* to a single Pub/Sub topic.
*/
public static Schema getSchemaFromPubSubTopic(String gcpProject, String pubsubTopic) {
DataCatalogClient client = null;
try {
client = DataCatalogClient.create();
} catch (IOException e) {
throw new RuntimeException("Unable to create a DataCatalogClient", e);
}
if (client == null) {
return null;
}
Entry entry = lookupPubSubEntry(client, pubsubTopic, gcpProject);
if (entry == null) {
// TODO(pabloem) Handle a failed entry lookup
return null;
}
return SchemaUtils.toBeamSchema(entry.getSchema());
}
use of com.google.cloud.datacatalog.v1.DataCatalogClient in project java-docs-samples by GoogleCloudPlatform.
the class CreateFilesetEntry method createEntry.
// Create Fileset Entry.
public static void createEntry(String projectId, String entryGroupId, String entryId) {
// Currently, Data Catalog stores metadata in the us-central1 region.
String location = "us-central1";
// the "close" method on the client to safely clean up any remaining background resources.
try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) {
// that will be used in step 3.
try {
dataCatalogClient.deleteEntry(EntryName.of(projectId, location, entryGroupId, entryId).toString());
} catch (PermissionDeniedException | NotFoundException e) {
// PermissionDeniedException or NotFoundException are thrown if
// Entry does not exist.
System.out.println("Entry does not exist.");
}
// that will be used in step 2.
try {
dataCatalogClient.deleteEntryGroup(EntryGroupName.of(projectId, location, entryGroupId).toString());
} catch (PermissionDeniedException | NotFoundException e) {
// PermissionDeniedException or NotFoundException are thrown if
// Entry Group does not exist.
System.out.println("Entry Group does not exist.");
}
// 2. Create an Entry Group.
// Construct the EntryGroup for the EntryGroup request.
EntryGroup entryGroup = EntryGroup.newBuilder().setDisplayName("My Fileset Entry Group").setDescription("This Entry Group consists of ....").build();
// Construct the EntryGroup request to be sent by the client.
CreateEntryGroupRequest entryGroupRequest = CreateEntryGroupRequest.newBuilder().setParent(LocationName.of(projectId, location).toString()).setEntryGroupId(entryGroupId).setEntryGroup(entryGroup).build();
// Use the client to send the API request.
EntryGroup entryGroupResponse = dataCatalogClient.createEntryGroup(entryGroupRequest);
System.out.printf("\nEntry Group created with name: %s\n", entryGroupResponse.getName());
// 3. Create a Fileset Entry.
// Construct the Entry for the Entry request.
Entry entry = Entry.newBuilder().setDisplayName("My Fileset").setDescription("This fileset consists of ....").setGcsFilesetSpec(GcsFilesetSpec.newBuilder().addFilePatterns("gs://cloud-samples-data/*").build()).setSchema(Schema.newBuilder().addColumns(ColumnSchema.newBuilder().setColumn("first_name").setDescription("First name").setMode("REQUIRED").setType("STRING").build()).addColumns(ColumnSchema.newBuilder().setColumn("last_name").setDescription("Last name").setMode("REQUIRED").setType("STRING").build()).addColumns(ColumnSchema.newBuilder().setColumn("addresses").setDescription("Addresses").setMode("REPEATED").setType("RECORD").addSubcolumns(ColumnSchema.newBuilder().setColumn("city").setDescription("City").setMode("NULLABLE").setType("STRING").build()).addSubcolumns(ColumnSchema.newBuilder().setColumn("state").setDescription("State").setMode("NULLABLE").setType("STRING").build()).build()).build()).setType(EntryType.FILESET).build();
// Construct the Entry request to be sent by the client.
CreateEntryRequest entryRequest = CreateEntryRequest.newBuilder().setParent(entryGroupResponse.getName()).setEntryId(entryId).setEntry(entry).build();
// Use the client to send the API request.
Entry entryResponse = dataCatalogClient.createEntry(entryRequest);
System.out.printf("\nEntry created with name: %s\n", entryResponse.getName());
} catch (AlreadyExistsException | IOException e) {
// AlreadyExistsException's are thrown if EntryGroup or Entry already exists.
// IOException's are thrown when unable to create the DataCatalogClient,
// for example an invalid Service Account path.
System.out.println("Error in create entry process:\n" + e.toString());
}
}
Aggregations