Examples with DataCatalogClient com.google.cloud.datacatalog.v1beta1.DataCatalogClient used on opensource projects

Example 1 with DataCatalogClient

use of com.google.cloud.datacatalog.v1beta1.DataCatalogClient in project DataflowTemplates by GoogleCloudPlatform.

the class DataCatalogSchemaUtils method getSchemasForEntryGroup.

/**
 * Retrieve all of the {@link Schema}s associated to {@link Entry}s in an {@link EntryGroup}.
 */
public static Map<String, Schema> getSchemasForEntryGroup(String gcpProject, String entryGroupId) {
    DataCatalogClient client = null;
    try {
        client = DataCatalogClient.create();
    } catch (IOException e) {
        throw new RuntimeException("Unable to create a DataCatalogClient", e);
    }
    if (client == null) {
        return null;
    }
    String formattedParent = DataCatalogClient.formatEntryGroupName(gcpProject, DEFAULT_LOCATION, entryGroupId);
    List<Entry> entries = new ArrayList<>();
    ListEntriesRequest request = ListEntriesRequest.newBuilder().setParent(formattedParent).build();
    while (true) {
        ListEntriesResponse response = client.listEntriesCallable().call(request);
        entries.addAll(response.getEntriesList());
        String nextPageToken = response.getNextPageToken();
        if (!Strings.isNullOrEmpty(nextPageToken)) {
            request = request.toBuilder().setPageToken(nextPageToken).build();
        } else {
            break;
        }
    }
    LOG.debug("Fetched entries: {}", entries);
    return entries.stream().collect(Collectors.toMap(Entry::getDescription, e -> SchemaUtils.toBeamSchema(e.getSchema())));
}

Example 2 with DataCatalogClient

use of com.google.cloud.datacatalog.v1beta1.DataCatalogClient in project DataflowTemplates by GoogleCloudPlatform.

the class DataCatalogSchemaUtils method getSchemaFromPubSubTopic.

/**
 * Retrieve the {@link Schema} associated to a Pub/Sub topics in an.
 *
 * <p>This method is to be used in multi-topic mode, where a single {@link Schema} is associated
 * to a single Pub/Sub topic.
 */
public static Schema getSchemaFromPubSubTopic(String gcpProject, String pubsubTopic) {
    DataCatalogClient client = null;
    try {
        client = DataCatalogClient.create();
    } catch (IOException e) {
        throw new RuntimeException("Unable to create a DataCatalogClient", e);
    }
    if (client == null) {
        return null;
    }
    Entry entry = lookupPubSubEntry(client, pubsubTopic, gcpProject);
    if (entry == null) {
        // TODO(pabloem) Handle a failed entry lookup
        return null;
    }
    return SchemaUtils.toBeamSchema(entry.getSchema());
}

Example 3 with DataCatalogClient

use of com.google.cloud.datacatalog.v1beta1.DataCatalogClient in project java-docs-samples by GoogleCloudPlatform.

the class CreateFilesetEntry method createEntry.

// Create Fileset Entry.
public static void createEntry(String projectId, String entryGroupId, String entryId) {
    // Currently, Data Catalog stores metadata in the us-central1 region.
    String location = "us-central1";
    // the "close" method on the client to safely clean up any remaining background resources.
    try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) {
        // that will be used in step 3.
        try {
            dataCatalogClient.deleteEntry(EntryName.of(projectId, location, entryGroupId, entryId).toString());
        } catch (PermissionDeniedException | NotFoundException e) {
            // PermissionDeniedException or NotFoundException are thrown if
            // Entry does not exist.
            System.out.println("Entry does not exist.");
        }
        // that will be used in step 2.
        try {
            dataCatalogClient.deleteEntryGroup(EntryGroupName.of(projectId, location, entryGroupId).toString());
        } catch (PermissionDeniedException | NotFoundException e) {
            // PermissionDeniedException or NotFoundException are thrown if
            // Entry Group does not exist.
            System.out.println("Entry Group does not exist.");
        }
        // 2. Create an Entry Group.
        // Construct the EntryGroup for the EntryGroup request.
        EntryGroup entryGroup = EntryGroup.newBuilder().setDisplayName("My Fileset Entry Group").setDescription("This Entry Group consists of ....").build();
        // Construct the EntryGroup request to be sent by the client.
        CreateEntryGroupRequest entryGroupRequest = CreateEntryGroupRequest.newBuilder().setParent(LocationName.of(projectId, location).toString()).setEntryGroupId(entryGroupId).setEntryGroup(entryGroup).build();
        // Use the client to send the API request.
        EntryGroup entryGroupResponse = dataCatalogClient.createEntryGroup(entryGroupRequest);
        System.out.printf("\nEntry Group created with name: %s\n", entryGroupResponse.getName());
        // 3. Create a Fileset Entry.
        // Construct the Entry for the Entry request.
        Entry entry = Entry.newBuilder().setDisplayName("My Fileset").setDescription("This fileset consists of ....").setGcsFilesetSpec(GcsFilesetSpec.newBuilder().addFilePatterns("gs://cloud-samples-data/*").build()).setSchema(Schema.newBuilder().addColumns(ColumnSchema.newBuilder().setColumn("first_name").setDescription("First name").setMode("REQUIRED").setType("STRING").build()).addColumns(ColumnSchema.newBuilder().setColumn("last_name").setDescription("Last name").setMode("REQUIRED").setType("STRING").build()).addColumns(ColumnSchema.newBuilder().setColumn("addresses").setDescription("Addresses").setMode("REPEATED").setType("RECORD").addSubcolumns(ColumnSchema.newBuilder().setColumn("city").setDescription("City").setMode("NULLABLE").setType("STRING").build()).addSubcolumns(ColumnSchema.newBuilder().setColumn("state").setDescription("State").setMode("NULLABLE").setType("STRING").build()).build()).build()).setType(EntryType.FILESET).build();
        // Construct the Entry request to be sent by the client.
        CreateEntryRequest entryRequest = CreateEntryRequest.newBuilder().setParent(entryGroupResponse.getName()).setEntryId(entryId).setEntry(entry).build();
        // Use the client to send the API request.
        Entry entryResponse = dataCatalogClient.createEntry(entryRequest);
        System.out.printf("\nEntry created with name: %s\n", entryResponse.getName());
    } catch (AlreadyExistsException | IOException e) {
        // AlreadyExistsException's are thrown if EntryGroup or Entry already exists.
        // IOException's are thrown when unable to create the DataCatalogClient,
        // for example an invalid Service Account path.
        System.out.println("Error in create entry process:\n" + e.toString());
    }
}

Example 4 with DataCatalogClient

use of com.google.cloud.datacatalog.v1beta1.DataCatalogClient in project DataflowTemplates by GoogleCloudPlatform.

the class DataCatalogSchemaUtils method lookupPubSubEntry.

static Entry lookupPubSubEntry(DataCatalogClient client, String pubsubTopic, String gcpProject) {
    String linkedResource = String.format(DATA_CATALOG_PUBSUB_URI_TEMPLATE, gcpProject, pubsubTopic);
    LOG.info("Looking up LinkedResource {}", linkedResource);
    LookupEntryRequest request = LookupEntryRequest.newBuilder().setLinkedResource(linkedResource).build();
    try {
        Entry entry = client.lookupEntry(request);
        return entry;
    } catch (ApiException e) {
        System.out.println("CANT LOOKUP ENTRY" + e.toString());
        e.printStackTrace();
        LOG.error("ApiException thrown by Data Catalog API:", e);
        return null;
    }
}