Examples with Schema com.google.cloud.datacatalog.v1beta1.Schema used on opensource projects

Example 56 with Schema

use of com.google.cloud.datacatalog.v1beta1.Schema in project molgenis-emx2 by molgenis.

the class TestTruncate method testTruncate.

@Test
public void testTruncate() {
    Database db = TestDatabaseFactory.getTestDatabase();
    Schema schema = db.dropCreateSchema(TestTruncate.class.getSimpleName());
    // create simple table, add data, and truncate
    Table table1 = schema.create(table("Table1", column("name").setPkey()));
    table1.insert(row("name", "a"));
    Assert.assertEquals(1, table1.retrieveRows().size());
    table1.truncate();
    Assert.assertEquals(0, table1.retrieveRows().size());
    // create with subclass
    Table table2 = schema.create(table("Table2").setInherit("Table1").add(column("col1")));
    table1.insert(row("name", "a"));
    table2.insert(row("name", "b", "col1", "checkb"));
    Assert.assertEquals(2, table1.retrieveRows().size());
    Assert.assertEquals(1, table2.retrieveRows().size());
    table1.truncate();
    Assert.assertEquals(1, table1.retrieveRows().size());
    Assert.assertEquals(1, table2.retrieveRows().size());
    table2.truncate();
    Assert.assertEquals(0, table1.retrieveRows().size());
    Assert.assertEquals(0, table2.retrieveRows().size());
    // create with subclass of a subclass
    Table table3 = schema.create(table("Table3").setInherit("Table2").add(column("col2")));
    table1.insert(row("name", "a"));
    table2.insert(row("name", "b", "col1", "checkb"));
    table3.insert(row("name", "c", "col1", "checkc", "col2", "checkc"));
    Assert.assertEquals(3, table1.retrieveRows().size());
    Assert.assertEquals(2, table2.retrieveRows().size());
    Assert.assertEquals(1, table3.retrieveRows().size());
    // leaves subclass?!!! is this expected behavior?
    table2.truncate();
    Assert.assertEquals(2, table1.retrieveRows().size());
    // !!!
    Assert.assertEquals(1, table2.retrieveRows().size());
    Assert.assertEquals(1, table3.retrieveRows().size());
    table3.truncate();
    Assert.assertEquals(1, table1.retrieveRows().size());
    Assert.assertEquals(0, table2.retrieveRows().size());
    Assert.assertEquals(0, table3.retrieveRows().size());
    table1.truncate();
    Assert.assertEquals(0, table1.retrieveRows().size());
    Assert.assertEquals(0, table2.retrieveRows().size());
    Assert.assertEquals(0, table3.retrieveRows().size());
}

Example 57 with Schema

use of com.google.cloud.datacatalog.v1beta1.Schema in project DataflowTemplates by GoogleCloudPlatform.

the class DataCatalogSchemaUtils method getSchemaFromPubSubTopic.

/**
 * Retrieve the {@link Schema} associated to a Pub/Sub topics in an.
 *
 * <p>This method is to be used in multi-topic mode, where a single {@link Schema} is associated
 * to a single Pub/Sub topic.
 */
public static Schema getSchemaFromPubSubTopic(String gcpProject, String pubsubTopic) {
    DataCatalogClient client = null;
    try {
        client = DataCatalogClient.create();
    } catch (IOException e) {
        throw new RuntimeException("Unable to create a DataCatalogClient", e);
    }
    if (client == null) {
        return null;
    }
    Entry entry = lookupPubSubEntry(client, pubsubTopic, gcpProject);
    if (entry == null) {
        // TODO(pabloem) Handle a failed entry lookup
        return null;
    }
    return SchemaUtils.toBeamSchema(entry.getSchema());
}

Example 58 with Schema

use of com.google.cloud.datacatalog.v1beta1.Schema in project DataflowTemplates by GoogleCloudPlatform.

the class DataCatalogSchemaUtils method getSchemasForEntryGroup.

/**
 * Retrieve all of the {@link Schema}s associated to {@link Entry}s in an {@link EntryGroup}.
 */
public static Map<String, Schema> getSchemasForEntryGroup(String gcpProject, String entryGroupId) {
    DataCatalogClient client = null;
    try {
        client = DataCatalogClient.create();
    } catch (IOException e) {
        throw new RuntimeException("Unable to create a DataCatalogClient", e);
    }
    if (client == null) {
        return null;
    }
    String formattedParent = DataCatalogClient.formatEntryGroupName(gcpProject, DEFAULT_LOCATION, entryGroupId);
    List<Entry> entries = new ArrayList<>();
    ListEntriesRequest request = ListEntriesRequest.newBuilder().setParent(formattedParent).build();
    while (true) {
        ListEntriesResponse response = client.listEntriesCallable().call(request);
        entries.addAll(response.getEntriesList());
        String nextPageToken = response.getNextPageToken();
        if (!Strings.isNullOrEmpty(nextPageToken)) {
            request = request.toBuilder().setPageToken(nextPageToken).build();
        } else {
            break;
        }
    }
    LOG.debug("Fetched entries: {}", entries);
    return entries.stream().collect(Collectors.toMap(Entry::getDescription, e -> SchemaUtils.toBeamSchema(e.getSchema())));
}

Example 59 with Schema

use of com.google.cloud.datacatalog.v1beta1.Schema in project DataflowTemplates by GoogleCloudPlatform.

the class SchemaUtils method fromBeamField.

private static ColumnSchema fromBeamField(org.apache.beam.sdk.schemas.Schema.Field beamField) {
    ColumnSchema.Builder columnBuilder = ColumnSchema.newBuilder();
    if (beamField.getType().getNullable()) {
        columnBuilder.setMode("NULLABLE");
    } else {
        columnBuilder.setMode("REQUIRED");
    }
    if (beamField.getType().getTypeName() == TypeName.ROW) {
        String columnType = "STRUCT";
        Schema subSchema = fromBeamSchema(beamField.getType().getRowSchema());
        return columnBuilder.setColumn(beamField.getName()).setType(columnType).addAllSubcolumns(subSchema.getColumnsList()).build();
    } else if (LOGICAL_FIELD_TYPES.inverse().containsKey(beamField.getType())) {
        String columnType = LOGICAL_FIELD_TYPES.inverse().get(beamField.getType());
        return columnBuilder.setColumn(beamField.getName()).setType(columnType).build();
    } else {
        String columnType = FIELD_TYPE_NAMES.inverse().get(beamField.getType().getTypeName());
        // TODO(pabloem): Include other characteristics of the field (e.g. required/nullable/etc).
        return columnBuilder.setColumn(beamField.getName()).setType(columnType).build();
    }
}

Example 60 with Schema

use of com.google.cloud.datacatalog.v1beta1.Schema in project java-pubsub by googleapis.

the class SchemaServiceClientTest method createSchemaExceptionTest2.

@Test
public void createSchemaExceptionTest2() throws Exception {
    StatusRuntimeException exception = new StatusRuntimeException(io.grpc.Status.INVALID_ARGUMENT);
    mockSchemaService.addException(exception);
    try {
        String parent = "parent-995424086";
        Schema schema = Schema.newBuilder().build();
        String schemaId = "schemaId-697673060";
        client.createSchema(parent, schema, schemaId);
        Assert.fail("No exception raised");
    } catch (InvalidArgumentException e) {
    // Expected exception.
    }
}