Examples with BigQueryProject bio.terra.service.tabulardata.google.BigQueryProject used on opensource projects

Example 1 with BigQueryProject

use of bio.terra.service.tabulardata.google.BigQueryProject in project jade-data-repo by DataBiosphere.

the class TestUtils method selectFromBigQueryDataset.

/**
 * Execute a SELECT query on BigQuery dataset.
 * @param bigQueryPdao pass in from the calling test class
 * @param datasetDao pass in from the calling test class
 * @param dataLocationService pass in from the calling test class
 * @param datasetName the name of the Data Repo dataset
 * @param tableName the name of Data Repo table
 * @param columns a comma-separated string of the columns to select (e.g. "name", "name, fileref")
 * @return the BigQuery TableResult
 */
public static TableResult selectFromBigQueryDataset(BigQueryPdao bigQueryPdao, DatasetDao datasetDao, DataLocationService dataLocationService, String datasetName, String tableName, String columns) throws Exception {
    String bqDatasetName = bigQueryPdao.prefixName(datasetName);
    BigQueryProject bigQueryProject = bigQueryProjectForDatasetName(datasetDao, dataLocationService, datasetName);
    String bigQueryProjectId = bigQueryProject.getProjectId();
    BigQuery bigQuery = bigQueryProject.getBigQuery();
    ST sqlTemplate = new ST(selectFromBigQueryDatasetTemplate);
    sqlTemplate.add("columns", columns);
    sqlTemplate.add("project", bigQueryProjectId);
    sqlTemplate.add("dataset", bqDatasetName);
    sqlTemplate.add("table", tableName);
    QueryJobConfiguration queryConfig = QueryJobConfiguration.newBuilder(sqlTemplate.render()).build();
    return bigQuery.query(queryConfig);
}

Example 2 with BigQueryProject

use of bio.terra.service.tabulardata.google.BigQueryProject in project jade-data-repo by DataBiosphere.

the class EncodeFileTest method getFileRefIdFromSnapshot.

private String getFileRefIdFromSnapshot(SnapshotSummaryModel snapshotSummary) throws InterruptedException {
    Snapshot snapshot = snapshotDao.retrieveSnapshotByName(snapshotSummary.getName());
    SnapshotDataProject dataProject = dataLocationService.getOrCreateProject(snapshot);
    BigQueryProject bigQueryProject = BigQueryProject.get(dataProject.getGoogleProjectId());
    StringBuilder builder = new StringBuilder().append("SELECT file_ref FROM `").append(dataProject.getGoogleProjectId()).append('.').append(snapshot.getName()).append(".file` AS T").append(" WHERE T.file_ref IS NOT NULL LIMIT 1");
    String sql = builder.toString();
    try {
        QueryJobConfiguration queryConfig = QueryJobConfiguration.newBuilder(sql).build();
        TableResult result = bigQueryProject.getBigQuery().query(queryConfig);
        FieldValueList row = result.iterateAll().iterator().next();
        FieldValue idValue = row.get(0);
        String drsUri = idValue.getStringValue();
        return drsUri;
    } catch (InterruptedException ie) {
        throw new PdaoException("get file ref id from snapshot unexpectedly interrupted", ie);
    }
}

Example 3 with BigQueryProject

use of bio.terra.service.tabulardata.google.BigQueryProject in project jade-data-repo by DataBiosphere.

the class SnapshotConnectedTest method getFileRefIdFromSnapshot.

// Technically a helper method, but so specific to testExcludeLockedFromSnapshotFileLookups, likely not re-useable
private String getFileRefIdFromSnapshot(SnapshotSummaryModel snapshotSummary) throws InterruptedException {
    Snapshot snapshot = snapshotDao.retrieveSnapshotByName(snapshotSummary.getName());
    SnapshotDataProject dataProject = dataLocationService.getOrCreateProject(snapshot);
    BigQueryProject bigQueryProject = BigQueryProject.get(dataProject.getGoogleProjectId());
    BigQuery bigQuery = bigQueryProject.getBigQuery();
    ST sqlTemplate = new ST(queryForRefIdTemplate);
    sqlTemplate.add("project", dataProject.getGoogleProjectId());
    sqlTemplate.add("snapshot", snapshot.getName());
    sqlTemplate.add("table", "tableA");
    QueryJobConfiguration queryConfig = QueryJobConfiguration.newBuilder(sqlTemplate.render()).build();
    TableResult result = bigQuery.query(queryConfig);
    FieldValueList row = result.iterateAll().iterator().next();
    FieldValue idValue = row.get(0);
    return idValue.getStringValue();
}

Example 4 with BigQueryProject

use of bio.terra.service.tabulardata.google.BigQueryProject in project jade-data-repo by DataBiosphere.

the class SnapshotConnectedTest method testMinimal.

@Test
public void testMinimal() throws Exception {
    DatasetSummaryModel datasetMinimalSummary = setupMinimalDataset();
    String datasetName = PDAO_PREFIX + datasetMinimalSummary.getName();
    BigQueryProject bigQueryProject = TestUtils.bigQueryProjectForDatasetName(datasetDao, dataLocationService, datasetMinimalSummary.getName());
    long datasetParticipants = queryForCount(datasetName, "participant", bigQueryProject);
    assertThat("dataset participants loaded properly", datasetParticipants, equalTo(2L));
    long datasetSamples = queryForCount(datasetName, "sample", bigQueryProject);
    assertThat("dataset samples loaded properly", datasetSamples, equalTo(5L));
    SnapshotRequestModel snapshotRequest = makeSnapshotTestRequest(datasetMinimalSummary, "dataset-minimal-snapshot.json");
    MockHttpServletResponse response = performCreateSnapshot(snapshotRequest, "");
    SnapshotSummaryModel summaryModel = validateSnapshotCreated(snapshotRequest, response);
    SnapshotModel snapshotModel = getTestSnapshot(summaryModel.getId(), snapshotRequest, datasetMinimalSummary);
    List<TableModel> tables = snapshotModel.getTables();
    Optional<TableModel> participantTable = tables.stream().filter(t -> t.getName().equals("participant")).findFirst();
    Optional<TableModel> sampleTable = tables.stream().filter(t -> t.getName().equals("sample")).findFirst();
    assertThat("participant table exists", participantTable.isPresent(), equalTo(true));
    assertThat("sample table exists", sampleTable.isPresent(), equalTo(true));
    long snapshotParticipants = queryForCount(summaryModel.getName(), "participant", bigQueryProject);
    assertThat("dataset participants loaded properly", snapshotParticipants, equalTo(1L));
    assertThat("participant row count matches expectation", participantTable.get().getRowCount(), equalTo(1));
    long snapshotSamples = queryForCount(summaryModel.getName(), "sample", bigQueryProject);
    assertThat("dataset samples loaded properly", snapshotSamples, equalTo(2L));
    assertThat("sample row count matches expectation", sampleTable.get().getRowCount(), equalTo(2));
    List<RelationshipModel> relationships = snapshotModel.getRelationships();
    assertThat("a relationship comes back", relationships.size(), equalTo(1));
    RelationshipModel relationshipModel = relationships.get(0);
    assertThat("relationship name is right", relationshipModel.getName(), equalTo("participant_sample"));
    assertThat("from table is right", relationshipModel.getFrom().getTable(), equalTo("participant"));
    assertThat("from column is right", relationshipModel.getFrom().getColumn(), equalTo("id"));
    assertThat("to table is right", relationshipModel.getTo().getTable(), equalTo("sample"));
    assertThat("to column is right", relationshipModel.getTo().getColumn(), equalTo("participant_id"));
}

Example 5 with BigQueryProject

use of bio.terra.service.tabulardata.google.BigQueryProject in project jade-data-repo by DataBiosphere.

the class SnapshotConnectedTest method testArrayStruct.

@Test
public void testArrayStruct() throws Exception {
    DatasetSummaryModel datasetArraySummary = setupArrayStructDataset();
    String datasetName = PDAO_PREFIX + datasetArraySummary.getName();
    BigQueryProject bigQueryProject = TestUtils.bigQueryProjectForDatasetName(datasetDao, dataLocationService, datasetArraySummary.getName());
    long datasetParticipants = queryForCount(datasetName, "participant", bigQueryProject);
    assertThat("dataset participants loaded properly", datasetParticipants, equalTo(2L));
    long datasetSamples = queryForCount(datasetName, "sample", bigQueryProject);
    assertThat("dataset samples loaded properly", datasetSamples, equalTo(5L));
    SnapshotRequestModel snapshotRequest = makeSnapshotTestRequest(datasetArraySummary, "snapshot-array-struct.json");
    MockHttpServletResponse response = performCreateSnapshot(snapshotRequest, "");
    SnapshotSummaryModel summaryModel = validateSnapshotCreated(snapshotRequest, response);
    getTestSnapshot(summaryModel.getId(), snapshotRequest, datasetArraySummary);
    long snapshotParticipants = queryForCount(summaryModel.getName(), "participant", bigQueryProject);
    assertThat("dataset participants loaded properly", snapshotParticipants, equalTo(2L));
    long snapshotSamples = queryForCount(summaryModel.getName(), "sample", bigQueryProject);
    assertThat("dataset samples loaded properly", snapshotSamples, equalTo(3L));
}