Examples with FieldValue com.google.cloud.bigquery.FieldValue used on opensource projects

Example 16 with FieldValue

use of com.google.cloud.bigquery.FieldValue in project workbench by all-of-us.

the class AuditController method auditBigQuery.

@Override
public ResponseEntity<AuditBigQueryResponse> auditBigQuery() {
    // We expect to only see queries run within Firecloud AoU projects, or for administrative
    // purposes within the CDR project itself.
    Set<String> cdrProjects = ImmutableList.copyOf(cdrVersionDao.findAll()).stream().map(v -> v.getBigqueryProject()).collect(Collectors.toSet());
    Set<String> whitelist = Sets.union(userDao.getAllUserProjects(), cdrProjects);
    Instant now = clock.instant();
    List<String> suffixes = IntStream.range(0, AUDIT_DAY_RANGE).mapToObj(i -> auditTableSuffix(now, i)).collect(Collectors.toList());
    int numBad = 0;
    int numQueries = 0;
    for (String cdrProjectId : cdrProjects) {
        QueryResult result = bigQueryService.executeQuery(QueryJobConfiguration.of(auditSql(cdrProjectId, suffixes)));
        Map<String, Integer> rm = bigQueryService.getResultMapper(result);
        for (List<FieldValue> row : result.iterateAll()) {
            String project_id = bigQueryService.getString(row, rm.get("client_project_id"));
            String email = bigQueryService.getString(row, rm.get("user_email"));
            long total = bigQueryService.getLong(row, rm.get("total"));
            if (bigQueryService.isNull(row, rm.get("client_project_id"))) {
                log.severe(String.format("AUDIT: (CDR project '%s') %d queries with missing project ID from user '%s'; " + "indicates an ACL misconfiguration, this user can access the CDR but is not a " + "project jobUser", cdrProjectId, total, email));
                numBad += total;
            } else if (!whitelist.contains(project_id)) {
                log.severe(String.format("AUDIT: (CDR project '%s') %d queries in unrecognized project '%s' from user '%s'", cdrProjectId, total, project_id, email));
                numBad += total;
            }
            numQueries += total;
        }
    }
    log.info(String.format("AUDIT: found audit issues with %d/%d BigQuery queries", numBad, numQueries));
    return ResponseEntity.ok(new AuditBigQueryResponse().numQueryIssues(numBad));
}

Example 17 with FieldValue

use of com.google.cloud.bigquery.FieldValue in project workbench by all-of-us.

the class FieldSetQueryBuilder method extractResults.

public Map<String, Object> extractResults(TableQueryAndConfig tableQueryAndConfig, List<FieldValue> row) {
    TableQuery tableQuery = tableQueryAndConfig.getTableQuery();
    List<ColumnConfig> columnConfigs = tableQuery.getColumns().stream().map(columnName -> tableQueryAndConfig.getColumn(columnName)).collect(Collectors.toList());
    Map<String, Object> results = new HashMap<>(tableQuery.getColumns().size());
    for (int i = 0; i < columnConfigs.size(); i++) {
        FieldValue fieldValue = row.get(i);
        ColumnConfig columnConfig = columnConfigs.get(i);
        if (!fieldValue.isNull()) {
            Object value;
            switch(columnConfig.type) {
                case DATE:
                    value = fieldValue.getStringValue();
                    break;
                case FLOAT:
                    value = fieldValue.getDoubleValue();
                    break;
                case INTEGER:
                    value = fieldValue.getLongValue();
                    break;
                case STRING:
                    value = fieldValue.getStringValue();
                    break;
                case TIMESTAMP:
                    value = DATE_TIME_FORMAT.print(fieldValue.getTimestampValue() / 1000L);
                    break;
                default:
                    throw new IllegalStateException("Unrecognized column type: " + columnConfig.type);
            }
            results.put(columnConfig.name, value);
        }
    }
    return results;
}

Example 18 with FieldValue

use of com.google.cloud.bigquery.FieldValue in project workbench by all-of-us.

the class CohortReviewControllerTest method createCohortReview.

@Test
public void createCohortReview() throws Exception {
    String definition = "{\"includes\":[{\"items\":[{\"type\":\"DEMO\",\"searchParameters\":" + "[{\"value\":\"Age\",\"subtype\":\"AGE\",\"conceptId\":null,\"attribute\":" + "{\"operator\":\"between\",\"operands\":[18,66]}}],\"modifiers\":[]}]}],\"excludes\":[]}";
    SearchRequest searchRequest = new Gson().fromJson(definition, SearchRequest.class);
    QueryResult queryResult = mock(QueryResult.class);
    Iterable testIterable = new Iterable() {

        @Override
        public Iterator iterator() {
            List<FieldValue> list = new ArrayList<>();
            list.add(null);
            return list.iterator();
        }
    };
    Map<String, Integer> rm = new HashMap<>();
    rm.put("person_id", 0);
    rm.put("birth_datetime", 1);
    rm.put("gender_concept_id", 2);
    rm.put("race_concept_id", 3);
    rm.put("ethnicity_concept_id", 4);
    when(workspaceService.enforceWorkspaceAccessLevel(namespace, name, WorkspaceAccessLevel.READER)).thenReturn(WorkspaceAccessLevel.OWNER);
    when(cohortReviewService.findCohortReview(cohortId, cdrVersionId)).thenReturn(createCohortReview(0, cohortId, cohortReviewId, cdrVersionId, null));
    when(cohortReviewService.findCohort(cohortId)).thenReturn(createCohort(cohortId, workspaceId, definition));
    when(cohortReviewService.validateMatchingWorkspace(namespace, name, workspaceId, WorkspaceAccessLevel.WRITER)).thenReturn(createWorkspace(workspaceId, namespace, name));
    when(participantCounter.buildParticipantIdQuery(new ParticipantCriteria(searchRequest), 200, 0L)).thenReturn(null);
    when(bigQueryService.filterBigQueryConfig(null)).thenReturn(null);
    when(bigQueryService.executeQuery(null)).thenReturn(queryResult);
    when(bigQueryService.getResultMapper(queryResult)).thenReturn(rm);
    when(queryResult.iterateAll()).thenReturn(testIterable);
    when(bigQueryService.getLong(null, 0)).thenReturn(0L);
    when(bigQueryService.getString(null, 1)).thenReturn("1");
    when(bigQueryService.getLong(null, 2)).thenReturn(0L);
    when(bigQueryService.getLong(null, 3)).thenReturn(0L);
    when(bigQueryService.getLong(null, 4)).thenReturn(0L);
    when(genderRaceEthnicityConceptProvider.get()).thenReturn(new GenderRaceEthnicityConcept(createGenderRaceEthnicityConcept()));
    doNothing().when(cohortReviewService).saveFullCohortReview(createCohortReview(1, cohortId, cohortReviewId, cdrVersionId, ReviewStatus.CREATED), Arrays.asList(createParticipantCohortStatus(cohortReviewId, 0, CohortStatus.NOT_REVIEWED)));
    when(cohortReviewService.findAll(isA(Long.class), isA(List.class), isA(PageRequest.class))).thenReturn(Arrays.asList(createParticipantCohortStatus(cohortReviewId, 0, CohortStatus.INCLUDED)));
    reviewController.createCohortReview(namespace, name, cohortId, cdrVersionId, new CreateReviewRequest().size(200));
    verify(cohortReviewService, times(1)).findCohortReview(cohortId, cdrVersionId);
    verify(cohortReviewService, times(1)).findCohort(cohortId);
    verify(cohortReviewService, times(1)).validateMatchingWorkspace(namespace, name, workspaceId, WorkspaceAccessLevel.WRITER);
    verify(participantCounter, times(1)).buildParticipantIdQuery(new ParticipantCriteria(searchRequest), 200, 0L);
    verify(bigQueryService, times(1)).filterBigQueryConfig(null);
    verify(bigQueryService, times(1)).executeQuery(null);
    verify(bigQueryService, times(1)).getResultMapper(queryResult);
    verify(bigQueryService, times(1)).getLong(null, 0);
    verify(bigQueryService, times(1)).getString(null, 1);
    verify(bigQueryService, times(1)).getLong(null, 2);
    verify(bigQueryService, times(1)).getLong(null, 3);
    verify(bigQueryService, times(1)).getLong(null, 4);
    verify(queryResult, times(1)).iterateAll();
    verify(cohortReviewService, times(1)).saveFullCohortReview(isA(CohortReview.class), isA(List.class));
    verify(genderRaceEthnicityConceptProvider, times(1)).get();
    verify(cohortReviewService).findAll(isA(Long.class), isA(List.class), isA(PageRequest.class));
    verifyNoMoreMockInteractions();
}

Example 19 with FieldValue

use of com.google.cloud.bigquery.FieldValue in project workbench by all-of-us.

the class AuditControllerTest method stubBigQueryCalls.

// TODO(RW-350): This stubbing is awful, improve this.
private void stubBigQueryCalls(String projectId, String email, long total) {
    QueryResult queryResult = mock(QueryResult.class);
    Iterable testIterable = new Iterable() {

        @Override
        public Iterator iterator() {
            List<FieldValue> list = new ArrayList<>();
            list.add(null);
            return list.iterator();
        }
    };
    Map<String, Integer> rm = ImmutableMap.<String, Integer>builder().put("client_project_id", 0).put("user_email", 1).put("total", 2).build();
    when(bigQueryService.executeQuery(any())).thenReturn(queryResult);
    when(bigQueryService.getResultMapper(queryResult)).thenReturn(rm);
    when(queryResult.iterateAll()).thenReturn(testIterable);
    when(bigQueryService.getString(null, 0)).thenReturn(projectId);
    when(bigQueryService.getString(null, 1)).thenReturn(email);
    when(bigQueryService.getLong(null, 2)).thenReturn(total);
}

Example 20 with FieldValue

use of com.google.cloud.bigquery.FieldValue in project google-cloud-java by GoogleCloudPlatform.

the class ITBigQuerySnippets method testInsertAllAndListTableData.

@Test
public void testInsertAllAndListTableData() throws IOException, InterruptedException {
    String tableName = "test_insert_all_and_list_table_data";
    String fieldName1 = "booleanField";
    String fieldName2 = "bytesField";
    String fieldName3 = "recordField";
    String fieldName4 = "stringField";
    TableId tableId = TableId.of(DATASET, tableName);
    Schema schema = Schema.of(Field.of(fieldName1, Type.bool()), Field.of(fieldName2, Type.bytes()), Field.of(fieldName3, Type.record(Field.of(fieldName4, Type.string()))));
    TableInfo table = TableInfo.of(tableId, StandardTableDefinition.of(schema));
    assertNotNull(bigquery.create(table));
    InsertAllResponse response = bigquerySnippets.insertAll(DATASET, tableName);
    assertFalse(response.hasErrors());
    assertTrue(response.getInsertErrors().isEmpty());
    Page<List<FieldValue>> listPage = bigquerySnippets.listTableDataFromId(DATASET, tableName);
    while (Iterators.size(listPage.iterateAll().iterator()) < 1) {
        Thread.sleep(500);
        listPage = bigquerySnippets.listTableDataFromId(DATASET, tableName);
    }
    List<FieldValue> row = listPage.getValues().iterator().next();
    assertEquals(true, row.get(0).getBooleanValue());
    assertArrayEquals(new byte[] { 0xA, 0xD, 0xD, 0xE, 0xD }, row.get(1).getBytesValue());
    assertEquals("Hello, World!", row.get(2).getRecordValue().get(0).getStringValue());
    assertTrue(bigquerySnippets.deleteTable(DATASET, tableName));
}