Examples with TableId com.google.cloud.bigquery.TableId used on opensource projects

Example 26 with TableId

use of com.google.cloud.bigquery.TableId in project google-cloud-java by GoogleCloudPlatform.

the class ITBigQueryTest method testQueryJob.

@Test
public void testQueryJob() throws InterruptedException, TimeoutException {
    String tableName = "test_query_job_table";
    String query = new StringBuilder().append("SELECT TimestampField, StringField, BooleanField FROM ").append(TABLE_ID.getTable()).toString();
    TableId destinationTable = TableId.of(DATASET, tableName);
    QueryJobConfiguration configuration = QueryJobConfiguration.newBuilder(query).setDefaultDataset(DatasetId.of(DATASET)).setDestinationTable(destinationTable).build();
    Job remoteJob = bigquery.create(JobInfo.of(configuration));
    remoteJob = remoteJob.waitFor();
    assertNull(remoteJob.getStatus().getError());
    QueryResponse response = bigquery.getQueryResults(remoteJob.getJobId());
    while (!response.jobCompleted()) {
        Thread.sleep(1000);
        response = bigquery.getQueryResults(response.getJobId());
    }
    assertFalse(response.hasErrors());
    assertEquals(QUERY_RESULT_SCHEMA, response.getResult().getSchema());
    int rowCount = 0;
    for (List<FieldValue> row : response.getResult().getValues()) {
        FieldValue timestampCell = row.get(0);
        FieldValue stringCell = row.get(1);
        FieldValue booleanCell = row.get(2);
        assertEquals(FieldValue.Attribute.PRIMITIVE, timestampCell.getAttribute());
        assertEquals(FieldValue.Attribute.PRIMITIVE, stringCell.getAttribute());
        assertEquals(FieldValue.Attribute.PRIMITIVE, booleanCell.getAttribute());
        assertEquals(1408452095220000L, timestampCell.getTimestampValue());
        assertEquals("stringValue", stringCell.getStringValue());
        assertEquals(false, booleanCell.getBooleanValue());
        rowCount++;
    }
    assertEquals(2, rowCount);
    assertTrue(bigquery.delete(DATASET, tableName));
    Job queryJob = bigquery.getJob(remoteJob.getJobId());
    JobStatistics.QueryStatistics statistics = queryJob.getStatistics();
    assertNotNull(statistics.getQueryPlan());
}

Example 27 with TableId

use of com.google.cloud.bigquery.TableId in project google-cloud-java by GoogleCloudPlatform.

the class ITBigQueryTest method testCreateExternalTable.

@Test
public void testCreateExternalTable() throws InterruptedException {
    String tableName = "test_create_external_table";
    TableId tableId = TableId.of(DATASET, tableName);
    ExternalTableDefinition externalTableDefinition = ExternalTableDefinition.of("gs://" + BUCKET + "/" + JSON_LOAD_FILE, TABLE_SCHEMA, FormatOptions.json());
    TableInfo tableInfo = TableInfo.of(tableId, externalTableDefinition);
    Table createdTable = bigquery.create(tableInfo);
    assertNotNull(createdTable);
    assertEquals(DATASET, createdTable.getTableId().getDataset());
    assertEquals(tableName, createdTable.getTableId().getTable());
    Table remoteTable = bigquery.getTable(DATASET, tableName);
    assertNotNull(remoteTable);
    assertTrue(remoteTable.getDefinition() instanceof ExternalTableDefinition);
    assertEquals(createdTable.getTableId(), remoteTable.getTableId());
    assertEquals(TABLE_SCHEMA, remoteTable.getDefinition().getSchema());
    QueryRequest request = QueryRequest.newBuilder("SELECT TimestampField, StringField, IntegerArrayField, BooleanField FROM " + DATASET + "." + tableName).setDefaultDataset(DatasetId.of(DATASET)).setMaxWaitTime(60000L).setPageSize(1000L).build();
    QueryResponse response = bigquery.query(request);
    while (!response.jobCompleted()) {
        response = bigquery.getQueryResults(response.getJobId());
        Thread.sleep(1000);
    }
    long integerValue = 0;
    int rowCount = 0;
    for (List<FieldValue> row : response.getResult().getValues()) {
        FieldValue timestampCell = row.get(0);
        FieldValue stringCell = row.get(1);
        FieldValue integerCell = row.get(2);
        FieldValue booleanCell = row.get(3);
        assertEquals(FieldValue.Attribute.PRIMITIVE, timestampCell.getAttribute());
        assertEquals(FieldValue.Attribute.PRIMITIVE, stringCell.getAttribute());
        assertEquals(FieldValue.Attribute.PRIMITIVE, integerCell.getAttribute());
        assertEquals(FieldValue.Attribute.PRIMITIVE, booleanCell.getAttribute());
        assertEquals(1408452095220000L, timestampCell.getTimestampValue());
        assertEquals("stringValue", stringCell.getStringValue());
        assertEquals(integerValue, integerCell.getLongValue());
        assertEquals(false, booleanCell.getBooleanValue());
        integerValue = ~integerValue & 0x1;
        rowCount++;
    }
    assertEquals(4, rowCount);
    assertTrue(remoteTable.delete());
}

Example 28 with TableId

use of com.google.cloud.bigquery.TableId in project google-cloud-java by GoogleCloudPlatform.

the class ITBigQueryTest method testCancelJob.

@Test
public void testCancelJob() throws InterruptedException, TimeoutException {
    String destinationTableName = "test_cancel_query_job_table";
    String query = "SELECT TimestampField, StringField, BooleanField FROM " + TABLE_ID.getTable();
    TableId destinationTable = TableId.of(DATASET, destinationTableName);
    QueryJobConfiguration configuration = QueryJobConfiguration.newBuilder(query).setDefaultDataset(DatasetId.of(DATASET)).setDestinationTable(destinationTable).build();
    Job remoteJob = bigquery.create(JobInfo.of(configuration));
    assertTrue(remoteJob.cancel());
    remoteJob = remoteJob.waitFor();
    assertNull(remoteJob.getStatus().getError());
}

Example 29 with TableId

use of com.google.cloud.bigquery.TableId in project presto by prestodb.

the class BigQuerySplitManager method createEmptyProjection.

private List<BigQuerySplit> createEmptyProjection(TableId tableId, int actualParallelism, Optional<String> filter) {
    try {
        long numberOfRows;
        if (filter.isPresent()) {
            // count the rows based on the filter
            String sql = bigQueryClient.createFormatSql(tableId, "COUNT(*)", new String[] { filter.get() });
            TableResult result = bigQueryClient.query(sql);
            numberOfRows = result.iterateAll().iterator().next().get(0).getLongValue();
        } else {
            // no filters, so we can take the value from the table info
            numberOfRows = bigQueryClient.getTable(tableId).getNumRows().longValue();
        }
        long rowsPerSplit = numberOfRows / actualParallelism;
        // need to be added to one of the split due to integer division
        long remainingRows = numberOfRows - (rowsPerSplit * actualParallelism);
        List<BigQuerySplit> splits = range(0, actualParallelism).mapToObj(ignored -> BigQuerySplit.emptyProjection(rowsPerSplit)).collect(toList());
        splits.set(0, BigQuerySplit.emptyProjection(rowsPerSplit + remainingRows));
        return splits;
    } catch (BigQueryException e) {
        throw new PrestoException(BIGQUERY_FAILED_TO_EXECUTE_QUERY, format("Failed to compute empty projection"), e);
    }
}

Example 30 with TableId

use of com.google.cloud.bigquery.TableId in project presto by prestodb.

the class BigQuerySplitManager method readFromBigQuery.

private ImmutableList<BigQuerySplit> readFromBigQuery(TableId tableId, Optional<List<ColumnHandle>> projectedColumns, int actualParallelism, Optional<String> filter) {
    List<ColumnHandle> columns = projectedColumns.orElse(ImmutableList.of());
    ImmutableList<String> projectedColumnsNames = columns.stream().map(column -> ((BigQueryColumnHandle) column).getName()).collect(toImmutableList());
    ReadSession readSession = new ReadSessionCreator(readSessionCreatorConfig, bigQueryClient, bigQueryStorageClientFactory).create(tableId, projectedColumnsNames, filter, actualParallelism);
    return readSession.getStreamsList().stream().map(stream -> BigQuerySplit.forStream(stream.getName(), readSession.getAvroSchema().getSchema(), columns)).collect(toImmutableList());
}