Examples with TableReference com.google.api.services.bigquery.model.TableReference used on opensource projects

Example 16 with TableReference

use of com.google.api.services.bigquery.model.TableReference in project beam by apache.

the class BigQueryHelpers method createTempTableReference.

static TableReference createTempTableReference(String projectId, String jobUuid) {
    String queryTempDatasetId = "temp_dataset_" + jobUuid;
    String queryTempTableId = "temp_table_" + jobUuid;
    TableReference queryTempTableRef = new TableReference().setProjectId(projectId).setDatasetId(queryTempDatasetId).setTableId(queryTempTableId);
    return queryTempTableRef;
}

Example 17 with TableReference

use of com.google.api.services.bigquery.model.TableReference in project beam by apache.

the class BigQueryTableRowIterator method executeQueryAndWaitForCompletion.

/**
   * Executes the specified query and returns a reference to the temporary BigQuery table created
   * to hold the results.
   *
   * @throws IOException if the query fails.
   */
private TableReference executeQueryAndWaitForCompletion() throws IOException, InterruptedException {
    checkState(projectId != null, "Unable to execute a query without a configured project id");
    checkState(queryConfig != null, "Unable to execute a query without a configured query");
    // Dry run query to get source table location
    Job dryRunJob = new Job().setConfiguration(new JobConfiguration().setQuery(queryConfig).setDryRun(true));
    JobStatistics jobStats = executeWithBackOff(client.jobs().insert(projectId, dryRunJob), String.format("Error when trying to dry run query %s.", queryConfig.toPrettyString())).getStatistics();
    // Let BigQuery to pick default location if the query does not read any tables.
    String location = null;
    @Nullable List<TableReference> tables = jobStats.getQuery().getReferencedTables();
    if (tables != null && !tables.isEmpty()) {
        Table table = getTable(tables.get(0));
        location = table.getLocation();
    }
    // Create a temporary dataset to store results.
    // Starting dataset name with an "_" so that it is hidden.
    Random rnd = new Random(System.currentTimeMillis());
    temporaryDatasetId = "_beam_temporary_dataset_" + rnd.nextInt(1000000);
    temporaryTableId = "beam_temporary_table_" + rnd.nextInt(1000000);
    createDataset(temporaryDatasetId, location);
    Job job = new Job();
    JobConfiguration config = new JobConfiguration();
    config.setQuery(queryConfig);
    job.setConfiguration(config);
    TableReference destinationTable = new TableReference();
    destinationTable.setProjectId(projectId);
    destinationTable.setDatasetId(temporaryDatasetId);
    destinationTable.setTableId(temporaryTableId);
    queryConfig.setDestinationTable(destinationTable);
    queryConfig.setAllowLargeResults(true);
    Job queryJob = executeWithBackOff(client.jobs().insert(projectId, job), String.format("Error when trying to execute the job for query %s.", queryConfig.toPrettyString()));
    JobReference jobId = queryJob.getJobReference();
    while (true) {
        Job pollJob = executeWithBackOff(client.jobs().get(projectId, jobId.getJobId()), String.format("Error when trying to get status of the job for query %s.", queryConfig.toPrettyString()));
        JobStatus status = pollJob.getStatus();
        if (status.getState().equals("DONE")) {
            // Job is DONE, but did not necessarily succeed.
            ErrorProto error = status.getErrorResult();
            if (error == null) {
                return pollJob.getConfiguration().getQuery().getDestinationTable();
            } else {
                // There will be no temporary table to delete, so null out the reference.
                temporaryTableId = null;
                throw new IOException(String.format("Executing query %s failed: %s", queryConfig.toPrettyString(), error.getMessage()));
            }
        }
        Uninterruptibles.sleepUninterruptibly(QUERY_COMPLETION_POLL_TIME.getMillis(), TimeUnit.MILLISECONDS);
    }
}

Example 18 with TableReference

use of com.google.api.services.bigquery.model.TableReference in project google-cloud-java by GoogleCloudPlatform.

the class DatasetInfo method setProjectId.

DatasetInfo setProjectId(String projectId) {
    Builder builder = toBuilder();
    builder.setDatasetId(getDatasetId().setProjectId(projectId));
    if (getAcl() != null) {
        List<Acl> acls = Lists.newArrayListWithCapacity(getAcl().size());
        for (Acl acl : getAcl()) {
            if (acl.getEntity().getType() == Acl.Entity.Type.VIEW) {
                Dataset.Access accessPb = acl.toPb();
                TableReference viewReferencePb = accessPb.getView();
                if (viewReferencePb.getProjectId() == null) {
                    viewReferencePb.setProjectId(projectId);
                }
                acls.add(Acl.of(new Acl.View(TableId.fromPb(viewReferencePb))));
            } else {
                acls.add(acl);
            }
        }
        builder.setAcl(acls);
    }
    return builder.build();
}

Example 19 with TableReference

use of com.google.api.services.bigquery.model.TableReference in project google-cloud-java by GoogleCloudPlatform.

the class HttpBigQueryRpc method patch.

@Override
public Table patch(Table table, Map<Option, ?> options) {
    try {
        // unset the type, as it is output only
        table.setType(null);
        TableReference reference = table.getTableReference();
        return bigquery.tables().patch(reference.getProjectId(), reference.getDatasetId(), reference.getTableId(), table).setFields(Option.FIELDS.getString(options)).execute();
    } catch (IOException ex) {
        throw translate(ex);
    }
}

Example 20 with TableReference

use of com.google.api.services.bigquery.model.TableReference in project components by Talend.

the class BigQueryOutputRuntime method expand.

@Override
public PDone expand(PCollection<IndexedRecord> in) {
    TableReference table = new TableReference();
    table.setProjectId(datastore.projectName.getValue());
    table.setDatasetId(dataset.bqDataset.getValue());
    table.setTableId(dataset.tableName.getValue());
    BigQueryIO.Write bigQueryIOPTransform = BigQueryIO.writeTableRows().to(table);
    bigQueryIOPTransform = setTableOperation(bigQueryIOPTransform);
    bigQueryIOPTransform = setWriteOperation(bigQueryIOPTransform);
    in.apply(ParDo.of(new IndexedRecordToTableRowFn())).apply(bigQueryIOPTransform);
    return PDone.in(in.getPipeline());
}