Examples with TableReference com.google.api.services.bigquery.model.TableReference used on opensource projects

Example 26 with TableReference

use of com.google.api.services.bigquery.model.TableReference in project beam by apache.

the class BigqueryClient method queryUnflattened.

/**
 * Performs a query without flattening results.
 */
@Nonnull
public List<TableRow> queryUnflattened(String query, String projectId, boolean typed) throws IOException, InterruptedException {
    Random rnd = new Random(System.currentTimeMillis());
    String temporaryDatasetId = "_dataflow_temporary_dataset_" + rnd.nextInt(1000000);
    String temporaryTableId = "dataflow_temporary_table_" + rnd.nextInt(1000000);
    TableReference tempTableReference = new TableReference().setProjectId(projectId).setDatasetId(temporaryDatasetId).setTableId(temporaryTableId);
    createNewDataset(projectId, temporaryDatasetId);
    createNewTable(projectId, temporaryDatasetId, new Table().setTableReference(tempTableReference));
    JobConfigurationQuery jcQuery = new JobConfigurationQuery().setFlattenResults(false).setAllowLargeResults(true).setDestinationTable(tempTableReference).setQuery(query);
    JobConfiguration jc = new JobConfiguration().setQuery(jcQuery);
    Job job = new Job().setConfiguration(jc);
    Job insertedJob = bqClient.jobs().insert(projectId, job).execute();
    GetQueryResultsResponse qResponse;
    do {
        qResponse = bqClient.jobs().getQueryResults(projectId, insertedJob.getJobReference().getJobId()).execute();
    } while (!qResponse.getJobComplete());
    final TableSchema schema = qResponse.getSchema();
    final List<TableRow> rows = qResponse.getRows();
    deleteDataset(projectId, temporaryDatasetId);
    return !typed ? rows : rows.stream().map(r -> getTypedTableRow(schema.getFields(), r)).collect(Collectors.toList());
}

Example 27 with TableReference

use of com.google.api.services.bigquery.model.TableReference in project beam by apache.

the class FakeJobService method runExtractJob.

private JobStatus runExtractJob(Job job, JobConfigurationExtract extract) throws InterruptedException, IOException {
    TableReference sourceTable = extract.getSourceTable();
    List<TableRow> rows = datasetService.getAllRows(sourceTable.getProjectId(), sourceTable.getDatasetId(), sourceTable.getTableId());
    TableSchema schema = datasetService.getTable(sourceTable).getSchema();
    List<Long> destinationFileCounts = Lists.newArrayList();
    for (String destination : extract.getDestinationUris()) {
        destinationFileCounts.add(writeRows(sourceTable.getTableId(), rows, schema, destination));
    }
    job.setStatistics(new JobStatistics().setExtract(new JobStatistics4().setDestinationUriFileCounts(destinationFileCounts)));
    return new JobStatus().setState("DONE");
}

Example 28 with TableReference

use of com.google.api.services.bigquery.model.TableReference in project beam by apache.

the class FakeDatasetService method createWriteStream.

@Override
public WriteStream createWriteStream(String tableUrn, Type type) throws IOException, InterruptedException {
    TableReference tableReference = BigQueryHelpers.parseTableUrn(BigQueryHelpers.stripPartitionDecorator(tableUrn));
    synchronized (tables) {
        TableContainer tableContainer = getTableContainer(tableReference.getProjectId(), tableReference.getDatasetId(), tableReference.getTableId());
        String streamName = UUID.randomUUID().toString();
        writeStreams.put(streamName, new Stream(streamName, tableContainer, type));
        return WriteStream.newBuilder().setName(streamName).build();
    }
}

Example 29 with TableReference

use of com.google.api.services.bigquery.model.TableReference in project beam by apache.

the class BigQueryStorageTableSource method getTargetTable.

@Override
@Nullable
protected Table getTargetTable(BigQueryOptions options) throws Exception {
    if (cachedTable.get() == null) {
        TableReference tableReference = tableReferenceProvider.get();
        if (Strings.isNullOrEmpty(tableReference.getProjectId())) {
            checkState(!Strings.isNullOrEmpty(options.getProject()), "No project ID set in %s or %s, cannot construct a complete %s", TableReference.class.getSimpleName(), BigQueryOptions.class.getSimpleName(), TableReference.class.getSimpleName());
            LOG.info("Project ID not set in {}. Using default project from {}.", TableReference.class.getSimpleName(), BigQueryOptions.class.getSimpleName());
            tableReference.setProjectId(options.getBigQueryProject() == null ? options.getProject() : options.getBigQueryProject());
        }
        Table table = bqServices.getDatasetService(options).getTable(tableReference);
        cachedTable.compareAndSet(null, table);
    }
    return cachedTable.get();
}

Example 30 with TableReference

use of com.google.api.services.bigquery.model.TableReference in project beam by apache.

the class BigQueryStorageTableSource method getTargetTableId.

@Override
protected String getTargetTableId(BigQueryOptions options) throws Exception {
    TableReference tableReference = tableReferenceProvider.get();
    if (Strings.isNullOrEmpty(tableReference.getProjectId())) {
        checkState(!Strings.isNullOrEmpty(options.getProject()), "No project ID set in %s or %s, cannot construct a complete %s", TableReference.class.getSimpleName(), BigQueryOptions.class.getSimpleName(), TableReference.class.getSimpleName());
        LOG.info("Project ID not set in {}. Using default project from {}.", TableReference.class.getSimpleName(), BigQueryOptions.class.getSimpleName());
        tableReference.setProjectId(options.getProject());
    }
    return String.format("projects/%s/datasets/%s/tables/%s", tableReference.getProjectId(), tableReference.getDatasetId(), tableReference.getTableId());
}