Examples with TableReference com.google.api.services.bigquery.model.TableReference used on opensource projects

Example 96 with TableReference

use of com.google.api.services.bigquery.model.TableReference in project beam by apache.

the class FakeDatasetService method createTable.

@Override
public void createTable(Table table) throws IOException {
    TableReference tableReference = table.getTableReference();
    validateWholeTableReference(tableReference);
    synchronized (tables) {
        Map<String, TableContainer> dataset = tables.get(tableReference.getProjectId(), tableReference.getDatasetId());
        if (dataset == null) {
            throwNotFound("Tried to get a dataset %s:%s, but no such table was set", tableReference.getProjectId(), tableReference.getDatasetId());
        }
        dataset.computeIfAbsent(tableReference.getTableId(), k -> {
            TableContainer tableContainer = new TableContainer(table);
            // Create the default stream.
            String streamName = String.format("projects/%s/datasets/%s/tables/%s/streams/_default", tableReference.getProjectId(), tableReference.getDatasetId(), BigQueryHelpers.stripPartitionDecorator(tableReference.getTableId()));
            writeStreams.put(streamName, new Stream(streamName, tableContainer, Type.COMMITTED));
            return tableContainer;
        });
    }
}

Example 97 with TableReference

use of com.google.api.services.bigquery.model.TableReference in project beam by apache.

the class BigQueryResourceNaming method createTempTableReference.

static TableReference createTempTableReference(String projectId, String jobUuid, Optional<String> tempDatasetIdOpt) {
    String tempDatasetId = tempDatasetIdOpt.orElse("temp_dataset_" + jobUuid);
    String queryTempTableId = "temp_table_" + jobUuid;
    return new TableReference().setProjectId(projectId).setDatasetId(tempDatasetId).setTableId(queryTempTableId);
}

Example 98 with TableReference

use of com.google.api.services.bigquery.model.TableReference in project beam by apache.

the class BigQuerySourceBase method extractFiles.

protected ExtractResult extractFiles(PipelineOptions options) throws Exception {
    BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
    TableReference tableToExtract = getTableToExtract(bqOptions);
    BigQueryServices.DatasetService datasetService = bqServices.getDatasetService(bqOptions);
    Table table = datasetService.getTable(tableToExtract);
    if (table == null) {
        throw new IOException(String.format("Cannot start an export job since table %s does not exist", BigQueryHelpers.toTableSpec(tableToExtract)));
    }
    TableSchema schema = table.getSchema();
    JobService jobService = bqServices.getJobService(bqOptions);
    String extractJobId = BigQueryResourceNaming.createJobIdPrefix(options.getJobName(), stepUuid, JobType.EXPORT);
    final String extractDestinationDir = resolveTempLocation(bqOptions.getTempLocation(), "BigQueryExtractTemp", stepUuid);
    String bqLocation = BigQueryHelpers.getDatasetLocation(datasetService, tableToExtract.getProjectId(), tableToExtract.getDatasetId());
    List<ResourceId> tempFiles = executeExtract(extractJobId, tableToExtract, jobService, bqOptions.getProject(), extractDestinationDir, bqLocation, useAvroLogicalTypes);
    return new ExtractResult(schema, tempFiles);
}

Example 99 with TableReference

use of com.google.api.services.bigquery.model.TableReference in project beam by apache.

the class BigQueryTableSource method getEstimatedSizeBytes.

@Override
public synchronized long getEstimatedSizeBytes(PipelineOptions options) throws Exception {
    if (tableSizeBytes.get() == null) {
        BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
        TableReference tableRef = tableDef.getTableReference(bqOptions);
        Table table = bqServices.getDatasetService(bqOptions).getTable(tableRef);
        Long numBytes = table.getNumBytes();
        if (table.getStreamingBuffer() != null && table.getStreamingBuffer().getEstimatedBytes() != null) {
            numBytes += table.getStreamingBuffer().getEstimatedBytes().longValue();
        }
        tableSizeBytes.compareAndSet(null, numBytes);
    }
    return tableSizeBytes.get();
}

Example 100 with TableReference

use of com.google.api.services.bigquery.model.TableReference in project beam by apache.

the class BigQueryUtils method toTableReference.

/**
 * @param fullTableId - Is one of the two forms commonly used to refer to bigquery tables in the
 *     beam codebase:
 *     <ul>
 *       <li>projects/{project_id}/datasets/{dataset_id}/tables/{table_id}
 *       <li>myproject:mydataset.mytable
 *       <li>myproject.mydataset.mytable
 *     </ul>
 *
 * @return a BigQueryTableIdentifier by parsing the fullTableId. If it cannot be parsed properly
 *     null is returned.
 */
@Nullable
public static TableReference toTableReference(String fullTableId) {
    // Try parsing the format:
    // "projects/{project_id}/datasets/{dataset_id}/tables/{table_id}"
    Matcher m = TABLE_RESOURCE_PATTERN.matcher(fullTableId);
    if (m.matches()) {
        return new TableReference().setProjectId(m.group("PROJECT")).setDatasetId(m.group("DATASET")).setTableId(m.group("TABLE"));
    }
    // If that failed, try the format:
    // "{project_id}:{dataset_id}.{table_id}" or
    // "{project_id}.{dataset_id}.{table_id}"
    m = SIMPLE_TABLE_PATTERN.matcher(fullTableId);
    if (m.matches()) {
        return new TableReference().setProjectId(m.group("PROJECT")).setDatasetId(m.group("DATASET")).setTableId(m.group("TABLE"));
    }
    return null;
}