Examples with FormatOptions com.google.cloud.bigquery.FormatOptions used on opensource projects

Example 1 with FormatOptions

use of com.google.cloud.bigquery.FormatOptions in project spring-cloud-gcp by spring-cloud.

the class BigQueryFileMessageHandler method handleRequestMessage.

@Override
protected Object handleRequestMessage(Message<?> message) {
    String tableName = this.tableNameExpression.getValue(this.evaluationContext, message, String.class);
    FormatOptions formatOptions = this.formatOptionsExpression.getValue(this.evaluationContext, message, FormatOptions.class);
    Schema schema = this.tableSchemaExpression.getValue(this.evaluationContext, message, Schema.class);
    Assert.notNull(tableName, "BigQuery table name must not be null.");
    Assert.notNull(formatOptions, "Data file formatOptions must not be null.");
    try (InputStream inputStream = convertToInputStream(message.getPayload())) {
        ListenableFuture<Job> jobFuture = this.bigQueryTemplate.writeDataToTable(tableName, inputStream, formatOptions, schema);
        if (this.sync) {
            return jobFuture.get(this.timeout.getSeconds(), TimeUnit.SECONDS);
        } else {
            return jobFuture;
        }
    } catch (FileNotFoundException e) {
        throw new MessageHandlingException(message, "Failed to find file to write to BigQuery in message handler: " + this, e);
    } catch (IOException e) {
        throw new MessageHandlingException(message, "Failed to write data to BigQuery tables in message handler: " + this, e);
    } catch (InterruptedException | ExecutionException | TimeoutException e) {
        throw new MessageHandlingException(message, "Failed to wait for BigQuery Job to complete in message handler: " + this, e);
    }
}

Example 2 with FormatOptions

use of com.google.cloud.bigquery.FormatOptions in project jade-data-repo by DataBiosphere.

the class BigQueryPdao method buildFormatOptions.

private FormatOptions buildFormatOptions(IngestRequestModel ingestRequest) {
    FormatOptions options;
    switch(ingestRequest.getFormat()) {
        case CSV:
            CsvOptions csvDefaults = FormatOptions.csv();
            options = CsvOptions.newBuilder().setFieldDelimiter(ingestRequest.getCsvFieldDelimiter() == null ? csvDefaults.getFieldDelimiter() : ingestRequest.getCsvFieldDelimiter()).setQuote(ingestRequest.getCsvQuote() == null ? csvDefaults.getQuote() : ingestRequest.getCsvQuote()).setSkipLeadingRows(ingestRequest.getCsvSkipLeadingRows() == null ? csvDefaults.getSkipLeadingRows() : ingestRequest.getCsvSkipLeadingRows()).setAllowQuotedNewLines(ingestRequest.isCsvAllowQuotedNewlines() == null ? csvDefaults.allowQuotedNewLines() : ingestRequest.isCsvAllowQuotedNewlines()).build();
            break;
        case JSON:
            options = FormatOptions.json();
            break;
        default:
            throw new PdaoException("Invalid format option: " + ingestRequest.getFormat());
    }
    return options;
}

Example 3 with FormatOptions

use of com.google.cloud.bigquery.FormatOptions in project spring-cloud-gcp by GoogleCloudPlatform.

the class BigQueryFileMessageHandler method handleRequestMessage.

@Override
protected Object handleRequestMessage(Message<?> message) {
    String tableName = this.tableNameExpression.getValue(this.evaluationContext, message, String.class);
    FormatOptions formatOptions = this.formatOptionsExpression.getValue(this.evaluationContext, message, FormatOptions.class);
    Schema schema = this.tableSchemaExpression.getValue(this.evaluationContext, message, Schema.class);
    Assert.notNull(tableName, "BigQuery table name must not be null.");
    Assert.notNull(formatOptions, "Data file formatOptions must not be null.");
    try (InputStream inputStream = convertToInputStream(message.getPayload())) {
        ListenableFuture<Job> jobFuture = this.bigQueryTemplate.writeDataToTable(tableName, inputStream, formatOptions, schema);
        if (this.sync) {
            return jobFuture.get(this.timeout.getSeconds(), TimeUnit.SECONDS);
        } else {
            return jobFuture;
        }
    } catch (FileNotFoundException e) {
        throw new MessageHandlingException(message, "Failed to find file to write to BigQuery in message handler: " + this, e);
    } catch (IOException e) {
        throw new MessageHandlingException(message, "Failed to write data to BigQuery tables in message handler: " + this, e);
    } catch (ExecutionException | TimeoutException e) {
        throw new MessageHandlingException(message, "Failed to wait for BigQuery Job to complete in message handler: " + this, e);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new MessageHandlingException(message, "Failed to wait for BigQuery Job (interrupted) in message handler: " + this, e);
    }
}

Example 4 with FormatOptions

use of com.google.cloud.bigquery.FormatOptions in project spark-bigquery-connector by GoogleCloudDataproc.

the class BigQueryIndirectDataSourceWriterContext method loadDataToBigQuery.

void loadDataToBigQuery(List<String> sourceUris) throws IOException {
    // Solving Issue #248
    List<String> optimizedSourceUris = SparkBigQueryUtil.optimizeLoadUriListForSpark(sourceUris);
    JobInfo.WriteDisposition writeDisposition = SparkBigQueryUtil.saveModeToWriteDisposition(saveMode);
    FormatOptions formatOptions = config.getIntermediateFormat().getFormatOptions();
    bigQueryClient.loadDataIntoTable(config, optimizedSourceUris, formatOptions, writeDisposition);
}