Examples with LoadJobConfiguration com.google.cloud.bigquery.LoadJobConfiguration used on opensource projects

Example 1 with LoadJobConfiguration

use of com.google.cloud.bigquery.LoadJobConfiguration in project google-cloud-java by GoogleCloudPlatform.

the class ITBigQueryTest method beforeClass.

@BeforeClass
public static void beforeClass() throws InterruptedException, TimeoutException {
    RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create();
    RemoteStorageHelper storageHelper = RemoteStorageHelper.create();
    bigquery = bigqueryHelper.getOptions().getService();
    storage = storageHelper.getOptions().getService();
    storage.create(BucketInfo.of(BUCKET));
    storage.create(BlobInfo.newBuilder(BUCKET, LOAD_FILE).setContentType("text/plain").build(), CSV_CONTENT.getBytes(StandardCharsets.UTF_8));
    storage.create(BlobInfo.newBuilder(BUCKET, JSON_LOAD_FILE).setContentType("application/json").build(), JSON_CONTENT.getBytes(StandardCharsets.UTF_8));
    DatasetInfo info = DatasetInfo.newBuilder(DATASET).setDescription(DESCRIPTION).setLabels(LABELS).build();
    bigquery.create(info);
    LoadJobConfiguration configuration = LoadJobConfiguration.newBuilder(TABLE_ID, "gs://" + BUCKET + "/" + JSON_LOAD_FILE, FormatOptions.json()).setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED).setSchema(TABLE_SCHEMA).build();
    Job job = bigquery.create(JobInfo.of(configuration));
    job = job.waitFor();
    assertNull(job.getStatus().getError());
}

Example 2 with LoadJobConfiguration

use of com.google.cloud.bigquery.LoadJobConfiguration in project google-cloud-java by GoogleCloudPlatform.

the class ITBigQueryTest method testInsertFromFile.

@Test
public void testInsertFromFile() throws InterruptedException, IOException, TimeoutException {
    String destinationTableName = "test_insert_from_file_table";
    TableId tableId = TableId.of(DATASET, destinationTableName);
    WriteChannelConfiguration configuration = WriteChannelConfiguration.newBuilder(tableId).setFormatOptions(FormatOptions.json()).setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED).setSchema(TABLE_SCHEMA).build();
    TableDataWriteChannel channel = bigquery.writer(configuration);
    try {
        channel.write(ByteBuffer.wrap(JSON_CONTENT.getBytes(StandardCharsets.UTF_8)));
    } finally {
        channel.close();
    }
    Job job = channel.getJob().waitFor();
    LoadStatistics statistics = job.getStatistics();
    assertEquals(1L, statistics.getInputFiles().longValue());
    assertEquals(2L, statistics.getOutputRows().longValue());
    LoadJobConfiguration jobConfiguration = job.getConfiguration();
    assertEquals(TABLE_SCHEMA, jobConfiguration.getSchema());
    assertNull(jobConfiguration.getSourceUris());
    assertNull(job.getStatus().getError());
    Page<List<FieldValue>> rows = bigquery.listTableData(tableId);
    int rowCount = 0;
    for (List<FieldValue> row : rows.getValues()) {
        FieldValue timestampCell = row.get(0);
        FieldValue stringCell = row.get(1);
        FieldValue integerArrayCell = row.get(2);
        FieldValue booleanCell = row.get(3);
        FieldValue bytesCell = row.get(4);
        FieldValue recordCell = row.get(5);
        FieldValue integerCell = row.get(6);
        FieldValue floatCell = row.get(7);
        assertEquals(FieldValue.Attribute.PRIMITIVE, timestampCell.getAttribute());
        assertEquals(FieldValue.Attribute.PRIMITIVE, stringCell.getAttribute());
        assertEquals(FieldValue.Attribute.REPEATED, integerArrayCell.getAttribute());
        assertEquals(FieldValue.Attribute.PRIMITIVE, booleanCell.getAttribute());
        assertEquals(FieldValue.Attribute.PRIMITIVE, bytesCell.getAttribute());
        assertEquals(FieldValue.Attribute.RECORD, recordCell.getAttribute());
        assertEquals(FieldValue.Attribute.PRIMITIVE, integerCell.getAttribute());
        assertEquals(FieldValue.Attribute.PRIMITIVE, floatCell.getAttribute());
        assertEquals(1408452095220000L, timestampCell.getTimestampValue());
        assertEquals("stringValue", stringCell.getStringValue());
        assertEquals(0, integerArrayCell.getRepeatedValue().get(0).getLongValue());
        assertEquals(1, integerArrayCell.getRepeatedValue().get(1).getLongValue());
        assertEquals(false, booleanCell.getBooleanValue());
        assertArrayEquals(BYTES, bytesCell.getBytesValue());
        assertEquals(-14182916000000L, recordCell.getRecordValue().get(0).getTimestampValue());
        assertTrue(recordCell.getRecordValue().get(1).isNull());
        assertEquals(1, recordCell.getRecordValue().get(2).getRepeatedValue().get(0).getLongValue());
        assertEquals(0, recordCell.getRecordValue().get(2).getRepeatedValue().get(1).getLongValue());
        assertEquals(true, recordCell.getRecordValue().get(3).getBooleanValue());
        assertEquals(3, integerCell.getLongValue());
        assertEquals(1.2, floatCell.getDoubleValue(), 0.0001);
        rowCount++;
    }
    assertEquals(2, rowCount);
    assertTrue(bigquery.delete(DATASET, destinationTableName));
}

Example 3 with LoadJobConfiguration

use of com.google.cloud.bigquery.LoadJobConfiguration in project google-cloud-java by GoogleCloudPlatform.

the class ITBigQueryTest method testExtractJob.

@Test
public void testExtractJob() throws InterruptedException, TimeoutException {
    String tableName = "test_export_job_table";
    TableId destinationTable = TableId.of(DATASET, tableName);
    LoadJobConfiguration configuration = LoadJobConfiguration.newBuilder(destinationTable, "gs://" + BUCKET + "/" + LOAD_FILE).setSchema(SIMPLE_SCHEMA).build();
    Job remoteLoadJob = bigquery.create(JobInfo.of(configuration));
    remoteLoadJob = remoteLoadJob.waitFor();
    assertNull(remoteLoadJob.getStatus().getError());
    ExtractJobConfiguration extractConfiguration = ExtractJobConfiguration.newBuilder(destinationTable, "gs://" + BUCKET + "/" + EXTRACT_FILE).setPrintHeader(false).build();
    Job remoteExtractJob = bigquery.create(JobInfo.of(extractConfiguration));
    remoteExtractJob = remoteExtractJob.waitFor();
    assertNull(remoteExtractJob.getStatus().getError());
    assertEquals(CSV_CONTENT, new String(storage.readAllBytes(BUCKET, EXTRACT_FILE), StandardCharsets.UTF_8));
    assertTrue(bigquery.delete(DATASET, tableName));
}