Examples with Job com.google.api.services.bigquery.model.Job used on opensource projects

Example 16 with Job

use of com.google.api.services.bigquery.model.Job in project google-cloud-java by GoogleCloudPlatform.

the class HttpBigQueryRpc method listJobs.

@Override
public Tuple<String, Iterable<Job>> listJobs(String projectId, Map<Option, ?> options) {
    try {
        JobList jobsList = bigquery.jobs().list(projectId).setAllUsers(Option.ALL_USERS.getBoolean(options)).setFields(Option.FIELDS.getString(options)).setStateFilter(Option.STATE_FILTER.<List<String>>get(options)).setMaxResults(Option.MAX_RESULTS.getLong(options)).setPageToken(Option.PAGE_TOKEN.getString(options)).setProjection(DEFAULT_PROJECTION).execute();
        Iterable<JobList.Jobs> jobs = jobsList.getJobs();
        return Tuple.of(jobsList.getNextPageToken(), Iterables.transform(jobs != null ? jobs : ImmutableList.<JobList.Jobs>of(), new Function<JobList.Jobs, Job>() {

            @Override
            public Job apply(JobList.Jobs jobPb) {
                JobStatus statusPb = jobPb.getStatus() != null ? jobPb.getStatus() : new JobStatus();
                if (statusPb.getState() == null) {
                    statusPb.setState(jobPb.getState());
                }
                if (statusPb.getErrorResult() == null) {
                    statusPb.setErrorResult(jobPb.getErrorResult());
                }
                return new Job().setConfiguration(jobPb.getConfiguration()).setId(jobPb.getId()).setJobReference(jobPb.getJobReference()).setKind(jobPb.getKind()).setStatistics(jobPb.getStatistics()).setStatus(statusPb).setUserEmail(jobPb.getUserEmail());
            }
        }));
    } catch (IOException ex) {
        throw translate(ex);
    }
}

Example 17 with Job

use of com.google.api.services.bigquery.model.Job in project google-cloud-java by GoogleCloudPlatform.

the class HttpBigQueryRpc method open.

@Override
public String open(JobConfiguration configuration) {
    try {
        Job loadJob = new Job().setConfiguration(configuration);
        StringBuilder builder = new StringBuilder().append(BASE_RESUMABLE_URI).append(options.getProjectId()).append("/jobs");
        GenericUrl url = new GenericUrl(builder.toString());
        url.set("uploadType", "resumable");
        JsonFactory jsonFactory = bigquery.getJsonFactory();
        HttpRequestFactory requestFactory = bigquery.getRequestFactory();
        HttpRequest httpRequest = requestFactory.buildPostRequest(url, new JsonHttpContent(jsonFactory, loadJob));
        httpRequest.getHeaders().set("X-Upload-Content-Value", "application/octet-stream");
        HttpResponse response = httpRequest.execute();
        return response.getHeaders().getLocation();
    } catch (IOException ex) {
        throw translate(ex);
    }
}

Example 18 with Job

use of com.google.api.services.bigquery.model.Job in project google-cloud-java by GoogleCloudPlatform.

the class JobInfo method toPb.

Job toPb() {
    Job jobPb = new Job();
    jobPb.setEtag(etag);
    jobPb.setId(generatedId);
    jobPb.setSelfLink(selfLink);
    jobPb.setUserEmail(userEmail);
    if (jobId != null) {
        jobPb.setJobReference(jobId.toPb());
    }
    if (status != null) {
        jobPb.setStatus(status.toPb());
    }
    if (statistics != null) {
        jobPb.setStatistics(statistics.toPb());
    }
    jobPb.setConfiguration(configuration.toPb());
    return jobPb;
}

Example 19 with Job

use of com.google.api.services.bigquery.model.Job in project beam by apache.

the class BigQueryQuerySource method executeQuery.

private void executeQuery(String executingProject, String jobId, TableReference destinationTable, JobService jobService) throws IOException, InterruptedException {
    JobReference jobRef = new JobReference().setProjectId(executingProject).setJobId(jobId);
    JobConfigurationQuery queryConfig = createBasicQueryConfig().setAllowLargeResults(true).setCreateDisposition("CREATE_IF_NEEDED").setDestinationTable(destinationTable).setPriority("BATCH").setWriteDisposition("WRITE_EMPTY");
    jobService.startQueryJob(jobRef, queryConfig);
    Job job = jobService.pollJob(jobRef, JOB_POLL_MAX_RETRIES);
    if (BigQueryHelpers.parseStatus(job) != Status.SUCCEEDED) {
        throw new IOException(String.format("Query job %s failed, status: %s.", jobId, BigQueryHelpers.statusToPrettyString(job.getStatus())));
    }
}

Example 20 with Job

use of com.google.api.services.bigquery.model.Job in project beam by apache.

the class WriteTables method load.

private void load(JobService jobService, DatasetService datasetService, String jobIdPrefix, TableReference ref, @Nullable TableSchema schema, List<String> gcsUris, WriteDisposition writeDisposition, CreateDisposition createDisposition, @Nullable String tableDescription) throws InterruptedException, IOException {
    JobConfigurationLoad loadConfig = new JobConfigurationLoad().setDestinationTable(ref).setSchema(schema).setSourceUris(gcsUris).setWriteDisposition(writeDisposition.name()).setCreateDisposition(createDisposition.name()).setSourceFormat("NEWLINE_DELIMITED_JSON");
    String projectId = ref.getProjectId();
    Job lastFailedLoadJob = null;
    for (int i = 0; i < BatchLoads.MAX_RETRY_JOBS; ++i) {
        String jobId = jobIdPrefix + "-" + i;
        JobReference jobRef = new JobReference().setProjectId(projectId).setJobId(jobId);
        jobService.startLoadJob(jobRef, loadConfig);
        Job loadJob = jobService.pollJob(jobRef, BatchLoads.LOAD_JOB_POLL_MAX_RETRIES);
        Status jobStatus = BigQueryHelpers.parseStatus(loadJob);
        switch(jobStatus) {
            case SUCCEEDED:
                if (tableDescription != null) {
                    datasetService.patchTableDescription(ref, tableDescription);
                }
                return;
            case UNKNOWN:
                throw new RuntimeException(String.format("UNKNOWN status of load job [%s]: %s.", jobId, BigQueryHelpers.jobToPrettyString(loadJob)));
            case FAILED:
                lastFailedLoadJob = loadJob;
                continue;
            default:
                throw new IllegalStateException(String.format("Unexpected status [%s] of load job: %s.", jobStatus, BigQueryHelpers.jobToPrettyString(loadJob)));
        }
    }
    throw new RuntimeException(String.format("Failed to create load job with id prefix %s, " + "reached max retries: %d, last failed load job: %s.", jobIdPrefix, BatchLoads.MAX_RETRY_JOBS, BigQueryHelpers.jobToPrettyString(lastFailedLoadJob)));
}