Examples with Job com.google.api.services.dataflow.model.Job used on opensource projects

Example 1 with Job

use of com.google.api.services.dataflow.model.Job in project beam by apache.

the class DataflowPipelineTranslator method translate.

/**
   * Translates a {@link Pipeline} into a {@code JobSpecification}.
   */
public JobSpecification translate(Pipeline pipeline, DataflowRunner runner, List<DataflowPackage> packages) {
    Translator translator = new Translator(pipeline, runner);
    Job result = translator.translate(packages);
    return new JobSpecification(result, Collections.unmodifiableMap(translator.stepNames));
}

Example 2 with Job

use of com.google.api.services.dataflow.model.Job in project beam by apache.

the class DataflowRunner method getJobIdFromName.

/**
 * Finds the id for the running job of the given name.
 */
private String getJobIdFromName(String jobName) {
    try {
        ListJobsResponse listResult;
        String token = null;
        do {
            listResult = dataflowClient.listJobs(token);
            token = listResult.getNextPageToken();
            for (Job job : listResult.getJobs()) {
                if (job.getName().equals(jobName) && MonitoringUtil.toState(job.getCurrentState()).equals(State.RUNNING)) {
                    return job.getId();
                }
            }
        } while (token != null);
    } catch (GoogleJsonResponseException e) {
        throw new RuntimeException("Got error while looking up jobs: " + (e.getDetails() != null ? e.getDetails().getMessage() : e), e);
    } catch (IOException e) {
        throw new RuntimeException("Got error while looking up jobs: ", e);
    }
    throw new IllegalArgumentException("Could not find running job named " + jobName);
}

Example 3 with Job

use of com.google.api.services.dataflow.model.Job in project beam by apache.

the class DataflowMetricsTest method testCachingMetricUpdates.

@Test
public void testCachingMetricUpdates() throws IOException {
    Job modelJob = new Job();
    modelJob.setCurrentState(State.RUNNING.toString());
    DataflowPipelineJob job = mock(DataflowPipelineJob.class);
    DataflowPipelineOptions options = mock(DataflowPipelineOptions.class);
    when(options.isStreaming()).thenReturn(false);
    when(job.getDataflowOptions()).thenReturn(options);
    when(job.getState()).thenReturn(State.DONE);
    job.jobId = JOB_ID;
    JobMetrics jobMetrics = new JobMetrics();
    jobMetrics.setMetrics(ImmutableList.of());
    DataflowClient dataflowClient = mock(DataflowClient.class);
    when(dataflowClient.getJobMetrics(JOB_ID)).thenReturn(jobMetrics);
    DataflowMetrics dataflowMetrics = new DataflowMetrics(job, dataflowClient);
    verify(dataflowClient, times(0)).getJobMetrics(JOB_ID);
    dataflowMetrics.allMetrics();
    verify(dataflowClient, times(1)).getJobMetrics(JOB_ID);
    dataflowMetrics.allMetrics();
    verify(dataflowClient, times(1)).getJobMetrics(JOB_ID);
}

Example 4 with Job

use of com.google.api.services.dataflow.model.Job in project beam by apache.

the class DataflowPipelineJob method getJobWithRetries.

/**
 * Attempts to get the underlying {@link Job}. Uses exponential backoff on failure up to the
 * maximum number of passed in attempts.
 *
 * @param backoff the {@link BackOff} used to control retries.
 * @param sleeper Object used to do the sleeps between attempts.
 * @return The underlying {@link Job} object.
 * @throws IOException When the maximum number of retries is exhausted, the last exception is
 *     thrown.
 */
private Job getJobWithRetries(BackOff backoff, Sleeper sleeper) throws IOException {
    // Retry loop ends in return or throw
    while (true) {
        try {
            Job job = dataflowClient.getJob(getJobId());
            State currentState = MonitoringUtil.toState(job.getCurrentState());
            if (currentState.isTerminal()) {
                terminalState = currentState;
                replacedByJob = new DataflowPipelineJob(dataflowClient, job.getReplacedByJobId(), dataflowOptions, transformStepNames, pipelineProto);
            }
            return job;
        } catch (IOException exn) {
            LOG.warn("There were problems getting current job status: {}.", exn.getMessage());
            LOG.debug("Exception information:", exn);
            if (!nextBackOff(sleeper, backoff)) {
                throw exn;
            }
        }
    }
}

Example 5 with Job

use of com.google.api.services.dataflow.model.Job in project beam by apache.

the class WorkerCustomSourcesTest method translateIOToCloudSource.

static com.google.api.services.dataflow.model.Source translateIOToCloudSource(BoundedSource<?> io, DataflowPipelineOptions options) throws Exception {
    DataflowPipelineTranslator translator = DataflowPipelineTranslator.fromOptions(options);
    Pipeline p = Pipeline.create(options);
    p.begin().apply(Read.from(io));
    // Note that we specifically perform this replacement since this is what the DataflowRunner
    // does and the DataflowRunner class does not expose a way to perform these replacements
    // without running the pipeline.
    p.replaceAll(Collections.singletonList(SplittableParDo.PRIMITIVE_BOUNDED_READ_OVERRIDE));
    DataflowRunner runner = DataflowRunner.fromOptions(options);
    SdkComponents sdkComponents = SdkComponents.create();
    RunnerApi.Environment defaultEnvironmentForDataflow = Environments.createDockerEnvironment("dummy-image-url");
    sdkComponents.registerEnvironment(defaultEnvironmentForDataflow);
    RunnerApi.Pipeline pipelineProto = PipelineTranslation.toProto(p, sdkComponents, true);
    Job workflow = translator.translate(p, pipelineProto, sdkComponents, runner, new ArrayList<DataflowPackage>()).getJob();
    Step step = workflow.getSteps().get(0);
    return stepToCloudSource(step);
}