Examples with Job com.google.api.services.dataflow.model.Job used on opensource projects

Example 16 with Job

use of com.google.api.services.dataflow.model.Job in project beam by apache.

the class DataflowPipelineJobTest method testCancelUnterminatedJobThatFails.

@Test
public void testCancelUnterminatedJobThatFails() throws IOException {
    Dataflow.Projects.Locations.Jobs.Get statusRequest = mock(Dataflow.Projects.Locations.Jobs.Get.class);
    Job statusResponse = new Job();
    statusResponse.setCurrentState("JOB_STATE_RUNNING");
    when(mockJobs.get(PROJECT_ID, REGION_ID, JOB_ID)).thenReturn(statusRequest);
    when(statusRequest.execute()).thenReturn(statusResponse);
    Dataflow.Projects.Locations.Jobs.Update update = mock(Dataflow.Projects.Locations.Jobs.Update.class);
    when(mockJobs.update(eq(PROJECT_ID), eq(REGION_ID), eq(JOB_ID), any(Job.class))).thenReturn(update);
    when(update.execute()).thenThrow(new IOException("Some random IOException"));
    DataflowPipelineJob job = new DataflowPipelineJob(DataflowClient.create(options), JOB_ID, options, null);
    thrown.expect(IOException.class);
    thrown.expectMessage("Failed to cancel job in state RUNNING, " + "please go to the Developers Console to cancel it manually:");
    job.cancel();
}

Example 17 with Job

use of com.google.api.services.dataflow.model.Job in project beam by apache.

the class DataflowRunner method getJobIdFromName.

/**
   * Finds the id for the running job of the given name.
   */
private String getJobIdFromName(String jobName) {
    try {
        ListJobsResponse listResult;
        String token = null;
        do {
            listResult = dataflowClient.listJobs(token);
            token = listResult.getNextPageToken();
            for (Job job : listResult.getJobs()) {
                if (job.getName().equals(jobName) && MonitoringUtil.toState(job.getCurrentState()).equals(State.RUNNING)) {
                    return job.getId();
                }
            }
        } while (token != null);
    } catch (GoogleJsonResponseException e) {
        throw new RuntimeException("Got error while looking up jobs: " + (e.getDetails() != null ? e.getDetails().getMessage() : e), e);
    } catch (IOException e) {
        throw new RuntimeException("Got error while looking up jobs: ", e);
    }
    throw new IllegalArgumentException("Could not find running job named " + jobName);
}

Example 18 with Job

use of com.google.api.services.dataflow.model.Job in project beam by apache.

the class DataflowRunnerTest method testUpdateAlreadyUpdatedPipeline.

@Test
public void testUpdateAlreadyUpdatedPipeline() throws IOException {
    DataflowPipelineOptions options = buildPipelineOptions();
    options.setUpdate(true);
    options.setJobName("oldJobName");
    Dataflow mockDataflowClient = options.getDataflowClient();
    Dataflow.Projects.Locations.Jobs.Create mockRequest = mock(Dataflow.Projects.Locations.Jobs.Create.class);
    when(mockDataflowClient.projects().locations().jobs().create(eq(PROJECT_ID), eq(REGION_ID), any(Job.class))).thenReturn(mockRequest);
    final Job resultJob = new Job();
    resultJob.setId("newid");
    // Return a different request id.
    resultJob.setClientRequestId("different_request_id");
    when(mockRequest.execute()).thenReturn(resultJob);
    Pipeline p = buildDataflowPipeline(options);
    thrown.expect(DataflowJobAlreadyUpdatedException.class);
    thrown.expect(new TypeSafeMatcher<DataflowJobAlreadyUpdatedException>() {

        @Override
        public void describeTo(Description description) {
            description.appendText("Expected job ID: " + resultJob.getId());
        }

        @Override
        protected boolean matchesSafely(DataflowJobAlreadyUpdatedException item) {
            return resultJob.getId().equals(item.getJob().getJobId());
        }
    });
    thrown.expectMessage("The job named oldjobname with id: oldJobId has already been updated " + "into job id: newid and cannot be updated again.");
    p.run();
}

Example 19 with Job

use of com.google.api.services.dataflow.model.Job in project beam by apache.

the class DataflowRunnerTest method testNonExistentStagingLocation.

@Test
public void testNonExistentStagingLocation() throws IOException {
    DataflowPipelineOptions options = buildPipelineOptions();
    options.setStagingLocation(NON_EXISTENT_BUCKET);
    thrown.expect(IllegalArgumentException.class);
    thrown.expectMessage(containsString("Output path does not exist or is not writeable: " + NON_EXISTENT_BUCKET));
    DataflowRunner.fromOptions(options);
    ArgumentCaptor<Job> jobCaptor = ArgumentCaptor.forClass(Job.class);
    Mockito.verify(mockJobs).create(eq(PROJECT_ID), eq(REGION_ID), jobCaptor.capture());
    assertValidJob(jobCaptor.getValue());
}

Example 20 with Job

use of com.google.api.services.dataflow.model.Job in project beam by apache.

the class DataflowRunnerTest method testTransformTranslatorMissing.

@Test
public void testTransformTranslatorMissing() throws IOException {
    DataflowPipelineOptions options = buildPipelineOptions();
    Pipeline p = Pipeline.create(options);
    p.apply(Create.of(Arrays.asList(1, 2, 3))).apply(new TestTransform());
    thrown.expect(IllegalStateException.class);
    thrown.expectMessage(Matchers.containsString("no translator registered"));
    DataflowPipelineTranslator.fromOptions(options).translate(p, DataflowRunner.fromOptions(options), Collections.<DataflowPackage>emptyList());
    ArgumentCaptor<Job> jobCaptor = ArgumentCaptor.forClass(Job.class);
    Mockito.verify(mockJobs).create(eq(PROJECT_ID), eq(REGION_ID), jobCaptor.capture());
    assertValidJob(jobCaptor.getValue());
}