use of com.google.cloud.scheduler.v1.Job in project cdap by caskdata.
the class DataprocRuntimeJobManager method launch.
@Override
public void launch(RuntimeJobInfo runtimeJobInfo) throws Exception {
String bucket = DataprocUtils.getBucketName(this.bucket);
ProgramRunInfo runInfo = runtimeJobInfo.getProgramRunInfo();
LOG.debug("Launching run {} with following configurations: cluster {}, project {}, region {}, bucket {}.", runInfo.getRun(), clusterName, projectId, region, bucket);
// TODO: CDAP-16408 use fixed directory for caching twill, application, artifact jars
File tempDir = Files.createTempDirectory("dataproc.launcher").toFile();
// on dataproc bucket the run root will be <bucket>/cdap-job/<runid>/. All the files for this run will be copied
// under that base dir.
String runRootPath = getPath(DataprocUtils.CDAP_GCS_ROOT, runInfo.getRun());
try {
// step 1: build twill.jar and launcher.jar and add them to files to be copied to gcs
List<LocalFile> localFiles = getRuntimeLocalFiles(runtimeJobInfo.getLocalizeFiles(), tempDir);
// step 2: upload all the necessary files to gcs so that those files are available to dataproc job
List<Future<LocalFile>> uploadFutures = new ArrayList<>();
for (LocalFile fileToUpload : localFiles) {
String targetFilePath = getPath(runRootPath, fileToUpload.getName());
uploadFutures.add(provisionerContext.execute(() -> uploadFile(bucket, targetFilePath, fileToUpload)).toCompletableFuture());
}
List<LocalFile> uploadedFiles = new ArrayList<>();
for (Future<LocalFile> uploadFuture : uploadFutures) {
uploadedFiles.add(uploadFuture.get());
}
// step 3: build the hadoop job request to be submitted to dataproc
SubmitJobRequest request = getSubmitJobRequest(runtimeJobInfo, uploadedFiles);
// step 4: submit hadoop job to dataproc
try {
Job job = getJobControllerClient().submitJob(request);
LOG.debug("Successfully submitted hadoop job {} to cluster {}.", job.getReference().getJobId(), clusterName);
} catch (AlreadyExistsException ex) {
// the job id already exists, ignore the job.
LOG.warn("The dataproc job {} already exists. Ignoring resubmission of the job.", request.getJob().getReference().getJobId());
}
DataprocUtils.emitMetric(provisionerContext, region, "provisioner.submitJob.response.count");
} catch (Exception e) {
// delete all uploaded gcs files in case of exception
DataprocUtils.deleteGCSPath(getStorageClient(), bucket, runRootPath);
DataprocUtils.emitMetric(provisionerContext, region, "provisioner.submitJob.response.count", e);
throw new Exception(String.format("Error while launching job %s on cluster %s", getJobId(runInfo), clusterName), e);
} finally {
// delete local temp directory
deleteDirectoryContents(tempDir);
}
}
use of com.google.cloud.scheduler.v1.Job in project java-docs-samples by GoogleCloudPlatform.
the class CustomRankingSearchJobs method searchCustomRankingJobs.
// Search Jobs using custom rankings.
public static void searchCustomRankingJobs(String projectId, String tenantId) throws IOException {
// the "close" method on the client to safely clean up any remaining background resources.
try (JobServiceClient jobServiceClient = JobServiceClient.create()) {
TenantName parent = TenantName.of(projectId, tenantId);
String domain = "www.example.com";
String sessionId = "Hashed session identifier";
String userId = "Hashed user identifier";
RequestMetadata requestMetadata = RequestMetadata.newBuilder().setDomain(domain).setSessionId(sessionId).setUserId(userId).build();
SearchJobsRequest.CustomRankingInfo.ImportanceLevel importanceLevel = SearchJobsRequest.CustomRankingInfo.ImportanceLevel.EXTREME;
String rankingExpression = "(someFieldLong + 25) * 0.25";
SearchJobsRequest.CustomRankingInfo customRankingInfo = SearchJobsRequest.CustomRankingInfo.newBuilder().setImportanceLevel(importanceLevel).setRankingExpression(rankingExpression).build();
String orderBy = "custom_ranking desc";
SearchJobsRequest request = SearchJobsRequest.newBuilder().setParent(parent.toString()).setRequestMetadata(requestMetadata).setCustomRankingInfo(customRankingInfo).setOrderBy(orderBy).build();
for (SearchJobsResponse.MatchingJob responseItem : jobServiceClient.searchJobs(request).iterateAll()) {
System.out.format("Job summary: %s%n", responseItem.getJobSummary());
System.out.format("Job title snippet: %s%n", responseItem.getJobTitleSnippet());
Job job = responseItem.getJob();
System.out.format("Job name: %s%n", job.getName());
System.out.format("Job title: %s%n", job.getTitle());
}
}
}
use of com.google.cloud.scheduler.v1.Job in project java-docs-samples by GoogleCloudPlatform.
the class JobSearchCreateJobCustomAttributes method createJob.
// Create Job with Custom Attributes.
public static void createJob(String projectId, String tenantId, String companyId, String requisitionId) throws IOException {
// the "close" method on the client to safely clean up any remaining background resources.
try (JobServiceClient jobServiceClient = JobServiceClient.create()) {
TenantName parent = TenantName.of(projectId, tenantId);
// Custom attribute can be string or numeric value, and can be filtered in search queries.
// https://cloud.google.com/talent-solution/job-search/docs/custom-attributes
CustomAttribute customAttribute = CustomAttribute.newBuilder().addStringValues("Internship").addStringValues("Apprenticeship").setFilterable(true).build();
Job job = Job.newBuilder().setCompany(companyId).setTitle("Software Developer I").setDescription("This is a description of this <i>wonderful</i> job!").putCustomAttributes("FOR_STUDENTS", customAttribute).setRequisitionId(requisitionId).setLanguageCode("en-US").build();
CreateJobRequest request = CreateJobRequest.newBuilder().setParent(parent.toString()).setJob(job).build();
Job response = jobServiceClient.createJob(request);
System.out.printf("Created job: %s\n", response.getName());
}
}
use of com.google.cloud.scheduler.v1.Job in project java-docs-samples by GoogleCloudPlatform.
the class JobSearchGetJob method getJob.
// Get Job.
public static void getJob(String projectId, String tenantId, String jobId) throws IOException {
// the "close" method on the client to safely clean up any remaining background resources.
try (JobServiceClient jobServiceClient = JobServiceClient.create()) {
JobName name = JobName.ofProjectTenantJobName(projectId, tenantId, jobId);
GetJobRequest request = GetJobRequest.newBuilder().setName(name.toString()).build();
Job response = jobServiceClient.getJob(request);
System.out.format("Job name: %s%n", response.getName());
System.out.format("Requisition ID: %s%n", response.getRequisitionId());
System.out.format("Title: %s%n", response.getTitle());
System.out.format("Description: %s%n", response.getDescription());
System.out.format("Posting language: %s%n", response.getLanguageCode());
for (String address : response.getAddressesList()) {
System.out.format("Address: %s%n", address);
}
for (String email : response.getApplicationInfo().getEmailsList()) {
System.out.format("Email: %s%n", email);
}
for (String websiteUri : response.getApplicationInfo().getUrisList()) {
System.out.format("Website: %s%n", websiteUri);
}
}
}
use of com.google.cloud.scheduler.v1.Job in project java-docs-samples by GoogleCloudPlatform.
the class CreateJobFromAdHoc method createJobFromAdHoc.
// Creates a job from an ad-hoc configuration.
public static void createJobFromAdHoc(String projectId, String location, String inputUri, String outputUri) throws IOException {
// once, and can be reused for multiple requests.
try (TranscoderServiceClient transcoderServiceClient = TranscoderServiceClient.create()) {
VideoStream videoStream0 = VideoStream.newBuilder().setH264(H264CodecSettings.newBuilder().setBitrateBps(550000).setFrameRate(60).setHeightPixels(360).setWidthPixels(640)).build();
VideoStream videoStream1 = VideoStream.newBuilder().setH264(H264CodecSettings.newBuilder().setBitrateBps(2500000).setFrameRate(60).setHeightPixels(720).setWidthPixels(1280)).build();
AudioStream audioStream0 = AudioStream.newBuilder().setCodec("aac").setBitrateBps(64000).build();
JobConfig config = JobConfig.newBuilder().addInputs(Input.newBuilder().setKey("input0").setUri(inputUri)).setOutput(Output.newBuilder().setUri(outputUri)).addElementaryStreams(ElementaryStream.newBuilder().setKey("video_stream0").setVideoStream(videoStream0)).addElementaryStreams(ElementaryStream.newBuilder().setKey("video_stream1").setVideoStream(videoStream1)).addElementaryStreams(ElementaryStream.newBuilder().setKey("audio_stream0").setAudioStream(audioStream0)).addMuxStreams(MuxStream.newBuilder().setKey("sd").setContainer("mp4").addElementaryStreams("video_stream0").addElementaryStreams("audio_stream0").build()).addMuxStreams(MuxStream.newBuilder().setKey("hd").setContainer("mp4").addElementaryStreams("video_stream1").addElementaryStreams("audio_stream0").build()).build();
var createJobRequest = CreateJobRequest.newBuilder().setJob(Job.newBuilder().setInputUri(inputUri).setOutputUri(outputUri).setConfig(config).build()).setParent(LocationName.of(projectId, location).toString()).build();
// Send the job creation request and process the response.
Job job = transcoderServiceClient.createJob(createJobRequest);
System.out.println("Job: " + job.getName());
}
}
Aggregations