Examples with JobId org.apache.hyracks.api.job.JobId used on opensource projects

Example 6 with JobId

use of org.apache.hyracks.api.job.JobId in project asterixdb by apache.

the class APIFramework method executeJobArray.

public void executeJobArray(IHyracksClientConnection hcc, JobSpecification[] specs, PrintWriter out) throws Exception {
    for (JobSpecification spec : specs) {
        spec.setMaxReattempts(0);
        JobId jobId = hcc.startJob(spec);
        long startTime = System.currentTimeMillis();
        hcc.waitForCompletion(jobId);
        long endTime = System.currentTimeMillis();
        double duration = (endTime - startTime) / 1000.00;
        out.println("<pre>Duration: " + duration + " sec</pre>");
    }
}

Example 7 with JobId

use of org.apache.hyracks.api.job.JobId in project asterixdb by apache.

the class AsterixHyracksIntegrationUtil method runJob.

public void runJob(JobSpecification spec) throws Exception {
    GlobalConfig.ASTERIX_LOGGER.info(spec.toJSON().toString());
    JobId jobId = hcc.startJob(spec, EnumSet.of(JobFlag.PROFILE_RUNTIME));
    GlobalConfig.ASTERIX_LOGGER.info(jobId.toString());
    hcc.waitForCompletion(jobId);
}

Example 8 with JobId

use of org.apache.hyracks.api.job.JobId in project asterixdb by apache.

the class CCNCFunctions method readPartitionId.

private static PartitionId readPartitionId(DataInputStream dis) throws IOException {
    long jobId = dis.readLong();
    int cdid = dis.readInt();
    int senderIndex = dis.readInt();
    int receiverIndex = dis.readInt();
    PartitionId pid = new PartitionId(new JobId(jobId), new ConnectorDescriptorId(cdid), senderIndex, receiverIndex);
    return pid;
}

Example 9 with JobId

use of org.apache.hyracks.api.job.JobId in project asterixdb by apache.

the class DataflowTest method testHDFSReadWriteOperators.

/**
     * Test a job with only HDFS read and writes.
     *
     * @throws Exception
     */
public void testHDFSReadWriteOperators() throws Exception {
    FileInputFormat.setInputPaths(conf, HDFS_INPUT_PATH);
    FileOutputFormat.setOutputPath(conf, new Path(HDFS_OUTPUT_PATH));
    conf.setInputFormat(TextInputFormat.class);
    Scheduler scheduler = new Scheduler(HyracksUtils.CC_HOST, HyracksUtils.TEST_HYRACKS_CC_CLIENT_PORT);
    InputSplit[] splits = conf.getInputFormat().getSplits(conf, numberOfNC * 4);
    String[] readSchedule = scheduler.getLocationConstraints(splits);
    JobSpecification jobSpec = new JobSpecification();
    RecordDescriptor recordDesc = new RecordDescriptor(new ISerializerDeserializer[] { new UTF8StringSerializerDeserializer() });
    String[] locations = new String[] { HyracksUtils.NC1_ID, HyracksUtils.NC1_ID, HyracksUtils.NC2_ID, HyracksUtils.NC2_ID };
    HDFSReadOperatorDescriptor readOperator = new HDFSReadOperatorDescriptor(jobSpec, recordDesc, conf, splits, readSchedule, new TextKeyValueParserFactory());
    PartitionConstraintHelper.addAbsoluteLocationConstraint(jobSpec, readOperator, locations);
    ExternalSortOperatorDescriptor sortOperator = new ExternalSortOperatorDescriptor(jobSpec, 10, new int[] { 0 }, new IBinaryComparatorFactory[] { RawBinaryComparatorFactory.INSTANCE }, recordDesc);
    PartitionConstraintHelper.addAbsoluteLocationConstraint(jobSpec, sortOperator, locations);
    HDFSWriteOperatorDescriptor writeOperator = new HDFSWriteOperatorDescriptor(jobSpec, conf, new TextTupleWriterFactory());
    PartitionConstraintHelper.addAbsoluteLocationConstraint(jobSpec, writeOperator, HyracksUtils.NC1_ID);
    jobSpec.connect(new OneToOneConnectorDescriptor(jobSpec), readOperator, 0, sortOperator, 0);
    jobSpec.connect(new MToNPartitioningMergingConnectorDescriptor(jobSpec, new FieldHashPartitionComputerFactory(new int[] { 0 }, new IBinaryHashFunctionFactory[] { RawBinaryHashFunctionFactory.INSTANCE }), new int[] { 0 }, new IBinaryComparatorFactory[] { RawBinaryComparatorFactory.INSTANCE }, null), sortOperator, 0, writeOperator, 0);
    jobSpec.addRoot(writeOperator);
    IHyracksClientConnection client = new HyracksConnection(HyracksUtils.CC_HOST, HyracksUtils.TEST_HYRACKS_CC_CLIENT_PORT);
    JobId jobId = client.startJob(jobSpec);
    client.waitForCompletion(jobId);
    Assert.assertEquals(true, checkResults());
}

Example 10 with JobId

use of org.apache.hyracks.api.job.JobId in project asterixdb by apache.

the class Groupby method main.

public static void main(String[] args) throws Exception {
    Options options = new Options();
    CmdLineParser parser = new CmdLineParser(options);
    if (args.length == 0) {
        parser.printUsage(System.err);
        return;
    }
    parser.parseArgument(args);
    IHyracksClientConnection hcc = new HyracksConnection(options.host, options.port);
    JobSpecification job;
    long start = System.currentTimeMillis();
    job = createJob(parseFileSplits(options.inFileSplits), parseFileSplits(options.outFileSplits), options.htSize, options.fileSize, options.frameLimit, options.frameSize, options.algo, options.outPlain);
    if (job != null) {
        System.out.print("CreateJobTime:" + (System.currentTimeMillis() - start));
        start = System.currentTimeMillis();
        JobId jobId = hcc.startJob(job);
        hcc.waitForCompletion(jobId);
        System.out.println("JobExecuteTime:" + (System.currentTimeMillis() - start));
    }
}