Examples with HyracksConnection org.apache.hyracks.api.client.HyracksConnection used on opensource projects

Example 6 with HyracksConnection

use of org.apache.hyracks.api.client.HyracksConnection in project asterixdb by apache.

the class WordCountMain method main.

public static void main(String[] args) throws Exception {
    Options options = new Options();
    CmdLineParser parser = new CmdLineParser(options);
    parser.parseArgument(args);
    IHyracksClientConnection hcc = new HyracksConnection(options.host, options.port);
    JobSpecification job = createJob(parseFileSplits(options.inFileSplits), parseFileSplits(options.outFileSplits), options.algo, options.htSize, options.memFrameLimit, options.format, options.frameSize);
    long start = System.currentTimeMillis();
    JobId jobId = hcc.startJob(job, options.runtimeProfiling ? EnumSet.of(JobFlag.PROFILE_RUNTIME) : EnumSet.noneOf(JobFlag.class));
    hcc.waitForCompletion(jobId);
    long end = System.currentTimeMillis();
    System.err.println(start + " " + end + " " + (end - start));
}

Example 7 with HyracksConnection

use of org.apache.hyracks.api.client.HyracksConnection in project asterixdb by apache.

the class DataflowTest method testHDFSReadWriteOperators.

/**
     * Test a job with only HDFS read and writes.
     *
     * @throws Exception
     */
@SuppressWarnings({ "rawtypes", "unchecked" })
public void testHDFSReadWriteOperators() throws Exception {
    FileInputFormat.setInputPaths(conf, HDFS_INPUT_PATH);
    FileOutputFormat.setOutputPath(conf, new Path(HDFS_OUTPUT_PATH));
    conf.setInputFormatClass(TextInputFormat.class);
    Scheduler scheduler = new Scheduler(HyracksUtils.CC_HOST, HyracksUtils.TEST_HYRACKS_CC_CLIENT_PORT);
    InputFormat inputFormat = ReflectionUtils.newInstance(conf.getInputFormatClass(), getConfiguration());
    List<InputSplit> splits = inputFormat.getSplits(conf);
    String[] readSchedule = scheduler.getLocationConstraints(splits);
    JobSpecification jobSpec = new JobSpecification();
    RecordDescriptor recordDesc = new RecordDescriptor(new ISerializerDeserializer[] { new UTF8StringSerializerDeserializer() });
    String[] locations = new String[] { HyracksUtils.NC1_ID, HyracksUtils.NC1_ID, HyracksUtils.NC2_ID, HyracksUtils.NC2_ID };
    HDFSReadOperatorDescriptor readOperator = new HDFSReadOperatorDescriptor(jobSpec, recordDesc, conf, splits, readSchedule, new TextKeyValueParserFactory());
    PartitionConstraintHelper.addAbsoluteLocationConstraint(jobSpec, readOperator, locations);
    ExternalSortOperatorDescriptor sortOperator = new ExternalSortOperatorDescriptor(jobSpec, 10, new int[] { 0 }, new IBinaryComparatorFactory[] { RawBinaryComparatorFactory.INSTANCE }, recordDesc);
    PartitionConstraintHelper.addAbsoluteLocationConstraint(jobSpec, sortOperator, locations);
    HDFSWriteOperatorDescriptor writeOperator = new HDFSWriteOperatorDescriptor(jobSpec, conf, new TextTupleWriterFactory());
    PartitionConstraintHelper.addAbsoluteLocationConstraint(jobSpec, writeOperator, HyracksUtils.NC1_ID);
    jobSpec.connect(new OneToOneConnectorDescriptor(jobSpec), readOperator, 0, sortOperator, 0);
    jobSpec.connect(new MToNPartitioningMergingConnectorDescriptor(jobSpec, new FieldHashPartitionComputerFactory(new int[] { 0 }, new IBinaryHashFunctionFactory[] { RawBinaryHashFunctionFactory.INSTANCE }), new int[] { 0 }, new IBinaryComparatorFactory[] { RawBinaryComparatorFactory.INSTANCE }, null), sortOperator, 0, writeOperator, 0);
    jobSpec.addRoot(writeOperator);
    IHyracksClientConnection client = new HyracksConnection(HyracksUtils.CC_HOST, HyracksUtils.TEST_HYRACKS_CC_CLIENT_PORT);
    JobId jobId = client.startJob(jobSpec);
    client.waitForCompletion(jobId);
    Assert.assertEquals(true, checkResults());
}

Example 8 with HyracksConnection

use of org.apache.hyracks.api.client.HyracksConnection in project asterixdb by apache.

the class PredistributedJobsTest method init.

@BeforeClass
public static void init() throws Exception {
    CCConfig ccConfig = new CCConfig();
    ccConfig.setClientListenAddress("127.0.0.1");
    ccConfig.setClientListenPort(39000);
    ccConfig.setClusterListenAddress("127.0.0.1");
    ccConfig.setClusterListenPort(39001);
    ccConfig.setProfileDumpPeriod(10000);
    FileUtils.deleteQuietly(new File(joinPath("target", "data")));
    FileUtils.copyDirectory(new File("data"), new File(joinPath("target", "data")));
    File outDir = new File("target" + File.separator + "ClusterController");
    outDir.mkdirs();
    File ccRoot = File.createTempFile(AbstractIntegrationTest.class.getName(), ".data", outDir);
    ccRoot.delete();
    ccRoot.mkdir();
    ccConfig.setRootDir(ccRoot.getAbsolutePath());
    ClusterControllerService ccBase = new ClusterControllerService(ccConfig);
    cc = Mockito.spy(ccBase);
    cc.start();
    NCConfig ncConfig1 = new NCConfig(NC1_ID);
    ncConfig1.setClusterAddress("localhost");
    ncConfig1.setClusterPort(39001);
    ncConfig1.setClusterListenAddress("127.0.0.1");
    ncConfig1.setDataListenAddress("127.0.0.1");
    ncConfig1.setResultListenAddress("127.0.0.1");
    ncConfig1.setResultSweepThreshold(5000);
    ncConfig1.setIODevices(new String[] { joinPath(System.getProperty("user.dir"), "target", "data", "device0") });
    NodeControllerService nc1Base = new NodeControllerService(ncConfig1);
    nc1 = Mockito.spy(nc1Base);
    nc1.start();
    NCConfig ncConfig2 = new NCConfig(NC2_ID);
    ncConfig2.setClusterAddress("localhost");
    ncConfig2.setClusterPort(39001);
    ncConfig2.setClusterListenAddress("127.0.0.1");
    ncConfig2.setDataListenAddress("127.0.0.1");
    ncConfig2.setResultListenAddress("127.0.0.1");
    ncConfig2.setResultSweepThreshold(5000);
    ncConfig2.setIODevices(new String[] { joinPath(System.getProperty("user.dir"), "target", "data", "device1") });
    NodeControllerService nc2Base = new NodeControllerService(ncConfig2);
    nc2 = Mockito.spy(nc2Base);
    nc2.start();
    hcc = new HyracksConnection(ccConfig.getClientListenAddress(), ccConfig.getClientListenPort());
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Starting CC in " + ccRoot.getAbsolutePath());
    }
}

Example 9 with HyracksConnection

use of org.apache.hyracks.api.client.HyracksConnection in project asterixdb by apache.

the class AsterixClientDriver method main.

public static void main(String[] args) throws Exception {
    AsterixClientConfig acc = new AsterixClientConfig();
    CmdLineParser cmdParser = new CmdLineParser(acc);
    try {
        cmdParser.parseArgument(args);
    } catch (Exception e) {
        cmdParser.printUsage(System.err);
        throw e;
    }
    if (acc.getArguments().isEmpty()) {
        System.err.println("Please specify the file containing the query.");
        return;
    }
    if (acc.getArguments().size() > 1) {
        System.err.print("Too many arguments. ");
        System.err.println("Only the file contained the query needs to be specified.");
        return;
    }
    boolean exec = new Boolean(acc.execute);
    IHyracksClientConnection hcc = exec ? new HyracksConnection("localhost", acc.hyracksPort) : null;
    AsterixJavaClient q = compileQuery(hcc, acc.getArguments().get(0), new Boolean(acc.optimize), new Boolean(acc.onlyPhysical), exec || new Boolean(acc.hyracksJob));
    if (exec) {
        q.execute();
    }
}

Example 10 with HyracksConnection

use of org.apache.hyracks.api.client.HyracksConnection in project asterixdb by apache.

the class SecondaryIndexBulkLoadExample method main.

public static void main(String[] args) throws Exception {
    Options options = new Options();
    CmdLineParser parser = new CmdLineParser(options);
    parser.parseArgument(args);
    IHyracksClientConnection hcc = new HyracksConnection(options.host, options.port);
    JobSpecification job = createJob(options);
    long start = System.currentTimeMillis();
    JobId jobId = hcc.startJob(job);
    hcc.waitForCompletion(jobId);
    long end = System.currentTimeMillis();
    System.err.println(start + " " + end + " " + (end - start));
}