Examples with HyracksConnection org.apache.hyracks.api.client.HyracksConnection used on opensource projects

Example 1 with HyracksConnection

use of org.apache.hyracks.api.client.HyracksConnection in project asterixdb by apache.

the class AbstractIntegrationTest method init.

@BeforeClass
public static void init() throws Exception {
    CCConfig ccConfig = new CCConfig();
    ccConfig.setClientListenAddress("127.0.0.1");
    ccConfig.setClientListenPort(39000);
    ccConfig.setClusterListenAddress("127.0.0.1");
    ccConfig.setClusterListenPort(39001);
    ccConfig.setProfileDumpPeriod(10000);
    FileUtils.deleteQuietly(new File("target" + File.separator + "data"));
    FileUtils.copyDirectory(new File("data"), new File(joinPath("target", "data")));
    File outDir = new File("target" + File.separator + "ClusterController");
    outDir.mkdirs();
    File ccRoot = File.createTempFile(AbstractIntegrationTest.class.getName(), ".data", outDir);
    ccRoot.delete();
    ccRoot.mkdir();
    ccConfig.setRootDir(ccRoot.getAbsolutePath());
    cc = new ClusterControllerService(ccConfig);
    cc.start();
    NCConfig ncConfig1 = new NCConfig(NC1_ID);
    ncConfig1.setClusterAddress("localhost");
    ncConfig1.setClusterPort(39001);
    ncConfig1.setClusterListenAddress("127.0.0.1");
    ncConfig1.setDataListenAddress("127.0.0.1");
    ncConfig1.setResultListenAddress("127.0.0.1");
    ncConfig1.setIODevices(new String[] { joinPath(System.getProperty("user.dir"), "target", "data", "device0") });
    nc1 = new NodeControllerService(ncConfig1);
    nc1.start();
    NCConfig ncConfig2 = new NCConfig(NC2_ID);
    ncConfig2.setClusterAddress("localhost");
    ncConfig2.setClusterPort(39001);
    ncConfig2.setClusterListenAddress("127.0.0.1");
    ncConfig2.setDataListenAddress("127.0.0.1");
    ncConfig2.setResultListenAddress("127.0.0.1");
    ncConfig2.setIODevices(new String[] { joinPath(System.getProperty("user.dir"), "target", "data", "device1") });
    nc2 = new NodeControllerService(ncConfig2);
    nc2.start();
    hcc = new HyracksConnection(ccConfig.getClientListenAddress(), ccConfig.getClientListenPort());
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Starting CC in " + ccRoot.getAbsolutePath());
    }
}

Example 2 with HyracksConnection

use of org.apache.hyracks.api.client.HyracksConnection in project asterixdb by apache.

the class DataflowTest method testHDFSReadWriteOperators.

/**
     * Test a job with only HDFS read and writes.
     *
     * @throws Exception
     */
public void testHDFSReadWriteOperators() throws Exception {
    FileInputFormat.setInputPaths(conf, HDFS_INPUT_PATH);
    FileOutputFormat.setOutputPath(conf, new Path(HDFS_OUTPUT_PATH));
    conf.setInputFormat(TextInputFormat.class);
    Scheduler scheduler = new Scheduler(HyracksUtils.CC_HOST, HyracksUtils.TEST_HYRACKS_CC_CLIENT_PORT);
    InputSplit[] splits = conf.getInputFormat().getSplits(conf, numberOfNC * 4);
    String[] readSchedule = scheduler.getLocationConstraints(splits);
    JobSpecification jobSpec = new JobSpecification();
    RecordDescriptor recordDesc = new RecordDescriptor(new ISerializerDeserializer[] { new UTF8StringSerializerDeserializer() });
    String[] locations = new String[] { HyracksUtils.NC1_ID, HyracksUtils.NC1_ID, HyracksUtils.NC2_ID, HyracksUtils.NC2_ID };
    HDFSReadOperatorDescriptor readOperator = new HDFSReadOperatorDescriptor(jobSpec, recordDesc, conf, splits, readSchedule, new TextKeyValueParserFactory());
    PartitionConstraintHelper.addAbsoluteLocationConstraint(jobSpec, readOperator, locations);
    ExternalSortOperatorDescriptor sortOperator = new ExternalSortOperatorDescriptor(jobSpec, 10, new int[] { 0 }, new IBinaryComparatorFactory[] { RawBinaryComparatorFactory.INSTANCE }, recordDesc);
    PartitionConstraintHelper.addAbsoluteLocationConstraint(jobSpec, sortOperator, locations);
    HDFSWriteOperatorDescriptor writeOperator = new HDFSWriteOperatorDescriptor(jobSpec, conf, new TextTupleWriterFactory());
    PartitionConstraintHelper.addAbsoluteLocationConstraint(jobSpec, writeOperator, HyracksUtils.NC1_ID);
    jobSpec.connect(new OneToOneConnectorDescriptor(jobSpec), readOperator, 0, sortOperator, 0);
    jobSpec.connect(new MToNPartitioningMergingConnectorDescriptor(jobSpec, new FieldHashPartitionComputerFactory(new int[] { 0 }, new IBinaryHashFunctionFactory[] { RawBinaryHashFunctionFactory.INSTANCE }), new int[] { 0 }, new IBinaryComparatorFactory[] { RawBinaryComparatorFactory.INSTANCE }, null), sortOperator, 0, writeOperator, 0);
    jobSpec.addRoot(writeOperator);
    IHyracksClientConnection client = new HyracksConnection(HyracksUtils.CC_HOST, HyracksUtils.TEST_HYRACKS_CC_CLIENT_PORT);
    JobId jobId = client.startJob(jobSpec);
    client.waitForCompletion(jobId);
    Assert.assertEquals(true, checkResults());
}

Example 3 with HyracksConnection

use of org.apache.hyracks.api.client.HyracksConnection in project asterixdb by apache.

the class Groupby method main.

public static void main(String[] args) throws Exception {
    Options options = new Options();
    CmdLineParser parser = new CmdLineParser(options);
    if (args.length == 0) {
        parser.printUsage(System.err);
        return;
    }
    parser.parseArgument(args);
    IHyracksClientConnection hcc = new HyracksConnection(options.host, options.port);
    JobSpecification job;
    long start = System.currentTimeMillis();
    job = createJob(parseFileSplits(options.inFileSplits), parseFileSplits(options.outFileSplits), options.htSize, options.fileSize, options.frameLimit, options.frameSize, options.algo, options.outPlain);
    if (job != null) {
        System.out.print("CreateJobTime:" + (System.currentTimeMillis() - start));
        start = System.currentTimeMillis();
        JobId jobId = hcc.startJob(job);
        hcc.waitForCompletion(jobId);
        System.out.println("JobExecuteTime:" + (System.currentTimeMillis() - start));
    }
}

Example 4 with HyracksConnection

use of org.apache.hyracks.api.client.HyracksConnection in project asterixdb by apache.

the class Join method main.

public static void main(String[] args) throws Exception {
    Options options = new Options();
    CmdLineParser parser = new CmdLineParser(options);
    if (args.length == 0) {
        parser.printUsage(System.err);
        return;
    }
    parser.parseArgument(args);
    IHyracksClientConnection hcc = new HyracksConnection(options.host, options.port);
    JobSpecification job = createJob(parseFileSplits(options.inFileCustomerSplits), parseFileSplits(options.inFileOrderSplits), parseFileSplits(options.outFileSplits), options.numJoinPartitions, options.algo, options.graceInputSize, options.graceRecordsPerFrame, options.graceFactor, options.memSize, options.tableSize, options.hasGroupBy, options.frameSize);
    if (job == null) {
        return;
    }
    long start = System.currentTimeMillis();
    JobId jobId = hcc.startJob(job, options.profile ? EnumSet.of(JobFlag.PROFILE_RUNTIME) : EnumSet.noneOf(JobFlag.class));
    hcc.waitForCompletion(jobId);
    long end = System.currentTimeMillis();
    System.err.println(start + " " + end + " " + (end - start));
}

Example 5 with HyracksConnection

use of org.apache.hyracks.api.client.HyracksConnection in project asterixdb by apache.

the class ClusterShutdownIT method runShutdown.

@Test
public void runShutdown() throws Exception {
    IHyracksClientConnection hcc = new HyracksConnection("localhost", 1098);
    hcc.stopCluster(false);
    //what happens here...
    closeTwice.expect(IPCException.class);
    closeTwice.expectMessage("Cannot send on a closed handle");
    hcc.stopCluster(false);
    ServerSocket c = null;
    ServerSocket s = null;
    try {
        c = new ServerSocket(1098);
        //we should be able to bind to this
        s = new ServerSocket(1099);
    //and we should be able to bind to this too
    } catch (Exception e) {
        LOGGER.severe(e.getMessage());
        throw e;
    } finally {
        s.close();
        c.close();
    }
}