Examples with ProcessingElement com.ibm.streamsx.rest.ProcessingElement used on opensource projects

Example 1 with ProcessingElement

use of com.ibm.streamsx.rest.ProcessingElement in project streamsx.topology by IBMStreams.

the class StreamsConnectionTest method validateOperators.

private void validateOperators() throws Exception {
    List<Operator> operators = job.getOperators();
    // there should be 3 operators for this test, ordered by name
    assertEquals(3, operators.size());
    List<ProcessingElement> jobpes = job.getPes();
    for (Operator op : operators) {
        assertSame(this.connection, op.getStreamsConnection());
        ProcessingElement pe = op.getPE();
        assertNotNull(pe);
        assertSame(this.connection, pe.getStreamsConnection());
        boolean inJobList = false;
        for (ProcessingElement pej : jobpes) {
            if (pej.getId().equals(pe.getId())) {
                inJobList = true;
                break;
            }
        }
        assertTrue("PE not in job list:" + pe.getId(), inJobList);
    }
    // the first operator will have an output port
    Operator op0 = operators.get(0);
    assertEquals("operator", op0.getResourceType());
    assertEquals("IntegerPeriodicMultiSource", op0.getName());
    assertEquals(0, op0.getIndexWithinJob());
    assertEquals("com.ibm.streamsx.topology.functional.java::FunctionPeriodicSource", op0.getOperatorKind());
    List<InputPort> inputSource = op0.getInputPorts();
    assertEquals(0, inputSource.size());
    List<OutputPort> outputSource = op0.getOutputPorts();
    assertEquals(1, outputSource.size());
    OutputPort opSource = outputSource.get(0);
    assertEquals(0, opSource.getIndexWithinOperator());
    assertEquals("operatorOutputPort", opSource.getResourceType());
    assertNameValid(opSource.getName());
    List<Metric> operatorMetrics = opSource.getMetrics();
    for (Metric m : operatorMetrics) {
        assertEquals(m.getMetricKind(), "counter");
        assertEquals(m.getMetricType(), "system");
        assertEquals(m.getResourceType(), "metric");
        assertNotNull(m.getName());
        assertNotNull(m.getDescription());
        assertTrue(m.getLastTimeRetrieved() > 0);
    }
    // this operator will have an input and an output port
    Operator op1 = operators.get(1);
    assertEquals("operator", op1.getResourceType());
    assertEquals("IntegerTransformInteger", op1.getName());
    assertEquals(1, op1.getIndexWithinJob());
    assertEquals("com.ibm.streamsx.topology.functional.java::Map", op1.getOperatorKind());
    List<InputPort> inputTransform = op1.getInputPorts();
    assertEquals(1, inputTransform.size());
    InputPort ip = inputTransform.get(0);
    assertNameValid(ip.getName());
    assertEquals(0, ip.getIndexWithinOperator());
    assertEquals("operatorInputPort", ip.getResourceType(), "operatorInputPort");
    List<Metric> inputPortMetrics = ip.getMetrics();
    for (Metric m : inputPortMetrics) {
        assertTrue("Unexpected metric kind for metric " + m.getName() + ": " + m.getMetricKind(), (m.getMetricKind().equals("counter")) || (m.getMetricKind().equals("gauge")) || (m.getMetricKind().equals("time")));
        assertEquals("system", m.getMetricType());
        assertEquals("metric", m.getResourceType());
        assertNotNull(m.getName());
        assertNotNull(m.getDescription());
        assertTrue(m.getLastTimeRetrieved() > 0);
    }
    List<OutputPort> outputTransform = op1.getOutputPorts();
    assertEquals(1, outputTransform.size());
    OutputPort opTransform = outputTransform.get(0);
    assertEquals(0, opTransform.getIndexWithinOperator());
    assertEquals("operatorOutputPort", opTransform.getResourceType());
    assertNameValid(opTransform.getName());
    assertNameValid(opTransform.getStreamName());
    List<Metric> outputPortMetrics = opTransform.getMetrics();
    for (Metric m : outputPortMetrics) {
        assertEquals("counter", m.getMetricKind());
        assertEquals("system", m.getMetricType());
        assertEquals("metric", m.getResourceType());
        assertNotNull(m.getName());
        assertNotNull(m.getDescription());
        assertTrue(m.getLastTimeRetrieved() > 0);
    }
}

Example 2 with ProcessingElement

use of com.ibm.streamsx.rest.ProcessingElement in project streamsx.topology by IBMStreams.

the class StreamsConnectionTest method checkResourceAllocation.

private static void checkResourceAllocation(ResourceAllocation ra, boolean app) throws IOException {
    if (ra == null)
        return;
    assertEquals("resourceAllocation", ra.getResourceType());
    if (app)
        assertTrue(ra.isApplicationResource());
    assertNotNull(ra.getSchedulerStatus());
    assertNotNull(ra.getStatus());
    Instance rai = ra.getInstance();
    for (ProcessingElement pe : ra.getPes()) {
        assertNotNull(pe);
        assertNotNull(pe.getStatus());
    }
    for (Job job : ra.getJobs()) {
        assertNotNull(job);
        assertNotNull(job.getStatus());
    }
    assertSame(rai, ra.getInstance());
    Resource r = ra.getResource();
    assertNotNull(r.getId());
    assertNotNull(r.getDisplayName());
    assertNotNull(r.getIpAddress());
    assertEquals("resource", r.getResourceType());
    for (Metric metric : r.getMetrics()) {
        assertTrue((metric.getMetricKind().equals("counter")) || (metric.getMetricKind().equals("gauge")));
        assertEquals("system", metric.getMetricType());
        assertEquals("metric", metric.getResourceType());
        assertNotNull(metric.getName());
        assertNotNull(metric.getDescription());
        assertTrue(metric.getLastTimeRetrieved() > 0);
    }
}

Example 3 with ProcessingElement

use of com.ibm.streamsx.rest.ProcessingElement in project streamsx.topology by IBMStreams.

the class StreamsConnectionTest method testJobObject.

@Test
public void testJobObject() throws Exception {
    setupJob();
    List<Job> jobs = instance.getJobs();
    // we should have at least one job
    assertTrue(jobs.size() > 0);
    boolean foundJob = false;
    for (Job j : jobs) {
        if (j.getId().equals(job.getId())) {
            foundJob = true;
            break;
        }
    }
    assertTrue(foundJob);
    assertSame(job.getStreamsConnection(), instance.getStreamsConnection());
    // get a specific job
    final Job job2 = instance.getJob(jobId);
    for (int i = 0; i < 3; i++) {
        // check a subset of info returned matches
        assertEquals(job.getId(), job2.getId());
        assertEquals(job.getName(), job2.getName());
        assertEquals(job.getHealth(), job2.getHealth());
        assertEquals(job.getApplicationName(), job2.getApplicationName());
        assertEquals(job.getJobGroup(), job2.getJobGroup());
        assertEquals(job.getStartedBy(), job2.getStartedBy());
        assertEquals(job.getStatus(), job2.getStatus());
        assertEquals("job", job2.getResourceType());
        assertEquals("job", job.getResourceType());
        Thread.sleep(400);
        job2.refresh();
    }
    // job is setup with 3 operators
    List<Operator> operators = job.getOperators();
    assertEquals(3, operators.size());
    // job is setup with 2 PEs
    List<ProcessingElement> pes = job.getPes();
    assertEquals(2, pes.size());
    checkResourceAllocations(job.getResourceAllocations(), true);
    validateOperators();
    validateProcessingElements();
}

Example 4 with ProcessingElement

use of com.ibm.streamsx.rest.ProcessingElement in project streamsx.topology by IBMStreams.

the class StreamsConnectionTest method validateProcessingElements.

private void validateProcessingElements() throws Exception {
    final List<ProcessingElement> pes = job.getPes();
    // there should be 2 processing element for this test
    assertEquals(2, pes.size());
    ProcessingElement pe1 = pes.get(0);
    assertEquals(0, pe1.getIndexWithinJob());
    assertTrue(pe1.getStatus().equals("running") || pe1.getStatus().equals("starting"));
    assertEquals("none", pe1.getStatusReason());
    assertTrue(pe1.getProcessId() != null);
    assertEquals("pe", pe1.getResourceType());
    // PE metrics
    List<Metric> peMetrics = pe1.getMetrics();
    for (int i = 0; i < 10; i++) {
        if (peMetrics.size() > 0) {
            break;
        }
        Thread.sleep(50);
        peMetrics = pe1.getMetrics();
    }
    assertTrue(peMetrics.size() > 0);
    for (Metric m : peMetrics) {
        assertTrue((m.getMetricKind().equals("counter")) || (m.getMetricKind().equals("gauge")));
        assertEquals("system", m.getMetricType());
        assertEquals("metric", m.getResourceType());
        assertNotNull(m.getName());
        assertNotNull(m.getDescription());
        assertTrue(m.getLastTimeRetrieved() > 0);
    }
    Metric m = peMetrics.get(0);
    long lastTime = m.getLastTimeRetrieved();
    Thread.sleep(3500);
    m.refresh();
    assertTrue(lastTime < m.getLastTimeRetrieved());
    String pid = pe1.getProcessId();
    pe1.refresh();
    assertEquals(pid, pe1.getProcessId());
    List<PEInputPort> inputPorts = pe1.getInputPorts();
    assertTrue(inputPorts.size() == 0);
    List<PEOutputPort> outputPorts = pe1.getOutputPorts();
    assertTrue(outputPorts.size() == 1);
    PEOutputPort op = outputPorts.get(0);
    assertEquals(0, op.getIndexWithinPE());
    assertEquals("peOutputPort", op.getResourceType());
    assertEquals("tcp", op.getTransportType());
    // PE Output Port metrics
    List<Metric> outputPortMetrics = op.getMetrics();
    assertTrue(outputPortMetrics.size() > 0);
    for (Metric opMetric : outputPortMetrics) {
        assertTrue((opMetric.getMetricKind().equals("counter")) || (opMetric.getMetricKind().equals("gauge")));
        assertEquals("system", opMetric.getMetricType());
        assertEquals("metric", opMetric.getResourceType());
        assertNotNull(opMetric.getName());
        assertNotNull(opMetric.getDescription());
        assertTrue(opMetric.getLastTimeRetrieved() > 0);
    }
    ProcessingElement pe2 = pes.get(1);
    assertEquals(1, pe2.getIndexWithinJob());
    assertEquals("running", pe2.getStatus());
    assertEquals("none", pe2.getStatusReason());
    assertTrue(pe2.getProcessId() != null);
    assertEquals("pe", pe2.getResourceType());
    List<PEOutputPort> PE2OutputPorts = pe2.getOutputPorts();
    assertTrue(PE2OutputPorts.size() == 0);
    List<PEInputPort> PE2inputPorts = pe2.getInputPorts();
    assertTrue(PE2inputPorts.size() == 1);
    // PE Input Port metrics
    PEInputPort ip = PE2inputPorts.get(0);
    List<Metric> inputPortMetrics = ip.getMetrics();
    assertTrue(inputPortMetrics.size() > 0);
    for (Metric ipMetric : inputPortMetrics) {
        assertTrue((ipMetric.getMetricKind().equals("counter")) || (ipMetric.getMetricKind().equals("gauge")));
        assertEquals("system", ipMetric.getMetricType());
        assertEquals("metric", ipMetric.getResourceType());
        assertNotNull(ipMetric.getName());
        assertNotNull(ipMetric.getDescription());
        assertTrue(ipMetric.getLastTimeRetrieved() > 0);
    }
    // operator for 2nd PE should point to the 3rd operator for job
    List<Operator> peOperators = pe2.getOperators();
    assertTrue(peOperators.size() == 1);
    List<Operator> jobOperators = job.getOperators();
    assertTrue(jobOperators.size() == 3);
    Operator peOp = peOperators.get(0);
    Operator jobOp = jobOperators.get(2);
    assertEquals(peOp.getName(), jobOp.getName());
    assertEquals(peOp.getIndexWithinJob(), jobOp.getIndexWithinJob());
    assertEquals(peOp.getResourceType(), jobOp.getResourceType());
    assertEquals(peOp.getOperatorKind(), jobOp.getOperatorKind());
    for (ProcessingElement pe : pes) {
        checkResourceAllocation(pe.getResourceAllocation(), true);
    }
}

Example 5 with ProcessingElement

use of com.ibm.streamsx.rest.ProcessingElement in project streamsx.topology by IBMStreams.

the class StreamsConnectionSample method main.

public static void main(String[] args) throws IOException {
    String userName = args[0];
    String authToken = args[1];
    String url = args[2];
    String instanceName = args[3];
    /*
         * Create the connection to the instance indicated
         */
    StreamsConnection sClient = StreamsConnection.createInstance(userName, authToken, url);
    /*
         * This option is only used to by-pass the certificate certification
         */
    if (args.length == 5 && "true".equals(args[4])) {
        sClient.allowInsecureHosts(true);
    }
    try {
        System.out.println("Instance: ");
        Instance instance = sClient.getInstance(instanceName);
        /*
             * From the Instance, get a list of jobs
             */
        List<Job> jobs = instance.getJobs();
        for (Job job : jobs) {
            System.out.println("Job: " + job.toString());
            /*
                 * For each job, get a list of operators
                 */
            List<Operator> operators = job.getOperators();
            for (Operator op : operators) {
                System.out.println("Operator: " + op.toString());
                List<Metric> metrics = op.getMetrics();
                /*
                     * For each operator, you can get a list of metrics, output
                     * ports and input ports
                     */
                for (Metric m : metrics) {
                    System.out.println("Metric: " + m.toString());
                }
                List<OutputPort> outP = op.getOutputPorts();
                for (OutputPort oport : outP) {
                    System.out.println("Output Port: " + oport.toString());
                    /*
                         * For each output port, you can get a list of metrics
                         */
                    for (Metric om : oport.getMetrics()) {
                        System.out.println("Output Port Metric: " + om.toString());
                    }
                }
                List<InputPort> inP = op.getInputPorts();
                /*
                     * For each input port, get a list of metrics
                     */
                for (InputPort ip : inP) {
                    System.out.println("Input Port: " + ip.toString());
                    for (Metric im : ip.getMetrics()) {
                        System.out.println("Input Port Metric: " + im.toString());
                    }
                }
            }
            /*
                 * For each job, get a list of processing elements
                 */
            for (ProcessingElement pe : job.getPes()) {
                System.out.println("ProcessingElement:" + pe.toString());
            }
        }
        try {
            /*
                 * Get a specific job in the instance
                 */
            instance.getJob("99999");
        } catch (RESTException e) {
            /*
                 * This shows what is available in the RESTException should
                 * something fail
                 */
            System.out.println("Status Code: " + e.getStatusCode());
            System.out.println("Message Id: " + e.getStreamsErrorMessageId());
            System.out.println("MessageAsJson: " + e.getStreamsErrorMessageAsJson().toString());
            System.out.println("Message: " + e.getMessage());
        }
    } catch (Exception e) {
        e.printStackTrace();
    }
}