Examples with Operator com.ibm.streamsx.rest.Operator used on opensource projects

Example 1 with Operator

use of com.ibm.streamsx.rest.Operator in project streamsx.topology by IBMStreams.

the class ParallelOperatorsTest method testParallelOperators.

@Test
public void testParallelOperators() throws Exception {
    assumeNotNull(System.getenv("STREAMS_REST_URL"));
    setupJob();
    /*
         * Note: the order of the operators is
         * 
         * IntegerPeriodicMultiSource
         * IntegerPeriodicMultiSource_parallel.Outer
         * IntegerPeriodicMultiSource_parallel.Outer_parallel.Inner
         * IntegerPeriodicMultiSource_parallel.Outer_parallel.Inner
         * IntegerPeriodicMultiSource_parallel.Outer
         * IntegerPeriodicMultiSource_parallel.Outer_parallel.Inner
         * IntegerPeriodicMultiSource_parallel.Outer_parallel.Inner
         * IntegerPeriodicMultiSource_parallel.Outer
         * IntegerPeriodicMultiSource_parallel.Outer_parallel.Inner
         * IntegerPeriodicMultiSource_parallel.Outer_parallel.Inner
         * ZIntegerTransformInteger

         * 
         */
    List<Operator> operators = helper.job.getOperators();
    Iterator<Operator> curr = operators.iterator();
    Operator op = null;
    assertTrue("Missing source operator", curr.hasNext());
    op = curr.next();
    assertEquals("IntegerPeriodicMultiSource", op.getLogicalName());
    for (int outer = 0; outer < 3; ++outer) {
        assertTrue("Missing expected outer parallel operator " + outer, curr.hasNext());
        op = curr.next();
        assertEquals("IntegerPeriodicMultiSource_parallel.Outer", op.getLogicalName());
        assertFalse("Logical name should not match name for outer parallel operator " + outer, op.getName().equals(op.getLogicalName()));
        for (int inner = 0; inner < 2; ++inner) {
            String id = "[" + outer + "," + inner + "]";
            assertTrue("Missing expected inner parallel operator " + id, curr.hasNext());
            op = curr.next();
            assertEquals("IntegerPeriodicMultiSource_parallel.Outer_parallel.Inner", op.getLogicalName());
            assertFalse("Logical name should not match name for inner parallel operator " + id, op.getName().equals(op.getLogicalName()));
        }
    }
    assertTrue("Missing final operator", curr.hasNext());
    op = curr.next();
    assertEquals("ZIntegerTransformInteger", op.getLogicalName());
}

Example 2 with Operator

use of com.ibm.streamsx.rest.Operator in project streamsx.topology by IBMStreams.

the class MetricConditionChecker method findConditionMetrics.

private void findConditionMetrics() throws IOException {
    job.refresh();
    for (Operator op : job.getOperators()) {
        for (Metric metric : op.getMetrics()) {
            final String metricName = metric.getName();
            if (!metricName.startsWith(ConditionChecker.METRIC_PREFIX))
                continue;
            String conditionName = metricName.substring(metricName.lastIndexOf(':') + 1);
            if (conditions.containsKey(conditionName)) {
                MetricCondition<?> condition = conditions.get(conditionName);
                if (condition.hasMetrics())
                    continue;
                final String validName = ConditionChecker.metricName("valid", conditionName);
                final String seqName = ConditionChecker.metricName("seq", conditionName);
                final String failName = ConditionChecker.metricName("fail", conditionName);
                if (metricName.equals(validName))
                    condition.setValidMetric(metric);
                else if (metricName.equals(seqName))
                    condition.setSeqMetric(metric);
                else if (metricName.equals(failName))
                    condition.setFailMetric(metric);
            }
        }
    }
}

Example 3 with Operator

use of com.ibm.streamsx.rest.Operator in project streamsx.topology by IBMStreams.

the class StreamsConnectionTest method validateOperators.

private void validateOperators() throws Exception {
    List<Operator> operators = job.getOperators();
    // there should be 3 operators for this test, ordered by name
    assertEquals(3, operators.size());
    List<ProcessingElement> jobpes = job.getPes();
    for (Operator op : operators) {
        assertSame(this.connection, op.getStreamsConnection());
        ProcessingElement pe = op.getPE();
        assertNotNull(pe);
        assertSame(this.connection, pe.getStreamsConnection());
        boolean inJobList = false;
        for (ProcessingElement pej : jobpes) {
            if (pej.getId().equals(pe.getId())) {
                inJobList = true;
                break;
            }
        }
        assertTrue("PE not in job list:" + pe.getId(), inJobList);
    }
    // the first operator will have an output port
    Operator op0 = operators.get(0);
    assertEquals("operator", op0.getResourceType());
    assertEquals("IntegerPeriodicMultiSource", op0.getName());
    assertEquals(0, op0.getIndexWithinJob());
    assertEquals("com.ibm.streamsx.topology.functional.java::FunctionPeriodicSource", op0.getOperatorKind());
    List<InputPort> inputSource = op0.getInputPorts();
    assertEquals(0, inputSource.size());
    List<OutputPort> outputSource = op0.getOutputPorts();
    assertEquals(1, outputSource.size());
    OutputPort opSource = outputSource.get(0);
    assertEquals(0, opSource.getIndexWithinOperator());
    assertEquals("operatorOutputPort", opSource.getResourceType());
    assertNameValid(opSource.getName());
    List<Metric> operatorMetrics = opSource.getMetrics();
    for (Metric m : operatorMetrics) {
        assertEquals(m.getMetricKind(), "counter");
        assertEquals(m.getMetricType(), "system");
        assertEquals(m.getResourceType(), "metric");
        assertNotNull(m.getName());
        assertNotNull(m.getDescription());
        assertTrue(m.getLastTimeRetrieved() > 0);
    }
    // this operator will have an input and an output port
    Operator op1 = operators.get(1);
    assertEquals("operator", op1.getResourceType());
    assertEquals("IntegerTransformInteger", op1.getName());
    assertEquals(1, op1.getIndexWithinJob());
    assertEquals("com.ibm.streamsx.topology.functional.java::Map", op1.getOperatorKind());
    List<InputPort> inputTransform = op1.getInputPorts();
    assertEquals(1, inputTransform.size());
    InputPort ip = inputTransform.get(0);
    assertNameValid(ip.getName());
    assertEquals(0, ip.getIndexWithinOperator());
    assertEquals("operatorInputPort", ip.getResourceType(), "operatorInputPort");
    List<Metric> inputPortMetrics = ip.getMetrics();
    for (Metric m : inputPortMetrics) {
        assertTrue("Unexpected metric kind for metric " + m.getName() + ": " + m.getMetricKind(), (m.getMetricKind().equals("counter")) || (m.getMetricKind().equals("gauge")) || (m.getMetricKind().equals("time")));
        assertEquals("system", m.getMetricType());
        assertEquals("metric", m.getResourceType());
        assertNotNull(m.getName());
        assertNotNull(m.getDescription());
        assertTrue(m.getLastTimeRetrieved() > 0);
    }
    List<OutputPort> outputTransform = op1.getOutputPorts();
    assertEquals(1, outputTransform.size());
    OutputPort opTransform = outputTransform.get(0);
    assertEquals(0, opTransform.getIndexWithinOperator());
    assertEquals("operatorOutputPort", opTransform.getResourceType());
    assertNameValid(opTransform.getName());
    assertNameValid(opTransform.getStreamName());
    List<Metric> outputPortMetrics = opTransform.getMetrics();
    for (Metric m : outputPortMetrics) {
        assertEquals("counter", m.getMetricKind());
        assertEquals("system", m.getMetricType());
        assertEquals("metric", m.getResourceType());
        assertNotNull(m.getName());
        assertNotNull(m.getDescription());
        assertTrue(m.getLastTimeRetrieved() > 0);
    }
}

Example 4 with Operator

use of com.ibm.streamsx.rest.Operator in project streamsx.topology by IBMStreams.

the class StreamsConnectionTest method testJobObject.

@Test
public void testJobObject() throws Exception {
    setupJob();
    List<Job> jobs = instance.getJobs();
    // we should have at least one job
    assertTrue(jobs.size() > 0);
    boolean foundJob = false;
    for (Job j : jobs) {
        if (j.getId().equals(job.getId())) {
            foundJob = true;
            break;
        }
    }
    assertTrue(foundJob);
    assertSame(job.getStreamsConnection(), instance.getStreamsConnection());
    // get a specific job
    final Job job2 = instance.getJob(jobId);
    for (int i = 0; i < 3; i++) {
        // check a subset of info returned matches
        assertEquals(job.getId(), job2.getId());
        assertEquals(job.getName(), job2.getName());
        assertEquals(job.getHealth(), job2.getHealth());
        assertEquals(job.getApplicationName(), job2.getApplicationName());
        assertEquals(job.getJobGroup(), job2.getJobGroup());
        assertEquals(job.getStartedBy(), job2.getStartedBy());
        assertEquals(job.getStatus(), job2.getStatus());
        assertEquals("job", job2.getResourceType());
        assertEquals("job", job.getResourceType());
        Thread.sleep(400);
        job2.refresh();
    }
    // job is setup with 3 operators
    List<Operator> operators = job.getOperators();
    assertEquals(3, operators.size());
    // job is setup with 2 PEs
    List<ProcessingElement> pes = job.getPes();
    assertEquals(2, pes.size());
    checkResourceAllocations(job.getResourceAllocations(), true);
    validateOperators();
    validateProcessingElements();
}

Example 5 with Operator

use of com.ibm.streamsx.rest.Operator in project streamsx.topology by IBMStreams.

the class StreamsConnectionTest method validateProcessingElements.

private void validateProcessingElements() throws Exception {
    final List<ProcessingElement> pes = job.getPes();
    // there should be 2 processing element for this test
    assertEquals(2, pes.size());
    ProcessingElement pe1 = pes.get(0);
    assertEquals(0, pe1.getIndexWithinJob());
    assertTrue(pe1.getStatus().equals("running") || pe1.getStatus().equals("starting"));
    assertEquals("none", pe1.getStatusReason());
    assertTrue(pe1.getProcessId() != null);
    assertEquals("pe", pe1.getResourceType());
    // PE metrics
    List<Metric> peMetrics = pe1.getMetrics();
    for (int i = 0; i < 10; i++) {
        if (peMetrics.size() > 0) {
            break;
        }
        Thread.sleep(50);
        peMetrics = pe1.getMetrics();
    }
    assertTrue(peMetrics.size() > 0);
    for (Metric m : peMetrics) {
        assertTrue((m.getMetricKind().equals("counter")) || (m.getMetricKind().equals("gauge")));
        assertEquals("system", m.getMetricType());
        assertEquals("metric", m.getResourceType());
        assertNotNull(m.getName());
        assertNotNull(m.getDescription());
        assertTrue(m.getLastTimeRetrieved() > 0);
    }
    Metric m = peMetrics.get(0);
    long lastTime = m.getLastTimeRetrieved();
    Thread.sleep(3500);
    m.refresh();
    assertTrue(lastTime < m.getLastTimeRetrieved());
    String pid = pe1.getProcessId();
    pe1.refresh();
    assertEquals(pid, pe1.getProcessId());
    List<PEInputPort> inputPorts = pe1.getInputPorts();
    assertTrue(inputPorts.size() == 0);
    List<PEOutputPort> outputPorts = pe1.getOutputPorts();
    assertTrue(outputPorts.size() == 1);
    PEOutputPort op = outputPorts.get(0);
    assertEquals(0, op.getIndexWithinPE());
    assertEquals("peOutputPort", op.getResourceType());
    assertEquals("tcp", op.getTransportType());
    // PE Output Port metrics
    List<Metric> outputPortMetrics = op.getMetrics();
    assertTrue(outputPortMetrics.size() > 0);
    for (Metric opMetric : outputPortMetrics) {
        assertTrue((opMetric.getMetricKind().equals("counter")) || (opMetric.getMetricKind().equals("gauge")));
        assertEquals("system", opMetric.getMetricType());
        assertEquals("metric", opMetric.getResourceType());
        assertNotNull(opMetric.getName());
        assertNotNull(opMetric.getDescription());
        assertTrue(opMetric.getLastTimeRetrieved() > 0);
    }
    ProcessingElement pe2 = pes.get(1);
    assertEquals(1, pe2.getIndexWithinJob());
    assertEquals("running", pe2.getStatus());
    assertEquals("none", pe2.getStatusReason());
    assertTrue(pe2.getProcessId() != null);
    assertEquals("pe", pe2.getResourceType());
    List<PEOutputPort> PE2OutputPorts = pe2.getOutputPorts();
    assertTrue(PE2OutputPorts.size() == 0);
    List<PEInputPort> PE2inputPorts = pe2.getInputPorts();
    assertTrue(PE2inputPorts.size() == 1);
    // PE Input Port metrics
    PEInputPort ip = PE2inputPorts.get(0);
    List<Metric> inputPortMetrics = ip.getMetrics();
    assertTrue(inputPortMetrics.size() > 0);
    for (Metric ipMetric : inputPortMetrics) {
        assertTrue((ipMetric.getMetricKind().equals("counter")) || (ipMetric.getMetricKind().equals("gauge")));
        assertEquals("system", ipMetric.getMetricType());
        assertEquals("metric", ipMetric.getResourceType());
        assertNotNull(ipMetric.getName());
        assertNotNull(ipMetric.getDescription());
        assertTrue(ipMetric.getLastTimeRetrieved() > 0);
    }
    // operator for 2nd PE should point to the 3rd operator for job
    List<Operator> peOperators = pe2.getOperators();
    assertTrue(peOperators.size() == 1);
    List<Operator> jobOperators = job.getOperators();
    assertTrue(jobOperators.size() == 3);
    Operator peOp = peOperators.get(0);
    Operator jobOp = jobOperators.get(2);
    assertEquals(peOp.getName(), jobOp.getName());
    assertEquals(peOp.getIndexWithinJob(), jobOp.getIndexWithinJob());
    assertEquals(peOp.getResourceType(), jobOp.getResourceType());
    assertEquals(peOp.getOperatorKind(), jobOp.getOperatorKind());
    for (ProcessingElement pe : pes) {
        checkResourceAllocation(pe.getResourceAllocation(), true);
    }
}