Examples with CollectionSetBuilder org.opennms.netmgt.collection.support.builder.CollectionSetBuilder used on opensource projects

Example 1 with CollectionSetBuilder

use of org.opennms.netmgt.collection.support.builder.CollectionSetBuilder in project opennms by OpenNMS.

the class NewtsPersisterIT method canPersist.

@Test
public void canPersist() throws InterruptedException {
    ServiceParameters params = new ServiceParameters(Collections.emptyMap());
    RrdRepository repo = new RrdRepository();
    // Only the last element of the path matters here
    repo.setRrdBaseDir(Paths.get("a", "path", "that", "ends", "with", "snmp").toFile());
    Persister persister = m_persisterFactory.createPersister(params, repo);
    int nodeId = 1;
    CollectionAgent agent = mock(CollectionAgent.class);
    when(agent.getStorageResourcePath()).thenReturn(ResourcePath.get(Integer.toString(nodeId)));
    NodeLevelResource nodeLevelResource = new NodeLevelResource(nodeId);
    // Build a collection set with a single sample
    Timestamp now = Timestamp.now();
    CollectionSet collectionSet = new CollectionSetBuilder(agent).withNumericAttribute(nodeLevelResource, "metrics", "metric", 900, AttributeType.GAUGE).withTimestamp(now.asDate()).build();
    // Persist
    collectionSet.visit(persister);
    // Wait for the sample(s) to be flushed
    Thread.sleep(5 * 1000);
    // Fetch the (persisted) sample
    Resource resource = new Resource("snmp:1:metrics");
    Timestamp end = Timestamp.now();
    Results<Sample> samples = m_sampleRepository.select(Context.DEFAULT_CONTEXT, resource, Optional.of(now), Optional.of(end));
    assertEquals(1, samples.getRows().size());
    Row<Sample> row = samples.getRows().iterator().next();
    assertEquals(900, row.getElement("metric").getValue().doubleValue(), 0.00001);
}

Example 2 with CollectionSetBuilder

use of org.opennms.netmgt.collection.support.builder.CollectionSetBuilder in project opennms by OpenNMS.

the class Sftp3gppXmlCollectionHandler method collect.

/* (non-Javadoc)
     * @see org.opennms.protocols.xml.collector.XmlCollectionHandler#collect(org.opennms.netmgt.collectd.CollectionAgent, org.opennms.protocols.xml.config.XmlDataCollection, java.util.Map)
     */
@Override
public CollectionSet collect(CollectionAgent agent, XmlDataCollection collection, Map<String, Object> parameters) throws CollectionException {
    String status = "finished";
    // Create a new collection set.
    CollectionSetBuilder builder = new CollectionSetBuilder(agent);
    // TODO We could be careful when handling exceptions because parsing exceptions will be treated different from connection or retrieval exceptions
    DateTime startTime = new DateTime();
    Sftp3gppUrlConnection connection = null;
    try {
        ResourcePath resourcePath = ResourcePath.get(ResourceTypeUtils.SNMP_DIRECTORY, Integer.toString(agent.getNodeId()));
        for (XmlSource source : collection.getXmlSources()) {
            if (!source.getUrl().startsWith(Sftp3gppUrlHandler.PROTOCOL)) {
                throw new CollectionException("The 3GPP SFTP Collection Handler can only use the protocol " + Sftp3gppUrlHandler.PROTOCOL);
            }
            final String urlStr = source.getUrl();
            final Request request = source.getRequest();
            URL url = UrlFactory.getUrl(urlStr, request);
            String lastFile = Sftp3gppUtils.getLastFilename(getResourceStorageDao(), getServiceName(), resourcePath, url.getPath());
            connection = (Sftp3gppUrlConnection) url.openConnection();
            if (lastFile == null) {
                lastFile = connection.get3gppFileName();
                LOG.debug("collect(single): retrieving file from {}{}{} from {}", url.getPath(), File.separatorChar, lastFile, agent.getHostAddress());
                Document doc = getXmlDocument(urlStr, request);
                fillCollectionSet(agent, builder, source, doc);
                Sftp3gppUtils.setLastFilename(getResourceStorageDao(), getServiceName(), resourcePath, url.getPath(), lastFile);
                Sftp3gppUtils.deleteFile(connection, lastFile);
            } else {
                connection.connect();
                List<String> files = connection.getFileList();
                long lastTs = connection.getTimeStampFromFile(lastFile);
                boolean collected = false;
                for (String fileName : files) {
                    if (connection.getTimeStampFromFile(fileName) > lastTs) {
                        LOG.debug("collect(multiple): retrieving file {} from {}", fileName, agent.getHostAddress());
                        InputStream is = connection.getFile(fileName);
                        try {
                            Document doc = getXmlDocument(is, request);
                            IOUtils.closeQuietly(is);
                            fillCollectionSet(agent, builder, source, doc);
                        } finally {
                            IOUtils.closeQuietly(is);
                        }
                        Sftp3gppUtils.setLastFilename(getResourceStorageDao(), getServiceName(), resourcePath, url.getPath(), fileName);
                        Sftp3gppUtils.deleteFile(connection, fileName);
                        collected = true;
                    }
                }
                if (!collected) {
                    LOG.warn("collect: could not find any file after {} on {}", lastFile, agent);
                }
            }
        }
        return builder.build();
    } catch (Exception e) {
        status = "failed";
        throw new CollectionException(e.getMessage(), e);
    } finally {
        DateTime endTime = new DateTime();
        LOG.debug("collect: {} collection {}: duration: {} ms", status, collection.getName(), endTime.getMillis() - startTime.getMillis());
        UrlFactory.disconnect(connection);
    }
}

Example 3 with CollectionSetBuilder

use of org.opennms.netmgt.collection.support.builder.CollectionSetBuilder in project opennms by OpenNMS.

the class AbstractXmlCollectionHandler method collect.

/* (non-Javadoc)
     * @see org.opennms.protocols.xml.collector.XmlCollectionHandler#collect(org.opennms.netmgt.collectd.CollectionAgent, org.opennms.protocols.xml.config.XmlDataCollection, java.util.Map)
     */
@Override
public CollectionSet collect(CollectionAgent agent, XmlDataCollection collection, Map<String, Object> parameters) throws CollectionException {
    String status = "finished";
    CollectionSetBuilder builder = new CollectionSetBuilder(agent);
    DateTime startTime = new DateTime();
    try {
        LOG.debug("collect: looping sources for collection {}", collection.getName());
        for (XmlSource source : collection.getXmlSources()) {
            final String urlStr = source.getUrl();
            final Request request = source.getRequest();
            LOG.debug("collect: starting source url '{}' collection with request: {}", urlStr, request);
            fillCollectionSet(urlStr, request, agent, builder, source);
            LOG.debug("collect: finished source url '{}' collection with {} resources", urlStr, builder.getNumResources());
        }
        return builder.build();
    } catch (Exception e) {
        status = "failed";
        throw new CollectionException(e.getMessage(), e);
    } finally {
        DateTime endTime = new DateTime();
        LOG.debug("collect: {} collection {}: duration: {} ms", status, collection.getName(), endTime.getMillis() - startTime.getMillis());
    }
}

Example 4 with CollectionSetBuilder

use of org.opennms.netmgt.collection.support.builder.CollectionSetBuilder in project opennms by OpenNMS.

the class Sftp3gppVTDXmlCollectionHandler method collect.

@Override
public CollectionSet collect(CollectionAgent agent, XmlDataCollection collection, Map<String, Object> parameters) throws CollectionException {
    String status = "finished";
    // Create a new collection set.
    CollectionSetBuilder builder = new CollectionSetBuilder(agent);
    // TODO We could be careful when handling exceptions because parsing exceptions will be treated different from connection or retrieval exceptions
    DateTime startTime = new DateTime();
    Sftp3gppUrlConnection connection = null;
    try {
        // FIXME: Does not support storeByFS
        ResourcePath resourcePath = ResourcePath.get(Integer.toString(agent.getNodeId()));
        for (XmlSource source : collection.getXmlSources()) {
            if (!source.getUrl().startsWith(Sftp3gppUrlHandler.PROTOCOL)) {
                throw new CollectionException("The 3GPP SFTP Collection Handler can only use the protocol " + Sftp3gppUrlHandler.PROTOCOL);
            }
            final String urlStr = source.getUrl();
            final Request request = source.getRequest();
            URL url = UrlFactory.getUrl(source.getUrl(), source.getRequest());
            String lastFile = Sftp3gppUtils.getLastFilename(getResourceStorageDao(), getServiceName(), resourcePath, url.getPath());
            connection = (Sftp3gppUrlConnection) url.openConnection();
            if (lastFile == null) {
                lastFile = connection.get3gppFileName();
                LOG.debug("collect(single): retrieving file from {}{}{} from {}", url.getPath(), File.separatorChar, lastFile, agent.getHostAddress());
                VTDNav doc = getVTDXmlDocument(urlStr, request);
                fillCollectionSet(agent, builder, source, doc);
                Sftp3gppUtils.setLastFilename(getResourceStorageDao(), getServiceName(), resourcePath, url.getPath(), lastFile);
                Sftp3gppUtils.deleteFile(connection, lastFile);
            } else {
                connection.connect();
                List<String> files = connection.getFileList();
                long lastTs = connection.getTimeStampFromFile(lastFile);
                boolean collected = false;
                for (String fileName : files) {
                    if (connection.getTimeStampFromFile(fileName) > lastTs) {
                        LOG.debug("collect(multiple): retrieving file {} from {}", fileName, agent.getHostAddress());
                        InputStream is = connection.getFile(fileName);
                        try {
                            VTDNav doc = getVTDXmlDocument(is, request);
                            fillCollectionSet(agent, builder, source, doc);
                        } finally {
                            IOUtils.closeQuietly(is);
                        }
                        Sftp3gppUtils.setLastFilename(getResourceStorageDao(), getServiceName(), resourcePath, url.getPath(), fileName);
                        Sftp3gppUtils.deleteFile(connection, fileName);
                        collected = true;
                    }
                }
                if (!collected) {
                    LOG.warn("collect: could not find any file after {} on {}", lastFile, agent);
                }
            }
        }
        return builder.build();
    } catch (Exception e) {
        status = "failed";
        throw new CollectionException(e.getMessage(), e);
    } finally {
        DateTime endTime = new DateTime();
        LOG.debug("collect: {} collection {}: duration: {} ms", status, collection.getName(), endTime.getMillis() - startTime.getMillis());
        UrlFactory.disconnect(connection);
    }
}

Example 5 with CollectionSetBuilder

use of org.opennms.netmgt.collection.support.builder.CollectionSetBuilder in project opennms by OpenNMS.

the class CollectionSetDTOTest method data.

@Parameters
public static Collection<Object[]> data() throws ParseException {
    CollectionAgent collectionAgent = mock(CollectionAgent.class);
    NodeLevelResource nodeLevelResource = new NodeLevelResource(1);
    InterfaceLevelResource interfaceLevelResource = new InterfaceLevelResource(nodeLevelResource, "eth0");
    ResourceType rt = mock(ResourceType.class, RETURNS_DEEP_STUBS);
    when(rt.getName()).thenReturn("Charles");
    when(rt.getStorageStrategy().getClazz()).thenReturn(MockStorageStrategy.class.getCanonicalName());
    when(rt.getPersistenceSelectorStrategy().getClazz()).thenReturn(MockPersistenceSelectorStrategy.class.getCanonicalName());
    DeferredGenericTypeResource deferredGenericTypeResource = new DeferredGenericTypeResource(nodeLevelResource, "Charles", "id");
    GenericTypeResource genericTypeResource = new GenericTypeResource(nodeLevelResource, rt, "idx");
    genericTypeResource.setTimestamp(new Date(0));
    ResourceTypeMapper.getInstance().setResourceTypeMapper((name) -> rt);
    // For complete coverage make sure that there is at least one attribute
    // for every different resource type, and that every different type
    // of attribute is represented at least once
    CollectionSet collectionSet = new CollectionSetBuilder(collectionAgent).withTimestamp(new Date(0)).withNumericAttribute(nodeLevelResource, "ucd-sysstat", "CpuRawIdle", 99, AttributeType.GAUGE).withNumericAttribute(interfaceLevelResource, "mib2-X-interfaces", "ifHCInOctets", 1001, AttributeType.COUNTER).withStringAttribute(interfaceLevelResource, "mib2-X-interfaces", "ifDescr", "LAN").withIdentifiedNumericAttribute(deferredGenericTypeResource, "net-snmp-disk", "ns-dsk1", 1024, AttributeType.GAUGE, "some-oid").withIdentifiedNumericAttribute(genericTypeResource, "net-snmp-disk", "ns-dskTotal", 1024, AttributeType.GAUGE, "some-oid").build();
    return Arrays.asList(new Object[][] { { collectionSet, "<collection-set status=\"SUCCEEDED\" timestamp=\"" + StringUtils.iso8601OffsetString(new Date(0), ZoneId.systemDefault(), ChronoUnit.SECONDS) + "\">\n" + "   <agent type=\"0\" store-by-fs=\"false\" node-id=\"0\" sys-up-time=\"0\"/>\n" + "   <collection-resource>\n" + "      <node-level-resource node-id=\"1\"/>\n" + "      <numeric-attribute group=\"ucd-sysstat\" name=\"CpuRawIdle\" type=\"gauge\" value=\"99\"/>\n" + "   </collection-resource>\n" + "   <collection-resource>\n" + "      <interface-level-resource if-name=\"eth0\">\n" + "         <node-level-resource node-id=\"1\"/>\n" + "      </interface-level-resource>\n" + "      <numeric-attribute group=\"mib2-X-interfaces\" name=\"ifHCInOctets\" type=\"counter\" value=\"1001\"/>\n" + "      <string-attribute group=\"mib2-X-interfaces\" name=\"ifDescr\" type=\"string\" value=\"LAN\"/>\n" + "   </collection-resource>\n" + "   <collection-resource>\n" + "      <generic-type-resource name=\"Charles\" instance=\"id\">\n" + "         <node-level-resource node-id=\"1\"/>\n" + "      </generic-type-resource>\n" + "      <numeric-attribute group=\"net-snmp-disk\" name=\"ns-dsk1\" type=\"gauge\" identifier=\"some-oid\" value=\"1024\"/>\n" + "   </collection-resource>\n" + "   <collection-resource>\n" + "      <generic-type-resource name=\"Charles\" instance=\"idx\" timestamp=\"" + StringUtils.iso8601OffsetString(new Date(0), ZoneId.systemDefault(), ChronoUnit.SECONDS) + "\">\n" + "         <node-level-resource node-id=\"1\"/>\n" + "      </generic-type-resource>\n" + "      <numeric-attribute group=\"net-snmp-disk\" name=\"ns-dskTotal\" type=\"gauge\" identifier=\"some-oid\" value=\"1024\"/>\n" + "   </collection-resource>\n" + "</collection-set>" } });
}