Examples with Series com.srotya.sidewinder.core.storage.Series used on opensource projects

Example 6 with Series

use of com.srotya.sidewinder.core.storage.Series in project sidewinder by srotya.

the class DatabaseOpsApi method querySeries.

@Path("/{" + DB_NAME + "}/query")
@POST
@Produces({ MediaType.APPLICATION_JSON })
@Consumes({ MediaType.TEXT_PLAIN })
public String querySeries(@PathParam(DatabaseOpsApi.DB_NAME) String dbName, String query) {
    try {
        String[] queryParts = query.split("<=?");
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
        long endTs = System.currentTimeMillis();
        long startTs = endTs;
        String startTime = queryParts[0];
        String endTime = queryParts[2];
        if (startTime.contains("now")) {
            String[] split = startTime.split("-");
            int offset = Integer.parseInt(split[1].charAt(0) + "");
            startTs = startTs - (offset * 3600 * 1000);
        } else if (startTime.matches("\\d+")) {
            startTs = Long.parseLong(startTime);
            endTs = Long.parseLong(endTime);
        } else {
            startTs = sdf.parse(startTime).getTime();
            endTs = sdf.parse(endTime).getTime();
        }
        // cpu.load.host=v1.domain=test\.com=>derivative,10,mean
        TargetSeries tagSeries = MiscUtils.extractTargetFromQuery(query);
        List<Series> points = storageEngine.queryDataPoints(dbName, tagSeries.getMeasurementName(), tagSeries.getFieldName(), startTs, endTs, tagSeries.getTagFilter(), null, tagSeries.getAggregationFunction());
        return new Gson().toJson(points);
    } catch (ItemNotFoundException e) {
        throw new NotFoundException(e);
    } catch (BadRequestException e) {
        throw e;
    } catch (Exception e) {
        e.printStackTrace();
        throw new InternalServerErrorException(e);
    }
}

Example 7 with Series

use of com.srotya.sidewinder.core.storage.Series in project sidewinder by srotya.

the class MeasurementOpsApi method getSeries.

public List<Number[]> getSeries(@PathParam(DatabaseOpsApi.DB_NAME) String dbName, @PathParam(MEASUREMENT) String measurementName, @QueryParam("field") String valueFieldName, @DefaultValue("now-1h") @QueryParam(START_TIME) String startTime, @QueryParam(END_TIME) String endTime) {
    if (valueFieldName == null) {
        throw new BadRequestException("Must specify a value field");
    }
    try {
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        long endTs = System.currentTimeMillis();
        long startTs = endTs;
        if (startTime.contains("now")) {
            String[] split = startTime.split("-");
            int offset = Integer.parseInt(split[1].charAt(0) + "");
            startTs = startTs - (offset * 3600 * 1000);
        } else {
            startTs = sdf.parse(startTime).getTime();
            endTs = sdf.parse(endTime).getTime();
        }
        List<Series> points = engine.queryDataPoints(dbName, measurementName, valueFieldName, startTs, endTs, null);
        List<Number[]> response = new ArrayList<>();
        for (Series entry : points) {
            for (DataPoint dataPoint : entry.getDataPoints()) {
                if (entry.isFp()) {
                    response.add(new Number[] { dataPoint.getLongValue(), dataPoint.getTimestamp() });
                } else {
                    response.add(new Number[] { dataPoint.getValue(), dataPoint.getTimestamp() });
                }
            }
        }
        return response;
    } catch (ItemNotFoundException e) {
        throw new NotFoundException(e);
    } catch (Exception e) {
        throw new InternalServerErrorException(e);
    }
}

Example 8 with Series

use of com.srotya.sidewinder.core.storage.Series in project sidewinder by srotya.

the class WindowedFunction method apply.

@Override
public Series apply(Series t) {
    Series output = new Series(t.getMeasurementName(), t.getValueFieldName(), t.getTags());
    output.setFp(t.isFp());
    SortedMap<Long, List<DataPoint>> map = new TreeMap<>();
    for (DataPoint dataPoint : t.getDataPoints()) {
        try {
            long bucket = (dataPoint.getTimestamp() / getTimeWindow()) * getTimeWindow();
            List<DataPoint> list = map.get(bucket);
            if (list == null) {
                list = new ArrayList<>();
                map.put(bucket, list);
            }
            list.add(dataPoint);
        } catch (Exception e) {
            System.err.println("Exception :" + getTimeWindow());
        }
    }
    output.setDataPoints(apply(map, t.isFp()));
    return output;
}

Example 9 with Series

use of com.srotya.sidewinder.core.storage.Series in project sidewinder by srotya.

the class TestDiskStorageEngine method testAddAndReadDataPointsWithTagFilters.

@Test
public void testAddAndReadDataPointsWithTagFilters() throws Exception {
    DiskStorageEngine engine = new DiskStorageEngine();
    MiscUtils.delete(new File("target/db5/"));
    HashMap<String, String> map = new HashMap<>();
    map.put("metadata.dir", "target/db5/mdq");
    map.put("index.dir", "target/db5/index");
    map.put("data.dir", "target/db5/data");
    map.put(StorageEngine.PERSISTENCE_DISK, "true");
    engine.configure(map, bgTasks);
    long curr = 1497720452566L;
    String dbName = "test";
    String measurementName = "cpu";
    String valueFieldName = "value";
    String tag = "host123123";
    for (int i = 1; i <= 3; i++) {
        engine.writeDataPoint(MiscUtils.buildDataPoint(dbName, measurementName, valueFieldName, Arrays.asList(tag + "=" + i, tag + "=" + (i + 1)), curr + i, 2 * i));
    }
    assertEquals(1, engine.getAllMeasurementsForDb(dbName).size());
    SimpleTagFilter filter1 = new SimpleTagFilter(FilterType.EQUALS, "host123123", "1");
    SimpleTagFilter filter2 = new SimpleTagFilter(FilterType.EQUALS, "host123123", "2");
    List<Series> queryDataPoints = engine.queryDataPoints(dbName, measurementName, valueFieldName, curr, curr + 3, new ComplexTagFilter(ComplexFilterType.OR, Arrays.asList(filter1, filter2)), null, null);
    assertEquals(2, queryDataPoints.size());
    int i = 1;
    assertEquals(1, queryDataPoints.iterator().next().getDataPoints().size());
    queryDataPoints.sort(new Comparator<Series>() {

        @Override
        public int compare(Series o1, Series o2) {
            return o1.getTags().toString().compareTo(o2.getTags().toString());
        }
    });
    for (Series list : queryDataPoints) {
        for (DataPoint dataPoint : list.getDataPoints()) {
            assertEquals(curr + i, dataPoint.getTimestamp());
            i++;
        }
    }
    Set<String> tags = engine.getTagKeysForMeasurement(dbName, measurementName);
    assertEquals(new HashSet<>(Arrays.asList(tag)), tags);
    assertEquals(new HashSet<>(Arrays.asList("1", "2", "3", "4")), engine.getTagValuesForMeasurement(dbName, measurementName, tag));
    Set<String> fieldsForMeasurement = engine.getFieldsForMeasurement(dbName, measurementName);
    assertEquals(new HashSet<>(Arrays.asList(valueFieldName)), fieldsForMeasurement);
    try {
        engine.getTagKeysForMeasurement(dbName + "1", measurementName);
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.getTagKeysForMeasurement(dbName, measurementName + "1");
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.getFieldsForMeasurement(dbName + "1", measurementName);
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.getFieldsForMeasurement(dbName, measurementName + "1");
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    engine.disconnect();
}

Example 10 with Series

use of com.srotya.sidewinder.core.storage.Series in project sidewinder by srotya.

the class TestDiskStorageEngine method testAddAndReadDataPoints.

@Test
public void testAddAndReadDataPoints() throws Exception {
    DiskStorageEngine engine = new DiskStorageEngine();
    MiscUtils.delete(new File("target/db19/"));
    HashMap<String, String> map = new HashMap<>();
    map.put("metadata.dir", "target/db19/mdq");
    map.put("index.dir", "target/db19/index");
    map.put("data.dir", "target/db19/data");
    map.put(StorageEngine.PERSISTENCE_DISK, "true");
    engine.configure(map, bgTasks);
    long curr = System.currentTimeMillis();
    String dbName = "test";
    String measurementName = "cpu";
    String valueFieldName = "value";
    try {
        engine.writeDataPoint(dbName, measurementName, valueFieldName, null, curr, 2 * 0);
        fail("Must reject the above datapoint due to missing tags");
    } catch (Exception e) {
    }
    String tag = "host=123123";
    for (int i = 1; i <= 3; i++) {
        engine.writeDataPoint(MiscUtils.buildDataPoint(dbName, measurementName, valueFieldName, Arrays.asList(tag), curr + i, 2 * i));
    }
    assertEquals(1, engine.getAllMeasurementsForDb(dbName).size());
    List<Series> queryDataPoints = engine.queryDataPoints(dbName, measurementName, valueFieldName, curr, curr + 3, null);
    assertEquals(1, queryDataPoints.size());
    int i = 1;
    assertEquals(3, queryDataPoints.iterator().next().getDataPoints().size());
    List<List<DataPoint>> output = new ArrayList<>();
    for (Series series : queryDataPoints) {
        output.add(series.getDataPoints());
    }
    for (List<DataPoint> list : output) {
        for (DataPoint dataPoint : list) {
            assertEquals(curr + i, dataPoint.getTimestamp());
            i++;
        }
    }
    Set<String> tags = engine.getTagKeysForMeasurement(dbName, measurementName);
    assertEquals(new HashSet<>(Arrays.asList("host")), tags);
    Set<String> fieldsForMeasurement = engine.getFieldsForMeasurement(dbName, measurementName);
    assertEquals(new HashSet<>(Arrays.asList(valueFieldName)), fieldsForMeasurement);
    try {
        engine.getTagKeysForMeasurement(dbName + "1", measurementName);
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.getTagKeysForMeasurement(dbName, measurementName + "1");
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.getFieldsForMeasurement(dbName + "1", measurementName);
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.getFieldsForMeasurement(dbName, measurementName + "1");
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    engine.disconnect();
}