use of org.graylog2.plugin.streams.DefaultStream in project graylog2-server by Graylog2.
the class FunctionsSnippetsTest method registerFunctions.
@BeforeClass
@SuppressForbidden("Allow using default thread factory")
public static void registerFunctions() {
final Map<String, Function<?>> functions = commonFunctions();
functions.put(BooleanConversion.NAME, new BooleanConversion());
functions.put(DoubleConversion.NAME, new DoubleConversion());
functions.put(LongConversion.NAME, new LongConversion());
functions.put(StringConversion.NAME, new StringConversion());
functions.put(MapConversion.NAME, new MapConversion());
// message related functions
functions.put(HasField.NAME, new HasField());
functions.put(SetField.NAME, new SetField());
functions.put(SetFields.NAME, new SetFields());
functions.put(RenameField.NAME, new RenameField());
functions.put(RemoveField.NAME, new RemoveField());
functions.put(DropMessage.NAME, new DropMessage());
functions.put(CreateMessage.NAME, new CreateMessage());
functions.put(CloneMessage.NAME, new CloneMessage());
functions.put(TrafficAccountingSize.NAME, new TrafficAccountingSize());
// route to stream mocks
final StreamService streamService = mock(StreamService.class);
otherStream = mock(Stream.class, "some stream id2");
when(otherStream.isPaused()).thenReturn(false);
when(otherStream.getTitle()).thenReturn("some name");
when(otherStream.getId()).thenReturn("id2");
when(streamService.loadAll()).thenReturn(Lists.newArrayList(defaultStream, otherStream));
when(streamService.loadAllEnabled()).thenReturn(Lists.newArrayList(defaultStream, otherStream));
streamCacheService = new StreamCacheService(eventBus, streamService, null);
streamCacheService.startAsync().awaitRunning();
final Provider<Stream> defaultStreamProvider = () -> defaultStream;
functions.put(RouteToStream.NAME, new RouteToStream(streamCacheService, defaultStreamProvider));
functions.put(RemoveFromStream.NAME, new RemoveFromStream(streamCacheService, defaultStreamProvider));
lookupTableService = mock(LookupTableService.class, RETURNS_DEEP_STUBS);
lookupTable = spy(LookupTable.class);
when(lookupTableService.getTable(anyString())).thenReturn(lookupTable);
lookupServiceFunction = new LookupTableService.Function(lookupTableService, "table");
when(lookupTableService.newBuilder().lookupTable(anyString()).build()).thenReturn(lookupServiceFunction);
// input related functions
// TODO needs mock
// functions.put(FromInput.NAME, new FromInput());
// generic functions
functions.put(RegexMatch.NAME, new RegexMatch());
functions.put(RegexReplace.NAME, new RegexReplace());
// string functions
functions.put(Abbreviate.NAME, new Abbreviate());
functions.put(Capitalize.NAME, new Capitalize());
functions.put(Concat.NAME, new Concat());
functions.put(Contains.NAME, new Contains());
functions.put(EndsWith.NAME, new EndsWith());
functions.put(Lowercase.NAME, new Lowercase());
functions.put(Substring.NAME, new Substring());
functions.put(Swapcase.NAME, new Swapcase());
functions.put(Uncapitalize.NAME, new Uncapitalize());
functions.put(Uppercase.NAME, new Uppercase());
functions.put(KeyValue.NAME, new KeyValue());
functions.put(Join.NAME, new Join());
functions.put(Split.NAME, new Split());
functions.put(StartsWith.NAME, new StartsWith());
functions.put(Replace.NAME, new Replace());
functions.put(Length.NAME, new Length());
functions.put(FirstNonNull.NAME, new FirstNonNull());
final ObjectMapper objectMapper = new ObjectMapperProvider().get();
functions.put(JsonParse.NAME, new JsonParse(objectMapper));
functions.put(SelectJsonPath.NAME, new SelectJsonPath(objectMapper));
functions.put(DateConversion.NAME, new DateConversion());
functions.put(Now.NAME, new Now());
functions.put(FlexParseDate.NAME, new FlexParseDate());
functions.put(ParseDate.NAME, new ParseDate());
functions.put(ParseUnixMilliseconds.NAME, new ParseUnixMilliseconds());
functions.put(FormatDate.NAME, new FormatDate());
functions.put(Years.NAME, new Years());
functions.put(Months.NAME, new Months());
functions.put(Weeks.NAME, new Weeks());
functions.put(Days.NAME, new Days());
functions.put(Hours.NAME, new Hours());
functions.put(Minutes.NAME, new Minutes());
functions.put(Seconds.NAME, new Seconds());
functions.put(Millis.NAME, new Millis());
functions.put(PeriodParseFunction.NAME, new PeriodParseFunction());
functions.put(CRC32.NAME, new CRC32());
functions.put(CRC32C.NAME, new CRC32C());
functions.put(MD5.NAME, new MD5());
functions.put(Murmur3_32.NAME, new Murmur3_32());
functions.put(Murmur3_128.NAME, new Murmur3_128());
functions.put(SHA1.NAME, new SHA1());
functions.put(SHA256.NAME, new SHA256());
functions.put(SHA512.NAME, new SHA512());
functions.put(Base16Encode.NAME, new Base16Encode());
functions.put(Base16Decode.NAME, new Base16Decode());
functions.put(Base32Encode.NAME, new Base32Encode());
functions.put(Base32Decode.NAME, new Base32Decode());
functions.put(Base32HumanEncode.NAME, new Base32HumanEncode());
functions.put(Base32HumanDecode.NAME, new Base32HumanDecode());
functions.put(Base64Encode.NAME, new Base64Encode());
functions.put(Base64Decode.NAME, new Base64Decode());
functions.put(Base64UrlEncode.NAME, new Base64UrlEncode());
functions.put(Base64UrlDecode.NAME, new Base64UrlDecode());
functions.put(IpAddressConversion.NAME, new IpAddressConversion());
functions.put(CidrMatch.NAME, new CidrMatch());
functions.put(IsNull.NAME, new IsNull());
functions.put(IsNotNull.NAME, new IsNotNull());
functions.put(SyslogPriorityConversion.NAME, new SyslogPriorityConversion());
functions.put(SyslogPriorityToStringConversion.NAME, new SyslogPriorityToStringConversion());
functions.put(SyslogFacilityConversion.NAME, new SyslogFacilityConversion());
functions.put(SyslogLevelConversion.NAME, new SyslogLevelConversion());
functions.put(UrlConversion.NAME, new UrlConversion());
functions.put(UrlDecode.NAME, new UrlDecode());
functions.put(UrlEncode.NAME, new UrlEncode());
functions.put(IsBoolean.NAME, new IsBoolean());
functions.put(IsNumber.NAME, new IsNumber());
functions.put(IsDouble.NAME, new IsDouble());
functions.put(IsLong.NAME, new IsLong());
functions.put(IsString.NAME, new IsString());
functions.put(IsCollection.NAME, new IsCollection());
functions.put(IsList.NAME, new IsList());
functions.put(IsMap.NAME, new IsMap());
functions.put(IsDate.NAME, new IsDate());
functions.put(IsPeriod.NAME, new IsPeriod());
functions.put(IsIp.NAME, new IsIp());
functions.put(IsJson.NAME, new IsJson());
functions.put(IsUrl.NAME, new IsUrl());
final GrokPatternService grokPatternService = mock(GrokPatternService.class);
final GrokPattern greedyPattern = GrokPattern.create("GREEDY", ".*");
Set<GrokPattern> patterns = Sets.newHashSet(greedyPattern, GrokPattern.create("GREEDY", ".*"), GrokPattern.create("BASE10NUM", "(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\\.[0-9]+)?)|(?:\\.[0-9]+)))"), GrokPattern.create("NUMBER", "(?:%{BASE10NUM:UNWANTED})"), GrokPattern.create("UNDERSCORE", "(?<test_field>test)"), GrokPattern.create("NUM", "%{BASE10NUM}"));
when(grokPatternService.loadAll()).thenReturn(patterns);
when(grokPatternService.loadByName("GREEDY")).thenReturn(Optional.of(greedyPattern));
final EventBus clusterBus = new EventBus();
final GrokPatternRegistry grokPatternRegistry = new GrokPatternRegistry(clusterBus, grokPatternService, Executors.newScheduledThreadPool(1));
functions.put(GrokMatch.NAME, new GrokMatch(grokPatternRegistry));
functions.put(GrokExists.NAME, new GrokExists(grokPatternRegistry));
functions.put(MetricCounterIncrement.NAME, new MetricCounterIncrement(metricRegistry));
functions.put(LookupSetValue.NAME, new LookupSetValue(lookupTableService));
functions.put(LookupClearKey.NAME, new LookupClearKey(lookupTableService));
functions.put(LookupSetStringList.NAME, new LookupSetStringList(lookupTableService));
functions.put(LookupAddStringList.NAME, new LookupAddStringList(lookupTableService));
functions.put(LookupRemoveStringList.NAME, new LookupRemoveStringList(lookupTableService));
functionRegistry = new FunctionRegistry(functions);
}
use of org.graylog2.plugin.streams.DefaultStream in project graylog2-server by Graylog2.
the class BaseParserTest method evaluateRule.
@Nullable
protected Message evaluateRule(Rule rule, Consumer<Message> messageModifier) {
final Message message = new Message("hello test", "source", DateTime.now(DateTimeZone.UTC));
message.addStream(defaultStream);
messageModifier.accept(message);
return evaluateRule(rule, message);
}
use of org.graylog2.plugin.streams.DefaultStream in project graylog2-server by Graylog2.
the class StreamRouterEngine method match.
/**
* Returns a list of matching streams for the given message.
*
* @param message the message
* @return the list of matching streams
*/
public List<Stream> match(Message message) {
final Set<Stream> result = Sets.newHashSet();
final Set<String> blackList = Sets.newHashSet();
for (final Rule rule : rulesList) {
if (blackList.contains(rule.getStreamId())) {
continue;
}
final StreamRule streamRule = rule.getStreamRule();
final StreamRuleType streamRuleType = streamRule.getType();
final Stream.MatchingType matchingType = rule.getMatchingType();
if (!ruleTypesNotNeedingFieldPresence.contains(streamRuleType) && !message.hasField(streamRule.getField())) {
if (matchingType == Stream.MatchingType.AND) {
result.remove(rule.getStream());
// blacklist stream because it can't match anymore
blackList.add(rule.getStreamId());
}
continue;
}
final Stream stream;
if (streamRuleType != StreamRuleType.REGEX) {
stream = rule.match(message);
} else {
stream = rule.matchWithTimeOut(message, streamProcessingTimeout, TimeUnit.MILLISECONDS);
}
if (stream == null) {
if (matchingType == Stream.MatchingType.AND) {
result.remove(rule.getStream());
// blacklist stream because it can't match anymore
blackList.add(rule.getStreamId());
}
} else {
result.add(stream);
if (matchingType == Stream.MatchingType.OR) {
// blacklist stream because it is already matched
blackList.add(rule.getStreamId());
}
}
}
final Stream defaultStream = defaultStreamProvider.get();
boolean alreadyRemovedDefaultStream = false;
for (Stream stream : result) {
streamMetrics.markIncomingMeter(stream.getId());
if (stream.getRemoveMatchesFromDefaultStream()) {
if (alreadyRemovedDefaultStream || message.removeStream(defaultStream)) {
alreadyRemovedDefaultStream = true;
if (LOG.isTraceEnabled()) {
LOG.trace("Successfully removed default stream <{}> from message <{}>", defaultStream.getId(), message.getId());
}
} else {
if (LOG.isWarnEnabled()) {
LOG.warn("Couldn't remove default stream <{}> from message <{}>", defaultStream.getId(), message.getId());
}
}
}
}
// or someone removed it, in which case we don't mark it.
if (!alreadyRemovedDefaultStream) {
streamMetrics.markIncomingMeter(defaultStream.getId());
}
return ImmutableList.copyOf(result);
}
use of org.graylog2.plugin.streams.DefaultStream in project graylog2-server by Graylog2.
the class StreamRouterEngineTest method testRemoveFromAllMessages.
@Test
public void testRemoveFromAllMessages() throws Exception {
final StreamMock stream = getStreamMock("test");
final StreamRuleMock rule = new StreamRuleMock(ImmutableMap.of("_id", new ObjectId(), "field", "testfield", "type", StreamRuleType.PRESENCE.toInteger(), "stream_id", stream.getId()));
stream.setRemoveMatchesFromDefaultStream(true);
stream.setStreamRules(Collections.singletonList(rule));
final StreamRouterEngine engine = newEngine(Collections.singletonList(stream));
final Message message = getMessage();
message.addStream(defaultStream);
assertThat(message.getStreams()).containsExactly(defaultStream);
// Without testfield in the message.
assertThat(engine.match(message)).isEmpty();
// With field in the message.
message.addField("testfield", "testvalue");
assertThat(engine.match(message)).containsExactly(stream);
assertThat(message.getStreams()).doesNotContain(defaultStream);
}
use of org.graylog2.plugin.streams.DefaultStream in project graylog2-server by Graylog2.
the class DefaultStreamProvider method get.
@Override
public Stream get() {
Stream defaultStream = sharedInstance.get();
if (defaultStream != null) {
return defaultStream;
}
synchronized (this) {
defaultStream = sharedInstance.get();
if (defaultStream != null) {
return defaultStream;
}
int i = 0;
do {
try {
LOG.debug("Loading shared default stream instance");
defaultStream = service.load(Stream.DEFAULT_STREAM_ID);
} catch (NotFoundException ignored) {
if (i % 10 == 0) {
LOG.warn("Unable to load default stream, tried {} times, retrying every 500ms. Processing is blocked until this succeeds.", i + 1);
}
i++;
Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
}
} while (defaultStream == null);
sharedInstance.set(defaultStream);
}
return defaultStream;
}
Aggregations