Examples with StreamingEndpoint com.twitter.hbc.core.endpoint.StreamingEndpoint used on opensource projects

Example 1 with StreamingEndpoint

use of com.twitter.hbc.core.endpoint.StreamingEndpoint in project nifi by apache.

the class GetTwitter method onScheduled.

@OnScheduled
public void onScheduled(final ProcessContext context) throws MalformedURLException {
    final String endpointName = context.getProperty(ENDPOINT).getValue();
    final Authentication oauth = new OAuth1(context.getProperty(CONSUMER_KEY).getValue(), context.getProperty(CONSUMER_SECRET).getValue(), context.getProperty(ACCESS_TOKEN).getValue(), context.getProperty(ACCESS_TOKEN_SECRET).getValue());
    final ClientBuilder clientBuilder = new ClientBuilder();
    clientBuilder.name("GetTwitter[id=" + getIdentifier() + "]").authentication(oauth).eventMessageQueue(eventQueue).processor(new StringDelimitedProcessor(messageQueue));
    final String languageString = context.getProperty(LANGUAGES).getValue();
    final List<String> languages;
    if (languageString == null) {
        languages = null;
    } else {
        languages = new ArrayList<>();
        for (final String language : context.getProperty(LANGUAGES).getValue().split(",")) {
            languages.add(language.trim());
        }
    }
    final String host;
    final StreamingEndpoint streamingEndpoint;
    if (ENDPOINT_SAMPLE.getValue().equals(endpointName)) {
        host = Constants.STREAM_HOST;
        final StatusesSampleEndpoint sse = new StatusesSampleEndpoint();
        streamingEndpoint = sse;
        if (languages != null) {
            sse.languages(languages);
        }
    } else if (ENDPOINT_FIREHOSE.getValue().equals(endpointName)) {
        host = Constants.STREAM_HOST;
        final StatusesFirehoseEndpoint firehoseEndpoint = new StatusesFirehoseEndpoint();
        streamingEndpoint = firehoseEndpoint;
        if (languages != null) {
            firehoseEndpoint.languages(languages);
        }
    } else if (ENDPOINT_FILTER.getValue().equals(endpointName)) {
        host = Constants.STREAM_HOST;
        final StatusesFilterEndpoint filterEndpoint = new StatusesFilterEndpoint();
        final String followingString = context.getProperty(FOLLOWING).getValue();
        final List<Long> followingIds;
        if (followingString == null) {
            followingIds = Collections.emptyList();
        } else {
            followingIds = new ArrayList<>();
            for (final String split : followingString.split(",")) {
                final Long id = Long.parseLong(split.trim());
                followingIds.add(id);
            }
        }
        final String termString = context.getProperty(TERMS).getValue();
        final List<String> terms;
        if (termString == null) {
            terms = Collections.emptyList();
        } else {
            terms = new ArrayList<>();
            for (final String split : termString.split(",")) {
                terms.add(split.trim());
            }
        }
        if (!terms.isEmpty()) {
            filterEndpoint.trackTerms(terms);
        }
        if (!followingIds.isEmpty()) {
            filterEndpoint.followings(followingIds);
        }
        if (languages != null) {
            filterEndpoint.languages(languages);
        }
        final String locationString = context.getProperty(LOCATIONS).getValue();
        final List<Location> locations;
        if (locationString == null) {
            locations = Collections.emptyList();
        } else {
            locations = LocationUtil.parseLocations(locationString);
        }
        if (!locations.isEmpty()) {
            filterEndpoint.locations(locations);
        }
        streamingEndpoint = filterEndpoint;
    } else {
        throw new AssertionError("Endpoint was invalid value: " + endpointName);
    }
    clientBuilder.hosts(host).endpoint(streamingEndpoint);
    client = clientBuilder.build();
    client.connect();
}

Example 2 with StreamingEndpoint

use of com.twitter.hbc.core.endpoint.StreamingEndpoint in project flink by apache.

the class TwitterSource method run.

@Override
public void run(final SourceContext<String> ctx) throws Exception {
    LOG.info("Initializing Twitter Streaming API connection");
    StreamingEndpoint endpoint = initializer.createEndpoint();
    Authentication auth = new OAuth1(properties.getProperty(CONSUMER_KEY), properties.getProperty(CONSUMER_SECRET), properties.getProperty(TOKEN), properties.getProperty(TOKEN_SECRET));
    client = new ClientBuilder().name(properties.getProperty(CLIENT_NAME, "flink-twitter-source")).hosts(properties.getProperty(CLIENT_HOSTS, Constants.STREAM_HOST)).endpoint(endpoint).authentication(auth).processor(new HosebirdMessageProcessor() {

        public DelimitedStreamReader reader;

        @Override
        public void setup(InputStream input) {
            reader = new DelimitedStreamReader(input, Constants.DEFAULT_CHARSET, Integer.parseInt(properties.getProperty(CLIENT_BUFFER_SIZE, "50000")));
        }

        @Override
        public boolean process() throws IOException, InterruptedException {
            String line = reader.readLine();
            ctx.collect(line);
            return true;
        }
    }).build();
    client.connect();
    running = true;
    LOG.info("Twitter Streaming API connection established successfully");
    // just wait now
    while (running) {
        synchronized (waitLock) {
            waitLock.wait(100L);
        }
    }
}

Example 3 with StreamingEndpoint

use of com.twitter.hbc.core.endpoint.StreamingEndpoint in project ignite by apache.

the class TwitterStreamer method getClient.

/**
 * @param tweetQueue Tweet queue.
 * @return Client.
 */
protected Client getClient(BlockingQueue<String> tweetQueue) {
    StreamingEndpoint endpoint;
    HttpHosts hosts;
    switch(endpointUrl.toLowerCase()) {
        case StatusesFilterEndpoint.PATH:
            endpoint = new StatusesFilterEndpoint();
            hosts = HttpHosts.STREAM_HOST;
            break;
        case StatusesFirehoseEndpoint.PATH:
            endpoint = new StatusesFirehoseEndpoint();
            hosts = HttpHosts.STREAM_HOST;
            break;
        case StatusesSampleEndpoint.PATH:
            endpoint = new StatusesSampleEndpoint();
            hosts = HttpHosts.STREAM_HOST;
            break;
        case UserstreamEndpoint.PATH:
            endpoint = new UserstreamEndpoint();
            hosts = HttpHosts.USERSTREAM_HOST;
            break;
        case SitestreamEndpoint.PATH:
            String follow = apiParams.remove(SITE_USER_ID_KEY);
            List<Long> followers = Lists.newArrayList();
            for (String follower : Splitter.on(',').trimResults().omitEmptyStrings().split(follow)) followers.add(Long.valueOf(follower));
            endpoint = new SitestreamEndpoint(followers);
            hosts = HttpHosts.SITESTREAM_HOST;
            break;
        default:
            endpoint = new DefaultStreamingEndpoint(endpointUrl, HttpConstants.HTTP_GET, false);
            hosts = HttpHosts.STREAM_HOST;
    }
    for (Map.Entry<String, String> entry : apiParams.entrySet()) {
        endpoint.addPostParameter(entry.getKey(), entry.getValue());
    }
    return buildClient(tweetQueue, hosts, endpoint);
}