Examples with HashtagEntity twitter4j.HashtagEntity used on opensource projects

Example 1 with HashtagEntity

use of twitter4j.HashtagEntity in project druid by druid-io.

the class TwitterSpritzerFirehoseFactory method connect.

@Override
public Firehose connect(InputRowParser parser) throws IOException {
    final ConnectionLifeCycleListener connectionLifeCycleListener = new ConnectionLifeCycleListener() {

        @Override
        public void onConnect() {
            log.info("Connected_to_Twitter");
        }

        @Override
        public void onDisconnect() {
            log.info("Disconnect_from_Twitter");
        }

        /**
       * called before thread gets cleaned up
       */
        @Override
        public void onCleanUp() {
            log.info("Cleanup_twitter_stream");
        }
    };
    // ConnectionLifeCycleListener
    final TwitterStream twitterStream;
    final StatusListener statusListener;
    final int QUEUE_SIZE = 2000;
    /** This queue is used to move twitter events from the twitter4j thread to the druid ingest thread.   */
    final BlockingQueue<Status> queue = new ArrayBlockingQueue<Status>(QUEUE_SIZE);
    final long startMsec = System.currentTimeMillis();
    //
    //   set up Twitter Spritzer
    //
    twitterStream = new TwitterStreamFactory().getInstance();
    twitterStream.addConnectionLifeCycleListener(connectionLifeCycleListener);
    statusListener = new StatusListener() {

        // This is what really gets called to deliver stuff from twitter4j
        @Override
        public void onStatus(Status status) {
            // time to stop?
            if (Thread.currentThread().isInterrupted()) {
                throw new RuntimeException("Interrupted, time to stop");
            }
            try {
                boolean success = queue.offer(status, 15L, TimeUnit.SECONDS);
                if (!success) {
                    log.warn("queue too slow!");
                }
            } catch (InterruptedException e) {
                throw new RuntimeException("InterruptedException", e);
            }
        }

        @Override
        public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
        //log.info("Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
        }

        @Override
        public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            // This notice will be sent each time a limited stream becomes unlimited.
            // If this number is high and or rapidly increasing, it is an indication that your predicate is too broad, and you should consider a predicate with higher selectivity.
            log.warn("Got track limitation notice:" + numberOfLimitedStatuses);
        }

        @Override
        public void onScrubGeo(long userId, long upToStatusId) {
        //log.info("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
        }

        @Override
        public void onException(Exception ex) {
            ex.printStackTrace();
        }

        @Override
        public void onStallWarning(StallWarning warning) {
            System.out.println("Got stall warning:" + warning);
        }
    };
    twitterStream.addListener(statusListener);
    // creates a generic StatusStream
    twitterStream.sample();
    log.info("returned from sample()");
    return new Firehose() {

        private final Runnable doNothingRunnable = new Runnable() {

            public void run() {
            }
        };

        private long rowCount = 0L;

        private boolean waitIfmax = (getMaxEventCount() < 0L);

        private final Map<String, Object> theMap = new TreeMap<>();

        // DIY json parsing // private final ObjectMapper omapper = new ObjectMapper();
        private boolean maxTimeReached() {
            if (getMaxRunMinutes() <= 0) {
                return false;
            } else {
                return (System.currentTimeMillis() - startMsec) / 60000L >= getMaxRunMinutes();
            }
        }

        private boolean maxCountReached() {
            return getMaxEventCount() >= 0 && rowCount >= getMaxEventCount();
        }

        @Override
        public boolean hasMore() {
            if (maxCountReached() || maxTimeReached()) {
                return waitIfmax;
            } else {
                return true;
            }
        }

        @Override
        public InputRow nextRow() {
            // Interrupted to stop?
            if (Thread.currentThread().isInterrupted()) {
                throw new RuntimeException("Interrupted, time to stop");
            }
            // all done?
            if (maxCountReached() || maxTimeReached()) {
                if (waitIfmax) {
                    // sleep a long time instead of terminating
                    try {
                        log.info("reached limit, sleeping a long time...");
                        Thread.sleep(2000000000L);
                    } catch (InterruptedException e) {
                        throw new RuntimeException("InterruptedException", e);
                    }
                } else {
                // allow this event through, and the next hasMore() call will be false
                }
            }
            if (++rowCount % 1000 == 0) {
                log.info("nextRow() has returned %,d InputRows", rowCount);
            }
            Status status;
            try {
                status = queue.take();
            } catch (InterruptedException e) {
                throw new RuntimeException("InterruptedException", e);
            }
            theMap.clear();
            HashtagEntity[] hts = status.getHashtagEntities();
            String text = status.getText();
            theMap.put("text", (null == text) ? "" : text);
            theMap.put("htags", (hts.length > 0) ? Lists.transform(Arrays.asList(hts), new Function<HashtagEntity, String>() {

                @Nullable
                @Override
                public String apply(HashtagEntity input) {
                    return input.getText();
                }
            }) : ImmutableList.<String>of());
            long[] lcontrobutors = status.getContributors();
            List<String> contributors = new ArrayList<>();
            for (long contrib : lcontrobutors) {
                contributors.add(String.format("%d", contrib));
            }
            theMap.put("contributors", contributors);
            GeoLocation geoLocation = status.getGeoLocation();
            if (null != geoLocation) {
                double lat = status.getGeoLocation().getLatitude();
                double lon = status.getGeoLocation().getLongitude();
                theMap.put("lat", lat);
                theMap.put("lon", lon);
            } else {
                theMap.put("lat", null);
                theMap.put("lon", null);
            }
            if (status.getSource() != null) {
                Matcher m = sourcePattern.matcher(status.getSource());
                theMap.put("source", m.find() ? m.group(1) : status.getSource());
            }
            theMap.put("retweet", status.isRetweet());
            if (status.isRetweet()) {
                Status original = status.getRetweetedStatus();
                theMap.put("retweet_count", original.getRetweetCount());
                User originator = original.getUser();
                theMap.put("originator_screen_name", originator != null ? originator.getScreenName() : "");
                theMap.put("originator_follower_count", originator != null ? originator.getFollowersCount() : "");
                theMap.put("originator_friends_count", originator != null ? originator.getFriendsCount() : "");
                theMap.put("originator_verified", originator != null ? originator.isVerified() : "");
            }
            User user = status.getUser();
            final boolean hasUser = (null != user);
            theMap.put("follower_count", hasUser ? user.getFollowersCount() : 0);
            theMap.put("friends_count", hasUser ? user.getFriendsCount() : 0);
            theMap.put("lang", hasUser ? user.getLang() : "");
            // resolution in seconds, -1 if not available?
            theMap.put("utc_offset", hasUser ? user.getUtcOffset() : -1);
            theMap.put("statuses_count", hasUser ? user.getStatusesCount() : 0);
            theMap.put("user_id", hasUser ? String.format("%d", user.getId()) : "");
            theMap.put("screen_name", hasUser ? user.getScreenName() : "");
            theMap.put("location", hasUser ? user.getLocation() : "");
            theMap.put("verified", hasUser ? user.isVerified() : "");
            theMap.put("ts", status.getCreatedAt().getTime());
            List<String> dimensions = Lists.newArrayList(theMap.keySet());
            return new MapBasedInputRow(status.getCreatedAt().getTime(), dimensions, theMap);
        }

        @Override
        public Runnable commit() {
            // reuse the same object each time
            return doNothingRunnable;
        }

        @Override
        public void close() throws IOException {
            log.info("CLOSE twitterstream");
            // invokes twitterStream.cleanUp()
            twitterStream.shutdown();
        }
    };
}

Example 2 with HashtagEntity

use of twitter4j.HashtagEntity in project Talon-for-Twitter by klinker24.

the class TweetLinkUtils method getLinksInStatus.

private static String[] getLinksInStatus(String tweetTexts, UserMentionEntity[] users, HashtagEntity[] hashtags, URLEntity[] urls, MediaEntity[] medias) {
    String mUsers = "";
    for (UserMentionEntity name : users) {
        String n = name.getScreenName();
        if (n.length() > 1) {
            mUsers += n + "  ";
        }
    }
    String mHashtags = "";
    for (HashtagEntity hashtagEntity : hashtags) {
        String text = hashtagEntity.getText();
        if (text.length() > 1) {
            mHashtags += text + "  ";
        }
    }
    String expandedUrls = "";
    String compressedUrls = "";
    for (URLEntity entity : urls) {
        String url = entity.getExpandedURL();
        if (url.length() > 1) {
            expandedUrls += url + "  ";
            compressedUrls += entity.getURL() + "  ";
        }
    }
    String mediaExp = "";
    String mediaComp = "";
    String mediaDisplay = "";
    for (MediaEntity e : medias) {
        String url = e.getURL();
        if (url.length() > 1) {
            mediaComp += url + "  ";
            mediaExp += e.getExpandedURL() + "  ";
            mediaDisplay += e.getDisplayURL() + "  ";
        }
    }
    String[] sExpandedUrls;
    String[] sCompressedUrls;
    String[] sMediaExp;
    String[] sMediaComp;
    String[] sMediaDisplay;
    try {
        sCompressedUrls = compressedUrls.split("  ");
    } catch (Exception e) {
        sCompressedUrls = new String[0];
    }
    try {
        sExpandedUrls = expandedUrls.split("  ");
    } catch (Exception e) {
        sExpandedUrls = new String[0];
    }
    try {
        sMediaComp = mediaComp.split("  ");
    } catch (Exception e) {
        sMediaComp = new String[0];
    }
    try {
        sMediaExp = mediaExp.split("  ");
    } catch (Exception e) {
        sMediaExp = new String[0];
    }
    try {
        sMediaDisplay = mediaDisplay.split("  ");
    } catch (Exception e) {
        sMediaDisplay = new String[0];
    }
    String imageUrl = "";
    String otherUrl = "";
    for (int i = 0; i < sCompressedUrls.length; i++) {
        String comp = sCompressedUrls[i];
        String exp = sExpandedUrls[i];
        if (comp.length() > 1 && exp.length() > 1) {
            String str = exp.toLowerCase();
            try {
                String replacement = exp.replace("http://", "").replace("https://", "").replace("www.", "");
                boolean hasCom = replacement.contains(".com");
                replacement = replacement.substring(0, 30) + "...";
                if (hasCom && !replacement.contains(".com")) {
                    // the link was too long...
                    replacement = exp.replace("http://", "").replace("https://", "").replace("www.", "");
                    replacement = replacement.substring(0, replacement.indexOf(".com") + 6) + "...";
                }
                tweetTexts = tweetTexts.replace(comp, replacement);
            } catch (Exception e) {
                tweetTexts = tweetTexts.replace(comp, exp.replace("http://", "").replace("https://", "").replace("www.", ""));
            }
            if (str.contains("instag") && !str.contains("blog.insta")) {
                imageUrl = exp + "media/?size=l";
                otherUrl += exp + "  ";
            } else if (exp.toLowerCase().contains("youtub") && !(str.contains("channel") || str.contains("user") || str.contains("playlist"))) {
                // first get the youtube surfaceView code
                int start = exp.indexOf("v=") + 2;
                int end = exp.length();
                if (exp.substring(start).contains("&")) {
                    end = exp.indexOf("&");
                } else if (exp.substring(start).contains("?")) {
                    end = exp.indexOf("?");
                }
                try {
                    imageUrl = "http://img.youtube.com/vi/" + exp.substring(start, end) + "/hqdefault.jpg";
                } catch (Exception e) {
                    imageUrl = "http://img.youtube.com/vi/" + exp.substring(start, exp.length() - 1) + "/hqdefault.jpg";
                }
                otherUrl += exp + "  ";
            } else if (str.contains("youtu.be")) {
                // first get the youtube surfaceView code
                int start = exp.indexOf(".be/") + 4;
                int end = exp.length();
                if (exp.substring(start).contains("&")) {
                    end = exp.indexOf("&");
                } else if (exp.substring(start).contains("?")) {
                    end = exp.indexOf("?");
                }
                try {
                    imageUrl = "http://img.youtube.com/vi/" + exp.substring(start, end) + "/hqdefault.jpg";
                } catch (Exception e) {
                    imageUrl = "http://img.youtube.com/vi/" + exp.substring(start, exp.length() - 1) + "/hqdefault.jpg";
                }
                otherUrl += exp + "  ";
            } else if (str.contains("twitpic")) {
                int start = exp.indexOf(".com/") + 5;
                imageUrl = "http://twitpic.com/show/full/" + exp.substring(start).replace("/", "");
                otherUrl += exp + "  ";
            } else if (str.contains("i.imgur") && !str.contains("/a/") && !str.contains(".gifv")) {
                int start = exp.indexOf(".com/") + 5;
                imageUrl = "http://i.imgur.com/" + exp.replace("http://i.imgur.com/", "").replace(".jpg", "") + "l.jpg";
                imageUrl = imageUrl.replace("gallery/", "");
                otherUrl += exp + "  ";
            } else if (str.contains("imgur") && !str.contains("/a/") && !str.contains(".gifv")) {
                int start = exp.indexOf(".com/") + 6;
                imageUrl = "http://i.imgur.com/" + exp.replace("http://imgur.com/", "").replace(".jpg", "") + "l.jpg";
                imageUrl = imageUrl.replace("gallery/", "").replace("a/", "");
                otherUrl += exp + "  ";
            } else if (str.contains("pbs.twimg.com")) {
                imageUrl = exp;
                otherUrl += exp + "  ";
            } else if (str.contains("ow.ly/i/")) {
                imageUrl = "http://static.ow.ly/photos/original/" + exp.substring(exp.lastIndexOf("/")).replaceAll("/", "") + ".jpg";
                otherUrl += exp + "  ";
            } else if (str.contains("p.twipple.jp")) {
                imageUrl = "http://p.twipple.jp/show/large/" + exp.replace("p.twipple.jp/", "").replace("http://", "").replace("https://", "").replace("www.", "");
                otherUrl += exp + "  ";
            } else if (str.contains(".jpg") || str.contains(".png")) {
                imageUrl = exp;
                otherUrl += exp + "  ";
            } else if (str.contains("img.ly")) {
                imageUrl = exp.replace("https", "http").replace("http://img.ly/", "http://img.ly/show/large/");
                otherUrl += exp + "  ";
            } else {
                otherUrl += exp + "  ";
            }
        }
    }
    for (int i = 0; i < sMediaComp.length; i++) {
        String comp = sMediaComp[i];
        String exp = sMediaExp[i];
        if (comp.length() > 1 && exp.length() > 1) {
            try {
                String replacement = sMediaDisplay[i].replace("http://", "").replace("https://", "").replace("www.", "");
                boolean hasCom = replacement.contains(".com");
                replacement = replacement.substring(0, 22) + "...";
                if (hasCom && !replacement.contains(".com")) {
                    // the link was too long...
                    replacement = sMediaDisplay[i].replace("http://", "").replace("https://", "").replace("www.", "");
                    replacement = replacement.substring(0, replacement.indexOf(".com") + 6) + "...";
                }
                tweetTexts = tweetTexts.replace(comp, replacement);
            } catch (Exception e) {
                e.printStackTrace();
                tweetTexts = tweetTexts.replace(comp, sMediaDisplay[i].replace("http://", "").replace("https://", "").replace("www.", ""));
            }
            imageUrl = medias[0].getMediaURL();
            for (MediaEntity m : medias) {
                if (m.getType().equals("photo")) {
                    if (!imageUrl.contains(m.getMediaURL())) {
                        imageUrl += " " + m.getMediaURL();
                    }
                }
            }
            otherUrl += sMediaDisplay[i];
        }
    }
    return new String[] { tweetTexts, imageUrl, otherUrl, mHashtags, mUsers };
}

Example 3 with HashtagEntity

use of twitter4j.HashtagEntity in project twicalico by moko256.

the class TwitterStringUtils method setLinkedSequenceTo.

@SuppressLint("StaticFieldLeak")
public static void setLinkedSequenceTo(Status item, TextView textView) {
    Context context = textView.getContext();
    String tweet = item.getText();
    if (GlobalApplication.clientType == Type.MASTODON) {
        Spanned html = Html.fromHtml(tweet);
        int length = html.length();
        // Trim unless \n\n made by fromHtml() after post
        if (length == 3 && item.getMediaEntities().length > 0 && html.charAt(0) == ".".charAt(0)) {
            // If post has media only, context of post from Mastodon is "."
            textView.setText("");
            return;
        } else if (length >= 2) {
            html = (Spanned) html.subSequence(0, length - 2);
        }
        SpannableStringBuilder builder = convertUrlSpanToCustomTabs(html, context);
        textView.setText(builder);
        List<Emoji> list = ((StatusCacheMap.CachedStatus) item).getEmojis();
        if (list != null) {
            Matcher matcher = containsEmoji.matcher(builder);
            boolean matches = matcher.matches();
            int imageSize;
            if (matches) {
                imageSize = (int) Math.floor((textView.getTextSize() * 1.15) * 2.0);
            } else {
                imageSize = (int) Math.floor(textView.getTextSize() * 1.15);
            }
            new AsyncTask<Void, Void, Map<String, Drawable>>() {

                @Override
                protected Map<String, Drawable> doInBackground(Void... params) {
                    Map<String, Drawable> map = new ArrayMap<>();
                    GlideRequests glideRequests = GlideApp.with(context);
                    for (Emoji emoji : list) {
                        try {
                            Drawable value = glideRequests.load(emoji.getUrl()).submit().get();
                            value.setBounds(0, 0, imageSize, imageSize);
                            map.put(emoji.getShortCode(), value);
                        } catch (InterruptedException | ExecutionException e) {
                            e.printStackTrace();
                        }
                    }
                    return map;
                }

                @Override
                protected void onPostExecute(Map<String, Drawable> map) {
                    if (TextUtils.equals(builder, textView.getText())) {
                        boolean found = matches || matcher.find();
                        while (found) {
                            String shortCode = matcher.group(1);
                            Drawable drawable = map.get(shortCode);
                            if (drawable != null) {
                                builder.setSpan(new ImageSpan(drawable), matcher.start(), matcher.end(), Spanned.SPAN_EXCLUSIVE_EXCLUSIVE);
                            }
                            found = matcher.find();
                        }
                        textView.setText(builder);
                    }
                }
            }.execute();
        }
        return;
    }
    SpannableStringBuilder spannableStringBuilder = new SpannableStringBuilder(tweet);
    for (SymbolEntity symbolEntity : item.getSymbolEntities()) {
        spannableStringBuilder.setSpan(new ClickableSpan() {

            @Override
            public void onClick(View view) {
                context.startActivity(SearchResultActivity.getIntent(context, symbolEntity.getText()));
            }
        }, tweet.offsetByCodePoints(0, symbolEntity.getStart()), tweet.offsetByCodePoints(0, symbolEntity.getEnd()), Spanned.SPAN_INCLUSIVE_INCLUSIVE);
    }
    for (HashtagEntity hashtagEntity : item.getHashtagEntities()) {
        spannableStringBuilder.setSpan(new ClickableSpan() {

            @Override
            public void onClick(View view) {
                context.startActivity(SearchResultActivity.getIntent(context, "#" + hashtagEntity.getText()));
            }
        }, tweet.offsetByCodePoints(0, hashtagEntity.getStart()), tweet.offsetByCodePoints(0, hashtagEntity.getEnd()), Spanned.SPAN_INCLUSIVE_INCLUSIVE);
    }
    for (UserMentionEntity userMentionEntity : item.getUserMentionEntities()) {
        spannableStringBuilder.setSpan(new ClickableSpan() {

            @Override
            public void onClick(View view) {
                context.startActivity(ShowUserActivity.getIntent(context, userMentionEntity.getScreenName()));
            }
        }, tweet.offsetByCodePoints(0, userMentionEntity.getStart()), tweet.offsetByCodePoints(0, userMentionEntity.getEnd()), Spanned.SPAN_INCLUSIVE_INCLUSIVE);
    }
    boolean hasMedia = item.getMediaEntities().length > 0;
    List<URLEntity> urlEntities = new ArrayList<>(item.getURLEntities().length + (hasMedia ? 1 : 0));
    urlEntities.addAll(Arrays.asList(item.getURLEntities()));
    if (hasMedia) {
        urlEntities.add(item.getMediaEntities()[0]);
    }
    int tweetLength = tweet.codePointCount(0, tweet.length());
    int sp = 0;
    for (URLEntity entity : urlEntities) {
        String url = entity.getURL();
        String displayUrl = entity.getDisplayURL();
        int urlLength = url.codePointCount(0, url.length());
        int displayUrlLength = displayUrl.codePointCount(0, displayUrl.length());
        if (entity.getStart() <= tweetLength && entity.getEnd() <= tweetLength) {
            int dusp = displayUrlLength - urlLength;
            spannableStringBuilder.replace(tweet.offsetByCodePoints(0, entity.getStart()) + sp, tweet.offsetByCodePoints(0, entity.getEnd()) + sp, displayUrl);
            spannableStringBuilder.setSpan(new ClickableSpan() {

                @Override
                public void onClick(View view) {
                    AppCustomTabsKt.launchChromeCustomTabs(context, entity.getExpandedURL());
                }
            }, tweet.offsetByCodePoints(0, entity.getStart()) + sp, tweet.offsetByCodePoints(0, entity.getEnd()) + sp + dusp, Spanned.SPAN_INCLUSIVE_INCLUSIVE);
            sp += dusp;
        }
    }
    textView.setText(spannableStringBuilder);
}