Examples with ResourceItem com.virjar.vscrawler.core.resourcemanager.model.ResourceItem used on opensource projects

Example 1 with ResourceItem

use of com.virjar.vscrawler.core.resourcemanager.model.ResourceItem in project vscrawler by virjar.

the class UserManager2 method allocateUser.

@Override
public User allocateUser() {
    ResourceItem resourceItem = resourceManager.allocate(vsCrawlerContext.makeUserResourceTag());
    if (resourceItem == null) {
        return null;
    }
    JSONObject jsonObject = JSONObject.parseObject(resourceItem.getData());
    return JSONObject.toJavaObject(jsonObject, User.class);
}

Example 2 with ResourceItem

use of com.virjar.vscrawler.core.resourcemanager.model.ResourceItem in project vscrawler by virjar.

the class UserManager2ResourceLoader method loadResource.

@Override
public boolean loadResource(Collection<ResourceItem> resourceItems) {
    for (User user : userResourceFacade.importUser()) {
        ResourceItem resourceItem = new ResourceItem();
        resourceItem.setData(JSONObject.toJSONString(user));
        resourceItem.setKey(user.getUserID());
        resourceItems.add(resourceItem);
    }
    return false;
}

Example 3 with ResourceItem

use of com.virjar.vscrawler.core.resourcemanager.model.ResourceItem in project vscrawler by virjar.

the class JedisScoredQueueStore method addBatch.

@Override
public void addBatch(String queueID, Set<ResourceItem> resourceItems) {
    if (!lockQueue(queueID)) {
        return;
    }
    final Jedis jedis = jedisPool.getResource();
    final String dataKey = makeDataKey(queueID);
    try {
        final Set<String> hkeys = jedis.hkeys(dataKey);
        Set<ResourceItem> filterSet = Sets.filter(resourceItems, new Predicate<ResourceItem>() {

            @Override
            public boolean apply(ResourceItem input) {
                return !hkeys.contains(input.getKey());
            }
        });
        String poolQueueKey = makePoolQueueKey(queueID);
        for (ResourceItem resourceItem : filterSet) {
            jedis.hset(dataKey, resourceItem.getKey(), JSONObject.toJSONString(resourceItem));
            jedis.rpush(poolQueueKey, resourceItem.getKey());
        }
    } finally {
        IOUtils.closeQuietly(jedis);
        unLockQueue(queueID);
    }
}

Example 4 with ResourceItem

use of com.virjar.vscrawler.core.resourcemanager.model.ResourceItem in project vscrawler by virjar.

the class ResourceQueue method feedBack.

/**
 * 反馈某个资源的使用状况
 *
 * @param key  每个资源都应该有一个key
 * @param isOK 该资源状态,可用还是不可用
 */
public void feedBack(String key, boolean isOK) {
    boolean inLeaveQueue = false;
    ResourceItem resourceItem = scoredQueueStore.get(makePollingQueueID(), key);
    if (resourceItem == null) {
        resourceItem = blockingQueueStore.get(makeLeaveQueueID(), key);
        inLeaveQueue = true;
    }
    if (resourceItem == null) {
        VSCrawlerMonitor.recordOne(resourceQueueMonitorTag + "feedback_findKey_failed");
        // can not find resource for key,for resource always forbidden
        return;
    }
    double newScore = resourceItem.getScore() * (resourceSetting.getScoreFactory() - 1) + (isOK ? 1 : 0);
    resourceItem.setScore(newScore);
    resourceItem.setValidTimeStamp(0);
    resourceItem.setKey(key);
    if (inLeaveQueue) {
        forbiddenQueueStore.remove(makeLeaveQueueID(), key);
        scoredQueueStore.addFirst(makePollingQueueID(), resourceItem);
        return;
    }
    if (isOK) {
        scoredQueueStore.update(makePollingQueueID(), resourceItem);
        return;
    }
    long queueSize = scoredQueueStore.size(makePollingQueueID()) - 1;
    long index = (long) ((queueSize) * (resourceSetting.getScoreRatio() + (1 - resourceSetting.getScoreRatio()) * (1 - resourceItem.getScore())));
    scoredQueueStore.remove(makePollingQueueID(), key);
    boolean addSuccess = false;
    try {
        addSuccess = scoredQueueStore.addIndex(makePollingQueueID(), index, resourceItem);
    } catch (Exception e) {
        VSCrawlerMonitor.recordOne(resourceQueueMonitorTag + tag + "relocate_resource_sequence_failed");
        log.error("relocate resource sequence failed", e);
    }
    if (!addSuccess) {
        scoredQueueStore.addLast(makePollingQueueID(), resourceItem);
    }
}

Example 5 with ResourceItem

use of com.virjar.vscrawler.core.resourcemanager.model.ResourceItem in project vscrawler by virjar.

the class ResourceQueue method forbidden.

/**
 * 永久封禁某个资源
 *
 * @param key 每个资源都应该有一个key
 */
public void forbidden(String key) {
    ResourceItem resourceItem = scoredQueueStore.remove(makePollingQueueID(), key);
    if (resourceItem != null) {
        forbiddenQueueStore.add(makeForbiddenQueueID(), resourceItem);
        return;
    }
    resourceItem = blockingQueueStore.remove(makeLeaveQueueID(), key);
    if (resourceItem != null) {
        forbiddenQueueStore.add(makeForbiddenQueueID(), resourceItem);
    }
}