Examples with ResourceItem com.virjar.vscrawler.core.resourcemanager.model.ResourceItem used on opensource projects

Example 16 with ResourceItem

use of com.virjar.vscrawler.core.resourcemanager.model.ResourceItem in project vscrawler by virjar.

the class JedisQueueStore method queryAll.

@Override
public List<ResourceItem> queryAll(String queueID) {
    lockQueue(queueID);
    Jedis jedis = jedisPool.getResource();
    try {
        final Map<String, String> map = jedis.hgetAll(makeDataKey(queueID));
        return Lists.transform(jedis.lrange(makePoolQueueKey(queueID), 0, -1), new Function<String, ResourceItem>() {

            @Override
            public ResourceItem apply(String input) {
                return JSONObject.toJavaObject(JSONObject.parseObject(map.get(input)), ResourceItem.class);
            }
        });
    } finally {
        IOUtils.closeQuietly(jedis);
        unLockQueue(queueID);
    }
}

Example 17 with ResourceItem

use of com.virjar.vscrawler.core.resourcemanager.model.ResourceItem in project vscrawler by virjar.

the class ResourceQueue method forbidden.

/**
 * 永久封禁某个资源
 *
 * @param key 每个资源都应该有一个key
 */
public void forbidden(String key) {
    ResourceItem resourceItem = queue.remove(makePollingQueueID(), key);
    if (resourceItem != null) {
        queue.addLast(makeForbiddenQueueID(), resourceItem);
    }
    resourceItem = queue.remove(makeLeaveQueueID(), key);
    if (resourceItem != null) {
        queue.addLast(makeForbiddenQueueID(), resourceItem);
    }
}

Example 18 with ResourceItem

use of com.virjar.vscrawler.core.resourcemanager.model.ResourceItem in project vscrawler by virjar.

the class ResourceQueue method feedBack.

/**
 * 反馈某个资源的使用状况
 *
 * @param key  每个资源都应该有一个key
 * @param isOK 该资源状态,可用还是不可用
 */
public void feedBack(String key, boolean isOK) {
    boolean inLeaveQueue = false;
    ResourceItem resourceItem = queue.get(makePollingQueueID(), key);
    if (resourceItem == null) {
        resourceItem = queue.get(makeLeaveQueueID(), key);
        inLeaveQueue = true;
    }
    if (resourceItem == null) {
        // can not find resource for key,for resource always forbidden
        return;
    }
    double newScore = resourceItem.getScore() * (resourceSetting.getScoreFactory() - 1) + (isOK ? 1 : 0);
    resourceItem.setScore(newScore);
    resourceItem.setValidTimeStamp(0);
    resourceItem.setKey(key);
    if (inLeaveQueue) {
        queue.remove(makeLeaveQueueID(), key);
        queue.addFirst(makePollingQueueID(), resourceItem);
        return;
    }
    if (isOK) {
        queue.update(makePollingQueueID(), resourceItem);
        return;
    }
    long queueSize = queue.size(makePollingQueueID()) - 1;
    long index = (long) ((queueSize) * (resourceSetting.getScoreRatio() + (1 - resourceSetting.getScoreRatio()) * (1 - resourceItem.getScore())));
    queue.remove(makePollingQueueID(), key);
    boolean addSuccess = false;
    try {
        addSuccess = queue.addIndex(makePollingQueueID(), index, resourceItem);
    } catch (Exception e) {
    // ignore
    // TODO log
    }
    if (!addSuccess) {
        queue.addLast(makePollingQueueID(), resourceItem);
    }
}

Example 19 with ResourceItem

use of com.virjar.vscrawler.core.resourcemanager.model.ResourceItem in project vscrawler by virjar.

the class JedisScoredQueueStore method queryAll.

@Override
public List<ResourceItem> queryAll(String queueID) {
    lockQueue(queueID);
    Jedis jedis = jedisPool.getResource();
    try {
        final Map<String, String> map = jedis.hgetAll(makeDataKey(queueID));
        return Lists.transform(jedis.lrange(makePoolQueueKey(queueID), 0, -1), new Function<String, ResourceItem>() {

            @Override
            public ResourceItem apply(String input) {
                return JSONObject.toJavaObject(JSONObject.parseObject(map.get(input)), ResourceItem.class);
            }
        });
    } finally {
        IOUtils.closeQuietly(jedis);
        unLockQueue(queueID);
    }
}

Example 20 with ResourceItem

use of com.virjar.vscrawler.core.resourcemanager.model.ResourceItem in project vscrawler by virjar.

the class ResourceManager method allocate.

/**
 * 得到一个资源
 *
 * @param tag 资源tag,所有资源根据tag区分资源类型
 * @return 资源管理器当前分发的资源, 如果系统不能分发资源, 则该返回可能为null
 */
public ResourceItem allocate(String tag) {
    ResourceQueue queue = resourceQueueConcurrentMap.get(tag);
    if (queue == null) {
        log.error("no resource for tag:{}", tag);
        return null;
    }
    ResourceItem ret = queue.allocate();
    VSCrawlerMonitor.recordOne(resourceMangerTag + tag + "_allocate_" + (ret != null ? "success" : "failed"));
    return ret;
}