Examples with ResourceItem com.virjar.vscrawler.core.resourcemanager.model.ResourceItem used on opensource projects

Example 21 with ResourceItem

use of com.virjar.vscrawler.core.resourcemanager.model.ResourceItem in project vscrawler by virjar.

the class ResourceQueue method allocate.

/**
 * 得到一个资源
 *
 * @return 资源管理器当前分发的资源, 如果系统不能分发资源, 则该返回可能为null
 */
public ResourceItem allocate() {
    boolean hasQueryBlockingQueue = false;
    while (true) {
        ResourceItem resourceItem = scoredQueueStore.pop(makePollingQueueID());
        if (resourceItem == null && !hasQueryBlockingQueue) {
            resourceItem = blockingQueueStore.pop(makeLeaveQueueID());
            hasQueryBlockingQueue = true;
        }
        if (resourceItem == null && scoredQueueStore.size(makePollingQueueID()) == 0 && blockingQueueStore.size(makeLeaveQueueID()) == 0 && forbiddenQueueStore.size(makeForbiddenQueueID()) == 0) {
            loadResource();
            resourceItem = scoredQueueStore.pop(makePollingQueueID());
        }
        if (resourceItem == null) {
            // can not get available resource
            return null;
        }
        // check 该资源在未来才会生效,放置到blockingQueue
        if (resourceItem.getValidTimeStamp() > System.currentTimeMillis()) {
            blockingQueueStore.zadd(makeLeaveQueueID(), resourceItem);
            continue;
        }
        // check 通过,设置更新状态参数
        if (resourceSetting.isLock()) {
            resourceItem.setValidTimeStamp(System.currentTimeMillis() + resourceSetting.getLockForceLeaseDuration());
        }
        long queueSize = scoredQueueStore.size(makePollingQueueID());
        if (queueSize <= 3) {
            scoredQueueStore.addLast(makePollingQueueID(), resourceItem);
        } else {
            long index = (long) (resourceSetting.getScoreRatio() * queueSize);
            if (index > queueSize - 1) {
                index = queueSize - 1;
            }
            scoredQueueStore.addIndex(makePollingQueueID(), index, resourceItem);
        }
        return resourceItem;
    }
}

Example 22 with ResourceItem

use of com.virjar.vscrawler.core.resourcemanager.model.ResourceItem in project vscrawler by virjar.

the class ResourceQueue method unForbidden.

/**
 * 解除永久封禁
 *
 * @param key 每个资源都应该有一个key
 */
public void unForbidden(String key) {
    ResourceItem resourceItem = forbiddenQueueStore.get(makeForbiddenQueueID(), key);
    if (resourceItem == null) {
        VSCrawlerMonitor.recordOne(resourceQueueMonitorTag + tag + "_find_resource_meta_failed");
        log.warn("can not find resource meta data for tag:{} key:{}", tag, key);
        return;
    }
    resourceItem.setScore(0.5);
    scoredQueueStore.addFirst(makePollingQueueID(), resourceItem);
}

Example 23 with ResourceItem

use of com.virjar.vscrawler.core.resourcemanager.model.ResourceItem in project vscrawler by virjar.

the class JedisBlockingQueueStore method pop.

@Override
public ResourceItem pop(String queueID) {
    if (!lockQueue(queueID)) {
        throw new RuntimeException("failed to acquire resource queue for queueID: " + queueID);
    }
    Jedis jedis = jedisPool.getResource();
    try {
        ResourceItem resourceItem = poll(queueID);
        if (resourceItem == null) {
            return null;
        }
        jedis.zrem(makePoolQueueKey(queueID), resourceItem.getKey());
        return resourceItem;
    } finally {
        IOUtils.closeQuietly(jedis);
        unLockQueue(queueID);
    }
}

Example 24 with ResourceItem

use of com.virjar.vscrawler.core.resourcemanager.model.ResourceItem in project vscrawler by virjar.

the class RamBlockingQueueStore method remove.

@Override
public ResourceItem remove(String queueID, String key) {
    ResourceItem resourceItem = get(queueID, key);
    if (resourceItem == null) {
        return null;
    }
    SortedList<ResourceItemHolder> sortedList = createOrGet(queueID);
    ResourceItemHolder query = new ResourceItemHolder(resourceItem);
    sortedList.remove(query);
    return resourceItem;
}

Example 25 with ResourceItem

use of com.virjar.vscrawler.core.resourcemanager.model.ResourceItem in project vscrawler by virjar.

the class TreeSetTest method main.

public static void main(String[] args) {
    TreeSet<ResourceItemHolder> treeSet = Sets.newTreeSet();
    ResourceItem resourceItem = new ResourceItem();
    resourceItem.setKey("123");
    resourceItem.setData(resourceItem.getKey());
    resourceItem.setValidTimeStamp(100);
    treeSet.add(new ResourceItemHolder(resourceItem));
    resourceItem = new ResourceItem();
    resourceItem.setKey("12");
    resourceItem.setData(resourceItem.getKey());
    resourceItem.setValidTimeStamp(100);
    treeSet.add(new ResourceItemHolder(resourceItem));
    for (ResourceItemHolder resourceItemHolder : treeSet) {
        System.out.println(resourceItemHolder.resourceItem.getKey());
    }
}