Examples with ResourceItem com.virjar.vscrawler.core.resourcemanager.model.ResourceItem used on opensource projects

Example 6 with ResourceItem

use of com.virjar.vscrawler.core.resourcemanager.model.ResourceItem in project vscrawler by virjar.

the class ResourceQueue method updateResource.

public void updateResource(ResourceItem resourceItem) {
    String key = resourceItem.getKey();
    ResourceItem resourceItem1 = scoredQueueStore.get(makePollingQueueID(), key);
    if (resourceItem1 != null) {
        resourceItem1.setData(resourceItem.getData());
        scoredQueueStore.update(makePollingQueueID(), resourceItem1);
        return;
    }
    resourceItem1 = blockingQueueStore.get(makeLeaveQueueID(), key);
    if (resourceItem1 != null) {
        resourceItem1.setData(resourceItem.getData());
        blockingQueueStore.update(makePollingQueueID(), resourceItem1);
        return;
    }
    resourceItem1 = forbiddenQueueStore.get(makeForbiddenQueueID(), key);
    if (resourceItem1 != null) {
        resourceItem1.setData(resourceItem.getData());
        forbiddenQueueStore.update(makeForbiddenQueueID(), resourceItem1);
    }
}

Example 7 with ResourceItem

use of com.virjar.vscrawler.core.resourcemanager.model.ResourceItem in project vscrawler by virjar.

the class JedisBlockingQueueStore method remove.

@Override
public ResourceItem remove(String queueID, String key) {
    lockQueue(queueID);
    Jedis jedis = jedisPool.getResource();
    try {
        String data = jedis.hget(makeDataKey(queueID), key);
        if (StringUtils.isBlank(data)) {
            VSCrawlerMonitor.recordOne(queueID + "_find_meta_data_failed");
            return null;
        }
        ResourceItem ret = JSONObject.toJavaObject(JSON.parseObject(data), ResourceItem.class);
        jedis.hdel(makePoolQueueKey(queueID), key);
        jedis.zrem(makePoolQueueKey(queueID), key);
        return ret;
    } finally {
        IOUtils.closeQuietly(jedis);
        unLockQueue(queueID);
    }
}

Example 8 with ResourceItem

use of com.virjar.vscrawler.core.resourcemanager.model.ResourceItem in project vscrawler by virjar.

the class JedisSegmentScoredQueueStore method addBatch.

@Override
public void addBatch(String queueID, Set<ResourceItem> resourceItems) {
    if (!lockQueue(queueID)) {
        return;
    }
    @Cleanup Jedis jedis = jedisPool.getResource();
    final String dataKey = makeDataKey(queueID);
    try {
        final Set<String> hkeys = jedis.hkeys(dataKey);
        Set<ResourceItem> filterSet = Sets.filter(resourceItems, new Predicate<ResourceItem>() {

            @Override
            public boolean apply(ResourceItem input) {
                return !hkeys.contains(input.getKey());
            }
        });
        List<String> sliceQueue = sliceQueue(queueID);
        Set<String> newSlices = Sets.newHashSet();
        long index = size(queueID) + 1;
        String tailSlice = null;
        for (ResourceItem resourceItem : filterSet) {
            jedis.hset(dataKey, resourceItem.getKey(), JSONObject.toJSONString(resourceItem));
            String sliceID = String.valueOf(blockID(index));
            if (sliceID.equals(tailSlice) || sliceQueue.contains(sliceID)) {
                sliceID = sliceQueue.get(sliceQueue.size() - 1);
                tailSlice = sliceID;
            } else if (!newSlices.contains(sliceID)) {
                jedis.rpush(makeSliceQueueKey(queueID), sliceID);
                newSlices.add(sliceID);
            }
            jedis.rpush(makePoolQueueKey(queueID, sliceID), resourceItem.getKey());
            index++;
        }
    } finally {
        unLockQueue(queueID);
    }
}

Example 9 with ResourceItem

use of com.virjar.vscrawler.core.resourcemanager.model.ResourceItem in project vscrawler by virjar.

the class ResourceManagerTest method main.

public static void main(String[] args) {
    ResourceManager resourceManager = ResourceManagerFactory.create().registryResourceQueue(new ResourceQueue(tag, new RamQueueStorePlanner(), ResourceSetting.create().setLock(true).setLockForceLeaseDuration(100), new ResourceLoader() {

        @Override
        public boolean loadResource(Collection<ResourceItem> resourceItems) {
            for (int i = 0; i < 100; i++) {
                ResourceItem resourceItem = new ResourceItem();
                resourceItem.setKey("key_" + i);
                resourceItem.setData(resourceItem.getKey());
                resourceItems.add(resourceItem);
            }
            return false;
        }
    })).build();
    for (int i = 0; i < 1000000; i++) {
        ResourceItem resourceItem = resourceManager.allocate(tag);
        String data = "null";
        if (resourceItem != null) {
            feedBackRandom(tag, resourceManager, resourceItem);
            // feedBackAlways(tag, resourceManager, resourceItem);
            data = resourceItem.getData();
        }
        if (i % 100 == 0) {
            printQueueStatus(resourceManager, tag);
        }
    }
    printQueueStatus(resourceManager, tag);
}

Example 10 with ResourceItem

use of com.virjar.vscrawler.core.resourcemanager.model.ResourceItem in project vscrawler by virjar.

the class JedisQueueStore method addBatch.

@Override
public void addBatch(String queueID, Set<ResourceItem> resourceItems) {
    if (!lockQueue(queueID)) {
        return;
    }
    final Jedis jedis = jedisPool.getResource();
    final String dataKey = makeDataKey(queueID);
    try {
        final Set<String> hkeys = jedis.hkeys(dataKey);
        Set<ResourceItem> filterSet = Sets.filter(resourceItems, new Predicate<ResourceItem>() {

            @Override
            public boolean apply(ResourceItem input) {
                return !hkeys.contains(input.getKey());
            }
        });
        String poolQueueKey = makePoolQueueKey(queueID);
        for (ResourceItem resourceItem : filterSet) {
            jedis.hset(dataKey, resourceItem.getKey(), JSONObject.toJSONString(resourceItem));
            jedis.rpush(poolQueueKey, resourceItem.getKey());
        }
    } finally {
        IOUtils.closeQuietly(jedis);
        unLockQueue(queueID);
    }
}