Examples with ResourceManager com.virjar.vscrawler.core.resourcemanager.ResourceManager used on opensource projects

Example 1 with ResourceManager

use of com.virjar.vscrawler.core.resourcemanager.ResourceManager in project vscrawler by virjar.

the class ResourceManagerTest method main.

public static void main(String[] args) {
    ResourceManager resourceManager = ResourceManagerFactory.create().registryResourceQueue(new ResourceQueue(tag, new RamQueueStorePlanner(), ResourceSetting.create().setLock(true).setLockForceLeaseDuration(100), new ResourceLoader() {

        @Override
        public boolean loadResource(Collection<ResourceItem> resourceItems) {
            for (int i = 0; i < 100; i++) {
                ResourceItem resourceItem = new ResourceItem();
                resourceItem.setKey("key_" + i);
                resourceItem.setData(resourceItem.getKey());
                resourceItems.add(resourceItem);
            }
            return false;
        }
    })).build();
    for (int i = 0; i < 1000000; i++) {
        ResourceItem resourceItem = resourceManager.allocate(tag);
        String data = "null";
        if (resourceItem != null) {
            feedBackRandom(tag, resourceManager, resourceItem);
            // feedBackAlways(tag, resourceManager, resourceItem);
            data = resourceItem.getData();
        }
        if (i % 100 == 0) {
            printQueueStatus(resourceManager, tag);
        }
    }
    printQueueStatus(resourceManager, tag);
}

Example 2 with ResourceManager

use of com.virjar.vscrawler.core.resourcemanager.ResourceManager in project vscrawler by virjar.

the class ImeiTest method main.

public static void main(String[] args) {
    ResourceQueue resourceQueue = new ResourceQueue("android_imei", new RamQueueStorePlanner(), ResourceSetting.create().setLock(true), new ResourceLoader() {

        private BufferedReader reader = new BufferedReader(new InputStreamReader(ImeiTest.class.getResourceAsStream("/imei.txt")));

        private static final int batchSize = 100;

        private boolean closed = false;

        @Override
        public boolean loadResource(Collection<ResourceItem> resourceItems) {
            if (closed) {
                return false;
            }
            String line;
            int readSize = 0;
            try {
                while ((line = reader.readLine()) != null) {
                    ResourceItem resourceItem = new ResourceItem();
                    resourceItem.setData(line);
                    resourceItem.setKey(line.split(",")[0].trim());
                    resourceItems.add(resourceItem);
                    readSize++;
                    if (readSize > batchSize) {
                        return true;
                    }
                }
                IOUtils.closeQuietly(reader);
                closed = true;
                return false;
            } catch (IOException ioe) {
                ioe.printStackTrace();
                IOUtils.closeQuietly(reader);
                closed = true;
                return false;
            }
        }
    });
    ResourceManager resourceManager = ResourceManagerFactory.create().registryResourceQueue(resourceQueue).build();
    int allocatedTimes = 0;
    int notAllocatedTimes = 0;
    for (int i = 0; i < 1000; i++) {
        ResourceItem resourceItem = resourceManager.allocate("android_imei");
        if (resourceItem != null) {
            if (i < 50) {
                resourceManager.feedBack("android_imei", resourceItem.getKey(), true);
            } else if (i < 100) {
                resourceManager.feedBack("android_imei", resourceItem.getKey(), false);
            } else if (i < 150) {
                resourceManager.forbidden("android_imei", resourceItem.getKey());
            }
            allocatedTimes++;
            System.out.println(resourceItem.getData());
        } else {
            notAllocatedTimes++;
            System.out.println("none");
        }
    }
    System.out.println("allocatedTimes: " + allocatedTimes + "  notAllocatedTimes: " + notAllocatedTimes);
}