Search in sources :

Example 16 with Ignore

use of org.junit.Ignore in project druid by druid-io.

the class HyperLogLogCollectorTest method showErrorRate.

// Provides a nice printout of error rates as a function of cardinality
@Ignore
@Test
public void showErrorRate() throws Exception {
    HashFunction fn = Hashing.murmur3_128();
    Random random = new Random();
    double error = 0.0d;
    int count = 0;
    final int[] valsToCheck = { 10, 20, 50, 100, 1000, 2000, 5000, 10000, 20000, 50000, 100000, 1000000, 2000000, 10000000, Integer.MAX_VALUE };
    for (int numThings : valsToCheck) {
        long startTime = System.currentTimeMillis();
        HyperLogLogCollector collector = HyperLogLogCollector.makeLatestCollector();
        for (int i = 0; i < numThings; ++i) {
            if (i != 0 && i % 100000000 == 0) {
                ++count;
                error = computeError(error, count, i, startTime, collector);
            }
            collector.add(fn.hashLong(random.nextLong()).asBytes());
        }
        ++count;
        error = computeError(error, count, numThings, startTime, collector);
    }
}
Also used : HashFunction(com.google.common.hash.HashFunction) Random(java.util.Random) Ignore(org.junit.Ignore) Test(org.junit.Test)

Example 17 with Ignore

use of org.junit.Ignore in project druid by druid-io.

the class HyperLogLogCollectorTest method testFoldingwithDifferentOffsets.

// This test can help when finding potential combinations that are weird, but it's non-deterministic
@Ignore
// This test can help when finding potential combinations that are weird, but it's non-deterministic
@Test
public void testFoldingwithDifferentOffsets() throws Exception {
    // final Random random = new Random(37); // this seed will cause this test to fail because of slightly larger errors
    final Random random = new Random(0);
    for (int j = 0; j < 10; j++) {
        HyperLogLogCollector smallVals = HyperLogLogCollector.makeLatestCollector();
        HyperLogLogCollector bigVals = HyperLogLogCollector.makeLatestCollector();
        HyperLogLogCollector all = HyperLogLogCollector.makeLatestCollector();
        int numThings = 500000;
        for (int i = 0; i < numThings; i++) {
            byte[] hashedVal = fn.hashLong(random.nextLong()).asBytes();
            if (i < 1000) {
                smallVals.add(hashedVal);
            } else {
                bigVals.add(hashedVal);
            }
            all.add(hashedVal);
        }
        HyperLogLogCollector folded = HyperLogLogCollector.makeLatestCollector();
        folded.fold(smallVals);
        folded.fold(bigVals);
        final double expected = all.estimateCardinality();
        Assert.assertEquals(expected, folded.estimateCardinality(), expected * 0.025);
        Assert.assertEquals(numThings, folded.estimateCardinality(), numThings * 0.05);
    }
}
Also used : Random(java.util.Random) Ignore(org.junit.Ignore) Test(org.junit.Test)

Example 18 with Ignore

use of org.junit.Ignore in project webmagic by code4craft.

the class SeleniumTest method testSelenium.

@Ignore("need chrome driver")
@Test
public void testSelenium() {
    System.getProperties().setProperty("webdriver.chrome.driver", "/Users/yihua/Downloads/chromedriver");
    Map<String, Object> contentSettings = new HashMap<String, Object>();
    contentSettings.put("images", 2);
    Map<String, Object> preferences = new HashMap<String, Object>();
    preferences.put("profile.default_content_settings", contentSettings);
    DesiredCapabilities caps = DesiredCapabilities.chrome();
    caps.setCapability("chrome.prefs", preferences);
    caps.setCapability("chrome.switches", Arrays.asList("--user-data-dir=/Users/yihua/temp/chrome"));
    WebDriver webDriver = new ChromeDriver(caps);
    webDriver.get("http://huaban.com/");
    WebElement webElement = webDriver.findElement(By.xpath("/html"));
    System.out.println(webElement.getAttribute("outerHTML"));
    webDriver.close();
}
Also used : WebDriver(org.openqa.selenium.WebDriver) HashMap(java.util.HashMap) DesiredCapabilities(org.openqa.selenium.remote.DesiredCapabilities) ChromeDriver(org.openqa.selenium.chrome.ChromeDriver) WebElement(org.openqa.selenium.WebElement) Ignore(org.junit.Ignore) Test(org.junit.Test)

Example 19 with Ignore

use of org.junit.Ignore in project webmagic by code4craft.

the class WebDriverPoolTest method test.

@Ignore("need chrome driver")
@Test
public void test() {
    System.getProperties().setProperty("webdriver.chrome.driver", chromeDriverPath);
    WebDriverPool webDriverPool = new WebDriverPool(5);
    for (int i = 0; i < 5; i++) {
        try {
            WebDriver webDriver = webDriverPool.get();
            System.out.println(i);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
    webDriverPool.closeAll();
}
Also used : WebDriver(org.openqa.selenium.WebDriver) Ignore(org.junit.Ignore) Test(org.junit.Test)

Example 20 with Ignore

use of org.junit.Ignore in project webmagic by code4craft.

the class BloomFilterDuplicateRemoverTest method testMemory.

@Ignore("long time")
@Test
public void testMemory() throws Exception {
    int times = 5000000;
    DuplicateRemover duplicateRemover = new BloomFilterDuplicateRemover(times, 0.005);
    long freeMemory = Runtime.getRuntime().freeMemory();
    long time = System.currentTimeMillis();
    for (int i = 0; i < times; i++) {
        duplicateRemover.isDuplicate(new Request(String.valueOf(i)), null);
    }
    System.out.println("Time used by bloomfilter:" + (System.currentTimeMillis() - time));
    System.out.println("Memory used by bloomfilter:" + (freeMemory - Runtime.getRuntime().freeMemory()));
    duplicateRemover = new HashSetDuplicateRemover();
    System.gc();
    freeMemory = Runtime.getRuntime().freeMemory();
    time = System.currentTimeMillis();
    for (int i = 0; i < times; i++) {
        duplicateRemover.isDuplicate(new Request(String.valueOf(i)), null);
    }
    System.out.println("Time used by hashset:" + (System.currentTimeMillis() - time));
    System.out.println("Memory used by hashset:" + (freeMemory - Runtime.getRuntime().freeMemory()));
}
Also used : HashSetDuplicateRemover(us.codecraft.webmagic.scheduler.component.HashSetDuplicateRemover) Request(us.codecraft.webmagic.Request) DuplicateRemover(us.codecraft.webmagic.scheduler.component.DuplicateRemover) HashSetDuplicateRemover(us.codecraft.webmagic.scheduler.component.HashSetDuplicateRemover) Ignore(org.junit.Ignore) Test(org.junit.Test)

Aggregations

Ignore (org.junit.Ignore)5092 Test (org.junit.Test)4807 File (java.io.File)445 ArrayList (java.util.ArrayList)374 IOException (java.io.IOException)217 HashMap (java.util.HashMap)187 List (java.util.List)171 CountDownLatch (java.util.concurrent.CountDownLatch)118 Map (java.util.Map)103 LocalDate (java.time.LocalDate)94 Dataset (org.apache.jena.query.Dataset)93 InputStream (java.io.InputStream)89 Date (java.util.Date)88 Random (java.util.Random)85 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)83 Properties (java.util.Properties)78 DistributedTest (org.apache.geode.test.junit.categories.DistributedTest)78 HashSet (java.util.HashSet)71 ByteArrayOutputStream (java.io.ByteArrayOutputStream)70 ExecutorService (java.util.concurrent.ExecutorService)70