Examples with CounterStat com.facebook.airlift.stats.CounterStat used on opensource projects

Example 6 with CounterStat

use of com.facebook.airlift.stats.CounterStat in project presto by prestodb.

the class TestHiveSplitSource method testReaderWaitsForSplits.

@Test
public void testReaderWaitsForSplits() throws Exception {
    HiveSplitSource hiveSplitSource = HiveSplitSource.allAtOnce(SESSION, "database", "table", new CacheQuotaRequirement(GLOBAL, Optional.empty()), 10, 10, new DataSize(1, MEGABYTE), new TestingHiveSplitLoader(), EXECUTOR, new CounterStat());
    SettableFuture<ConnectorSplit> splits = SettableFuture.create();
    // create a thread that will get a split
    CountDownLatch started = new CountDownLatch(1);
    Thread getterThread = new Thread(() -> {
        try {
            started.countDown();
            List<ConnectorSplit> batch = getSplits(hiveSplitSource, 1);
            assertEquals(batch.size(), 1);
            splits.set(batch.get(0));
        } catch (Throwable e) {
            splits.setException(e);
        }
    });
    getterThread.start();
    try {
        // wait for the thread to be started
        assertTrue(started.await(10, SECONDS));
        // sleep for a bit, and assure the thread is blocked
        MILLISECONDS.sleep(200);
        assertTrue(!splits.isDone());
        // add a split
        hiveSplitSource.addToQueue(new TestSplit(33));
        // wait for thread to get the split
        ConnectorSplit split = splits.get(10, SECONDS);
        assertEquals(((HiveSplit) split).getPartitionDataColumnCount(), 33);
    } finally {
        // make sure the thread exits
        getterThread.interrupt();
    }
}

Example 7 with CounterStat

use of com.facebook.airlift.stats.CounterStat in project presto by prestodb.

the class TestHiveSplitSource method testEmptyBucket.

@Test(timeOut = 10_000)
public void testEmptyBucket() {
    final HiveSplitSource hiveSplitSource = HiveSplitSource.bucketed(SESSION, "database", "table", new CacheQuotaRequirement(GLOBAL, Optional.empty()), 10, 10, new DataSize(1, MEGABYTE), new TestingHiveSplitLoader(), EXECUTOR, new CounterStat());
    hiveSplitSource.addToQueue(new TestSplit(0, OptionalInt.of(2)));
    hiveSplitSource.noMoreSplits();
    assertEquals(getSplits(hiveSplitSource, OptionalInt.of(0), 10).size(), 0);
    assertEquals(getSplits(hiveSplitSource, OptionalInt.of(1), 10).size(), 0);
    assertEquals(getSplits(hiveSplitSource, OptionalInt.of(2), 10).size(), 1);
    assertEquals(getSplits(hiveSplitSource, OptionalInt.of(3), 10).size(), 0);
}

Example 8 with CounterStat

use of com.facebook.airlift.stats.CounterStat in project presto by prestodb.

the class TestHiveSplitSource method testRewindMultipleBuckets.

@Test
public void testRewindMultipleBuckets() {
    HiveSplitSource hiveSplitSource = HiveSplitSource.bucketedRewindable(SESSION, "database", "table", new CacheQuotaRequirement(GLOBAL, Optional.empty()), 10, new DataSize(1, MEGABYTE), new TestingHiveSplitLoader(), EXECUTOR, new CounterStat());
    for (int i = 0; i < 10; i++) {
        hiveSplitSource.addToQueue(new TestSplit(i, OptionalInt.of(1)));
        hiveSplitSource.addToQueue(new TestSplit(i, OptionalInt.of(2)));
        assertEquals(hiveSplitSource.getBufferedInternalSplitCount(), 2 * (i + 1));
    }
    hiveSplitSource.noMoreSplits();
    assertEquals(getSplits(hiveSplitSource, OptionalInt.of(1), 1).size(), 1);
    assertEquals(getSplits(hiveSplitSource, OptionalInt.of(2), 2).size(), 2);
    assertEquals(hiveSplitSource.getBufferedInternalSplitCount(), 17);
    // Rewind bucket 1 and test only bucket 1 is rewinded.
    hiveSplitSource.rewind(new HivePartitionHandle(1));
    assertEquals(hiveSplitSource.getBufferedInternalSplitCount(), 18);
    assertEquals(getSplits(hiveSplitSource, OptionalInt.of(1), 1).size(), 1);
    // Rewind bucket 2 and test only bucket 2 is rewinded.
    hiveSplitSource.rewind(new HivePartitionHandle(2));
    assertEquals(hiveSplitSource.getBufferedInternalSplitCount(), 19);
}

Example 9 with CounterStat

use of com.facebook.airlift.stats.CounterStat in project presto by prestodb.

the class TestHiveSplitSource method testOutstandingSplitCount.

@Test
public void testOutstandingSplitCount() {
    HiveSplitSource hiveSplitSource = HiveSplitSource.allAtOnce(SESSION, "database", "table", new CacheQuotaRequirement(TABLE, DEFAULT_QUOTA_SIZE), 10, 10, new DataSize(1, MEGABYTE), new TestingHiveSplitLoader(), EXECUTOR, new CounterStat());
    // add 10 splits
    for (int i = 0; i < 10; i++) {
        hiveSplitSource.addToQueue(new TestSplit(i));
        assertEquals(hiveSplitSource.getBufferedInternalSplitCount(), i + 1);
    }
    // remove 1 split
    assertEquals(getSplits(hiveSplitSource, 1).size(), 1);
    assertEquals(hiveSplitSource.getBufferedInternalSplitCount(), 9);
    // remove 4 splits
    assertEquals(getSplits(hiveSplitSource, 4).size(), 4);
    assertEquals(hiveSplitSource.getBufferedInternalSplitCount(), 5);
    // try to remove 20 splits, and verify we only got 5
    assertEquals(getSplits(hiveSplitSource, 20).size(), 5);
    assertEquals(hiveSplitSource.getBufferedInternalSplitCount(), 0);
}

Example 10 with CounterStat

use of com.facebook.airlift.stats.CounterStat in project presto by prestodb.

the class TestHiveSplitSource method testFail.

@Test
public void testFail() {
    HiveSplitSource hiveSplitSource = HiveSplitSource.allAtOnce(SESSION, "database", "table", new CacheQuotaRequirement(GLOBAL, Optional.empty()), 10, 10, new DataSize(1, MEGABYTE), new TestingHiveSplitLoader(), EXECUTOR, new CounterStat());
    // add some splits
    for (int i = 0; i < 5; i++) {
        hiveSplitSource.addToQueue(new TestSplit(i));
        assertEquals(hiveSplitSource.getBufferedInternalSplitCount(), i + 1);
    }
    // remove a split and verify
    assertEquals(getSplits(hiveSplitSource, 1).size(), 1);
    assertEquals(hiveSplitSource.getBufferedInternalSplitCount(), 4);
    // fail source
    hiveSplitSource.fail(new RuntimeException("test"));
    assertEquals(hiveSplitSource.getBufferedInternalSplitCount(), 4);
    // try to remove a split and verify we got the expected exception
    try {
        getSplits(hiveSplitSource, 1);
        fail("expected RuntimeException");
    } catch (RuntimeException e) {
        assertEquals(e.getMessage(), "test");
    }
    // 3 splits + poison
    assertEquals(hiveSplitSource.getBufferedInternalSplitCount(), 4);
    // attempt to add another split and verify it does not work
    hiveSplitSource.addToQueue(new TestSplit(99));
    // 3 splits + poison
    assertEquals(hiveSplitSource.getBufferedInternalSplitCount(), 4);
    // fail source again
    hiveSplitSource.fail(new RuntimeException("another failure"));
    // 3 splits + poison
    assertEquals(hiveSplitSource.getBufferedInternalSplitCount(), 4);
    // try to remove a split and verify we got the first exception
    try {
        getSplits(hiveSplitSource, 1);
        fail("expected RuntimeException");
    } catch (RuntimeException e) {
        assertEquals(e.getMessage(), "test");
    }
}