Search in sources :

Example 26 with MutableLong

use of org.apache.commons.lang3.mutable.MutableLong in project apex-malhar by apache.

the class WEQueryQueueManagerTest method testResetRead.

@Test
public void testResetRead() {
    final int numQueries = 3;
    WindowEndQueueManager<Query, Void> wqqm = new WindowEndQueueManager<>();
    wqqm.setup(null);
    wqqm.beginWindow(0);
    for (int qc = 0; qc < numQueries; qc++) {
        Query query = new MockQuery(Integer.toString(qc));
        wqqm.enqueue(query, null, new MutableLong(3L));
    }
    Query query = wqqm.dequeue().getQuery();
    Query query1 = wqqm.dequeue().getQuery();
    Assert.assertEquals("Query ids must equal.", "0", query.getId());
    Assert.assertEquals("Query ids must equal.", "1", query1.getId());
    wqqm.endWindow();
    wqqm.beginWindow(1);
    {
        int qc = 0;
        for (QueryBundle<Query, Void, MutableLong> tquery; (tquery = wqqm.dequeue()) != null; qc++) {
            Assert.assertEquals("Query ids must equal.", Integer.toString(qc), tquery.getQuery().getId());
        }
        Assert.assertEquals("The number of queries must match.", numQueries, qc);
    }
    wqqm.endWindow();
    wqqm.teardown();
}
Also used : MutableLong(org.apache.commons.lang3.mutable.MutableLong) Query(org.apache.apex.malhar.lib.appdata.schemas.Query) Test(org.junit.Test)

Example 27 with MutableLong

use of org.apache.commons.lang3.mutable.MutableLong in project apex-malhar by apache.

the class WEQueryQueueManagerTest method testExpirationReadAll.

@Test
public void testExpirationReadAll() {
    final int numQueries = 3;
    WindowEndQueueManager<Query, Void> wqqm = new WindowEndQueueManager<>();
    wqqm.setup(null);
    wqqm.beginWindow(0);
    for (int qc = 0; qc < numQueries; qc++) {
        Query query = new MockQuery(Integer.toString(qc));
        wqqm.enqueue(query, null, new MutableLong(2L));
    }
    wqqm.endWindow();
    wqqm.beginWindow(1);
    {
        int qc = 0;
        for (QueryBundle<Query, Void, MutableLong> qb; (qb = wqqm.dequeue()) != null; qc++) {
            Query query = qb.getQuery();
            Assert.assertEquals("Query ids must equal.", Integer.toString(qc), query.getId());
        }
        Assert.assertEquals("The number of queries must match.", numQueries, qc);
    }
    wqqm.endWindow();
    wqqm.beginWindow(2);
    Assert.assertEquals("There should be no queries now", null, wqqm.dequeue());
    wqqm.endWindow();
    wqqm.teardown();
}
Also used : MutableLong(org.apache.commons.lang3.mutable.MutableLong) Query(org.apache.apex.malhar.lib.appdata.schemas.Query) Test(org.junit.Test)

Example 28 with MutableLong

use of org.apache.commons.lang3.mutable.MutableLong in project apex-malhar by apache.

the class WEQueryQueueManagerTest method testSimpleAddRemove2.

@Test
public void testSimpleAddRemove2() {
    WindowEndQueueManager<Query, Void> wqqm = new WindowEndQueueManager<>();
    wqqm.setup(null);
    wqqm.beginWindow(0);
    Query query = new MockQuery("1");
    wqqm.enqueue(query, null, new MutableLong(1L));
    Query queryD = wqqm.dequeue().getQuery();
    QueryBundle<Query, Void, MutableLong> qb = wqqm.dequeue();
    Query queryD1 = qb == null ? null : qb.getQuery();
    Query query1 = new MockQuery("2");
    wqqm.enqueue(query1, null, new MutableLong(1L));
    Query query1D = wqqm.dequeue().getQuery();
    qb = wqqm.dequeue();
    Query query1D1 = qb == null ? null : qb.getQuery();
    wqqm.endWindow();
    wqqm.teardown();
    Assert.assertEquals("The queries must match.", query, queryD);
    Assert.assertEquals("The queries must match.", null, queryD1);
    Assert.assertEquals("The queries must match.", query1, query1D);
    Assert.assertEquals("The queries must match.", null, query1D1);
}
Also used : MutableLong(org.apache.commons.lang3.mutable.MutableLong) Query(org.apache.apex.malhar.lib.appdata.schemas.Query) Test(org.junit.Test)

Example 29 with MutableLong

use of org.apache.commons.lang3.mutable.MutableLong in project apex-malhar by apache.

the class QueryManagerAsynchronousTest method stressTest.

@Test
public void stressTest() throws Exception {
    final int totalTuples = 100000;
    final int batchSize = 100;
    final double waitMillisProb = .01;
    AppDataWindowEndQueueManager<MockQuery, Void> queueManager = new AppDataWindowEndQueueManager<MockQuery, Void>();
    DefaultOutputPort<String> outputPort = new DefaultOutputPort<String>();
    CollectorTestSink<MockResult> sink = new CollectorTestSink<MockResult>();
    TestUtils.setSink(outputPort, sink);
    MessageSerializerFactory msf = new MessageSerializerFactory(new ResultFormatter());
    QueryManagerAsynchronous<MockQuery, Void, MutableLong, MockResult> queryManagerAsynch = new QueryManagerAsynchronous<>(outputPort, queueManager, new NOPQueryExecutor(waitMillisProb), msf, Thread.currentThread());
    Thread producerThread = new Thread(new ProducerThread(queueManager, totalTuples, batchSize, waitMillisProb));
    producerThread.start();
    producerThread.setName("Producer Thread");
    long startTime = System.currentTimeMillis();
    queryManagerAsynch.setup(null);
    int numWindows = 0;
    for (; sink.collectedTuples.size() < totalTuples && ((System.currentTimeMillis() - startTime) < 60000); numWindows++) {
        queryManagerAsynch.beginWindow(numWindows);
        Thread.sleep(100);
        queryManagerAsynch.endWindow();
    }
    producerThread.stop();
    queryManagerAsynch.teardown();
    try {
        Thread.sleep(1000);
    } catch (InterruptedException e) {
    // Do Nothing
    }
    Assert.assertEquals(totalTuples, sink.collectedTuples.size());
}
Also used : ResultFormatter(org.apache.apex.malhar.lib.appdata.schemas.ResultFormatter) MessageSerializerFactory(org.apache.apex.malhar.lib.appdata.query.serde.MessageSerializerFactory) MutableLong(org.apache.commons.lang3.mutable.MutableLong) DefaultOutputPort(com.datatorrent.api.DefaultOutputPort) CollectorTestSink(org.apache.apex.malhar.lib.testbench.CollectorTestSink) Test(org.junit.Test)

Example 30 with MutableLong

use of org.apache.commons.lang3.mutable.MutableLong in project apex-malhar by apache.

the class StateTracker method bucketAccessed.

void bucketAccessed(long bucketId) {
    long now = System.currentTimeMillis();
    if (accessedBucketIds.add(bucketId) || now - lastUpdateAccessTime > updateAccessTimeInterval) {
        synchronized (bucketLastAccess) {
            for (long id : accessedBucketIds) {
                MutableLong lastAccessTime = bucketLastAccess.get(id);
                if (lastAccessTime != null) {
                    lastAccessTime.setValue(now);
                } else {
                    bucketLastAccess.put(id, new MutableLong(now));
                }
            }
        }
        accessedBucketIds.clear();
        lastUpdateAccessTime = now;
    }
}
Also used : MutableLong(org.apache.commons.lang3.mutable.MutableLong)

Aggregations

MutableLong (org.apache.commons.lang3.mutable.MutableLong)66 Test (org.junit.Test)45 IOException (java.io.IOException)11 Query (org.apache.apex.malhar.lib.appdata.schemas.Query)6 CollectorTestSink (org.apache.apex.malhar.lib.testbench.CollectorTestSink)5 PageCache (org.neo4j.io.pagecache.PageCache)4 File (java.io.File)3 Map (java.util.Map)3 KeyedWindowedOperatorImpl (org.apache.apex.malhar.lib.window.impl.KeyedWindowedOperatorImpl)3 MutableBoolean (org.apache.commons.lang3.mutable.MutableBoolean)3 ParseException (java.text.ParseException)2 ArrayList (java.util.ArrayList)2 TreeMap (java.util.TreeMap)2 ConsoleOutputOperator (org.apache.apex.malhar.lib.io.ConsoleOutputOperator)2 SpillableComplexComponentImpl (org.apache.apex.malhar.lib.state.spillable.SpillableComplexComponentImpl)2 Tuple (org.apache.apex.malhar.lib.window.Tuple)2 WindowOption (org.apache.apex.malhar.lib.window.WindowOption)2 WindowState (org.apache.apex.malhar.lib.window.WindowState)2 SumLong (org.apache.apex.malhar.lib.window.accumulation.SumLong)2 WindowedOperatorImpl (org.apache.apex.malhar.lib.window.impl.WindowedOperatorImpl)2