Examples with QueueFile com.squareup.tape2.QueueFile used on opensource projects

Example 1 with QueueFile

use of com.squareup.tape2.QueueFile in project java by wavefrontHQ.

the class ConcurrentShardedQueueFileTest method testConcurrency.

@Test
public void testConcurrency() throws Exception {
    File file = new File(File.createTempFile("proxyConcurrencyTest", null).getPath() + ".spool");
    ConcurrentShardedQueueFile queueFile = new ConcurrentShardedQueueFile(file.getCanonicalPath(), ".spool", 1024 * 1024, s -> new TapeQueueFile(new QueueFile.Builder(new File(s)).build()));
    Queue<Pair<Integer, Byte>> taskCheatSheet = new ArrayDeque<>();
    System.out.println(queueFile.shards.size());
    AtomicLong tasksGenerated = new AtomicLong();
    AtomicLong nanosAdd = new AtomicLong();
    AtomicLong nanosGet = new AtomicLong();
    while (queueFile.shards.size() < 4) {
        byte[] task = randomTask();
        queueFile.add(task);
        taskCheatSheet.add(Pair.of(task.length, task[0]));
        tasksGenerated.incrementAndGet();
    }
    AtomicBoolean done = new AtomicBoolean(false);
    AtomicBoolean fail = new AtomicBoolean(false);
    Runnable addTask = () -> {
        int delay = 0;
        while (!done.get() && !fail.get()) {
            try {
                byte[] task = randomTask();
                long start = System.nanoTime();
                queueFile.add(task);
                nanosAdd.addAndGet(System.nanoTime() - start);
                taskCheatSheet.add(Pair.of(task.length, task[0]));
                tasksGenerated.incrementAndGet();
                Thread.sleep(delay / 1000);
                delay++;
            } catch (Exception e) {
                e.printStackTrace();
                fail.set(true);
            }
        }
    };
    Runnable getTask = () -> {
        int delay = 2000;
        while (!taskCheatSheet.isEmpty() && !fail.get()) {
            try {
                long start = System.nanoTime();
                Pair<Integer, Byte> taskData = taskCheatSheet.remove();
                byte[] task = queueFile.peek();
                queueFile.remove();
                nanosGet.addAndGet(System.nanoTime() - start);
                if (taskData._1 != task.length) {
                    System.out.println("Data integrity fail! Expected: " + taskData._1 + " bytes, got " + task.length + " bytes");
                    fail.set(true);
                }
                for (byte b : task) {
                    if (taskData._2 != b) {
                        System.out.println("Data integrity fail! Expected " + taskData._2 + ", got " + b);
                        fail.set(true);
                    }
                }
                Thread.sleep(delay / 500);
                if (delay > 0)
                    delay--;
            } catch (Exception e) {
                e.printStackTrace();
                fail.set(true);
            }
        }
        done.set(true);
    };
    ExecutorService executor = Executors.newFixedThreadPool(2);
    long start = System.nanoTime();
    Future<?> addFuture = executor.submit(addTask);
    Future<?> getFuture = executor.submit(getTask);
    addFuture.get();
    getFuture.get();
    assertFalse(fail.get());
    System.out.println("Tasks generated: " + tasksGenerated.get());
    System.out.println("Real time (ms) = " + (System.nanoTime() - start) / 1_000_000);
    System.out.println("Add + remove time (ms) = " + (nanosGet.get() + nanosAdd.get()) / 1_000_000);
    System.out.println("Add time (ms) = " + nanosAdd.get() / 1_000_000);
    System.out.println("Remove time (ms) = " + nanosGet.get() / 1_000_000);
}

Example 2 with QueueFile

use of com.squareup.tape2.QueueFile in project java by wavefrontHQ.

the class TaskQueueFactoryImpl method createTaskQueue.

private <T extends DataSubmissionTask<T>> TaskQueue<T> createTaskQueue(@Nonnull HandlerKey handlerKey, int threadNum) {
    String fileName = bufferFile + "." + handlerKey.getEntityType().toString() + "." + handlerKey.getHandle() + "." + threadNum;
    String lockFileName = fileName + ".lck";
    String spoolFileName = fileName + ".spool";
    // iron-clad guarantee, but it works well in most cases.
    try {
        File lockFile = new File(lockFileName);
        if (lockFile.exists()) {
            Files.deleteIfExists(lockFile.toPath());
        }
        FileChannel channel = new RandomAccessFile(lockFile, "rw").getChannel();
        if (channel.tryLock() == null) {
            throw new OverlappingFileLockException();
        }
    } catch (SecurityException e) {
        logger.severe("Error writing to the buffer lock file " + lockFileName + " - please make sure write permissions are correct for this file path and restart the " + "proxy: " + e);
        return new TaskQueueStub<>();
    } catch (OverlappingFileLockException e) {
        logger.severe("Error requesting exclusive access to the buffer " + "lock file " + lockFileName + " - please make sure that no other processes " + "access this file and restart the proxy: " + e);
        return new TaskQueueStub<>();
    } catch (IOException e) {
        logger.severe("Error requesting access to buffer lock file " + lockFileName + " Channel is " + "closed or an I/O error has occurred - please restart the proxy: " + e);
        return new TaskQueueStub<>();
    }
    try {
        File buffer = new File(spoolFileName);
        if (purgeBuffer) {
            if (buffer.delete()) {
                logger.warning("Retry buffer has been purged: " + spoolFileName);
            }
        }
        BiConsumer<Integer, Long> statsUpdater = (bytes, millis) -> {
            bytesWritten.inc(bytes);
            ioTimeWrites.inc(millis);
        };
        com.wavefront.agent.queueing.QueueFile queueFile = disableSharding ? new ConcurrentQueueFile(new TapeQueueFile(new QueueFile.Builder(new File(spoolFileName)).build(), statsUpdater)) : new ConcurrentShardedQueueFile(spoolFileName, ".spool", shardSize * 1024 * 1024, s -> new TapeQueueFile(new QueueFile.Builder(new File(s)).build(), statsUpdater));
        // TODO: allow configurable compression types and levels
        return new InstrumentedTaskQueueDelegate<>(new FileBasedTaskQueue<>(queueFile, new RetryTaskConverter<T>(handlerKey.getHandle(), TaskConverter.CompressionType.LZ4)), "buffer", ImmutableMap.of("port", handlerKey.getHandle()), handlerKey.getEntityType());
    } catch (Exception e) {
        logger.severe("WF-006: Unable to open or create queue file " + spoolFileName + ": " + e.getMessage());
        return new TaskQueueStub<>();
    }
}