Examples with GridFileLock org.apache.ignite.testframework.GridFileLock used on opensource projects

Example 1 with GridFileLock

use of org.apache.ignite.testframework.GridFileLock in project ignite by apache.

the class GridDsiClient method main.

/**
 * Execute DSI load client.
 *
 * @param args Command line arguments, two required - first one is the number of threads,
 *      second one should point to the Spring XML configuration file.
 * @throws Exception If client fails.
 */
@SuppressWarnings("unchecked")
public static void main(String[] args) throws Exception {
    GridFileLock fileLock = GridLoadTestUtils.fileLock();
    // Get shared lock, allowing multiple instances.
    fileLock.lock(true);
    try {
        Ignition.start(args.length < 4 ? "modules/core/src/test/config/load/dsi-load-client.xml" : args[3]);
        Thread collector = null;
        Thread timer = null;
        try {
            g = Ignition.ignite("dsi");
            int noThreads = Integer.parseInt(args[0]);
            final int duration = args.length < 2 ? 0 : Integer.parseInt(args[1]);
            final String outputFileName = args.length < 3 ? null : args[2];
            X.println("Thread count: " + noThreads);
            Collection<ClusterNode> srvNodes = g.cluster().forPredicate(serverNode()).nodes();
            if (srvNodes.isEmpty()) {
                X.println("No server nodes available");
                System.exit(-1);
            }
            X.println("No of servers: " + srvNodes.size());
            int srvMaxNoTerminals = noThreads / srvNodes.size();
            if (srvMaxNoTerminals * srvNodes.size() != noThreads) {
                noThreads = srvMaxNoTerminals * srvNodes.size();
                X.println("Using " + noThreads + " threads instead to ensure equal distribution of terminals");
            }
            Collection<Callable<Object>> clients = new ArrayList<>(noThreads);
            // No 2 client should use the same simulator.
            HashMap<UUID, Collection<String>> terminals = (HashMap<UUID, Collection<String>>) g.cache("CLIENT_PARTITIONED_CACHE").get("terminals");
            if (terminals == null) {
                X.println(">>> Terminals map has not been initialized.");
                terminals = new HashMap<>(srvNodes.size());
                // Distribute terminals evenly across all servers.
                for (ClusterNode node : srvNodes) {
                    UUID srvrId = node.id();
                    X.println(">>> Node ID: " + srvrId);
                    Collection<String> list = terminals.get(srvrId);
                    if (list == null)
                        list = new ArrayList<>(0);
                    int terminalsPerSrv = 0;
                    // Terminal ID.
                    int tid = 0;
                    while (true) {
                        String terminalId = String.valueOf(++tid);
                        // Server partition cache.
                        if (!srvrId.equals(g.affinity("PARTITIONED_CACHE").mapKeyToNode(terminalId).id()))
                            continue;
                        if (terminalsPerSrv < srvMaxNoTerminals) {
                            list.add(terminalId);
                            clients.add(new GridDsiClient(terminalId, srvrId));
                            terminalsPerSrv++;
                            X.println("Terminal ID: " + terminalId);
                        } else
                            break;
                    }
                    terminals.put(srvrId, list);
                }
                g.cache("CLIENT_PARTITIONED_CACHE").put("terminals", terminals);
            } else {
                X.println(">>> Terminals map has been initialized.");
                for (Map.Entry<UUID, Collection<String>> e : terminals.entrySet()) {
                    X.println(">>> Node ID: " + e.getKey());
                    for (String s : e.getValue()) {
                        clients.add(new GridDsiClient(s, e.getKey()));
                        X.println("Terminal ID: " + s);
                    }
                }
            }
            if (duration > 0) {
                timer = new Thread(new Runnable() {

                    @Override
                    public void run() {
                        try {
                            Thread.sleep(duration * 1000);
                            finish.set(true);
                        } catch (InterruptedException ignored) {
                        // No-op.
                        }
                    }
                });
                timer.start();
            }
            collector = new Thread(new Runnable() {

                @SuppressWarnings({ "BusyWait", "InfiniteLoopStatement" })
                @Override
                public void run() {
                    long txPerSecond = -1;
                    long avgLatency = -1;
                    long maxSubmitTime = -1;
                    T3<Long, Integer, Integer> sst = null;
                    try {
                        while (!finish.get()) {
                            long cnt0 = txCnt.get();
                            long lt0 = latency.get();
                            Thread.sleep(UPDATE_INTERVAL_SEC * 1000);
                            long cnt1 = txCnt.get();
                            long lt1 = latency.get();
                            X.println(">>>");
                            txPerSecond = (cnt1 - cnt0) / UPDATE_INTERVAL_SEC;
                            X.println(">>> Transaction/s: " + txPerSecond);
                            avgLatency = (cnt1 - cnt0) > 0 ? (lt1 - lt0) / (cnt1 - cnt0) : -1;
                            X.println(">>> Avg Latency: " + (avgLatency >= 0 ? avgLatency + "ms" : "invalid"));
                            maxSubmitTime = submitTime.getAndSet(0);
                            X.println(">>> Max Submit Time: " + maxSubmitTime);
                            sst = srvStats;
                            if (sst != null)
                                X.println(String.format(">>> Server stats: [tx/sec=%d, nearSize=%d, dhtSize=%d]", sst.get1(), sst.get2(), sst.get3()));
                        }
                    } catch (InterruptedException ignored) {
                        X.println(">>> Interrupted.");
                        Thread.currentThread().interrupt();
                    }
                    // Output data to a file, if specified.
                    if (outputFileName != null) {
                        X.println("Writing client results to a file: " + outputFileName);
                        try {
                            GridLoadTestUtils.appendLineToFile(outputFileName, "%s,%d,%d,%d", GridLoadTestUtils.DATE_TIME_FORMAT.format(new Date()), txPerSecond, avgLatency, maxSubmitTime);
                        } catch (IOException e) {
                            X.println("Failed to write client results: ", e);
                        }
                        if (sst != null) {
                            String srvOutputFileName = outputFileName + "-server";
                            X.println("Writing server results to a file: " + srvOutputFileName);
                            try {
                                GridLoadTestUtils.appendLineToFile(srvOutputFileName, "%s,%d,%d,%d", GridLoadTestUtils.DATE_TIME_FORMAT.format(new Date()), sst.get1(), sst.get2(), sst.get3());
                            } catch (IOException e) {
                                X.println("Failed to write server results: ", e);
                            }
                        }
                    }
                }
            });
            collector.start();
            ExecutorService pool = Executors.newFixedThreadPool(noThreads);
            pool.invokeAll(clients);
            collector.interrupt();
            pool.shutdown();
        } finally {
            if (collector != null && !collector.isInterrupted())
                collector.interrupt();
            if (timer != null)
                timer.interrupt();
            Ignition.stopAll(true);
        }
    } finally {
        fileLock.close();
    }
}

Example 2 with GridFileLock

use of org.apache.ignite.testframework.GridFileLock in project ignite by apache.

the class GridJobExecutionLoadTestClient method main.

/**
 * @param args Args.
 * @throws Exception If failed.
 */
public static void main(String[] args) throws Exception {
    GridFileLock fileLock = GridLoadTestUtils.fileLock();
    fileLock.lock();
    try {
        final int noThreads = args.length > 0 ? Integer.parseInt(args[0]) : 64;
        final int duration = args.length > 1 ? Integer.parseInt(args[1]) : 0;
        final String outputFileName = args.length > 2 ? args[2] : null;
        X.println("Thread count: " + noThreads);
        g = G.start("modules/tests/config/jobs-load-client.xml");
        warmUp(noThreads);
        final Thread collector = new Thread(new Runnable() {

            @SuppressWarnings("BusyWait")
            @Override
            public void run() {
                GridCumulativeAverage avgTxPerSec = new GridCumulativeAverage();
                try {
                    while (!finish) {
                        Thread.sleep(UPDATE_INTERVAL_SEC * 1000);
                        long txPerSec = txCnt.sumThenReset() / UPDATE_INTERVAL_SEC;
                        X.println(">>>");
                        X.println(">>> Transactions/s: " + txPerSec);
                        avgTxPerSec.update(txPerSec);
                    }
                } catch (InterruptedException ignored) {
                    X.println(">>> Interrupted.");
                    Thread.currentThread().interrupt();
                }
                X.println(">>> Average Transactions/s: " + avgTxPerSec);
                if (outputFileName != null) {
                    try {
                        X.println("Writing results to file: " + outputFileName);
                        GridLoadTestUtils.appendLineToFile(outputFileName, "%s,%d", GridLoadTestUtils.DATE_TIME_FORMAT.format(new Date()), avgTxPerSec.get());
                    } catch (IOException e) {
                        X.error("Failed to output results to file.", e);
                    }
                }
            }
        });
        X.println("Running main test...");
        Thread timer = null;
        try {
            ExecutorService pool = Executors.newFixedThreadPool(noThreads);
            Collection<Callable<Object>> clients = new ArrayList<>(noThreads);
            for (int i = 0; i < noThreads; i++) clients.add(new GridJobExecutionLoadTestClient());
            collector.start();
            if (duration > 0) {
                timer = new Thread(new Runnable() {

                    @Override
                    public void run() {
                        try {
                            Thread.sleep(duration * 1000);
                            finish = true;
                        } catch (InterruptedException ignored) {
                            X.println(">>> Interrupted.");
                        }
                    }
                });
                timer.start();
            }
            pool.invokeAll(clients);
            collector.interrupt();
            pool.shutdown();
        } finally {
            if (collector != null && !collector.isInterrupted())
                collector.interrupt();
            if (timer != null)
                timer.interrupt();
            G.stopAll(true);
        }
    } finally {
        fileLock.close();
    }
}

Example 3 with GridFileLock

use of org.apache.ignite.testframework.GridFileLock in project ignite by apache.

the class GridJobExecutionSingleNodeSemaphoreLoadTest method main.

/**
 * @param args Command line arguments:
 *             1-st: Number of worker threads. Default equals to available CPU number / 2.
 *             2-nd: Concurrent tasks count. Default: 1024.
 *             3-rd: Test duration in seconds. 0 means infinite. Default: 0.
 *             4-th: File to output test results to.
 * @throws Exception If failed.
 */
public static void main(String[] args) throws Exception {
    GridFileLock fileLock = GridLoadTestUtils.fileLock();
    fileLock.lock();
    try {
        // Command line arguments.
        // 
        // NOTE: on MacOS better numbers are shown if public pool core and max sizes are
        // equal to CPU count. And producer threads count is equal to CPU count.
        // 
        int threadCnt = args.length > 0 ? Integer.parseInt(args[0]) : Runtime.getRuntime().availableProcessors() / 2;
        int taskCnt = args.length > 1 ? Integer.parseInt(args[1]) : 1024;
        final int duration = args.length > 2 ? Integer.parseInt(args[2]) : 0;
        final String outputFileName = args.length > 3 ? args[3] : null;
        final LongAdder execCnt = new LongAdder();
        try {
            final Ignite g = G.start("modules/tests/config/grid-job-load.xml");
            X.println("Thread count: " + threadCnt);
            X.println("Task count: " + taskCnt);
            X.println("Duration: " + duration);
            X.println("Warming up...");
            g.compute().execute(GridJobExecutionLoadTestTask.class, null);
            g.compute().execute(GridJobExecutionLoadTestTask.class, null);
            runTest(g, threadCnt, taskCnt, WARM_UP_DURATION, execCnt);
            System.gc();
            execCnt.reset();
            X.println("Running main test.");
            IgniteInternalFuture<Void> collectorFut = GridTestUtils.runAsync(new Callable<Void>() {

                @Override
                public Void call() throws Exception {
                    GridCumulativeAverage avgTasksPerSec = new GridCumulativeAverage();
                    try {
                        while (!Thread.currentThread().isInterrupted()) {
                            U.sleep(UPDATE_INTERVAL_SEC * 1000);
                            long curTasksPerSec = execCnt.sumThenReset() / UPDATE_INTERVAL_SEC;
                            X.println(">>> Tasks/s: " + curTasksPerSec);
                            avgTasksPerSec.update(curTasksPerSec);
                        }
                    } catch (IgniteInterruptedCheckedException ignored) {
                        X.println(">>> Interrupted.");
                        Thread.currentThread().interrupt();
                    }
                    X.println(">>> Average tasks/s: " + avgTasksPerSec);
                    if (outputFileName != null) {
                        X.println("Writing test results to a file: " + outputFileName);
                        try {
                            GridLoadTestUtils.appendLineToFile(outputFileName, "%s,%d", GridLoadTestUtils.DATE_TIME_FORMAT.format(new Date()), avgTasksPerSec.get());
                        } catch (IOException e) {
                            X.error("Failed to output to a file", e);
                        }
                    }
                    return null;
                }
            });
            runTest(g, threadCnt, taskCnt, duration * 1000, execCnt);
            X.println("All done, stopping.");
            collectorFut.cancel();
        } finally {
            G.stopAll(true);
        }
    } finally {
        fileLock.close();
    }
}

Example 4 with GridFileLock

use of org.apache.ignite.testframework.GridFileLock in project ignite by apache.

the class GridMergeSortLoadTest method main.

/**
 * Entry point for this test. Generates an input array of random integers,
 * starts the grid, and launches the job. The method then waits for job completion.
 *
 * @param args Program arguments.
 *      <ul>
 *          <li>
 *              <b>1-st argument:</b> absolute or relative path to the configuration
 *              file for the grid (optional).
 *          </li>
 *          <li>
 *              <b>2-nd argument:</b> size of the generated array (optional, default: {@code 100000}).
 *          </li>
 *          <li>
 *              <b>3-nd argument:</b> size of the generated array for "warm up" (optional, default: {@code 10000}).
 *          </li>
 *      </ul>
 * @throws IgniteCheckedException In case of error.
 * @throws IOException In case of file output error.
 */
public static void main(String[] args) throws IgniteCheckedException, IOException {
    GridFileLock fileLock = GridLoadTestUtils.fileLock();
    fileLock.lock();
    try {
        String outputFileName = args.length >= 1 ? args[0] : null;
        try (Ignite g = G.start(args.length >= 2 ? args[1] : "modules/core/src/test/config/load/merge-sort-base.xml")) {
            int arrRealSize = args.length > 1 ? Integer.parseInt(args[1]) : ARR_SIZE;
            int arrWarmupSize = args.length > 2 ? Integer.parseInt(args[2]) : ARR_SIZE;
            X.println("Test is being executed on the gird of size " + g.cluster().nodes().size() + ".");
            X.println("Performing warm up sorting of int[" + arrWarmupSize + "]...");
            sort(g, arrWarmupSize);
            X.println("Cleaning up after warm-up...");
            // Run GC on all nodes.
            g.compute().broadcast(new GridAbsClosure() {

                @Override
                public void apply() {
                    System.gc();
                }
            });
            X.println("Performing measured sorting of int[" + arrRealSize + "]...");
            long execTime = sort(g, arrRealSize);
            if (outputFileName != null)
                GridLoadTestUtils.appendLineToFile(outputFileName, "%s,%d", GridLoadTestUtils.DATE_TIME_FORMAT.format(new Date()), execTime / 1000);
        }
    } finally {
        fileLock.close();
    }
}

Example 5 with GridFileLock

use of org.apache.ignite.testframework.GridFileLock in project ignite by apache.

the class GridCacheBenchmark method main.

/**
 * @param args Arguments.
 * @throws Exception If failed.
 */
@SuppressWarnings("BusyWait")
public static void main(String[] args) throws Exception {
    GridFileLock fileLock = GridLoadTestUtils.fileLock();
    fileLock.lock();
    try {
        final String outputFileName = args.length > 0 ? args[0] : null;
        // try (Grid g = G.start("modules/core/src/test/config/load/cache-client-benchmark.xml")) {
        try (Ignite g = G.start("modules/core/src/test/config/load/cache-benchmark.xml")) {
            X.println("warmupTime=" + WARM_UP_TIME);
            X.println("putCnt=" + PUT_CNT);
            X.println("threadCnt=" + THREADS);
            X.println("testWrite=" + testWrite);
            final IgniteCache<Long, Long> cache = g.cache(CACHE);
            assert cache != null;
            cntr.set(0);
            final AtomicLong opCnt = new AtomicLong();
            X.println("Warming up (putx)...");
            GridLoadTestUtils.runMultithreadedInLoop(new Callable<Object>() {

                @Nullable
                @Override
                public Object call() throws Exception {
                    long keyVal = cntr.incrementAndGet();
                    cache.put(keyVal % 100000, keyVal);
                    long ops = opCnt.incrementAndGet();
                    if (ops % LOG_MOD == 0)
                        X.println(">>> Performed " + ops + " operations.");
                    return null;
                }
            }, THREADS, WARM_UP_TIME);
            cntr.set(0);
            opCnt.set(0);
            X.println("Warming up (get)...");
            GridLoadTestUtils.runMultithreadedInLoop(new Callable<Object>() {

                @Nullable
                @Override
                public Object call() throws Exception {
                    long keyVal = cntr.incrementAndGet();
                    Long old = cache.get(keyVal % 100000);
                    long ops = opCnt.incrementAndGet();
                    if (ops % LOG_MOD == 0)
                        X.println(">>> Performed " + ops + " operations, old=" + old + ", keyval=" + keyVal);
                    return null;
                }
            }, THREADS, WARM_UP_TIME);
            cache.clear();
            System.gc();
            cntr.set(0);
            opCnt.set(0);
            X.println("Starting Ignite cache putx() benchmark...");
            long durPutx = GridLoadTestUtils.measureTime(new Callable<Object>() {

                @Nullable
                @Override
                public Object call() throws Exception {
                    while (true) {
                        long keyVal = cntr.incrementAndGet();
                        if (keyVal >= PUT_CNT)
                            break;
                        cache.put(keyVal % 100000, keyVal);
                        long ops = opCnt.incrementAndGet();
                        if (ops % LOG_MOD == 0)
                            X.println(">>> Performed " + ops + " operations.");
                    }
                    return null;
                }
            }, THREADS);
            X.println(">>>");
            X.println(">> Ignite cache putx() benchmark results [duration=" + durPutx + " ms, tx/sec=" + (opCnt.get() * 1000 / durPutx) + ", total=" + opCnt.get() + ']');
            X.println(">>>");
            System.gc();
            cntr.set(0);
            opCnt.set(0);
            X.println("Starting Ignite cache get() benchmark...");
            long durGet = GridLoadTestUtils.measureTime(new Callable<Object>() {

                @Nullable
                @Override
                public Object call() throws Exception {
                    while (true) {
                        long keyVal = cntr.incrementAndGet();
                        if (keyVal >= PUT_CNT)
                            break;
                        Long old = cache.get(keyVal % 100000);
                        long ops = opCnt.incrementAndGet();
                        if (ops % LOG_MOD == 0)
                            X.println(">>> Performed " + ops + " operations, old=" + old + ", keyval=" + keyVal);
                    }
                    return null;
                }
            }, THREADS);
            X.println(">>>");
            X.println(">> Ignite cache get() benchmark results [duration=" + durGet + " ms, tx/sec=" + (opCnt.get() * 1000 / durGet) + ", total=" + opCnt.get() + ']');
            X.println(">>>");
            if (outputFileName != null)
                GridLoadTestUtils.appendLineToFile(outputFileName, "%s,%d,%d", GridLoadTestUtils.DATE_TIME_FORMAT.format(new Date()), durPutx, durGet);
        }
    } finally {
        fileLock.close();
    }
}