Examples with DataFileCollection com.swirlds.jasperdb.files.DataFileCollection used on opensource projects

Example 1 with DataFileCollection

use of com.swirlds.jasperdb.files.DataFileCollection in project hedera-services by hashgraph.

the class SmallVsBigFilesBench method setup.

@Setup(Level.Trial)
public void setup() throws Exception {
    accessibleReadDataItem = DataFileCollection.class.getDeclaredMethod("readDataItem", long.class);
    accessibleReadDataItem.setAccessible(true);
    // calculate number of files, number of data items per file
    numOfFiles = 512 / fileSizeGb;
    System.out.println("numOfFiles = " + numOfFiles);
    numOfDataItemsPerFile = (int) ((fileSizeGb * GB) / dataValueSize);
    System.out.println("numOfDataItemsPerFile = " + numOfDataItemsPerFile);
    long numOfDataItems = (long) numOfDataItemsPerFile * (long) numOfFiles;
    System.out.println("numOfDataItems = " + numOfDataItems);
    // create 1Tb of data
    Path dataDir = Path.of("jasperdb_fs" + fileSizeGb + "_bs" + dataValueSize);
    if (Files.isDirectory(dataDir)) {
        dataFileCollection = new DataFileCollection<>(dataDir, "jasperdb", new ByteArrayDataItemSerializer(), null);
    } else {
        // new
        Files.createDirectories(dataDir);
        // 
        dataFileCollection = new DataFileCollection<>(dataDir, "jasperdb", new ByteArrayDataItemSerializer(), null);
        // create files
        long START = System.currentTimeMillis();
        int count = 0;
        for (int f = 0; f < numOfFiles; f++) {
            dataFileCollection.startWriting();
            for (int i = 0; i < numOfDataItemsPerFile; i++, count++) {
                // create some random data to write
                byte[] bytes = new byte[dataValueSize];
                new Random(123456).nextBytes(bytes);
                dataFileCollection.storeDataItem(new DataBlob(count, bytes));
                if (count % 10_000_000 == 0)
                    System.out.printf("count = %,d\n", count);
            }
            System.out.println("writes done");
            long START2 = System.currentTimeMillis();
            dataFileCollection.endWriting(0, count);
            double timeTakenSeconds = (double) (System.currentTimeMillis() - START2) / 1000d;
            System.out.printf("endWriting %,.2f seconds\n", timeTakenSeconds);
        }
        long timeTaken = System.currentTimeMillis() - START;
        System.out.println("YAY!! finished writing all data");
        double timeTakenSeconds = (double) timeTaken / 1000d;
        System.out.printf("%,d in %,.2f seconds\n", count, timeTakenSeconds);
    }
}