Examples with Slice com.datatorrent.netlet.util.Slice used on opensource projects

Example 31 with Slice

use of com.datatorrent.netlet.util.Slice in project apex-malhar by apache.

the class ManagedStateImplTest method testCommitted.

@Test
public void testCommitted() {
    Slice one = ManagedStateTestUtils.getSliceFor("1");
    Slice two = ManagedStateTestUtils.getSliceFor("2");
    commitHelper(one, two);
    Bucket.DefaultBucket defaultBucket = (Bucket.DefaultBucket) testMeta.managedState.getBucket(0);
    Assert.assertEquals("value of one", one, defaultBucket.getCommittedData().firstEntry().getValue().get(one).getValue());
    Assert.assertNull("value of two", defaultBucket.getCommittedData().firstEntry().getValue().get(two));
    testMeta.managedState.teardown();
}

Example 32 with Slice

use of com.datatorrent.netlet.util.Slice in project apex-malhar by apache.

the class ManagedStateImplTest method testAsyncGetFromCheckpoint.

@Test
public void testAsyncGetFromCheckpoint() throws ExecutionException, InterruptedException {
    Slice one = ManagedStateTestUtils.getSliceFor("1");
    testMeta.managedState.setup(testMeta.operatorContext);
    long time = System.currentTimeMillis();
    testMeta.managedState.beginWindow(time);
    testMeta.managedState.put(0, one, one);
    testMeta.managedState.endWindow();
    testMeta.managedState.beforeCheckpoint(time);
    Future<Slice> valFuture = testMeta.managedState.getAsync(0, one);
    Assert.assertEquals("value of one", one, valFuture.get());
    testMeta.managedState.teardown();
}

Example 33 with Slice

use of com.datatorrent.netlet.util.Slice in project apex-malhar by apache.

the class ManagedTimeStateImplTest method testAsyncGetWithTime.

@Test
public void testAsyncGetWithTime() throws ExecutionException, InterruptedException {
    Slice one = ManagedStateTestUtils.getSliceFor("1");
    testMeta.managedState.setup(testMeta.operatorContext);
    long time = System.currentTimeMillis();
    testMeta.managedState.beginWindow(0);
    testMeta.managedState.put(0, time, one, one);
    Future<Slice> valFuture = testMeta.managedState.getAsync(0, time, one);
    Slice value = valFuture.get();
    Assert.assertEquals("value of one", one, value);
    testMeta.managedState.teardown();
}

Example 34 with Slice

use of com.datatorrent.netlet.util.Slice in project apex-malhar by apache.

the class BucketsFileSystem method loadBucketMetaFile.

/**
 * Loads the bucket meta-file. This should be entered only after acquiring the lock on {@link #timeBucketsMeta}.
 *
 * @param bucketId bucket id
 * @param dis      data input stream
 * @throws IOException
 */
private void loadBucketMetaFile(long bucketId, DataInputStream dis) throws IOException {
    LOG.debug("Loading bucket meta-file {}", bucketId);
    int metaDataVersion = dis.readInt();
    if (metaDataVersion == META_FILE_VERSION) {
        int numberOfEntries = dis.readInt();
        for (int i = 0; i < numberOfEntries; i++) {
            long timeBucketId = dis.readLong();
            long dataSize = dis.readLong();
            long lastTransferredWindow = dis.readLong();
            MutableTimeBucketMeta tbm = new MutableTimeBucketMeta(bucketId, timeBucketId);
            int sizeOfFirstKey = dis.readInt();
            byte[] firstKeyBytes = new byte[sizeOfFirstKey];
            dis.readFully(firstKeyBytes, 0, firstKeyBytes.length);
            tbm.updateTimeBucketMeta(lastTransferredWindow, dataSize, new Slice(firstKeyBytes));
            timeBucketsMeta.put(bucketId, timeBucketId, tbm);
        }
    }
}

Example 35 with Slice

use of com.datatorrent.netlet.util.Slice in project apex-malhar by apache.

the class BucketsFileSystem method writeBucketData.

/**
 * Saves data to a bucket. The data consists of key/values of all time-buckets of a particular bucket.
 *
 * @param windowId        window id
 * @param bucketId        bucket id
 * @param data            data of all time-buckets
 * @throws IOException
 */
protected void writeBucketData(long windowId, long bucketId, Map<Slice, Bucket.BucketedValue> data, long latestPurgedTimeBucket) throws IOException {
    Table<Long, Slice, Bucket.BucketedValue> timeBucketedKeys = TreeBasedTable.create(Ordering.<Long>natural(), managedStateContext.getKeyComparator());
    for (Map.Entry<Slice, Bucket.BucketedValue> entry : data.entrySet()) {
        long timeBucketId = entry.getValue().getTimeBucket();
        if (timeBucketId <= latestPurgedTimeBucket) {
            continue;
        }
        timeBucketedKeys.put(timeBucketId, entry.getKey(), entry.getValue());
    }
    for (long timeBucket : timeBucketedKeys.rowKeySet()) {
        BucketsFileSystem.MutableTimeBucketMeta tbm = getMutableTimeBucketMeta(bucketId, timeBucket);
        if (tbm == null) {
            tbm = new MutableTimeBucketMeta(bucketId, timeBucket);
        }
        addBucketName(bucketId);
        long dataSize = 0;
        Slice firstKey = null;
        FileAccess.FileWriter fileWriter;
        String tmpFileName = getTmpFileName();
        if (tbm.getLastTransferredWindowId() == -1) {
            // A new time bucket so we append all the key/values to the new file
            fileWriter = getWriter(bucketId, tmpFileName);
            for (Map.Entry<Slice, Bucket.BucketedValue> entry : timeBucketedKeys.row(timeBucket).entrySet()) {
                Slice key = entry.getKey();
                Slice value = entry.getValue().getValue();
                dataSize += key.length;
                dataSize += value.length;
                fileWriter.append(key, value);
                if (firstKey == null) {
                    firstKey = key;
                }
            }
        } else {
            // the time bucket existed so we need to read the file and then re-write it
            TreeMap<Slice, Slice> fileData = new TreeMap<>(managedStateContext.getKeyComparator());
            FileAccess.FileReader fileReader = getReader(bucketId, getFileName(timeBucket));
            fileReader.readFully(fileData);
            fileReader.close();
            for (Map.Entry<Slice, Bucket.BucketedValue> entry : timeBucketedKeys.row(timeBucket).entrySet()) {
                fileData.put(entry.getKey(), entry.getValue().getValue());
            }
            fileWriter = getWriter(bucketId, tmpFileName);
            for (Map.Entry<Slice, Slice> entry : fileData.entrySet()) {
                Slice key = entry.getKey();
                Slice value = entry.getValue();
                dataSize += key.length;
                dataSize += value.length;
                fileWriter.append(key, value);
                if (firstKey == null) {
                    firstKey = key;
                }
            }
        }
        fileWriter.close();
        rename(bucketId, tmpFileName, getFileName(timeBucket));
        tbm.updateTimeBucketMeta(windowId, dataSize, firstKey);
        updateTimeBuckets(tbm);
    }
    updateBucketMetaFile(bucketId);
}