Examples with RealLiveTable org.nustaq.reallive.api.RealLiveTable used on opensource projects

Example 1 with RealLiveTable

use of org.nustaq.reallive.api.RealLiveTable in project kontraktor by RuedigerMoeller.

the class DataClient method export.

/**
 * @param directory
 */
public IPromise export(String directory) {
    Promise res = new Promise();
    // use separate thread to enable slowly, blocking processing
    Actors.exec.execute(() -> {
        File d = new File(directory);
        if (d.exists() && (!d.isDirectory() || !d.canWrite())) {
            res.reject(new RuntimeException("cannot write to " + d + " or not a directory"));
            return;
        } else {
            d.mkdirs();
        }
        FSTConfiguration writeConf = FSTConfiguration.createDefaultConfiguration();
        Arrays.stream(config.getSchema()).forEach(desc -> {
            try {
                DataOutputStream fout = new DataOutputStream(new FileOutputStream(new File(d, desc.getName() + ".oos")));
                CountDownLatch pl = new CountDownLatch(shards.length);
                for (int i = 0; i < shards.length; i++) {
                    TableSpaceActor shard = shards[i];
                    Log.Info(this, "exporting shard " + i + " table " + desc.getName());
                    try {
                        RealLiveTable table = shard.getTableAsync(desc.getName()).await(60_000);
                        table.forEach(rec -> true, (rec, err) -> {
                            if (rec != null) {
                                try {
                                    // write marker to enable recovery in case of corruption
                                    synchronized (fout) {
                                        fout.write(31);
                                        fout.write(32);
                                        fout.write(33);
                                        fout.write(34);
                                        byte[] b = writeConf.asByteArray(rec);
                                        fout.writeInt(b.length);
                                        fout.write(b);
                                    }
                                } catch (IOException e) {
                                    Log.Error(this, e);
                                }
                            } else if (err != null) {
                                Log.Warn(this, "error during export " + err);
                                pl.countDown();
                            } else {
                                // fin
                                pl.countDown();
                            }
                        });
                    } catch (Exception e) {
                        Log.Error(this, "export failure " + desc.getName() + " shard " + i);
                    }
                }
                try {
                    boolean succ = pl.await(5, TimeUnit.MINUTES);
                    if (!succ)
                        Log.Error(this, "export timed out on table " + desc.getName());
                    try {
                        fout.close();
                    } catch (IOException e) {
                        Log.Error(this, e);
                    }
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            } catch (FileNotFoundException e) {
                Log.Error(this, e);
            }
        });
        res.complete();
    });
    return res;
}

Example 2 with RealLiveTable

use of org.nustaq.reallive.api.RealLiveTable in project kontraktor by RuedigerMoeller.

the class TableSpaceSharding method createOrLoadTable.

@Override
public IPromise<RealLiveTable> createOrLoadTable(TableDescription desc) {
    Promise<RealLiveTable> res = new Promise();
    ArrayList<IPromise<RealLiveTable>> results = new ArrayList();
    for (int i = 0; i < shards.length; i++) {
        TableSpaceActor shard = shards[i];
        IPromise<RealLiveTable> table = shard.createOrLoadTable(desc.clone().shardNo(i));
        Promise p = new Promise();
        results.add(p);
        final int finalI = i;
        table.then((r, e) -> {
            if (e == null)
                Log.Info(this, "table creation: " + desc.getName() + " " + finalI);
            else if (e instanceof Throwable)
                Log.Info(this, (Throwable) e, "failed table creation: " + desc.getName() + " " + finalI);
            else
                Log.Info(this, "failed table creation: " + desc.getName() + " " + finalI + " " + e);
            p.complete(r, e);
        });
    }
    List<IPromise<RealLiveTable>> tables = Actors.all(results).await();
    RealLiveTable[] tableShards = new RealLiveTable[tables.size()];
    boolean errors = false;
    for (int i = 0; i < tables.size(); i++) {
        if (tables.get(i).get() == null) {
            res.reject(tables.get(i).getError());
            errors = true;
            break;
        } else {
            // tables[i].get().shardNo();
            int sno = i;
            if (tableShards[sno] != null) {
                res.reject("shard " + sno + " is present more than once");
                errors = true;
                break;
            }
            tableShards[sno] = tables.get(i).get();
        }
    }
    if (!errors) {
        TableSharding ts = new TableSharding(func, tableShards, desc);
        tableMap.put(desc.getName(), ts);
        tableDescriptionMap.put(desc.getName(), desc);
        res.resolve(ts);
    }
    return res;
}

Example 3 with RealLiveTable

use of org.nustaq.reallive.api.RealLiveTable in project kontraktor by RuedigerMoeller.

the class DataClient method processSharded.

public void processSharded(String tableName, RLPredicate<Record> predicate, int shardNo, Callback<Record> cb) {
    TableSpaceActor shard = shards[shardNo];
    shard.getTableAsync(tableName).then(t -> {
        RealLiveTable table = t;
        table.forEach(predicate, cb);
    });
}

Example 4 with RealLiveTable

use of org.nustaq.reallive.api.RealLiveTable in project kontraktor by RuedigerMoeller.

the class UpdateMixin method updateRec.

/**
 * update or add
 * @param table
 * @param key
 * @param keyVals
 */
default IPromise updateRec(String table, String key, Object[] keyVals) {
    if (keyVals.getClass() != Object[].class) {
        Object[] tmp = new Object[keyVals.length];
        for (int i = 0; i < tmp.length; i++) {
            tmp[i] = keyVals[i];
        }
        keyVals = tmp;
    }
    RealLiveTable tab = getTable(table);
    if (tab == null)
        return Actors.reject("table not found " + table);
    tab.update(key, keyVals);
    return Actors.resolve();
}

Example 5 with RealLiveTable

use of org.nustaq.reallive.api.RealLiveTable in project kontraktor by RuedigerMoeller.

the class UpdateMixin method removeRec.

default IPromise removeRec(String table, String key) {
    RealLiveTable tab = getTable(table);
    if (tab == null)
        return Actors.reject("table not found " + table);
    tab.remove(key);
    return Actors.resolve();
}