Examples with TableSpaceActor org.nustaq.reallive.impl.tablespace.TableSpaceActor used on opensource projects

Example 1 with TableSpaceActor

use of org.nustaq.reallive.impl.tablespace.TableSpaceActor in project kontraktor by RuedigerMoeller.

the class DataClient method export.

/**
 * @param directory
 */
public IPromise export(String directory) {
    Promise res = new Promise();
    // use separate thread to enable slowly, blocking processing
    Actors.exec.execute(() -> {
        File d = new File(directory);
        if (d.exists() && (!d.isDirectory() || !d.canWrite())) {
            res.reject(new RuntimeException("cannot write to " + d + " or not a directory"));
            return;
        } else {
            d.mkdirs();
        }
        FSTConfiguration writeConf = FSTConfiguration.createDefaultConfiguration();
        Arrays.stream(config.getSchema()).forEach(desc -> {
            try {
                DataOutputStream fout = new DataOutputStream(new FileOutputStream(new File(d, desc.getName() + ".oos")));
                CountDownLatch pl = new CountDownLatch(shards.length);
                for (int i = 0; i < shards.length; i++) {
                    TableSpaceActor shard = shards[i];
                    Log.Info(this, "exporting shard " + i + " table " + desc.getName());
                    try {
                        RealLiveTable table = shard.getTableAsync(desc.getName()).await(60_000);
                        table.forEach(rec -> true, (rec, err) -> {
                            if (rec != null) {
                                try {
                                    // write marker to enable recovery in case of corruption
                                    synchronized (fout) {
                                        fout.write(31);
                                        fout.write(32);
                                        fout.write(33);
                                        fout.write(34);
                                        byte[] b = writeConf.asByteArray(rec);
                                        fout.writeInt(b.length);
                                        fout.write(b);
                                    }
                                } catch (IOException e) {
                                    Log.Error(this, e);
                                }
                            } else if (err != null) {
                                Log.Warn(this, "error during export " + err);
                                pl.countDown();
                            } else {
                                // fin
                                pl.countDown();
                            }
                        });
                    } catch (Exception e) {
                        Log.Error(this, "export failure " + desc.getName() + " shard " + i);
                    }
                }
                try {
                    boolean succ = pl.await(5, TimeUnit.MINUTES);
                    if (!succ)
                        Log.Error(this, "export timed out on table " + desc.getName());
                    try {
                        fout.close();
                    } catch (IOException e) {
                        Log.Error(this, e);
                    }
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            } catch (FileNotFoundException e) {
                Log.Error(this, e);
            }
        });
        res.complete();
    });
    return res;
}

Example 2 with TableSpaceActor

use of org.nustaq.reallive.impl.tablespace.TableSpaceActor in project kontraktor by RuedigerMoeller.

the class DataClient method connect.

public IPromise connect(DataCfg config, TableSpaceActor[] shards, ServiceActor hostingService) {
    this.config = config;
    this.hostingService = hostingService;
    this.shards = shards;
    syncTableAccess = new HashMap();
    tableSharding = new TableSpaceSharding(shards, key -> Math.abs(key.hashCode()) % shards.length);
    tableSharding.init().await();
    TableDescription[] schema = config.getSchema();
    return all(schema.length, i -> {
        Promise p = new Promise();
        tableSharding.createOrLoadTable(schema[i]).then((r, e) -> {
            if (r != null) {
                syncTableAccess.put(schema[i].getName(), r);
            }
            p.complete(r, e);
        });
        return p;
    });
}

Example 3 with TableSpaceActor

use of org.nustaq.reallive.impl.tablespace.TableSpaceActor in project kontraktor by RuedigerMoeller.

the class TableSpaceTest method simpleRemoteNoServer.

@Test
public void simpleRemoteNoServer() {
    TableSpaceActor remoteTS = (TableSpaceActor) new TCPConnectable(TableSpaceActor.class, "localhost", 5432).connect((disc, err) -> System.out.println("client disc " + disc + " " + err)).await();
    Assert.assertTrue(runSimpleTest(remoteTS, () -> createTableDescription()) == EXPECT_SIMPLECOUNT);
}

Example 4 with TableSpaceActor

use of org.nustaq.reallive.impl.tablespace.TableSpaceActor in project kontraktor by RuedigerMoeller.

the class Basic method testTableSpace.

@Test
public void testTableSpace() {
    TableSpaceActor ts = Actors.AsActor(TableSpaceActor.class);
    ts.init();
    ts.createOrLoadTable(new TableDescription("blogs").numEntries(500_000).storageType(TableDescription.StorageType.TEMP).sizeMB(500)).await();
    ts.createOrLoadTable(new TableDescription("articles").numEntries(500_000 * 10).storageType(TableDescription.StorageType.TEMP).sizeMB(500 * 10)).await();
    RealLiveTable blogs = ts.getTableAsync("blogs").await();
    RealLiveTable articles = ts.getTableAsync("articles").await();
    int numBP = 30_000;
    for (int i = 0; i < numBP; i++) {
        MapRecord blogEntry = MapRecord.New("blog" + i);
        blogEntry.put("title", "Java is beautiful").put("description", "High performance Java, realtime distributed computing, other stuff").put("tags", "#tech #java #distributed #actors #concurrency #webcomponents #realtime").put("author", "R.Moeller").put("likes", 199).put("reads", 3485).put("sequence", i);
        blogs.addRecord(blogEntry);
    }
    System.out.println("finished blogs");
    for (int i = 0; i < numBP * 10; i++) {
        MapRecord article = MapRecord.New("art" + i);
        // article
        // .putRecord("title", "WebComponents BLabla")
        // .putRecord("description", "High performance Java, realtime distributed computing, other stuff")
        // .putRecord("tags", "#tech #java #distributed #actors #concurrency #webcomponents #realtime")
        // .putRecord("author", "R.Moeller")
        // .putRecord("likes", 199)
        // .putRecord("reads", 3485)
        // .putRecord("blog", "blog"+i)
        // .putRecord("date", new Date() );
        article.put("tags", new String[] { "#tech", "#java2", "#distributed", "#actors", "#concurrency", "#webcomponents", "#realtime" }).put("likes", 199).put("reads", 3485).put("blog", "blog" + i).put("date", System.currentTimeMillis());
        articles.addRecord(article);
    }
    System.out.println("finished articles");
    // while( true )
    {
        long tim = System.currentTimeMillis();
        AtomicInteger counter = new AtomicInteger(0);
        blogs.forEachWithSpore(new FilterSpore(record -> record.getKey().indexOf("99") >= 0).setForEach((r, e) -> {
        }));
        blogs.ping().await();
        System.out.println("time blo " + (System.currentTimeMillis() - tim));
        System.out.println("hits:" + counter.get());
        counter.set(0);
        tim = System.currentTimeMillis();
        articles.forEachWithSpore(new FilterSpore(record -> record.getKey().indexOf("999") >= 0).setForEach((r, e) -> {
        }));
        articles.ping().await();
        System.out.println("time art " + (System.currentTimeMillis() - tim));
        System.out.println("hits:" + counter.get());
    }
// ts.shutDown();
}

Example 5 with TableSpaceActor

use of org.nustaq.reallive.impl.tablespace.TableSpaceActor in project kontraktor by RuedigerMoeller.

the class TableSpaceTest method simple.

@Test
public void simple() {
    TableSpaceActor ts = Actors.AsActor(TableSpaceActor.class);
    ts.init();
    Assert.assertTrue(runSimpleTest(ts, () -> createTableDescription()) == EXPECT_SIMPLECOUNT);
    ts.shutDown().await();
}