Examples with JournalStructure com.questdb.store.factory.configuration.JournalStructure used on opensource projects

Example 1 with JournalStructure

use of com.questdb.store.factory.configuration.JournalStructure in project questdb by bluestreak01.

the class JoinQueryTest method generateJoinData.

private static void generateJoinData() throws JournalException, NumericException {
    try (JournalWriter customers = FACTORY_CONTAINER.getFactory().writer(new JournalStructure("customers").$int("customerId").$str("customerName").$str("contactName").$str("address").$str("city").$str("postalCode").$sym("country").$ts())) {
        try (JournalWriter categories = FACTORY_CONTAINER.getFactory().writer(new JournalStructure("categories").$sym("category").index().buckets(100).$str("description").$ts())) {
            try (JournalWriter employees = FACTORY_CONTAINER.getFactory().writer(new JournalStructure("employees").$str("employeeId").index().buckets(2048).$str("firstName").$str("lastName").$date("birthday").$ts())) {
                try (JournalWriter orderDetails = FACTORY_CONTAINER.getFactory().writer(new JournalStructure("orderDetails").$int("orderDetailId").$int("orderId").$int("productId").$int("quantity").$ts())) {
                    try (JournalWriter orders = FACTORY_CONTAINER.getFactory().writer(new JournalStructure("orders").$int("orderId").$int("customerId").index().$int("productId").$str("employeeId").index().$ts("orderDate").$sym("shipper").$())) {
                        try (JournalWriter products = FACTORY_CONTAINER.getFactory().writer(new JournalStructure("products").$int("productId").$str("productName").$sym("supplier").index().buckets(100).$sym("category").index().buckets(100).$double("price").$ts())) {
                            try (JournalWriter shippers = FACTORY_CONTAINER.getFactory().writer(new JournalStructure("shippers").$sym("shipper").$str("phone").$ts())) {
                                try (JournalWriter suppliers = FACTORY_CONTAINER.getFactory().writer(new JournalStructure("suppliers").$sym("supplier").buckets(100).$str("contactName").$str("address").$str("city").$str("postalCode").$sym("country").index().$str("phone").$ts())) {
                                    final Rnd rnd = new Rnd();
                                    long time = DateFormatUtils.parseDateTime("2015-07-10T00:00:00.000Z");
                                    // statics
                                    int countryCount = 196;
                                    ObjList<String> countries = new ObjList<>();
                                    for (int i = 0; i < countryCount; i++) {
                                        countries.add(rnd.nextString(rnd.nextInt() & 15));
                                    }
                                    IntHashSet blackList = new IntHashSet();
                                    // customers
                                    int customerCount = 10000;
                                    for (int i = 0; i < customerCount; i++) {
                                        if (rnd.nextPositiveInt() % 100 == 0) {
                                            blackList.add(i);
                                        }
                                        JournalEntryWriter w = customers.entryWriter();
                                        w.putInt(0, i);
                                        w.putStr(1, rnd.nextChars(rnd.nextInt() & 15));
                                        w.putStr(2, rnd.nextChars(rnd.nextInt() & 31));
                                        w.putStr(4, rnd.nextChars(rnd.nextInt() & 63));
                                        w.putStr(5, rnd.nextChars(rnd.nextInt() & 15));
                                        w.putSym(6, countries.getQuick(rnd.nextPositiveInt() % 196));
                                        w.putDate(7, time++);
                                        w.append();
                                    }
                                    customers.commit();
                                    // categories
                                    for (int i = 0; i < 100; i++) {
                                        JournalEntryWriter w = categories.entryWriter();
                                        w.putSym(0, rnd.nextChars(rnd.nextInt() & 15));
                                        w.putStr(1, rnd.nextChars(rnd.nextInt() & 63));
                                        w.putDate(2, time++);
                                        w.append();
                                    }
                                    categories.commit();
                                    // employees
                                    int employeeCount = 2000;
                                    for (int i = 0; i < employeeCount; i++) {
                                        JournalEntryWriter w = employees.entryWriter();
                                        w.putStr(0, rnd.nextChars(rnd.nextInt() & 7));
                                        w.putStr(1, rnd.nextChars(rnd.nextInt() & 15));
                                        w.putStr(2, rnd.nextChars(rnd.nextInt() & 15));
                                        w.putDate(3, 0);
                                        w.putDate(4, time++);
                                        w.append();
                                    }
                                    employees.commit();
                                    // suppliers
                                    for (int i = 0; i < 100; i++) {
                                        JournalEntryWriter w = suppliers.entryWriter();
                                        w.putSym(0, rnd.nextChars(rnd.nextInt() & 15));
                                        w.putStr(1, rnd.nextChars(rnd.nextInt() & 15));
                                        w.putStr(2, rnd.nextChars(rnd.nextInt() & 15));
                                        w.putStr(3, rnd.nextChars(rnd.nextInt() & 15));
                                        w.putStr(4, rnd.nextChars(rnd.nextInt() & 7));
                                        w.putSym(5, countries.getQuick(rnd.nextPositiveInt() % countryCount));
                                        w.putStr(6, rnd.nextChars(rnd.nextInt() & 15));
                                        w.putDate(7, time++);
                                        w.append();
                                    }
                                    suppliers.commit();
                                    MMappedSymbolTable categoryTab = categories.getSymbolTable("category");
                                    int categoryTabSize = categoryTab.size();
                                    MMappedSymbolTable supplierTab = suppliers.getSymbolTable("supplier");
                                    int supplierTabSize = supplierTab.size();
                                    // products
                                    int productCount = 2000;
                                    for (int i = 0; i < productCount; i++) {
                                        JournalEntryWriter w = products.entryWriter();
                                        w.putInt(0, i);
                                        w.putStr(1, rnd.nextChars(rnd.nextInt() & 15));
                                        w.putSym(2, supplierTab.value(rnd.nextPositiveInt() % supplierTabSize));
                                        w.putSym(3, categoryTab.value(rnd.nextPositiveInt() % categoryTabSize));
                                        w.putDouble(4, rnd.nextDouble());
                                        w.putDate(5, time++);
                                        w.append();
                                    }
                                    products.commit();
                                    // shippers
                                    for (int i = 0; i < 20; i++) {
                                        JournalEntryWriter w = shippers.entryWriter();
                                        w.putSym(0, rnd.nextChars(rnd.nextInt() & 15));
                                        w.putStr(1, rnd.nextChars(rnd.nextInt() & 7));
                                        w.append();
                                    }
                                    shippers.commit();
                                    MMappedSymbolTable shipperTab = shippers.getSymbolTable("shipper");
                                    int shipperTabSize = shipperTab.size();
                                    int d = 0;
                                    for (int i = 0; i < 100000; i++) {
                                        int customerId = rnd.nextPositiveInt() % customerCount;
                                        if (blackList.contains(customerId)) {
                                            continue;
                                        }
                                        int orderId = rnd.nextPositiveInt();
                                        JournalEntryWriter w = orders.entryWriter(time++);
                                        w.putInt(0, orderId);
                                        w.putInt(1, customerId);
                                        w.putInt(2, rnd.nextPositiveInt() % productCount);
                                        w.putStr(3, employees.getPartition(0, true).getFlyweightStr(rnd.nextPositiveLong() % employeeCount, 0));
                                        w.putSym(5, shipperTab.value(rnd.nextPositiveInt() % shipperTabSize));
                                        w.append();
                                        int k = (rnd.nextInt() & 3) + 1;
                                        for (int n = 0; n < k; n++) {
                                            JournalEntryWriter dw = orderDetails.entryWriter();
                                            dw.putInt(0, ++d);
                                            dw.putInt(1, orderId);
                                            dw.putInt(2, rnd.nextPositiveInt() % productCount);
                                            dw.putInt(3, (rnd.nextInt() & 3) + 1);
                                            dw.append();
                                        }
                                    }
                                    orders.commit();
                                    orderDetails.commit();
                                }
                            }
                        }
                    }
                }
            }
        }
    }
}

Example 2 with JournalStructure

use of com.questdb.store.factory.configuration.JournalStructure in project questdb by bluestreak01.

the class QueryCompiler method createWriter.

public JournalWriter createWriter(Factory factory, ParsedModel model) throws ParserException, JournalException {
    if (model.getModelType() != ParsedModel.CREATE_JOURNAL) {
        throw new IllegalArgumentException("create table statement expected");
    }
    clearState();
    CreateJournalModel cm = (CreateJournalModel) model;
    final String name = cm.getName().token;
    switch(factory.getConfiguration().exists(name)) {
        case JournalConfiguration.EXISTS:
            throw QueryError.$(cm.getName().position, "Journal already exists");
        case JournalConfiguration.EXISTS_FOREIGN:
            throw QueryError.$(cm.getName().position, "Name is reserved");
        default:
            break;
    }
    JournalStructure struct = cm.getStruct();
    RecordSource rs;
    QueryModel queryModel = cm.getQueryModel();
    if (queryModel != null) {
        rs = compile(queryModel, factory);
    } else {
        rs = null;
    }
    if (struct == null) {
        assert rs != null;
        RecordMetadata metadata = rs.getMetadata();
        CharSequenceObjHashMap<ColumnCastModel> castModels = cm.getColumnCastModels();
        // validate cast models
        for (int i = 0, n = castModels.size(); i < n; i++) {
            ColumnCastModel castModel = castModels.valueQuick(i);
            if (metadata.getColumnIndexQuiet(castModel.getName().token) == -1) {
                throw QueryError.invalidColumn(castModel.getName().position, castModel.getName().token);
            }
        }
        struct = createStructure(name, metadata, castModels);
    }
    try {
        validateAndSetTimestamp(struct, cm.getTimestamp());
        validateAndSetPartitionBy(struct, cm.getPartitionBy());
        ExprNode recordHint = cm.getRecordHint();
        if (recordHint != null) {
            try {
                struct.recordCountHint(Numbers.parseInt(recordHint.token));
            } catch (NumericException e) {
                throw QueryError.$(recordHint.position, "Bad int");
            }
        }
        ObjList<ColumnIndexModel> columnIndexModels = cm.getColumnIndexModels();
        for (int i = 0, n = columnIndexModels.size(); i < n; i++) {
            ColumnIndexModel cim = columnIndexModels.getQuick(i);
            ExprNode nn = cim.getName();
            ColumnMetadata m = struct.getColumnMetadata(nn.token);
            if (m == null) {
                throw QueryError.invalidColumn(nn.position, nn.token);
            }
            switch(m.getType()) {
                case ColumnType.INT:
                case ColumnType.LONG:
                case ColumnType.SYMBOL:
                case ColumnType.STRING:
                    m.indexed = true;
                    m.distinctCountHint = cim.getBuckets();
                    break;
                default:
                    throw QueryError.$(nn.position, "Type index not supported");
            }
        }
        JournalWriter w = factory.writer(struct);
        w.setSequentialAccess(true);
        if (rs != null) {
            try {
                copy(factory, rs, w);
            } catch (Throwable e) {
                w.close();
                throw e;
            }
        }
        return w;
    } catch (Throwable e) {
        Misc.free(rs);
        throw e;
    }
}

Example 3 with JournalStructure

use of com.questdb.store.factory.configuration.JournalStructure in project questdb by bluestreak01.

the class AppendObjectBlobs method main.

public static void main(String[] args) throws JournalException, IOException {
    final String dirToIndex = args[1];
    JournalConfiguration configuration = new JournalConfigurationBuilder().build(args[0]);
    try (Factory factory = new Factory(configuration, 1000, 1, 0)) {
        try (JournalWriter writer = factory.writer(new JournalStructure("files") {

            {
                $sym("name").index();
                $bin("data");
                $ts();
            }
        })) {
            long t = System.currentTimeMillis();
            int count = processDir(writer, new File(dirToIndex));
            System.out.println("Added " + count + " files in " + (System.currentTimeMillis() - t) + " ms.");
        }
    }
}

Example 4 with JournalStructure

use of com.questdb.store.factory.configuration.JournalStructure in project questdb by bluestreak01.

the class AppendRawTimeSeries method main.

public static void main(String[] args) throws JournalException, ParserException {
    if (args.length < 1) {
        System.out.println("Usage: AppendRawTimeSeries <path>");
        System.exit(1);
    }
    final String location = args[0];
    // factory can be reused in application and must be explicitly closed when no longer needed.
    try (Factory factory = new Factory(location, 1000, 1, 0)) {
        // to populate it.
        try (JournalWriter writer = factory.writer(new JournalStructure("customers").$int("id").$str("name").$ts("updateDate").$())) {
            Rnd rnd = new Rnd();
            long timestamp = System.currentTimeMillis();
            for (int i = 0; i < 1000000; i++) {
                // enforce timestamp order
                JournalEntryWriter ew = writer.entryWriter(timestamp);
                // columns accessed by index
                ew.putInt(0, rnd.nextPositiveInt());
                ew.putStr(1, rnd.nextChars(25));
                // increment timestamp by 30 seconds
                timestamp += 30000;
                // append record to journal
                ew.append();
            }
            // commit all records at once
            // there is no limit on how many records can be in the same transaction
            writer.commit();
        }
    }
}

Example 5 with JournalStructure

use of com.questdb.store.factory.configuration.JournalStructure in project questdb by bluestreak01.

the class GenericAppendPerfTest method testAppend.

@Test
public void testAppend() throws Exception {
    try (JournalWriter wg = getFactory().writer(new JournalStructure("qq") {

        {
            $sym("sym").index().buckets(20);
            $double("bid");
            $double("ask");
            $int("bidSize");
            $int("askSize");
            $sym("ex").buckets(1);
            $sym("mode").buckets(1);
            recordCountHint(TEST_DATA_SIZE);
        }
    })) {
        long t = System.nanoTime();
        TestUtils.generateQuoteDataGeneric(wg, TEST_DATA_SIZE, DateFormatUtils.parseDateTime("2013-10-05T10:00:00.000Z"), 1000);
        wg.commit();
        long result = System.nanoTime() - t;
        LOG.info().$("generic append (1M): ").$(TimeUnit.NANOSECONDS.toMillis(result) / 2).$("ms").$();
    }
}