Examples with OIntentMassiveInsert com.orientechnologies.orient.core.intent.OIntentMassiveInsert used on opensource projects

Example 11 with OIntentMassiveInsert

use of com.orientechnologies.orient.core.intent.OIntentMassiveInsert in project orientdb by orientechnologies.

the class LocalCreateVertexSpeedTest method init.

@Override
@Test(enabled = false)
public void init() {
    final OrientGraphFactory factory = new OrientGraphFactory(System.getProperty("url"));
    factory.setStandardElementConstraints(false);
    if (factory.exists())
        factory.drop();
    database = factory.getNoTx();
    database.createVertexType("Account");
    database.getRawGraph().declareIntent(new OIntentMassiveInsert());
}

Example 12 with OIntentMassiveInsert

use of com.orientechnologies.orient.core.intent.OIntentMassiveInsert in project orientdb by orientechnologies.

the class GiantFileTest method storeFileData.

private static void storeFileData(final ODocument fileDoc, final File file) throws Exception {
    // To avoid overwriting a stored file, DataChunks must be null.
    final List<ORID> existingChunks = fileDoc.field("DataChunks");
    if (existingChunks != null) {
        final String fileName = fileDoc.field("FileName");
        throw new RuntimeException("File record already has data; overwrite not allowed! fileName: " + fileName);
    }
    // TODO: is this assumption ok?
    // Get the currently open database for this thread and set intent.
    final ODatabase database = ODatabaseRecordThreadLocal.INSTANCE.get();
    database.declareIntent(new OIntentMassiveInsert());
    // Insert File data.
    final long fileSize = file.length();
    final FileInputStream in = new FileInputStream(file);
    try {
        final int CHUNK_SIZE = 81920;
        int bufferedBytes;
        final byte[] buffer = new byte[CHUNK_SIZE];
        byte currentPercent = 0;
        final int fullChunks = (int) (fileSize / CHUNK_SIZE);
        final long fullChunksSize = fullChunks * CHUNK_SIZE;
        final int totalChunks;
        if (fileSize > fullChunksSize) {
            totalChunks = fullChunks + 1;
        } else {
            totalChunks = fullChunks;
        }
        final List<ORID> chunkRids = new ArrayList<ORID>(totalChunks);
        // Make only one ORecordBytes instance and reuse it for every chunk,
        // to reduce heap garbage.
        final ORecordBytes chunk = new ORecordBytes();
        // Handle the full chunks.
        for (int page = 0; page < fullChunks; page++) {
            // Read a full chunk of data from the file into a buffer.
            bufferedBytes = 0;
            while (bufferedBytes < buffer.length) {
                final int bytesRead = in.read(buffer, bufferedBytes, buffer.length - bufferedBytes);
                if (bytesRead == -1) {
                    throw new Exception("Reached end of file prematurely. (File changed while reading?) fileName=" + file.getAbsolutePath());
                }
                bufferedBytes += bytesRead;
            }
            // Save the chunk to the database.
            final long saveStartTime = System.currentTimeMillis();
            chunk.reset(buffer);
            chunk.save();
            final long saveMs = System.currentTimeMillis() - saveStartTime;
            // Log the amount of time taken by the save.
            System.out.printf("Saved chunk %d in %d ms.\n", page, saveMs);
            // Save the chunk's record ID in the list.
            // Have to copy() the ORID or else every chunk in the list gets the same last ORID.
            // This is because we are using the chunk.reset(); approach to reduce garbage objects.
            chunkRids.add(chunk.getIdentity().copy());
            // Only report progress if it has changed.
            final byte percent = (byte) ((page + 1) * 100 / totalChunks);
            if (percent > currentPercent) {
                System.out.printf("Progress: %d%%\n", percent);
                currentPercent = percent;
            }
        }
        // Handle the final partial chunk (if any).
        if (fullChunks < totalChunks) {
            final int remainder = (int) (fileSize - fullChunksSize);
            // Read the remaining data from the file into a buffer.
            bufferedBytes = 0;
            while (bufferedBytes < remainder) {
                final int bytesRead = in.read(buffer, bufferedBytes, remainder - bufferedBytes);
                if (bytesRead == -1) {
                    throw new Exception("Reached end of file prematurely. (File changed while reading?) fileName=" + file.getAbsolutePath());
                }
                bufferedBytes += bytesRead;
            }
            // Save the chunk to the database.
            final long saveStartTime = System.currentTimeMillis();
            chunk.reset(Arrays.copyOf(buffer, remainder));
            chunk.save();
            final long saveMs = System.currentTimeMillis() - saveStartTime;
            // Log the amount of time taken by the save.
            System.out.printf("Saved partial chunk %d in %d ms.\n", fullChunks, saveMs);
            // Save the chunk's record ID in the list.
            chunkRids.add(chunk.getIdentity());
        }
        // Should be no more data, so validate this.
        final int b = in.read();
        if (b != -1) {
            throw new Exception("File changed while saving to database! fileName=" + file.getAbsolutePath());
        }
        // Report 100% progress if we haven't already.
        if (currentPercent < 100) {
            System.out.println("Progress: 100%");
        }
        // Save the list of chunk references.
        final long saveChunkListStartTime = System.currentTimeMillis();
        fileDoc.field("DataChunks", chunkRids);
        fileDoc.save();
        final long saveChunkListMs = System.currentTimeMillis() - saveChunkListStartTime;
        // Log the amount of time taken to save the list of chunk RIDs.
        System.out.printf("Saved list of %d chunk RIDs in %d ms.\n", chunkRids.size(), saveChunkListMs);
    } finally {
        database.declareIntent(null);
        in.close();
    }
}

Example 13 with OIntentMassiveInsert

use of com.orientechnologies.orient.core.intent.OIntentMassiveInsert in project orientdb by orientechnologies.

the class LocalCreateAsynchDocumentSpeedTest method init.

@Override
public void init() {
    Orient.instance().getProfiler().startRecording();
    OGlobalConfiguration.NETWORK_SOCKET_BUFFER_SIZE.setValue(10000000);
    database = new ODatabaseDocumentTx(System.getProperty("url")).open("admin", "admin");
    record = database.newInstance();
    database.declareIntent(new OIntentMassiveInsert());
    database.begin(TXTYPE.NOTX);
}

Example 14 with OIntentMassiveInsert

use of com.orientechnologies.orient.core.intent.OIntentMassiveInsert in project orientdb by orientechnologies.

the class LocalCreateBinarySpeedTest method init.

@Override
public void init() {
    Orient.instance().getProfiler().startRecording();
    database = new ODatabaseDocumentTx(System.getProperty("url")).open("admin", "admin");
    record = new ORecordBytes();
    database.declareIntent(new OIntentMassiveInsert());
    database.begin(TXTYPE.NOTX);
    Random rnd = new Random();
    recordContent = new byte[RECORD_SIZE];
    for (int i = 0; i < RECORD_SIZE; ++i) recordContent[i] = (byte) rnd.nextInt(256);
}

Example 15 with OIntentMassiveInsert

use of com.orientechnologies.orient.core.intent.OIntentMassiveInsert in project orientdb by orientechnologies.

the class OCommandExecutorSQLCreateLink method execute.

/**
   * Execute the CREATE LINK.
   */
public Object execute(final Map<Object, Object> iArgs) {
    if (destField == null)
        throw new OCommandExecutionException("Cannot execute the command because it has not been parsed yet");
    final ODatabaseDocumentInternal database = getDatabase();
    if (!(database.getDatabaseOwner() instanceof ODatabaseDocument))
        throw new OCommandSQLParsingException("This command supports only the database type ODatabaseDocumentTx and type '" + database.getClass() + "' was found");
    final ODatabaseDocument db = (ODatabaseDocument) database.getDatabaseOwner();
    final OClass sourceClass = database.getMetadata().getSchema().getClass(sourceClassName);
    if (sourceClass == null)
        throw new OCommandExecutionException("Source class '" + sourceClassName + "' not found");
    final OClass destClass = database.getMetadata().getSchema().getClass(destClassName);
    if (destClass == null)
        throw new OCommandExecutionException("Destination class '" + destClassName + "' not found");
    Object value;
    String cmd = "select from ";
    if (!ODocumentHelper.ATTRIBUTE_RID.equals(destField)) {
        cmd = "select from " + destClassName + " where " + destField + " = ";
    }
    List<ODocument> result;
    ODocument target;
    Object oldValue;
    long total = 0;
    if (linkName == null)
        // NO LINK NAME EXPRESSED: OVERWRITE THE SOURCE FIELD
        linkName = sourceField;
    boolean multipleRelationship;
    if (linkType != null)
        // DETERMINE BASED ON FORCED TYPE
        multipleRelationship = linkType == OType.LINKSET || linkType == OType.LINKLIST;
    else
        multipleRelationship = false;
    long totRecords = db.countClass(sourceClass.getName());
    long currRecord = 0;
    if (progressListener != null)
        progressListener.onBegin(this, totRecords, false);
    database.declareIntent(new OIntentMassiveInsert());
    try {
        // BROWSE ALL THE RECORDS OF THE SOURCE CLASS
        for (ODocument doc : db.browseClass(sourceClass.getName())) {
            value = doc.field(sourceField);
            if (value != null) {
                if (value instanceof ODocument || value instanceof ORID) {
                // ALREADY CONVERTED
                } else if (value instanceof Collection<?>) {
                // TODO
                } else {
                    // SEARCH THE DESTINATION RECORD
                    target = null;
                    if (!ODocumentHelper.ATTRIBUTE_RID.equals(destField) && value instanceof String)
                        if (((String) value).length() == 0)
                            value = null;
                        else
                            value = "'" + value + "'";
                    result = database.<OCommandRequest>command(new OSQLSynchQuery<ODocument>(cmd + value)).execute();
                    if (result == null || result.size() == 0)
                        value = null;
                    else if (result.size() > 1)
                        throw new OCommandExecutionException("Cannot create link because multiple records was found in class '" + destClass.getName() + "' with value " + value + " in field '" + destField + "'");
                    else {
                        target = result.get(0);
                        value = target;
                    }
                    if (target != null && inverse) {
                        // INVERSE RELATIONSHIP
                        oldValue = target.field(linkName);
                        if (oldValue != null) {
                            if (!multipleRelationship)
                                multipleRelationship = true;
                            Collection<ODocument> coll;
                            if (oldValue instanceof Collection) {
                                // ADD IT IN THE EXISTENT COLLECTION
                                coll = (Collection<ODocument>) oldValue;
                                target.setDirty();
                            } else {
                                // CREATE A NEW COLLECTION FOR BOTH
                                coll = new ArrayList<ODocument>(2);
                                target.field(linkName, coll);
                                coll.add((ODocument) oldValue);
                            }
                            coll.add(doc);
                        } else {
                            if (linkType != null)
                                if (linkType == OType.LINKSET) {
                                    value = new ORecordLazySet(target);
                                    ((Set<OIdentifiable>) value).add(doc);
                                } else if (linkType == OType.LINKLIST) {
                                    value = new ORecordLazyList(target);
                                    ((ORecordLazyList) value).add(doc);
                                } else
                                    // IGNORE THE TYPE, SET IT AS LINK
                                    value = doc;
                            else
                                value = doc;
                            target.field(linkName, value);
                        }
                        target.save();
                    } else {
                        // SET THE REFERENCE
                        doc.field(linkName, value);
                        doc.save();
                    }
                    total++;
                }
            }
            if (progressListener != null)
                progressListener.onProgress(this, currRecord, currRecord * 100f / totRecords);
        }
        if (total > 0) {
            if (inverse) {
                // REMOVE THE OLD PROPERTY IF ANY
                OProperty prop = destClass.getProperty(linkName);
                if (prop != null)
                    destClass.dropProperty(linkName);
                if (linkType == null)
                    linkType = multipleRelationship ? OType.LINKSET : OType.LINK;
                // CREATE THE PROPERTY
                destClass.createProperty(linkName, linkType, sourceClass);
            } else {
                // REMOVE THE OLD PROPERTY IF ANY
                OProperty prop = sourceClass.getProperty(linkName);
                if (prop != null)
                    sourceClass.dropProperty(linkName);
                // CREATE THE PROPERTY
                sourceClass.createProperty(linkName, OType.LINK, destClass);
            }
        }
        if (progressListener != null)
            progressListener.onCompletition(this, true);
    } catch (Exception e) {
        if (progressListener != null)
            progressListener.onCompletition(this, false);
        throw OException.wrapException(new OCommandExecutionException("Error on creation of links"), e);
    } finally {
        database.declareIntent(null);
    }
    return total;
}