Examples with SolrInputField org.apache.solr.common.SolrInputField used on opensource projects

Example 21 with SolrInputField

use of org.apache.solr.common.SolrInputField in project lucene-solr by apache.

the class TestStressCloudBlindAtomicUpdates method f.

public static SolrInputField f(String fieldName, Object... values) {
    SolrInputField f = new SolrInputField(fieldName);
    f.setValue(values);
    // if the Map with the "inc" operation is inside of a collection - even if it's the only "value") ...
    if (1 == values.length) {
        f.setValue(values[0]);
    } else {
        f.setValue(values);
    }
    return f;
}

Example 22 with SolrInputField

use of org.apache.solr.common.SolrInputField in project lucene-solr by apache.

the class AtomicUpdateDocumentMerger method doInc.

protected void doInc(SolrInputDocument toDoc, SolrInputField sif, Object fieldVal) {
    SolrInputField numericField = toDoc.get(sif.getName());
    SchemaField sf = schema.getField(sif.getName());
    if (numericField != null || sf.getDefaultValue() != null) {
        // TODO: fieldtype needs externalToObject?
        String oldValS = (numericField != null) ? numericField.getFirstValue().toString() : sf.getDefaultValue().toString();
        BytesRefBuilder term = new BytesRefBuilder();
        sf.getType().readableToIndexed(oldValS, term);
        Object oldVal = sf.getType().toObject(sf, term.get());
        String fieldValS = fieldVal.toString();
        Number result;
        if (oldVal instanceof Long) {
            result = ((Long) oldVal).longValue() + Long.parseLong(fieldValS);
        } else if (oldVal instanceof Float) {
            result = ((Float) oldVal).floatValue() + Float.parseFloat(fieldValS);
        } else if (oldVal instanceof Double) {
            result = ((Double) oldVal).doubleValue() + Double.parseDouble(fieldValS);
        } else {
            // int, short, byte
            result = ((Integer) oldVal).intValue() + Integer.parseInt(fieldValS);
        }
        toDoc.setField(sif.getName(), result);
    } else {
        toDoc.setField(sif.getName(), fieldVal);
    }
}

Example 23 with SolrInputField

use of org.apache.solr.common.SolrInputField in project lucene-solr by apache.

the class AtomicUpdateDocumentMerger method doRemoveRegex.

protected void doRemoveRegex(SolrInputDocument toDoc, SolrInputField sif, Object valuePatterns) {
    final String name = sif.getName();
    final SolrInputField existingField = toDoc.get(name);
    if (existingField != null) {
        final Collection<Object> valueToRemove = new HashSet<>();
        final Collection<Object> original = existingField.getValues();
        final Collection<Pattern> patterns = preparePatterns(valuePatterns);
        for (Object value : original) {
            for (Pattern pattern : patterns) {
                final Matcher m = pattern.matcher(value.toString());
                if (m.matches()) {
                    valueToRemove.add(value);
                }
            }
        }
        original.removeAll(valueToRemove);
        toDoc.setField(name, original);
    }
}

Example 24 with SolrInputField

use of org.apache.solr.common.SolrInputField in project lucene-solr by apache.

the class DistributedUpdateProcessor method versionAdd.

/**
   * @return whether or not to drop this cmd
   * @throws IOException If there is a low-level I/O error.
   */
protected boolean versionAdd(AddUpdateCommand cmd) throws IOException {
    BytesRef idBytes = cmd.getIndexedId();
    if (idBytes == null) {
        super.processAdd(cmd);
        return false;
    }
    if (vinfo == null) {
        if (AtomicUpdateDocumentMerger.isAtomicUpdate(cmd)) {
            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Atomic document updates are not supported unless <updateLog/> is configured");
        } else {
            super.processAdd(cmd);
            return false;
        }
    }
    // This is only the hash for the bucket, and must be based only on the uniqueKey (i.e. do not use a pluggable hash here)
    int bucketHash = Hash.murmurhash3_x86_32(idBytes.bytes, idBytes.offset, idBytes.length, 0);
    // at this point, there is an update we need to try and apply.
    // we may or may not be the leader.
    // Find any existing version in the document
    // TODO: don't reuse update commands any more!
    long versionOnUpdate = cmd.getVersion();
    if (versionOnUpdate == 0) {
        SolrInputField versionField = cmd.getSolrInputDocument().getField(CommonParams.VERSION_FIELD);
        if (versionField != null) {
            Object o = versionField.getValue();
            versionOnUpdate = o instanceof Number ? ((Number) o).longValue() : Long.parseLong(o.toString());
        } else {
            // Find the version
            String versionOnUpdateS = req.getParams().get(CommonParams.VERSION_FIELD);
            versionOnUpdate = versionOnUpdateS == null ? 0 : Long.parseLong(versionOnUpdateS);
        }
    }
    boolean isReplayOrPeersync = (cmd.getFlags() & (UpdateCommand.REPLAY | UpdateCommand.PEER_SYNC)) != 0;
    boolean leaderLogic = isLeader && !isReplayOrPeersync;
    boolean forwardedFromCollection = cmd.getReq().getParams().get(DISTRIB_FROM_COLLECTION) != null;
    VersionBucket bucket = vinfo.bucket(bucketHash);
    long dependentVersionFound = -1;
    // this update depends), before entering the synchronized block
    if (!leaderLogic && cmd.isInPlaceUpdate()) {
        dependentVersionFound = waitForDependentUpdates(cmd, versionOnUpdate, isReplayOrPeersync, bucket);
        if (dependentVersionFound == -1) {
            // it means the document has been deleted by now at the leader. drop this update
            return true;
        }
    }
    vinfo.lockForUpdate();
    try {
        synchronized (bucket) {
            //just in case anyone is waiting let them know that we have a new update
            bucket.notifyAll();
            // we obtain the version when synchronized and then do the add so we can ensure that
            // if version1 < version2 then version1 is actually added before version2.
            // even if we don't store the version field, synchronizing on the bucket
            // will enable us to know what version happened first, and thus enable
            // realtime-get to work reliably.
            // TODO: if versions aren't stored, do we need to set on the cmd anyway for some reason?
            // there may be other reasons in the future for a version on the commands
            boolean checkDeleteByQueries = false;
            if (versionsStored) {
                long bucketVersion = bucket.highest;
                if (leaderLogic) {
                    if (forwardedFromCollection && ulog.getState() == UpdateLog.State.ACTIVE) {
                        // forwarded from a collection but we are not buffering so strip original version and apply our own
                        // see SOLR-5308
                        log.info("Removing version field from doc: " + cmd.getPrintableId());
                        cmd.solrDoc.remove(CommonParams.VERSION_FIELD);
                        versionOnUpdate = 0;
                    }
                    boolean updated = getUpdatedDocument(cmd, versionOnUpdate);
                    // leaders can also be in buffering state during "migrate" API call, see SOLR-5308
                    if (forwardedFromCollection && ulog.getState() != UpdateLog.State.ACTIVE && isReplayOrPeersync == false) {
                        // we're not in an active state, and this update isn't from a replay, so buffer it.
                        log.info("Leader logic applied but update log is buffering: " + cmd.getPrintableId());
                        cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);
                        ulog.add(cmd);
                        return true;
                    }
                    if (versionOnUpdate != 0) {
                        Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());
                        long foundVersion = lastVersion == null ? -1 : lastVersion;
                        if (versionOnUpdate == foundVersion || (versionOnUpdate < 0 && foundVersion < 0) || (versionOnUpdate == 1 && foundVersion > 0)) {
                        // we're ok if versions match, or if both are negative (all missing docs are equal), or if cmd
                        // specified it must exist (versionOnUpdate==1) and it does.
                        } else {
                            throw new SolrException(ErrorCode.CONFLICT, "version conflict for " + cmd.getPrintableId() + " expected=" + versionOnUpdate + " actual=" + foundVersion);
                        }
                    }
                    long version = vinfo.getNewClock();
                    cmd.setVersion(version);
                    cmd.getSolrInputDocument().setField(CommonParams.VERSION_FIELD, version);
                    bucket.updateHighest(version);
                } else {
                    // The leader forwarded us this update.
                    cmd.setVersion(versionOnUpdate);
                    if (ulog.getState() != UpdateLog.State.ACTIVE && isReplayOrPeersync == false) {
                        // we're not in an active state, and this update isn't from a replay, so buffer it.
                        cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);
                        ulog.add(cmd);
                        return true;
                    }
                    if (cmd.isInPlaceUpdate()) {
                        long prev = cmd.prevVersion;
                        Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());
                        if (lastVersion == null || Math.abs(lastVersion) < prev) {
                            // this was checked for (in waitForDependentUpdates()) before entering the synchronized block.
                            // So we shouldn't be here, unless what must've happened is:
                            // by the time synchronization block was entered, the prev update was deleted by DBQ. Since
                            // now that update is not in index, the vinfo.lookupVersion() is possibly giving us a version 
                            // from the deleted list (which might be older than the prev update!) 
                            UpdateCommand fetchedFromLeader = fetchFullUpdateFromLeader(cmd, versionOnUpdate);
                            if (fetchedFromLeader instanceof DeleteUpdateCommand) {
                                log.info("In-place update of {} failed to find valid lastVersion to apply to, and the document" + " was deleted at the leader subsequently.", idBytes.utf8ToString());
                                versionDelete((DeleteUpdateCommand) fetchedFromLeader);
                                return true;
                            } else {
                                assert fetchedFromLeader instanceof AddUpdateCommand;
                                // Newer document was fetched from the leader. Apply that document instead of this current in-place update.
                                log.info("In-place update of {} failed to find valid lastVersion to apply to, forced to fetch full doc from leader: {}", idBytes.utf8ToString(), fetchedFromLeader);
                                // Make this update to become a non-inplace update containing the full document obtained from the leader
                                cmd.solrDoc = ((AddUpdateCommand) fetchedFromLeader).solrDoc;
                                cmd.prevVersion = -1;
                                cmd.setVersion((long) cmd.solrDoc.getFieldValue(CommonParams.VERSION_FIELD));
                                assert cmd.isInPlaceUpdate() == false;
                            }
                        } else {
                            if (lastVersion != null && Math.abs(lastVersion) > prev) {
                                // this means we got a newer full doc update and in that case it makes no sense to apply the older
                                // inplace update. Drop this update
                                log.info("Update was applied on version: " + prev + ", but last version I have is: " + lastVersion + ". Dropping current update.");
                                return true;
                            } else {
                                // We're good, we should apply this update. First, update the bucket's highest.
                                if (bucketVersion != 0 && bucketVersion < versionOnUpdate) {
                                    bucket.updateHighest(versionOnUpdate);
                                }
                            }
                        }
                    }
                    if (!cmd.isInPlaceUpdate()) {
                        // if we aren't the leader, then we need to check that updates were not re-ordered
                        if (bucketVersion != 0 && bucketVersion < versionOnUpdate) {
                            // we're OK... this update has a version higher than anything we've seen
                            // in this bucket so far, so we know that no reordering has yet occurred.
                            bucket.updateHighest(versionOnUpdate);
                        } else {
                            // there have been updates higher than the current update.  we need to check
                            // the specific version for this id.
                            Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());
                            if (lastVersion != null && Math.abs(lastVersion) >= versionOnUpdate) {
                                // This update is a repeat, or was reordered.  We need to drop this update.
                                log.debug("Dropping add update due to version {}", idBytes.utf8ToString());
                                return true;
                            }
                            // also need to re-apply newer deleteByQuery commands
                            checkDeleteByQueries = true;
                        }
                    }
                    if (replicaType == Replica.Type.TLOG && (cmd.getFlags() & UpdateCommand.REPLAY) == 0) {
                        cmd.setFlags(cmd.getFlags() | UpdateCommand.IGNORE_INDEXWRITER);
                    }
                }
            }
            boolean willDistrib = isLeader && nodes != null && nodes.size() > 0;
            SolrInputDocument clonedDoc = null;
            if (willDistrib && cloneRequiredOnLeader) {
                clonedDoc = cmd.solrDoc.deepCopy();
            }
            // TODO: possibly set checkDeleteByQueries as a flag on the command?
            doLocalAdd(cmd);
            if (willDistrib && cloneRequiredOnLeader) {
                cmd.solrDoc = clonedDoc;
            }
        }
    // end synchronized (bucket)
    } finally {
        vinfo.unlockForUpdate();
    }
    return false;
}

Example 25 with SolrInputField

use of org.apache.solr.common.SolrInputField in project lucene-solr by apache.

the class PreAnalyzedUpdateProcessor method mutate.

@Override
protected SolrInputField mutate(SolrInputField src) {
    SchemaField sf = schema.getFieldOrNull(src.getName());
    if (sf == null) {
        // remove this field
        return null;
    }
    FieldType type = PreAnalyzedField.createFieldType(sf);
    if (type == null) {
        // neither indexed nor stored - skip
        return null;
    }
    SolrInputField res = new SolrInputField(src.getName());
    for (Object o : src) {
        if (o == null) {
            continue;
        }
        Field pre = (Field) parser.createField(sf, o);
        if (pre != null) {
            res.addValue(pre);
        } else {
            // restore the original value
            log.warn("Could not parse field {} - using original value as is: {}", src.getName(), o);
            res.addValue(o);
        }
    }
    return res;
}