use of com.simiacryptus.mindseye.opt.line.LineSearchPoint in project MindsEye by SimiaCryptus.
the class TrustRegionStrategy method orient.
@Nonnull
@Override
public LineSearchCursor orient(@Nonnull final Trainable subject, final PointSample origin, final TrainingMonitor monitor) {
history.add(0, origin);
while (history.size() > maxHistory) {
history.remove(history.size() - 1);
}
final SimpleLineSearchCursor cursor = inner.orient(subject, origin, monitor);
return new LineSearchCursorBase() {
@Nonnull
@Override
public CharSequence getDirectionType() {
return cursor.getDirectionType() + "+Trust";
}
@Nonnull
@Override
public DeltaSet<Layer> position(final double alpha) {
reset();
@Nonnull final DeltaSet<Layer> adjustedPosVector = cursor.position(alpha);
project(adjustedPosVector, new TrainingMonitor());
return adjustedPosVector;
}
@Nonnull
public DeltaSet<Layer> project(@Nonnull final DeltaSet<Layer> deltaIn, final TrainingMonitor monitor) {
final DeltaSet<Layer> originalAlphaDerivative = cursor.direction;
@Nonnull final DeltaSet<Layer> newAlphaDerivative = originalAlphaDerivative.copy();
deltaIn.getMap().forEach((layer, buffer) -> {
@Nullable final double[] delta = buffer.getDelta();
if (null == delta)
return;
final double[] currentPosition = buffer.target;
@Nullable final double[] originalAlphaD = originalAlphaDerivative.get(layer, currentPosition).getDelta();
@Nullable final double[] newAlphaD = newAlphaDerivative.get(layer, currentPosition).getDelta();
@Nonnull final double[] proposedPosition = ArrayUtil.add(currentPosition, delta);
final TrustRegion region = getRegionPolicy(layer);
if (null != region) {
final Stream<double[]> zz = history.stream().map((@Nonnull final PointSample x) -> {
final DoubleBuffer<Layer> d = x.weights.getMap().get(layer);
@Nullable final double[] z = null == d ? null : d.getDelta();
return z;
});
final double[] projectedPosition = region.project(zz.filter(x -> null != x).toArray(i -> new double[i][]), proposedPosition);
if (projectedPosition != proposedPosition) {
for (int i = 0; i < projectedPosition.length; i++) {
delta[i] = projectedPosition[i] - currentPosition[i];
}
@Nonnull final double[] normal = ArrayUtil.subtract(projectedPosition, proposedPosition);
final double normalMagSq = ArrayUtil.dot(normal, normal);
// normalMagSq));
if (0 < normalMagSq) {
final double a = ArrayUtil.dot(originalAlphaD, normal);
if (a != -1) {
@Nonnull final double[] tangent = ArrayUtil.add(originalAlphaD, ArrayUtil.multiply(normal, -a / normalMagSq));
for (int i = 0; i < tangent.length; i++) {
newAlphaD[i] = tangent[i];
}
// double newAlphaDerivSq = ArrayUtil.dot(tangent, tangent);
// double originalAlphaDerivSq = ArrayUtil.dot(originalAlphaD, originalAlphaD);
// assert(newAlphaDerivSq <= originalAlphaDerivSq);
// assert(Math.abs(ArrayUtil.dot(tangent, normal)) <= 1e-4);
// monitor.log(String.format("%s: normalMagSq = %s, newAlphaDerivSq = %s, originalAlphaDerivSq = %s", layer, normalMagSq, newAlphaDerivSq, originalAlphaDerivSq));
}
}
}
}
});
return newAlphaDerivative;
}
@Override
public void reset() {
cursor.reset();
}
@Nonnull
@Override
public LineSearchPoint step(final double alpha, final TrainingMonitor monitor) {
cursor.reset();
@Nonnull final DeltaSet<Layer> adjustedPosVector = cursor.position(alpha);
@Nonnull final DeltaSet<Layer> adjustedGradient = project(adjustedPosVector, monitor);
adjustedPosVector.accumulate(1);
@Nonnull final PointSample sample = subject.measure(monitor).setRate(alpha);
return new LineSearchPoint(sample, adjustedGradient.dot(sample.delta));
}
@Override
public void _free() {
cursor.freeRef();
}
};
}
use of com.simiacryptus.mindseye.opt.line.LineSearchPoint in project MindsEye by SimiaCryptus.
the class OwlQn method orient.
@Nonnull
@Override
public LineSearchCursor orient(final Trainable subject, @Nonnull final PointSample measurement, final TrainingMonitor monitor) {
@Nonnull final SimpleLineSearchCursor gradient = (SimpleLineSearchCursor) inner.orient(subject, measurement, monitor);
@Nonnull final DeltaSet<Layer> searchDirection = gradient.direction.copy();
@Nonnull final DeltaSet<Layer> orthant = new DeltaSet<Layer>();
for (@Nonnull final Layer layer : getLayers(gradient.direction.getMap().keySet())) {
final double[] weights = gradient.direction.getMap().get(layer).target;
@Nullable final double[] delta = gradient.direction.getMap().get(layer).getDelta();
@Nullable final double[] searchDir = searchDirection.get(layer, weights).getDelta();
@Nullable final double[] suborthant = orthant.get(layer, weights).getDelta();
for (int i = 0; i < searchDir.length; i++) {
final int positionSign = sign(weights[i]);
final int directionSign = sign(delta[i]);
suborthant[i] = 0 == positionSign ? directionSign : positionSign;
searchDir[i] += factor_L1 * (weights[i] < 0 ? -1.0 : 1.0);
if (sign(searchDir[i]) != directionSign) {
searchDir[i] = delta[i];
}
}
assert null != searchDir;
}
return new SimpleLineSearchCursor(subject, measurement, searchDirection) {
@Nonnull
@Override
public LineSearchPoint step(final double alpha, final TrainingMonitor monitor) {
origin.weights.stream().forEach(d -> d.restore());
@Nonnull final DeltaSet<Layer> currentDirection = direction.copy();
direction.getMap().forEach((layer, buffer) -> {
if (null == buffer.getDelta())
return;
@Nullable final double[] currentDelta = currentDirection.get(layer, buffer.target).getDelta();
for (int i = 0; i < buffer.getDelta().length; i++) {
final double prevValue = buffer.target[i];
final double newValue = prevValue + buffer.getDelta()[i] * alpha;
if (sign(prevValue) != 0 && sign(prevValue) != sign(newValue)) {
currentDelta[i] = 0;
buffer.target[i] = 0;
} else {
buffer.target[i] = newValue;
}
}
});
@Nonnull final PointSample measure = subject.measure(monitor).setRate(alpha);
return new LineSearchPoint(measure, currentDirection.dot(measure.delta));
}
}.setDirectionType("OWL/QN");
}
use of com.simiacryptus.mindseye.opt.line.LineSearchPoint in project MindsEye by SimiaCryptus.
the class QQN method orient.
@Override
public LineSearchCursor orient(@Nonnull final Trainable subject, @Nonnull final PointSample origin, @Nonnull final TrainingMonitor monitor) {
inner.addToHistory(origin, monitor);
final SimpleLineSearchCursor lbfgsCursor = inner.orient(subject, origin, monitor);
final DeltaSet<Layer> lbfgs = lbfgsCursor.direction;
@Nonnull final DeltaSet<Layer> gd = origin.delta.scale(-1.0);
final double lbfgsMag = lbfgs.getMagnitude();
final double gdMag = gd.getMagnitude();
if (Math.abs(lbfgsMag - gdMag) / (lbfgsMag + gdMag) > 1e-2) {
@Nonnull final DeltaSet<Layer> scaledGradient = gd.scale(lbfgsMag / gdMag);
monitor.log(String.format("Returning Quadratic Cursor %s GD, %s QN", gdMag, lbfgsMag));
gd.freeRef();
return new LineSearchCursorBase() {
@Nonnull
@Override
public CharSequence getDirectionType() {
return CURSOR_NAME;
}
@Override
public DeltaSet<Layer> position(final double t) {
if (!Double.isFinite(t))
throw new IllegalArgumentException();
return scaledGradient.scale(t - t * t).add(lbfgs.scale(t * t));
}
@Override
public void reset() {
lbfgsCursor.reset();
}
@Nonnull
@Override
public LineSearchPoint step(final double t, @Nonnull final TrainingMonitor monitor) {
if (!Double.isFinite(t))
throw new IllegalArgumentException();
reset();
position(t).accumulate(1);
@Nonnull final PointSample sample = subject.measure(monitor).setRate(t);
// monitor.log(String.format("delta buffers %d %d %d %d %d", sample.delta.apply.size(), origin.delta.apply.size(), lbfgs.apply.size(), gd.apply.size(), scaledGradient.apply.size()));
inner.addToHistory(sample, monitor);
@Nonnull final DeltaSet<Layer> tangent = scaledGradient.scale(1 - 2 * t).add(lbfgs.scale(2 * t));
return new LineSearchPoint(sample, tangent.dot(sample.delta));
}
@Override
public void _free() {
scaledGradient.freeRef();
lbfgsCursor.freeRef();
}
};
} else {
gd.freeRef();
return lbfgsCursor;
}
}
Aggregations