From 7e36b834128b438e98ff82ddbc2d6ac3d651a190 Mon Sep 17 00:00:00 2001 From: alisterde Date: Sun, 24 May 2020 17:41:18 +0100 Subject: [PATCH 01/16] abstract class LineSearchBasedOptimiser --- pints/_optimisers/__init__.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pints/_optimisers/__init__.py b/pints/_optimisers/__init__.py index 4cb19588d..57a1c02db 100644 --- a/pints/_optimisers/__init__.py +++ b/pints/_optimisers/__init__.py @@ -279,6 +279,29 @@ def set_hyper_parameters(self, x): self.set_population_size(x[0]) +class LineSearchBasedOptimiser(Optimiser): + """ + Base class for optimisers that incorporate a line search + within their algorithm. + + Extends :class:`Optimiser`. + """ + + def __init__(self, x0, sigma0=None, boundaries=None): + super(LineSearchBasedOptimiser, self).__init__(x0, sigma0, boundaries) + + self._evaluator = None + + def _set_function_evaluator(self, function): + + f = function + if self.needs_sensitivities: + f = f.evaluateS1 + + # Create evaluator object + self._evaluator = pints.SequentialEvaluator(f) + + class OptimisationController(object): """ Finds the parameter values that minimise an :class:`ErrorMeasure` or @@ -337,6 +360,8 @@ def __init__( elif not issubclass(method, pints.Optimiser): raise ValueError('Method must be subclass of pints.Optimiser.') self._optimiser = method(x0, sigma0, boundaries) + if issubclass(method, pints.LineSearchBasedOptimiser): + self._optimiser._set_function_evaluator(self._function) # Check if sensitivities are required self._needs_sensitivities = self._optimiser.needs_sensitivities() From ab50fa89f08e19e332b64d6d233e39859a91a9eb Mon Sep 17 00:00:00 2001 From: alisterde Date: Sun, 24 May 2020 18:01:23 +0100 Subject: [PATCH 02/16] :construction: lbfgs algorithm --- examples/optimisation/bfgs_trial.ipynb | 827 +++++++++++++++++++ pints/__init__.py | 5 +- pints/_optimisers/_bfgs_linesearch.py | 1051 ++++++++++++++++++++++++ pints/_optimisers/_bfgs_scipy.py | 399 +++++++++ 4 files changed, 2281 insertions(+), 1 deletion(-) create mode 100644 examples/optimisation/bfgs_trial.ipynb create mode 100644 pints/_optimisers/_bfgs_linesearch.py create mode 100644 pints/_optimisers/_bfgs_scipy.py diff --git a/examples/optimisation/bfgs_trial.ipynb b/examples/optimisation/bfgs_trial.ipynb new file mode 100644 index 000000000..10394190c --- /dev/null +++ b/examples/optimisation/bfgs_trial.ipynb @@ -0,0 +1,827 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "os.chdir(\"../..\")\n", + "import pints\n", + "import pints.toy" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using the attempted Hager-Zhang linesearch implimentation" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "tags": [ + "outputPrepend" + ] + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": "ps: 59.33333333333348 propose alpha: 5.447714322654104e-10 propose points: [ 4.50854165 450.00004005]\n60 61 1.9e+07 0:00.2\n\nHessian updates: 0 line steps: 60.333333333333485 propose alpha: 5.447684887856562e-10 propose points: [ 4.50851735 450.00004005]\n\nHessian updates: 0 line steps: 61.33333333333349 propose alpha: 5.447655453038777e-10 propose points: [ 4.50849304 450.00004005]\n\nHessian updates: 0 line steps: 62.3333333333335 propose alpha: 5.447626018200718e-10 propose points: [ 4.50846873 450.00004005]\n\nHessian updates: 0 line steps: 63.333333333333506 propose alpha: 5.447596583342355e-10 propose points: [ 4.50844443 450.00004004]\n\nHessian updates: 0 line steps: 64.3333333333335 propose alpha: 5.447567148463658e-10 propose points: [ 4.50842012 450.00004004]\n\nHessian updates: 0 line steps: 65.33333333333348 propose alpha: 5.447537713564598e-10 propose points: [ 4.50839582 450.00004004]\n\nHessian updates: 0 line steps: 66.33333333333347 propose alpha: 5.447508278645143e-10 propose points: [ 4.50837151 450.00004004]\n\nHessian updates: 0 line steps: 67.33333333333346 propose alpha: 5.447478843705264e-10 propose points: [ 4.5083472 450.00004004]\n\nHessian updates: 0 line steps: 68.33333333333344 propose alpha: 5.447449408744931e-10 propose points: [ 4.5083229 450.00004004]\n\nHessian updates: 0 line steps: 69.33333333333343 propose alpha: 5.447419973764114e-10 propose points: [ 4.50829859 450.00004004]\n\nHessian updates: 0 line steps: 70.33333333333341 propose alpha: 5.447390538762783e-10 propose points: [ 4.50827428 450.00004004]\n\nHessian updates: 0 line steps: 71.3333333333334 propose alpha: 5.447361103740907e-10 propose points: [ 4.50824998 450.00004004]\n\nHessian updates: 0 line steps: 72.33333333333339 propose alpha: 5.447331668698458e-10 propose points: [ 4.50822567 450.00004004]\n\nHessian updates: 0 line steps: 73.33333333333337 propose alpha: 5.447302233635402e-10 propose points: [ 4.50820136 450.00004004]\n\nHessian updates: 0 line steps: 74.33333333333336 propose alpha: 5.447272798551713e-10 propose points: [ 4.50817706 450.00004004]\n\nHessian updates: 0 line steps: 75.33333333333334 propose alpha: 5.447243363447359e-10 propose points: [ 4.50815275 450.00004004]\n\nHessian updates: 0 line steps: 76.33333333333333 propose alpha: 5.447213928322309e-10 propose points: [ 4.50812844 450.00004004]\n\nHessian updates: 0 line steps: 77.33333333333331 propose alpha: 5.447184493176534e-10 propose points: [ 4.50810414 450.00004004]\n\nHessian updates: 0 line steps: 78.3333333333333 propose alpha: 5.447155058010004e-10 propose points: [ 4.50807983 450.00004004]\n\nHessian updates: 0 line steps: 79.33333333333329 propose alpha: 5.44712562282269e-10 propose points: [ 4.50805552 450.00004004]\n80 81 1.9e+07 0:00.2\n\nHessian updates: 0 line steps: 80.33333333333327 propose alpha: 5.447096187614559e-10 propose points: [ 4.50803122 450.00004004]\n\nHessian updates: 0 line steps: 81.33333333333326 propose alpha: 5.447066752385582e-10 propose points: [ 4.50800691 450.00004004]\n\nHessian updates: 0 line steps: 82.33333333333324 propose alpha: 5.44703731713573e-10 propose points: [ 4.5079826 450.00004004]\n\nHessian updates: 0 line steps: 83.33333333333323 propose alpha: 5.447007881864974e-10 propose points: [ 4.5079583 450.00004004]\n\nHessian updates: 0 line steps: 84.33333333333321 propose alpha: 5.44697844657328e-10 propose points: [ 4.50793399 450.00004004]\n\nHessian updates: 0 line steps: 85.3333333333332 propose alpha: 5.446949011260622e-10 propose points: [ 4.50790968 450.00004004]\n\nHessian updates: 0 line steps: 86.33333333333319 propose alpha: 5.446919575926966e-10 propose points: [ 4.50788538 450.00004004]\n\nHessian updates: 0 line steps: 87.33333333333317 propose alpha: 5.446890140572285e-10 propose points: [ 4.50786107 450.00004004]\n\nHessian updates: 0 line steps: 88.33333333333316 propose alpha: 5.446860705196548e-10 propose points: [ 4.50783676 450.00004004]\n\nHessian updates: 0 line steps: 89.33333333333314 propose alpha: 5.446831269799725e-10 propose points: [ 4.50781246 450.00004004]\n\nHessian updates: 0 line steps: 90.33333333333313 propose alpha: 5.446801834381785e-10 propose points: [ 4.50778815 450.00004004]\n\nHessian updates: 0 line steps: 91.33333333333312 propose alpha: 5.446772398942698e-10 propose points: [ 4.50776384 450.00004004]\n\nHessian updates: 0 line steps: 92.3333333333331 propose alpha: 5.446742963482434e-10 propose points: [ 4.50773954 450.00004004]\n\nHessian updates: 0 line steps: 93.33333333333309 propose alpha: 5.446713528000964e-10 propose points: [ 4.50771523 450.00004004]\n\nHessian updates: 0 line steps: 94.33333333333307 propose alpha: 5.446684092498257e-10 propose points: [ 4.50769092 450.00004004]\n\nHessian updates: 0 line steps: 95.33333333333306 propose alpha: 5.446654656974282e-10 propose points: [ 4.50766662 450.00004004]\n\nHessian updates: 0 line steps: 96.33333333333304 propose alpha: 5.44662522142901e-10 propose points: [ 4.50764231 450.00004004]\n\nHessian updates: 0 line steps: 97.33333333333303 propose alpha: 5.446595785862412e-10 propose points: [ 4.507618 450.00004004]\n\nHessian updates: 0 line steps: 98.33333333333302 propose alpha: 5.446566350274456e-10 propose points: [ 4.5075937 450.00004004]\n\nHessian updates: 0 line steps: 99.333333333333 propose alpha: 5.446536914665113e-10 propose points: [ 4.50756939 450.00004004]\n100 101 1.9e+07 0:00.2\n\nHessian updates: 0 line steps: 100.33333333333299 propose alpha: 5.446507479034352e-10 propose points: [ 4.50754508 450.00004004]\n\nHessian updates: 0 line steps: 101.33333333333297 propose alpha: 5.446478043382142e-10 propose points: [ 4.50752077 450.00004004]\n\nHessian updates: 0 line steps: 102.33333333333296 propose alpha: 5.446448607708455e-10 propose points: [ 4.50749647 450.00004004]\n\nHessian updates: 0 line steps: 103.33333333333294 propose alpha: 5.44641917201326e-10 propose points: [ 4.50747216 450.00004004]\n\nHessian updates: 0 line steps: 104.33333333333293 propose alpha: 5.446389736296527e-10 propose points: [ 4.50744785 450.00004004]\n\nHessian updates: 0 line steps: 105.33333333333292 propose alpha: 5.446360300558226e-10 propose points: [ 4.50742355 450.00004004]\n\nHessian updates: 0 line steps: 106.3333333333329 propose alpha: 5.446330864798326e-10 propose points: [ 4.50739924 450.00004004]\n\nHessian updates: 0 line steps: 107.33333333333289 propose alpha: 5.446301429016798e-10 propose points: [ 4.50737493 450.00004004]\n\nHessian updates: 0 line steps: 108.33333333333287 propose alpha: 5.44627199321361e-10 propose points: [ 4.50735063 450.00004004]\n\nHessian updates: 0 line steps: 109.33333333333286 propose alpha: 5.446242557388735e-10 propose points: [ 4.50732632 450.00004003]\n\nHessian updates: 0 line steps: 110.33333333333285 propose alpha: 5.44621312154214e-10 propose points: [ 4.50730201 450.00004003]\n\nHessian updates: 0 line steps: 111.33333333333283 propose alpha: 5.446183685673796e-10 propose points: [ 4.5072777 450.00004003]\n\nHessian updates: 0 line steps: 112.33333333333282 propose alpha: 5.446154249783673e-10 propose points: [ 4.5072534 450.00004003]\n\nHessian updates: 0 line steps: 113.3333333333328 propose alpha: 5.446124813871741e-10 propose points: [ 4.50722909 450.00004003]\n\nHessian updates: 0 line steps: 114.33333333333279 propose alpha: 5.446095377937969e-10 propose points: [ 4.50720478 450.00004003]\n\nHessian updates: 0 line steps: 115.33333333333277 propose alpha: 5.446065941982327e-10 propose points: [ 4.50718047 450.00004003]\n\nHessian updates: 0 line steps: 116.33333333333276 propose alpha: 5.446036506004786e-10 propose points: [ 4.50715617 450.00004003]\n\nHessian updates: 0 line steps: 117.33333333333275 propose alpha: 5.446007070005316e-10 propose points: [ 4.50713186 450.00004003]\n\nHessian updates: 0 line steps: 118.33333333333273 propose alpha: 5.445977633983885e-10 propose points: [ 4.50710755 450.00004003]\n\nHessian updates: 0 line steps: 119.33333333333272 propose alpha: 5.445948197940464e-10 propose points: [ 4.50708325 450.00004003]\n120 121 1.9e+07 0:00.3\n\nHessian updates: 0 line steps: 120.3333333333327 propose alpha: 5.445918761875024e-10 propose points: [ 4.50705894 450.00004003]\n\nHessian updates: 0 line steps: 121.33333333333269 propose alpha: 5.445889325787533e-10 propose points: [ 4.50703463 450.00004003]\n\nHessian updates: 0 line steps: 122.33333333333267 propose alpha: 5.445859889677962e-10 propose points: [ 4.50701032 450.00004003]\n\nHessian updates: 0 line steps: 123.33333333333266 propose alpha: 5.44583045354628e-10 propose points: [ 4.50698602 450.00004003]\n\nHessian updates: 0 line steps: 124.33333333333265 propose alpha: 5.445801017392457e-10 propose points: [ 4.50696171 450.00004003]\n\nHessian updates: 0 line steps: 125.33333333333263 propose alpha: 5.445771581216464e-10 propose points: [ 4.5069374 450.00004003]\n\nHessian updates: 0 line steps: 126.33333333333262 propose alpha: 5.445742145018268e-10 propose points: [ 4.50691309 450.00004003]\n\nHessian updates: 0 line steps: 127.3333333333326 propose alpha: 5.445712708797842e-10 propose points: [ 4.50688879 450.00004003]\n\nHessian updates: 0 line steps: 128.3333333333326 propose alpha: 5.445683272555156e-10 propose points: [ 4.50686448 450.00004003]\n\nHessian updates: 0 line steps: 129.33333333333263 propose alpha: 5.445653836290176e-10 propose points: [ 4.50684017 450.00004003]\n\nHessian updates: 0 line steps: 130.33333333333266 propose alpha: 5.445624400002877e-10 propose points: [ 4.50681586 450.00004003]\n\nHessian updates: 0 line steps: 131.3333333333327 propose alpha: 5.445594963693226e-10 propose points: [ 4.50679156 450.00004003]\n\nHessian updates: 0 line steps: 132.33333333333272 propose alpha: 5.445565527361194e-10 propose points: [ 4.50676725 450.00004003]\n\nHessian updates: 0 line steps: 133.33333333333275 propose alpha: 5.445536091006749e-10 propose points: [ 4.50674294 450.00004003]\n\nHessian updates: 0 line steps: 134.33333333333277 propose alpha: 5.445506654629862e-10 propose points: [ 4.50671863 450.00004003]\n\nHessian updates: 0 line steps: 135.3333333333328 propose alpha: 5.445477218230503e-10 propose points: [ 4.50669433 450.00004003]\n\nHessian updates: 0 line steps: 136.33333333333283 propose alpha: 5.44544778180864e-10 propose points: [ 4.50667002 450.00004003]\n\nHessian updates: 0 line steps: 137.33333333333286 propose alpha: 5.445418345364246e-10 propose points: [ 4.50664571 450.00004003]\n\nHessian updates: 0 line steps: 138.3333333333329 propose alpha: 5.445388908897289e-10 propose points: [ 4.5066214 450.00004003]\n\nHessian updates: 0 line steps: 139.33333333333292 propose alpha: 5.44535947240774e-10 propose points: [ 4.5065971 450.00004003]\n140 141 1.9e+07 0:00.3\n\nHessian updates: 0 line steps: 140.33333333333294 propose alpha: 5.445330035895567e-10 propose points: [ 4.50657279 450.00004003]\n\nHessian updates: 0 line steps: 141.33333333333297 propose alpha: 5.445300599360741e-10 propose points: [ 4.50654848 450.00004003]\n\nHessian updates: 0 line steps: 142.333333333333 propose alpha: 5.445271162803231e-10 propose points: [ 4.50652417 450.00004003]\n\nHessian updates: 0 line steps: 143.33333333333303 propose alpha: 5.445241726223008e-10 propose points: [ 4.50649986 450.00004003]\n\nHessian updates: 0 line steps: 144.33333333333306 propose alpha: 5.445212289620042e-10 propose points: [ 4.50647556 450.00004003]\n\nHessian updates: 0 line steps: 145.3333333333331 propose alpha: 5.445182852994302e-10 propose points: [ 4.50645125 450.00004003]\n\nHessian updates: 0 line steps: 146.33333333333312 propose alpha: 5.445153416345758e-10 propose points: [ 4.50642694 450.00004003]\n\nHessian updates: 0 line steps: 147.33333333333314 propose alpha: 5.445123979674381e-10 propose points: [ 4.50640263 450.00004003]\n\nHessian updates: 0 line steps: 148.33333333333317 propose alpha: 5.44509454298014e-10 propose points: [ 4.50637833 450.00004003]\n\nHessian updates: 0 line steps: 149.3333333333332 propose alpha: 5.445065106263004e-10 propose points: [ 4.50635402 450.00004003]\n\nHessian updates: 0 line steps: 150.33333333333323 propose alpha: 5.445035669522944e-10 propose points: [ 4.50632971 450.00004003]\n\nHessian updates: 0 line steps: 151.33333333333326 propose alpha: 5.445006232759929e-10 propose points: [ 4.5063054 450.00004003]\n\nHessian updates: 0 line steps: 152.3333333333333 propose alpha: 5.444976795973929e-10 propose points: [ 4.50628109 450.00004003]\n\nHessian updates: 0 line steps: 153.33333333333331 propose alpha: 5.444947359164915e-10 propose points: [ 4.50625679 450.00004003]\n\nHessian updates: 0 line steps: 154.33333333333334 propose alpha: 5.444917922332856e-10 propose points: [ 4.50623248 450.00004003]\n\nHessian updates: 0 line steps: 155.33333333333337 propose alpha: 5.444888485477721e-10 propose points: [ 4.50620817 450.00004002]\n\nHessian updates: 0 line steps: 156.3333333333334 propose alpha: 5.444859048599481e-10 propose points: [ 4.50618386 450.00004002]\n\nHessian updates: 0 line steps: 157.33333333333343 propose alpha: 5.444829611698105e-10 propose points: [ 4.50615955 450.00004002]\n\nHessian updates: 0 line steps: 158.33333333333346 propose alpha: 5.444800174773563e-10 propose points: [ 4.50613525 450.00004002]\n\nHessian updates: 0 line steps: 159.33333333333348 propose alpha: 5.444770737825826e-10 propose points: [ 4.50611094 450.00004002]\n160 161 1.9e+07 0:00.3\n\nHessian updates: 0 line steps: 160.3333333333335 propose alpha: 5.444741300854863e-10 propose points: [ 4.50608663 450.00004002]\n\nHessian updates: 0 line steps: 161.33333333333354 propose alpha: 5.444711863860644e-10 propose points: [ 4.50606232 450.00004002]\n\nHessian updates: 0 line steps: 162.33333333333357 propose alpha: 5.444682426843138e-10 propose points: [ 4.50603801 450.00004002]\n\nHessian updates: 0 line steps: 163.3333333333336 propose alpha: 5.444652989802316e-10 propose points: [ 4.50601371 450.00004002]\n\nHessian updates: 0 line steps: 164.33333333333363 propose alpha: 5.444623552738148e-10 propose points: [ 4.5059894 450.00004002]\n\nHessian updates: 0 line steps: 165.33333333333366 propose alpha: 5.444594115650603e-10 propose points: [ 4.50596509 450.00004002]\n\nHessian updates: 0 line steps: 166.33333333333368 propose alpha: 5.444564678539651e-10 propose points: [ 4.50594078 450.00004002]\n\nHessian updates: 0 line steps: 167.3333333333337 propose alpha: 5.444535241405262e-10 propose points: [ 4.50591647 450.00004002]\n\nHessian updates: 0 line steps: 168.33333333333374 propose alpha: 5.444505804247405e-10 propose points: [ 4.50589216 450.00004002]\n\nHessian updates: 0 line steps: 169.33333333333377 propose alpha: 5.444476367066052e-10 propose points: [ 4.50586786 450.00004002]\n\nHessian updates: 0 line steps: 170.3333333333338 propose alpha: 5.444446929861171e-10 propose points: [ 4.50584355 450.00004002]\n\nHessian updates: 0 line steps: 171.33333333333383 propose alpha: 5.444417492632732e-10 propose points: [ 4.50581924 450.00004002]\n\nHessian updates: 0 line steps: 172.33333333333385 propose alpha: 5.444388055380704e-10 propose points: [ 4.50579493 450.00004002]\n\nHessian updates: 0 line steps: 173.33333333333388 propose alpha: 5.444358618105059e-10 propose points: [ 4.50577062 450.00004002]\n\nHessian updates: 0 line steps: 174.3333333333339 propose alpha: 5.444329180805766e-10 propose points: [ 4.50574632 450.00004002]\n\nHessian updates: 0 line steps: 175.33333333333394 propose alpha: 5.444299743482794e-10 propose points: [ 4.50572201 450.00004002]\n\nHessian updates: 0 line steps: 176.33333333333397 propose alpha: 5.444270306136114e-10 propose points: [ 4.5056977 450.00004002]\n\nHessian updates: 0 line steps: 177.333333333334 propose alpha: 5.444240868765695e-10 propose points: [ 4.50567339 450.00004002]\n\nHessian updates: 0 line steps: 178.33333333333402 propose alpha: 5.444211431371507e-10 propose points: [ 4.50564908 450.00004002]\n\nHessian updates: 0 line steps: 179.33333333333405 propose alpha: 5.444181993953519e-10 propose points: [ 4.50562477 450.00004002]\n180 181 1.9e+07 0:00.4\n\nHessian updates: 0 line steps: 180.33333333333408 propose alpha: 5.444152556511703e-10 propose points: [ 4.50560046 450.00004002]\n\nHessian updates: 0 line steps: 181.3333333333341 propose alpha: 5.444123119046026e-10 propose points: [ 4.50557616 450.00004002]\n\nHessian updates: 0 line steps: 182.33333333333414 propose alpha: 5.44409368155646e-10 propose points: [ 4.50555185 450.00004002]\n\nHessian updates: 0 line steps: 183.33333333333417 propose alpha: 5.444064244042975e-10 propose points: [ 4.50552754 450.00004002]\n\nHessian updates: 0 line steps: 184.3333333333342 propose alpha: 5.44403480650554e-10 propose points: [ 4.50550323 450.00004002]\n\nHessian updates: 0 line steps: 185.33333333333422 propose alpha: 5.444005368944125e-10 propose points: [ 4.50547892 450.00004002]\n\nHessian updates: 0 line steps: 186.33333333333425 propose alpha: 5.443975931358698e-10 propose points: [ 4.50545461 450.00004002]\n\nHessian updates: 0 line steps: 187.33333333333428 propose alpha: 5.443946493749232e-10 propose points: [ 4.5054303 450.00004002]\n\nHessian updates: 0 line steps: 188.3333333333343 propose alpha: 5.443917056115695e-10 propose points: [ 4.505406 450.00004002]\n\nHessian updates: 0 line steps: 189.33333333333434 propose alpha: 5.443887618458057e-10 propose points: [ 4.50538169 450.00004002]\n\nHessian updates: 0 line steps: 190.33333333333437 propose alpha: 5.443858180776289e-10 propose points: [ 4.50535738 450.00004002]\n\nHessian updates: 0 line steps: 191.3333333333344 propose alpha: 5.443828743070359e-10 propose points: [ 4.50533307 450.00004002]\n\nHessian updates: 0 line steps: 192.33333333333442 propose alpha: 5.443799305340237e-10 propose points: [ 4.50530876 450.00004002]\n\nHessian updates: 0 line steps: 193.33333333333445 propose alpha: 5.443769867585894e-10 propose points: [ 4.50528445 450.00004002]\n\nHessian updates: 0 line steps: 194.33333333333448 propose alpha: 5.4437404298073e-10 propose points: [ 4.50526014 450.00004002]\n\nHessian updates: 0 line steps: 195.3333333333345 propose alpha: 5.443710992004422e-10 propose points: [ 4.50523584 450.00004002]\n\nHessian updates: 0 line steps: 196.33333333333454 propose alpha: 5.443681554177234e-10 propose points: [ 4.50521153 450.00004002]\n\nHessian updates: 0 line steps: 197.33333333333456 propose alpha: 5.443652116325703e-10 propose points: [ 4.50518722 450.00004002]\n\nHessian updates: 0 line steps: 198.3333333333346 propose alpha: 5.4436226784498e-10 propose points: [ 4.50516291 450.00004002]\n\nHessian updates: 0 line steps: 199.33333333333462 propose alpha: 5.443593240549494e-10 propose points: [ 4.5051386 450.00004002]\n200 201 1.9e+07 0:00.4\n201 201 1.9e+07 0:00.4\nHalting: No significant change for 200 iterations.\nScore at true solution: \n106262.41486040733\nFound solution: True parameters:\n 1.00000000000000002e-02 1.49999999999999994e-02\n 4.50000000000000000e+02 5.00000000000000000e+02\n" + } + ], + "source": [ + "# Load a forward model\n", + "model = pints.toy.LogisticModel()\n", + "\n", + "# Create some toy data\n", + "real_parameters = [0.015, 500]\n", + "times = np.linspace(0, 1000, 1000)\n", + "values = model.simulate(real_parameters, times)\n", + "\n", + "# Add noise\n", + "values += np.random.normal(0, 10, values.shape)\n", + "\n", + "# Create an object with links to the model and time series\n", + "problem = pints.SingleOutputProblem(model, times, values)\n", + "\n", + "# Select a score function\n", + "score = pints.SumOfSquaresError(problem)\n", + "\n", + "# Perform an optimization\n", + "x0 = [0.01, 450]\n", + "opt = pints.OptimisationController(\n", + " score,\n", + " x0,\n", + " method=pints.BFGS\n", + ")\n", + "opt.set_max_unchanged_iterations(200)\n", + "opt.set_max_iterations(1000)\n", + "found_parameters, found_value = opt.run()\n", + "\n", + "# Show score of true solution\n", + "print('Score at true solution: ')\n", + "print(score(real_parameters))\n", + "\n", + "# Compare parameters with original\n", + "print('Found solution: True parameters:' )\n", + "for k, x in enumerate(found_parameters):\n", + " print(pints.strfloat(x) + ' ' + pints.strfloat(real_parameters[k]))" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "tags": [ + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend", + "outputPrepend" + ] + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": "ropose alpha: 0.9853004251050581 propose points: [ 0.05805812 -0.70685582 2.31663456]\nalpha_initial: 1.9706008502101162\nNumber of accepted steps: 928\nstep sized alpha: 41.19133925117548 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 929 line steps: 0 propose alpha: 41.19133925117548 propose points: [ 0.05804951 -0.70683788 2.31627516]\nalpha_initial: 82.38267850235096\nNumber of accepted steps: 929\nstep sized alpha: 0.9837835985315283 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 930 line steps: 0 propose alpha: 0.9837835985315283 propose points: [ 0.05807305 -0.70688811 2.31627209]\nalpha_initial: 1.9675671970630566\nNumber of accepted steps: 930\nstep sized alpha: 42.1802912347368 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 931 line steps: 0 propose alpha: 42.1802912347368 propose points: [ 0.05806474 -0.70687113 2.31591937]\nalpha_initial: 84.3605824694736\nNumber of accepted steps: 931\nstep sized alpha: 0.9822161226427419 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 932 line steps: 0 propose alpha: 0.9822161226427419 propose points: [ 0.05808755 -0.7069198 2.31591649]\nalpha_initial: 1.9644322452854839\nNumber of accepted steps: 932\nstep sized alpha: 43.19236875916908 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 933 line steps: 0 propose alpha: 43.19236875916908 propose points: [ 0.05807992 -0.70690358 2.31557068]\nalpha_initial: 86.38473751833816\nNumber of accepted steps: 933\nstep sized alpha: 0.9806273291958192 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 934 line steps: 0 propose alpha: 0.9806273291958192 propose points: [ 0.05810199 -0.70695071 2.31556798]\nalpha_initial: 1.9612546583916384\nNumber of accepted steps: 934\nstep sized alpha: 44.280002900249485 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 935 line steps: 0 propose alpha: 44.280002900249485 propose points: [ 0.05809443 -0.70693552 2.3152289 ]\nalpha_initial: 88.56000580049897\nNumber of accepted steps: 935\nstep sized alpha: 0.9789710383899488 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 936 line steps: 0 propose alpha: 0.9789710383899488 propose points: [ 0.0581158 -0.70698111 2.31522639]\n940 941 36718.74 3:40.2\nalpha_initial: 1.9579420767798976\nNumber of accepted steps: 936\nstep sized alpha: 45.38326240469792 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 937 line steps: 0 propose alpha: 45.38326240469792 propose points: [ 0.05810926 -0.70696648 2.31489437]\nalpha_initial: 90.76652480939585\nNumber of accepted steps: 937\nstep sized alpha: 0.9772736517059862 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 938 line steps: 0 propose alpha: 0.9772736517059862 propose points: [ 0.05812988 -0.70701055 2.31489202]\nalpha_initial: 1.9545473034119725\nNumber of accepted steps: 938\nstep sized alpha: 46.66076684351502 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 939 line steps: 0 propose alpha: 46.66076684351502 propose points: [ 0.05812278 -0.70699725 2.31456626]\nalpha_initial: 93.32153368703004\nNumber of accepted steps: 939\nstep sized alpha: 0.9754247000926022 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 940 line steps: 0 propose alpha: 0.9754247000926022 propose points: [ 0.05814275 -0.70703984 2.31456408]\nalpha_initial: 1.9508494001852044\nNumber of accepted steps: 940\nstep sized alpha: 47.77994076855586 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 941 line steps: 0 propose alpha: 47.77994076855586 propose points: [ 0.05813782 -0.7070265 2.31424617]\nalpha_initial: 95.55988153711172\nNumber of accepted steps: 941\nstep sized alpha: 0.9736747154152247 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 942 line steps: 0 propose alpha: 0.9736747154152247 propose points: [ 0.05815699 -0.70706757 2.31424415]\nalpha_initial: 1.9473494308304493\nNumber of accepted steps: 942\nstep sized alpha: 49.39344559848598 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 943 line steps: 0 propose alpha: 49.39344559848598 propose points: [ 0.05814922 -0.70705662 2.31393131]\nalpha_initial: 98.78689119697196\nNumber of accepted steps: 943\nstep sized alpha: 0.9714355070860714 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 944 line steps: 0 propose alpha: 0.9714355070860714 propose points: [ 0.05816788 -0.70709626 2.31392946]\nalpha_initial: 1.9428710141721428\nNumber of accepted steps: 944\nstep sized alpha: 50.178225674483606 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 945 line steps: 0 propose alpha: 50.178225674483606 propose points: [ 0.05816672 -0.70708308 2.3136274 ]\nalpha_initial: 100.35645134896721\nNumber of accepted steps: 945\nstep sized alpha: 0.9695061100904564 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 946 line steps: 0 propose alpha: 0.9695061100904564 propose points: [ 0.05818432 -0.70712114 2.31362567]\nalpha_initial: 1.9390122201809128\nNumber of accepted steps: 946\nstep sized alpha: 52.71079127859243 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 947 line steps: 0 propose alpha: 52.71079127859243 propose points: [ 0.05817109 -0.70711482 2.31332441]\nalpha_initial: 105.42158255718486\nNumber of accepted steps: 947\nstep sized alpha: 0.9645884702708936 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 948 line steps: 0 propose alpha: 0.9645884702708936 propose points: [ 0.05818871 -0.70715155 2.31332286]\nalpha_initial: 1.9291769405417871\nNumber of accepted steps: 948\nstep sized alpha: 50.553520962228234 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 949 line steps: 0 propose alpha: 50.553520962228234 propose points: [ 0.058201 -0.70713366 2.31304911]\nalpha_initial: 101.10704192445647\nNumber of accepted steps: 949\nstep sized alpha: 0.9558473081233407 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 950 line steps: 0 propose alpha: 0.9558473081233407 propose points: [ 0.05821625 -0.7071681 2.31304754]\nalpha_initial: 1.9116946162466815\nNumber of accepted steps: 950\nstep sized alpha: 52.84968946407165 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 951 line steps: 0 propose alpha: 52.84968946407165 propose points: [ 0.05817747 -0.70717588 2.31277594]\nalpha_initial: 105.6993789281433\nNumber of accepted steps: 951\nstep sized alpha: 0.9109547852317396 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 952 line steps: 0 propose alpha: 0.9109547852317396 propose points: [ 0.05819422 -0.70720776 2.31277447]\nalpha_initial: 1.8219095704634791\nNumber of accepted steps: 952\nstep sized alpha: 33.38194674986785 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 953 line steps: 0 propose alpha: 33.38194674986785 propose points: [ 0.05823593 -0.7071765 2.31261153]\nalpha_initial: 66.7638934997357\nNumber of accepted steps: 953\nstep sized alpha: 0.8284770353382566 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 954 line steps: 0 propose alpha: 0.8284770353382566 propose points: [ 0.05824577 -0.70720143 2.31260926]\nalpha_initial: 1.6569540706765131\nNumber of accepted steps: 954\nstep sized alpha: 26.556301410258673 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 955 line steps: 0 propose alpha: 26.556301410258673 propose points: [ 0.05820501 -0.70720799 2.31248365]\nalpha_initial: 53.112602820517345\nNumber of accepted steps: 955\nstep sized alpha: 0.8744500722765555 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 956 line steps: 0 propose alpha: 0.8744500722765555 propose points: [ 0.05821597 -0.70722912 2.3124812 ]\n960 961 36718.74 3:45.6\nalpha_initial: 1.748900144553111\nNumber of accepted steps: 956\nstep sized alpha: 23.439578485715582 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 957 line steps: 0 propose alpha: 23.439578485715582 propose points: [ 0.05822971 -0.70720982 2.31237319]\nalpha_initial: 46.879156971431165\nNumber of accepted steps: 957\nstep sized alpha: 0.9726783551307459 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 958 line steps: 0 propose alpha: 0.9726783551307459 propose points: [ 0.05823881 -0.70723022 2.3123707 ]\nalpha_initial: 1.9453567102614917\nNumber of accepted steps: 958\nstep sized alpha: 23.836450682939443 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 959 line steps: 0 propose alpha: 23.836450682939443 propose points: [ 0.05822621 -0.70722338 2.3122635 ]\nalpha_initial: 47.672901365878886\nNumber of accepted steps: 959\nstep sized alpha: 1.0100609012192157 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 960 line steps: 0 propose alpha: 1.0100609012192157 propose points: [ 0.05823573 -0.70724333 2.31226111]\nalpha_initial: 2.0201218024384313\nNumber of accepted steps: 960\nstep sized alpha: 24.603027303527533 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 961 line steps: 0 propose alpha: 24.603027303527533 propose points: [ 0.05823442 -0.70723149 2.3121532 ]\nalpha_initial: 49.206054607055066\nNumber of accepted steps: 961\nstep sized alpha: 1.0146568855352263 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 962 line steps: 0 propose alpha: 1.0146568855352263 propose points: [ 0.05824362 -0.70725135 2.31215091]\nalpha_initial: 2.0293137710704525\nNumber of accepted steps: 962\nstep sized alpha: 25.681859551539237 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 963 line steps: 0 propose alpha: 25.681859551539237 propose points: [ 0.0582373 -0.70724218 2.31204115]\nalpha_initial: 51.36371910307847\nNumber of accepted steps: 963\nstep sized alpha: 1.0130834315484123 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 964 line steps: 0 propose alpha: 1.0130834315484123 propose points: [ 0.05824656 -0.70726188 2.31203897]\nalpha_initial: 2.0261668630968246\nNumber of accepted steps: 964\nstep sized alpha: 26.840743781642743 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 965 line steps: 0 propose alpha: 26.840743781642743 propose points: [ 0.05824303 -0.70725174 2.31192732]\nalpha_initial: 53.681487563285486\nNumber of accepted steps: 965\nstep sized alpha: 1.0103905622892726 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 966 line steps: 0 propose alpha: 1.0103905622892726 propose points: [ 0.05825216 -0.70727134 2.31192525]\nalpha_initial: 2.020781124578545\nNumber of accepted steps: 966\nstep sized alpha: 28.191241579177273 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 967 line steps: 0 propose alpha: 28.191241579177273 propose points: [ 0.05824721 -0.70726222 2.31181124]\nalpha_initial: 56.382483158354546\nNumber of accepted steps: 967\nstep sized alpha: 1.007211291311263 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 968 line steps: 0 propose alpha: 1.007211291311263 propose points: [ 0.05825634 -0.70728171 2.31180928]\nalpha_initial: 2.014422582622526\nNumber of accepted steps: 968\nstep sized alpha: 29.72480561652972 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 969 line steps: 0 propose alpha: 29.72480561652972 propose points: [ 0.05825258 -0.70727241 2.31169257]\nalpha_initial: 59.44961123305944\nNumber of accepted steps: 969\nstep sized alpha: 1.0037472948306263 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 970 line steps: 0 propose alpha: 1.0037472948306263 propose points: [ 0.05826164 -0.70729182 2.31169073]\nalpha_initial: 2.0074945896612526\nNumber of accepted steps: 970\nstep sized alpha: 31.466011061357214 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 971 line steps: 0 propose alpha: 31.466011061357214 propose points: [ 0.05825725 -0.70728322 2.31157098]\nalpha_initial: 62.93202212271443\nNumber of accepted steps: 971\nstep sized alpha: 1.000087711218419 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 972 line steps: 0 propose alpha: 1.000087711218419 propose points: [ 0.05826629 -0.70730253 2.31156927]\nalpha_initial: 2.000175422436838\nNumber of accepted steps: 972\nstep sized alpha: 33.446108038120684 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 973 line steps: 0 propose alpha: 33.446108038120684 propose points: [ 0.05826276 -0.70729399 2.3114461 ]\nalpha_initial: 66.89221607624137\nNumber of accepted steps: 973\nstep sized alpha: 0.9961264224882628 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 974 line steps: 0 propose alpha: 0.9961264224882628 propose points: [ 0.05827174 -0.70731322 2.31144451]\nalpha_initial: 1.9922528449765255\nNumber of accepted steps: 974\nstep sized alpha: 35.78301193570644 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 975 line steps: 0 propose alpha: 35.78301193570644 propose points: [ 0.05826774 -0.70730539 2.31131728]\nalpha_initial: 71.56602387141288\nNumber of accepted steps: 975\nstep sized alpha: 0.9917693742743835 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 976 line steps: 0 propose alpha: 0.9917693742743835 propose points: [ 0.0582767 -0.70732454 2.31131582]\n980 981 36718.74 3:50.4\nalpha_initial: 1.983538748548767\nNumber of accepted steps: 976\nstep sized alpha: 38.49840024184752 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 977 line steps: 0 propose alpha: 38.49840024184752 propose points: [ 0.05827366 -0.70731681 2.31118395]\nalpha_initial: 76.99680048369504\nNumber of accepted steps: 977\nstep sized alpha: 0.9869979906592825 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 978 line steps: 0 propose alpha: 0.9869979906592825 propose points: [ 0.05828258 -0.70733591 2.31118263]\nalpha_initial: 1.973995981318565\nNumber of accepted steps: 978\nstep sized alpha: 41.797048018246215 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 979 line steps: 0 propose alpha: 41.797048018246215 propose points: [ 0.05827881 -0.70732914 2.3110451 ]\nalpha_initial: 83.59409603649243\nNumber of accepted steps: 979\nstep sized alpha: 0.9816152808443387 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 980 line steps: 0 propose alpha: 0.9816152808443387 propose points: [ 0.05828774 -0.70734819 2.31104392]\nalpha_initial: 1.9632305616886774\nNumber of accepted steps: 980\nstep sized alpha: 45.67253981776184 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 981 line steps: 0 propose alpha: 45.67253981776184 propose points: [ 0.05828572 -0.70734131 2.31090006]\nalpha_initial: 91.34507963552367\nNumber of accepted steps: 981\nstep sized alpha: 0.9755848179245643 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 982 line steps: 0 propose alpha: 0.9755848179245643 propose points: [ 0.05829457 -0.70736034 2.31089902]\nalpha_initial: 1.9511696358491286\nNumber of accepted steps: 982\nstep sized alpha: 50.613261706813304 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 983 line steps: 0 propose alpha: 50.613261706813304 propose points: [ 0.05829031 -0.70735534 2.31074701]\nalpha_initial: 101.22652341362661\nNumber of accepted steps: 983\nstep sized alpha: 0.9683582998152838 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 984 line steps: 0 propose alpha: 0.9683582998152838 propose points: [ 0.05829927 -0.70737433 2.31074613]\nalpha_initial: 1.9367165996305675\nNumber of accepted steps: 984\nstep sized alpha: 55.96523971136157 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 985 line steps: 0 propose alpha: 55.96523971136157 propose points: [ 0.05830066 -0.70736761 2.31058668]\nalpha_initial: 111.93047942272314\nNumber of accepted steps: 985\nstep sized alpha: 0.9594635667765983 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 986 line steps: 0 propose alpha: 0.9594635667765983 propose points: [ 0.05830932 -0.70738657 2.31058596]\nalpha_initial: 1.9189271335531966\nNumber of accepted steps: 986\nstep sized alpha: 63.235663883354555 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 987 line steps: 0 propose alpha: 63.235663883354555 propose points: [ 0.0582983 -0.70738689 2.310416 ]\nalpha_initial: 126.47132776670911\nNumber of accepted steps: 987\nstep sized alpha: 0.9404807259871993 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 988 line steps: 0 propose alpha: 0.9404807259871993 propose points: [ 0.05830751 -0.70740548 2.31041544]\nalpha_initial: 1.8809614519743987\nNumber of accepted steps: 988\nstep sized alpha: 57.2908705844076 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 989 line steps: 0 propose alpha: 57.2908705844076 propose points: [ 0.0583285 -0.70739124 2.31027116]\nalpha_initial: 114.5817411688152\nNumber of accepted steps: 989\nstep sized alpha: 0.8678577511508032 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 990 line steps: 0 propose alpha: 0.8678577511508032 propose points: [ 0.0583351 -0.70740799 2.31027047]\nalpha_initial: 1.7357155023016064\nNumber of accepted steps: 990\nstep sized alpha: 36.64845187727479 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 991 line steps: 0 propose alpha: 36.64845187727479 propose points: [ 0.05829955 -0.7074201 2.31018323]\nalpha_initial: 73.29690375454958\nNumber of accepted steps: 991\nstep sized alpha: 0.6739339523784038 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 992 line steps: 0 propose alpha: 0.6739339523784038 propose points: [ 0.05830615 -0.7074311 2.31018207]\nalpha_initial: 1.3478679047568076\nNumber of accepted steps: 992\nstep sized alpha: 17.2163699544149 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 993 line steps: 0 propose alpha: 17.2163699544149 propose points: [ 0.05832221 -0.70741722 2.31014225]\nalpha_initial: 34.4327399088298\nNumber of accepted steps: 993\nstep sized alpha: 0.8335795547119987 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 994 line steps: 0 propose alpha: 0.8335795547119987 propose points: [ 0.05832627 -0.70742607 2.3101408 ]\nalpha_initial: 1.6671591094239975\nNumber of accepted steps: 994\nstep sized alpha: 17.215946503275195 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 995 line steps: 0 propose alpha: 17.215946503275195 propose points: [ 0.05832296 -0.70742128 2.31010169]\n1000 1000 36718.74 3:55.2\nHalting: Maximum number of iterations (1000) reached.\nScore at true solution: \n36894.34103855895\nFound solution: True parameters:\n 5.83229645034749195e-02 1.00000000000000006e-01\n-7.07421276762396634e-01 5.00000000000000000e-01\n 2.31010169210899896e+00 3.00000000000000000e+00\n" + } + ], + "source": [ + "# Load a forward model\n", + "\n", + "f = pints.toy.FitzhughNagumoModel()\n", + "real_parameters = f.suggested_parameters()\n", + "# [0.1, 0.5, 3. ]\n", + "times =f.suggested_times()\n", + "values = f.simulate(real_parameters, times)\n", + "\n", + "# Add noise\n", + "values += np.random.normal(0, 10, values.shape)\n", + "\n", + "# Create an object with links to the model and time series\n", + "problem = pints.MultiOutputProblem(f, times, values)\n", + "\n", + "# Select a score function\n", + "score = pints.SumOfSquaresError(problem)\n", + "\n", + "# Perform an optimization\n", + "x0 = [0.2, 0.3, 2.5]\n", + "opt = pints.OptimisationController(\n", + " score,\n", + " x0,\n", + " method=pints.BFGS\n", + ")\n", + "opt.set_max_unchanged_iterations(200)\n", + "opt.set_max_iterations(1000)\n", + "found_parameters, found_value = opt.run()\n", + "\n", + "# Show score of true solution\n", + "print('Score at true solution: ')\n", + "print(score(real_parameters))\n", + "\n", + "# Compare parameters with original\n", + "print('Found solution: True parameters:' )\n", + "for k, x in enumerate(found_parameters):\n", + " print(pints.strfloat(x) + ' ' + pints.strfloat(real_parameters[k]))\n", + "\n", + "# [0.1, 0.5, 3. ] real\n", + "# [0.2, 0.3, 2.5] starting" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## now using scipy line search" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": "Minimising error measure\nUsing Broyden–Fletcher–Goldfarb–Shanno (BFGS)\nRunning in sequential mode.\nIter. Eval. Best Time m:s\n0 1 1.91e+07 0:00.0\n\n----------------------------------------\nUnexpected termination.\nCurrent best score: 19146011.016347833\nCurrent best position:\n 1.00000000000000002e-02\n 4.50000000000000000e+02\n----------------------------------------\n" + }, + { + "output_type": "error", + "ename": "TypeError", + "evalue": "unsupported operand type(s) for *: 'NoneType' and 'float'", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_max_unchanged_iterations\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m200\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_max_iterations\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m \u001b[0mfound_parameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfound_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0;31m# Show score of true solution\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/pints/pints-clone/pints/pints/_optimisers/__init__.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 540\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0mrunning\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 541\u001b[0m \u001b[0;31m# Get points\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 542\u001b[0;31m \u001b[0mxs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_optimiser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 543\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 544\u001b[0m \u001b[0;31m# Calculate scores\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/pints/pints-clone/pints/pints/_optimisers/_bfgs_scipy.py\u001b[0m in \u001b[0;36mask\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 245\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_proposed_alpha\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 246\u001b[0m \u001b[0;31m# print('alpha: ', self._proposed_alpha)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 247\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_proposed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_current\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_proposed_alpha\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_px\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 248\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 249\u001b[0m \u001b[0;31m# Running, and ready for tell now\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mTypeError\u001b[0m: unsupported operand type(s) for *: 'NoneType' and 'float'" + ] + } + ], + "source": [ + "# Load a forward model\n", + "model = pints.toy.LogisticModel()\n", + "\n", + "# Create some toy data\n", + "real_parameters = [0.015, 500]\n", + "times = np.linspace(0, 1000, 1000)\n", + "values = model.simulate(real_parameters, times)\n", + "\n", + "# Add noise\n", + "values += np.random.normal(0, 10, values.shape)\n", + "\n", + "# Create an object with links to the model and time series\n", + "problem = pints.SingleOutputProblem(model, times, values)\n", + "\n", + "# Select a score function\n", + "score = pints.SumOfSquaresError(problem)\n", + "\n", + "# Perform an optimization\n", + "x0 = [0.01, 450]\n", + "opt = pints.OptimisationController(\n", + " score,\n", + " x0,\n", + " method=pints.BFGS_scipy\n", + ")\n", + "opt.set_max_unchanged_iterations(200)\n", + "opt.set_max_iterations(1000)\n", + "found_parameters, found_value = opt.run()\n", + "\n", + "# Show score of true solution\n", + "print('Score at true solution: ')\n", + "print(score(real_parameters))\n", + "\n", + "# Compare parameters with original\n", + "print('Found solution: True parameters:' )\n", + "for k, x in enumerate(found_parameters):\n", + " print(pints.strfloat(x) + ' ' + pints.strfloat(real_parameters[k]))" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": "Minimising error measure\nUsing Broyden–Fletcher–Goldfarb–Shanno (BFGS)\nRunning in sequential mode.\nIter. Eval. Best Time m:s\n0 1 34916.69 0:00.1\n1 2 34498.91 0:00.7\n2 3 34481.78 0:00.9\n3 4 34478.34 0:01.1\n20 21 34233.43 0:06.2\n40 41 34172.38 0:11.5\n\n----------------------------------------\nUnexpected termination.\nCurrent best score: 34161.946320079565\nCurrent best position:\n-5.44201477518764937e-02\n-5.50556538864840572e-01\n 2.83944303459274172e+00\n----------------------------------------\n" + }, + { + "output_type": "error", + "ename": "KeyboardInterrupt", + "evalue": "", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 23\u001b[0m \u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_max_unchanged_iterations\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m200\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_max_iterations\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m \u001b[0mfound_parameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfound_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[0;31m# Show score of true solution\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/pints/pints-clone/pints/pints/_optimisers/__init__.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 540\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0mrunning\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 541\u001b[0m \u001b[0;31m# Get points\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 542\u001b[0;31m \u001b[0mxs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_optimiser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 543\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 544\u001b[0m \u001b[0;31m# Calculate scores\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/pints/pints-clone/pints/pints/_optimisers/_bfgs_scipy.py\u001b[0m in \u001b[0;36mask\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 216\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 217\u001b[0m \u001b[0;31m# line search using an algorithm from scipy to meet wolfe condtions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 218\u001b[0;31m results = line_search_wolfe2(f=self.__objective_function,\n\u001b[0m\u001b[1;32m 219\u001b[0m \u001b[0mmyfprime\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__gradients_function\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 220\u001b[0m \u001b[0mxk\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_current\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpk\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_px\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/envs/pints/lib/python3.8/site-packages/scipy/optimize/linesearch.py\u001b[0m in \u001b[0;36mline_search_wolfe2\u001b[0;34m(f, myfprime, xk, pk, gfk, old_fval, old_old_fval, args, c1, c2, amax, extra_condition, maxiter)\u001b[0m\n\u001b[1;32m 307\u001b[0m \u001b[0mextra_condition2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 308\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 309\u001b[0;31m alpha_star, phi_star, old_fval, derphi_star = scalar_search_wolfe2(\n\u001b[0m\u001b[1;32m 310\u001b[0m \u001b[0mphi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mderphi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mold_fval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mold_old_fval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mderphi0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mc1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mc2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mamax\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 311\u001b[0m extra_condition2, maxiter=maxiter)\n", + "\u001b[0;32m~/anaconda3/envs/pints/lib/python3.8/site-packages/scipy/optimize/linesearch.py\u001b[0m in \u001b[0;36mscalar_search_wolfe2\u001b[0;34m(phi, derphi, phi0, old_phi0, derphi0, c1, c2, amax, extra_condition, maxiter)\u001b[0m\n\u001b[1;32m 435\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 436\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 437\u001b[0;31m \u001b[0mderphi_a1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mderphi\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0malpha1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 438\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mabs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mderphi_a1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0mc2\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mderphi0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 439\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mextra_condition\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0malpha1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mphi_a1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/envs/pints/lib/python3.8/site-packages/scipy/optimize/linesearch.py\u001b[0m in \u001b[0;36mderphi\u001b[0;34m(alpha)\u001b[0m\n\u001b[1;32m 288\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mderphi\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0malpha\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 289\u001b[0m \u001b[0mgc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 290\u001b[0;31m \u001b[0mgval\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfprime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxk\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0malpha\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mpk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# store for later use\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 291\u001b[0m \u001b[0mgval_alpha\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0malpha\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 292\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgval\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/pints/pints-clone/pints/pints/_optimisers/_bfgs_scipy.py\u001b[0m in \u001b[0;36m__gradients_function\u001b[0;34m(self, point_alpha)\u001b[0m\n\u001b[1;32m 195\u001b[0m '''\n\u001b[1;32m 196\u001b[0m \u001b[0;31m#point_alpha = self._current + alpha * self._px\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 197\u001b[0;31m \u001b[0mfs_alpha\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_evaluator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpoint_alpha\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 198\u001b[0m \u001b[0mf_alpha\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdfdx_alpha\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfs_alpha\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 199\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/pints/pints-clone/pints/pints/_evaluation.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(self, positions)\u001b[0m\n\u001b[1;32m 104\u001b[0m \u001b[0;34m'The argument `positions` must be a sequence of input values'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 105\u001b[0m ' to the evaluator\\'s function.')\n\u001b[0;32m--> 106\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_evaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpositions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 107\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 108\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_evaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpositions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/pints/pints-clone/pints/pints/_evaluation.py\u001b[0m in \u001b[0;36m_evaluate\u001b[0;34m(self, positions)\u001b[0m\n\u001b[1;32m 396\u001b[0m \u001b[0mscores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpositions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 397\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpositions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 398\u001b[0;31m \u001b[0mscores\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 399\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mscores\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 400\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/pints/pints-clone/pints/pints/_error_measures.py\u001b[0m in \u001b[0;36mevaluateS1\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m 333\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mevaluateS1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 334\u001b[0m \u001b[0;34m\"\"\" See :meth:`ErrorMeasure.evaluateS1()`. \"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 335\u001b[0;31m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_problem\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluateS1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 336\u001b[0m \u001b[0mdy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_times\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_outputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_parameters\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 337\u001b[0m \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_values\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/pints/pints-clone/pints/pints/_core.py\u001b[0m in \u001b[0;36mevaluateS1\u001b[0;34m(self, parameters)\u001b[0m\n\u001b[1;32m 277\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;32mclass\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mForwardModelS1\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0minterface\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 278\u001b[0m \"\"\"\n\u001b[0;32m--> 279\u001b[0;31m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msimulateS1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_times\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 280\u001b[0m return (\n\u001b[1;32m 281\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_times\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_outputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/pints/pints-clone/pints/pints/toy/_toy_classes.py\u001b[0m in \u001b[0;36msimulateS1\u001b[0;34m(self, parameters, times)\u001b[0m\n\u001b[1;32m 227\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0msimulateS1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 228\u001b[0m \u001b[0;34m\"\"\" See :meth:`pints.ForwardModelS1.simulateS1()`. \"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 229\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_simulate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m~/Documents/pints/pints-clone/pints/pints/toy/_toy_classes.py\u001b[0m in \u001b[0;36m_simulate\u001b[0;34m(self, parameters, times, sensitivities)\u001b[0m\n\u001b[1;32m 214\u001b[0m \u001b[0my0\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_params\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mn_outputs\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mn_outputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 215\u001b[0m \u001b[0my0\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mn_outputs\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_y0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 216\u001b[0;31m result = scipy.integrate.odeint(\n\u001b[0m\u001b[1;32m 217\u001b[0m self._rhs_S1, y0, times, (parameters,))\n\u001b[1;32m 218\u001b[0m \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mn_outputs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/envs/pints/lib/python3.8/site-packages/scipy/integrate/odepack.py\u001b[0m in \u001b[0;36modeint\u001b[0;34m(func, y0, t, args, Dfun, col_deriv, full_output, ml, mu, rtol, atol, tcrit, h0, hmax, hmin, ixpr, mxstep, mxhnil, mxordn, mxords, printmessg, tfirst)\u001b[0m\n\u001b[1;32m 240\u001b[0m \u001b[0mt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 241\u001b[0m \u001b[0my0\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 242\u001b[0;31m output = _odepack.odeint(func, y0, t, args, Dfun, col_deriv, ml, mu,\n\u001b[0m\u001b[1;32m 243\u001b[0m \u001b[0mfull_output\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrtol\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0matol\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtcrit\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhmax\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhmin\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 244\u001b[0m \u001b[0mixpr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmxstep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmxhnil\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmxordn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmxords\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/pints/pints-clone/pints/pints/toy/_toy_classes.py\u001b[0m in \u001b[0;36m_rhs_S1\u001b[0;34m(self, y_and_dydp, t, p)\u001b[0m\n\u001b[1;32m 173\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdydp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjacobian\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 174\u001b[0m np.transpose(self._dfdp(y, t, p)))\n\u001b[0;32m--> 175\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdydt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0md_dydp_dt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 176\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 177\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0msimulate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m<__array_function__ internals>\u001b[0m in \u001b[0;36mconcatenate\u001b[0;34m(*args, **kwargs)\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "f = pints.toy.FitzhughNagumoModel()\n", + "real_parameters = f.suggested_parameters()\n", + "# [0.1, 0.5, 3. ]\n", + "times =f.suggested_times()\n", + "values = f.simulate(real_parameters, times)\n", + "\n", + "# Add noise\n", + "values += np.random.normal(0, 10, values.shape)\n", + "\n", + "# Create an object with links to the model and time series\n", + "problem = pints.MultiOutputProblem(f, times, values)\n", + "\n", + "# Select a score function\n", + "score = pints.SumOfSquaresError(problem)\n", + "\n", + "# Perform an optimization\n", + "x0 = [0.2, 0.3, 2.5]\n", + "opt = pints.OptimisationController(\n", + " score,\n", + " x0,\n", + " method=pints.BFGS_scipy\n", + ")\n", + "opt.set_max_unchanged_iterations(200)\n", + "opt.set_max_iterations(1000)\n", + "found_parameters, found_value = opt.run()\n", + "\n", + "# Show score of true solution\n", + "print('Score at true solution: ')\n", + "print(score(real_parameters))\n", + "\n", + "# Compare parameters with original\n", + "print('Found solution: True parameters:' )\n", + "for k, x in enumerate(found_parameters):\n", + " print(pints.strfloat(x) + ' ' + pints.strfloat(real_parameters[k]))\n", + "\n", + "# [0.1, 0.5, 3. ] real\n", + "# [0.2, 0.3, 2.5] starting" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": "False\n" + } + ], + "source": [ + "print( 5 < 4 | 0 > 1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.8.2 64-bit ('pints': conda)", + "language": "python", + "name": "python38264bitpintsconda8d0b754a30424fdd8045d82b54ea71e7" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.2-final" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/pints/__init__.py b/pints/__init__.py index 55cc96049..a3fe61c2e 100644 --- a/pints/__init__.py +++ b/pints/__init__.py @@ -166,7 +166,8 @@ def version(formatted=False): optimise, Optimiser, PopulationBasedOptimiser, - TriangleWaveTransform, + LineSearchBasedOptimiser, + TriangleWaveTransform ) from ._optimisers._cmaes import CMAES from ._optimisers._cmaes_bare import BareCMAES @@ -175,6 +176,8 @@ def version(formatted=False): from ._optimisers._pso import PSO from ._optimisers._snes import SNES from ._optimisers._xnes import XNES +from ._optimisers._bfgs_scipy import BFGS_scipy +from ._optimisers._bfgs_linesearch import BFGS # diff --git a/pints/_optimisers/_bfgs_linesearch.py b/pints/_optimisers/_bfgs_linesearch.py new file mode 100644 index 000000000..507dae302 --- /dev/null +++ b/pints/_optimisers/_bfgs_linesearch.py @@ -0,0 +1,1051 @@ +# +# Broyden–Fletcher–Goldfarb–Shanno algorithm +# +# This file is part of PINTS (https://github.com/pints-team/pints/) which is +# released under the BSD 3-clause license. See accompanying LICENSE.md for +# copyright notice and full license details. +# +from __future__ import absolute_import, division +from __future__ import print_function, unicode_literals +import numpy as np +from numpy.linalg import norm +import pints + + +class BFGS(pints.LineSearchBasedOptimiser): + """ + Broyden–Fletcher–Goldfarb–Shanno algorithm [2], [3], [4] + + The Hager-Zhang line search algorithm [1] is implemented in this class + # TODO: when this is working move everything to an abstract class + + [1] Hager, W. W.; Zhang, H. Algorithm 851: CG_DESCENT, + a Conjugate Gradient Method with Guaranteed Descent. + ACM Trans. Math. Softw. 2006, 32 (1), 113–137. + https://doi.org/10.1145/1132973.1132979. + + [2] Liu, D. C.; Nocedal, J. + On the Limited Memory BFGS Method for Large Scale Optimization. + Mathematical Programming 1989, 45 (1), + 503–528. https://doi.org/10.1007/BF01589116. + + [3] Nocedal, J. Updating Quasi-Newton Matrices with Limited Storage. + Math. Comp. 1980, 35 (151), 773–782. + https://doi.org/10.1090/S0025-5718-1980-0572855-7. + + [4] Nash, S. G.; Nocedal, J. A Numerical Study of the Limited Memory + BFGS Method and the Truncated-Newton Method for Large Scale Optimization. + SIAM J. Optim. 1991, 1 (3), 358–372. https://doi.org/10.1137/0801023. + + """ + + def __init__(self, x0, sigma0=None, boundaries=None): + super(BFGS, self).__init__(x0, sigma0, boundaries) + + # Set optimiser state + self._running = False + self._ready_for_tell = False + + # Best solution found + self._xbest = self._x0 + self._fbest = float('inf') + + # Number of iterations run + self._iterations = 0 + + # Parameters for wolfe conditions on line search + + # As c1 approaches 0 and c2 approaches 1, the line search + # terminates more quickly. + self._c1 = 1E-4 # Parameter for Armijo condition rule, 0 < c1 < 0.5 + self._c2 = 0.9 # Parameter for curvature condition rule, c1 < c2 < 1.0 + + # boundary values of alpha + self._minimum_alpha = 0.0 + self._maximum_alpha = float("inf") + self._proposed_alpha = 0.001 # same default value as used in stan + + self.__first_update_step_not_completed = True + self.__update_step_not_completed = True + self.__performing_line_search = False + + # Increase allowed between accepted positions when using approximate + # wolfe conditions, this takes into acccount machine error and + # insures decreasing. + self.epsilon = 1E-6 + + # range (0, 1), used in the ``self.__update()`` and + # ``self.__initial_bracket()`` when the potential intervals violate + # the opposite slope condition (see function definition) + self.theta = 0.5 + + self.__gamma = 0.66 + + # range (0, 1) small factor used in initial guess of step size + self.__ps_0 = 0.01 + # range (0, 1) small factor used in subsequent guesses of step size + self.__ps_1 = 0.1 + # range (1, inf) factor used in subsequent guesses of step size + self.__ps_2 = 2.0 + + self.__current_alpha = None + + # approximate inverse hessian + # initial the identity is used + self._B = np.identity(self._n_parameters) + + # newton direction + self._px = None + + # maximum number of correction matrices stored + self._m = 5 + + # Storage for vectors constructing the correction matrix + # this is the advised way of storing them. + self._S = np.zeros(shape=(self._n_parameters, self._m)) + self._Y = np.zeros(shape=(self._n_parameters, self._m)) + + # number of accepted steps/ newton direction updates + self.__k = 0 + + # number of steps in the current line search iteration + self.__j = 0 + self.__logic_steps_left = 0 + + # maximum number of line search steps before a successful point + + # Current point, score, and gradient + self._current = self._x0 + self._current_f = None + self._current_dfdx = None + + # Proposed next point (read-only, so can be passed to user) + self._proposed = self._x0 + self._proposed.setflags(write=False) + + self.__convergence = False + + # logic for passing to tell at the right moments + self.__1st_wolfe_check_needed = False + self.__1st_wolfe_check_done = False + self.__2nd_wolfe_check_needed = False + self.__2nd_wolfe_check_done = False + self.__need_update = False + self.__converged_ls = False + + def fbest(self): + """ See :meth:`Optimiser.fbest()`. """ + return self._fbest + + def wolfe_line_search_parameters(self): + """ + Returns the wolfe line search parameters this optimiser is using + as a vector ``[c1, c2]``. + As c1 approaches 0 and c2 approaches 1, the line search terminates + more quickly. ``c1`` is the parameter for the Armijo condition + rule, ``0 < c1 < 0.5``. ``c2 `` is the parameter for the + curvature condition rule, ``c1 < c2 < 1.0``. + """ + return (self._c1, self._c2) + + def max_correction_matrice_storage(self): + """ + Returns ``m``, the maximum number of correction matrice for + calculating the inverse hessian used and stored + """ + return self._m + + def name(self): + """ See :meth:`Optimiser.name()`. """ + return 'Broyden–Fletcher–Goldfarb–Shanno (BFGS)' + + def needs_sensitivities(self): + """ See :meth:`Optimiser.needs_sensitivities()`. """ + return True + + def n_hyper_parameters(self): + """ See :meth:`pints.TunableMethod.n_hyper_parameters()`. """ + return 2 + + def running(self): + """ See :meth:`Optimiser.running()`. """ + return self._running + + def set_hyper_parameters(self, x): + """ + See :meth:`pints.TunableMethod.set_hyper_parameters()`. + + The hyper-parameter vector is ``[c1, c2, m]``. + ``c1`` is the parameter for the Armijo condition rule, + ``0 < c1 < 0.5``. + ``c2`` is the parameter for the curvature condition rule, + ``c1 < c2 < 1.0``. + ``m`` is the number of maximum number of correction matrices + that can be stored for the LM-BFGS update. + """ + + self.__set_wolfe_line_search_parameters(x[0], x[1]) + + def __set_wolfe_line_search_parameters(self, c1: float, c2: float): + """ + Sets the parameters for the wolfe conditions. + + Parameters + ---------- + c1: float + Parameter for the Armijo condition rule, ``0 < c1 < 0.5``. + + c2: float + Parameter for the curvature condition rule, ``c1 < c2 < 1.0``. + """ + if(0 < c1 < 0.5 and c1 < c2 < 1.0): + self._c1 = c1 + self._c2 = c2 + else: + cs = self.wolfe_line_search_parameters() + print('Invalid wolfe line search parameters!!!') + print('0 < c1 < 0.5 and c1 < c2 < 1.0') + print('using default parameters: c1 = ', cs[0], ' c2 = ', cs[1]) + + def __set_max_correction_matrice_storage(self, m: int): + """ + Sets the parameters for the wolfe conditions. + + Parameters + ---------- + m: int + The maximum number of correction matrice for calculating the + inverse hessian used and stored. + """ + if(m == int(m)): + self._m = m + self._S = np.zeros(self._n_parameters, m) + self._Y = np.zeros(self._n_parameters, m) + else: + print('Invalid value of m!!!\nm must be an integer') + print('using default parameters: m = ', self._m) + + def xbest(self): + """ See :meth:`Optimiser.xbest()`. """ + return self._xbest + + def ask(self): + """ See :meth:`Optimiser.ask()`. """ + + # print('') + # print('in ask') + # print('') + + if not self._running: + self._proposed = np.asarray(self._x0) + else: + + if self.__j == 0: + # working out an initial stepsize value alpha + alpha_0 = self.__initialising(k=self.__k, + alpha_k0=self.__current_alpha) + print('alpha_initial: ', alpha_0) + # Creating an initial bracketing interval of alpha values + # satisfying the opposite slope condition (see function + # docstring) beginning with initial guess [0, alpha_initial]. + bracket = (self.__initial_bracket(c=alpha_0)) + self._minimum_alpha = bracket[0] + self._maximum_alpha = bracket[1] + self._updated_minimum_alpha = self._minimum_alpha + self._updated_maximum_alpha = self._maximum_alpha + + # Looping while wolfe conditions don't need to be checked + # and the line search hasn't converged. + while (~(self.__1st_wolfe_check_needed) & + ~(self.__2nd_wolfe_check_needed) & ~(self.__converged_ls)): + + # secant squared step of the line search + + # *************************************************************** + # a, b = self.__secant2(self._minimum_alpha, + # self._maximum_alpha) + a = self._updated_minimum_alpha + b = self._updated_maximum_alpha + c = self._proposed_alpha + + # checking if the bracketing interval has converged + self.__converged_ls = self.__very_close(a, b) + self.__logic_steps_left = 'not started' + if self.__converged_ls: + self.__logic_steps_left = ' converged in ls ' + # if converged is True don't do anything more. + + # ************ beginning of secant squared see [1] ************ + + # Preforming a secant to propose a value of alpha. + if (~(self.__1st_wolfe_check_done) & + ~(self.__2nd_wolfe_check_done) & ~(self.__converged_ls)): + + # step S1 in [1] + self._proposed_alpha = self.__secant_for_alpha(a, b) + # passing to tell to check wolfe conditions + self.__1st_wolfe_check_needed = True + + # checking the proposed point is in range + if self._proposed_alpha < a or self._proposed_alpha > b: + # If the point is out of range there is no need to + # check wolfe conditions. + self.__1st_wolfe_check_needed = False + self.__1st_wolfe_check_done = True + + self.__logic_steps_left = 2 + self.__j += 1 / 3 + + elif (self.__1st_wolfe_check_done & + ~(self.__2nd_wolfe_check_done) & ~(self.__converged_ls)): + + # (typically) updating one side of the + # bracketing interval + A, B = self.__update(a, b, c) + # end of step S1 in [1] + + # (typically) updating the otherside side of + # the bracketing interval + if c == B: + # S2 in [1] + # Preforming a secant to propose a value of alpha. + self._proposed_alpha = self.__secant_for_alpha(b, B) + + # checking the proposed point is in range + if self._proposed_alpha < A | self._proposed_alpha > B: + # If the point is out of range there is no need to + # check wolfe conditions. + self.__2nd_wolfe_check_needed = False + self.__2nd_wolfe_check_done = True + else: + self.__2nd_wolfe_check_needed = True + self.__need_update = True + elif c == A: + # S3 in [1] + # Preforming a secant to propose a value of alpha. + self._proposed_alpha = self.__secant_for_alpha(a, A) + + # checking the proposed point is in range + if self._proposed_alpha < A | self._proposed_alpha > B: + # If the point is out of range there is no need to + # check wolfe conditions. + self.__2nd_wolfe_check_needed = False + self.__2nd_wolfe_check_done = True + else: + self.__2nd_wolfe_check_needed = True + self.__need_update = True + else: + # No new point has been proposed therefore there + # is no need to check the wolfe conditions. + self.__2nd_wolfe_check_needed = False + self.__2nd_wolfe_check_done = True + + self._updated_minimum_alpha = A + self._updated_maximum_alpha = B + + self.__logic_steps_left = 1 + self.__j += 1 / 3 + + elif (self.__1st_wolfe_check_done & + self.__2nd_wolfe_check_done & ~(self.__converged_ls)): + + # S4 in [1], this is only preformed if S2 or S3 was done + # and the propsed point was in range. + if self.__need_update: + a, b = self.__update(a, b, c) + self.__need_update = False + + # ***************** end of secant squared ***************** + + # preforming steps L2 from [1] + # determing whether a bisection step should be preformed + # i.e if self.__secant_for_interval() didn't shrink the + # bracketing interval by the propotion self.__gamma + new_width = b - a + old_width = (self._maximum_alpha - self._minimum_alpha) + # print('lower_alpha: ', self._updated_minimum_alpha, + # 'updated_upper_alpha: ', self._updated_maximum_alpha) + # print('_maximum_alpha: ', self._minimum_alpha, + # ' maximum alpha: ', self._maximum_alpha) + if new_width > self.__gamma * old_width: + # preforming bisection + #print('preforming bisection') + c = (a + b) / 2.0 + a, b = self.__update(a=a, b=b, c=c) + #print('bisected_lower: ',a,' bisected upper: ', b) + #print('finished bisection') + + # preforming steps L3 from [1] + # updating bracketing interval + self._minimum_alpha, self._maximum_alpha = a, b + self.__j += 1 / 3 + self.__logic_steps_left = 0 + + # reset logic + self.__1st_wolfe_check_needed = False + self.__1st_wolfe_check_done = False + self.__2nd_wolfe_check_needed = False + self.__2nd_wolfe_check_done = False + + # print('line step loops: ', self.__j, + # ' logic steps left: ', self.__logic_steps_left) + # print('lower_alpha: ', self._updated_minimum_alpha, + # 'updated_upper_alpha: ', self._updated_maximum_alpha) + # print('_maximum_alpha: ', self._minimum_alpha, + # ' maximum alpha: ', self._maximum_alpha) + # print('converged: ', + # self.__very_close(self._updated_minimum_alpha, + # self._updated_maximum_alpha)) + # *********************** CONTINUE LOOP *********************** + + if self.__converged_ls: + self._proposed = (self._current + + self._updated_maximum_alpha * self._px) + else: + self._proposed = (self._current + + self._proposed_alpha * self._px) + + # Running, and ready for tell now + self._ready_for_tell = True + self._running = True + + # print('') + # print('finished ask') + # print('') + # Return proposed points (just the one) in the search space to evaluate + return [self._proposed] + + def tell(self, reply): + """ See :meth:`Optimiser.tell()`. """ + + # Check ask-tell pattern + if not self._ready_for_tell: + raise Exception('ask() not called before tell()') + self._ready_for_tell = False + + # Unpack reply + proposed_f, proposed_dfdx = reply[0] + proposed_f = proposed_f + proposed_dfdx = np.asarray(proposed_dfdx) + + # We need the evaluation of the gradients before we can start the BFGS, + # the first tell gets these. + if self._current_f is None: + + # Move to proposed point + self._current = self._proposed + self._current_f = np.asarray(proposed_f) + self._current_dfdx = np.asarray(proposed_dfdx) + + # Update newton direction + # FIXME: is this right for inital newton direction??? + # if it isn't a desecnt direction the line searches will fail + # i.e return alpha = none + self._px = - np.matmul(self._B, self._current_dfdx) + + # resetting the number of steps in the current + # line search iteration. + self.__j = 0 + + # Checking if exact wolfe conditions. + proposed_grad = np.matmul(np.transpose(proposed_dfdx), self._px) + + wolfe_curvature = (proposed_grad >= + self._c2 * + np.matmul(np.transpose(self._current_dfdx), + self._px)) + + exact_wolfe_suff_dec = (self._c1 * + np.matmul(np.transpose(self._current_dfdx), + self._px) + >= proposed_f - self._current_f) + + exact_wolfe = (exact_wolfe_suff_dec and wolfe_curvature) + + # Checking if approximate wolfe conditions are meet. + approx_wolfe_suff_dec = ((2.0 * self._c1 - 1.0) * + np.matmul(np.transpose(self._current_dfdx), + self._px) >= proposed_grad) + + approx_wolfe_applies = proposed_f <= self._current_f + self.epsilon + + approximate_wolfe = (approx_wolfe_suff_dec and wolfe_curvature + and approx_wolfe_applies) + + # If wolfe conditions meet the line search is stopped + # and the hessian matrix and newton direction are updated by the + # L-BFGS/BFGS approximation of the hessian described in reference [2] + # [3], and [4]. If the line search has converged we also accept the + # steps and update. + if exact_wolfe or approximate_wolfe or self.__converged_ls: + + print('Number of accepted steps: ', self.__k) + print('step sized alpha: ', self._proposed_alpha, ' accepted') + print('updating Hessian and changing newton direction') + + # Updating inverse hessian. + + # identity matrix + I = np.identity(self._n_parameters) + + # We do this if we haven't exhausted existing memory yet, this is + # identical to the BFGS algorithm + if self.__k <= self._m - 1: + k = self.__k + # Defining the next column. + self._S[:, k] = self._proposed - self._current + self._Y[:, k] = proposed_dfdx - self._current_dfdx + + # Defining B_0. Scaling taken from [4]. + self._B = ((np.matmul(np.transpose(self._Y[:, k]), + self._S[:, k]) + / (norm(self._Y[:, k], ord=2) ** 2)) * I) + + # Updating inverse hessian. + for k in range(self.__k + 1): + + V = (I - np.matmul(self._Y[:, k], + np.transpose(self._S[:, k])) + / np.matmul(np.transpose(self._Y[:, k]), + self._S[:, k])) + + self._B = np.matmul(np.transpose(V), np.matmul(self._B, V)) + self._B += (np.matmul(self._S[:, k], + np.transpose(self._S[:, k])) + / np.matmul(np.transpose(self._Y[:, k]), + self._S[:, k])) + + # We have exhausted the limited memory and now enter + # the LM-BFGS algorithm + else: + + m = self._m - 1 + # Shifting everything one column to the left. + self._S[:, 0:m] = self._S[:, 1:self._m] + self._Y[:, 0:m] = self._Y[:, 1:self._m] + + # Renewing last column. + self._S[:, m] = self._proposed - self._current + self._Y[:, m] = proposed_dfdx - self._current_dfdx + + # Defining B_0. Scaling taken from [4]. + self._B = ((np.matmul(np.transpose(self._Y[:, m]), + self._S[:, m]) + / (norm(self._Y[:, m], ord=2) ** 2)) * I) + + # Updating inverse hessian. + for k in range(self._m): + + V = (I - np.matmul(self._Y[:, k], + np.transpose(self._S[:, k])) + / np.matmul(np.transpose(self._Y[:, k]), + self._S[:, k])) + + self._B = np.matmul(np.transpose(V), np.matmul(self._B, V)) + self._B += (np.matmul(self._S[:, k], + np.transpose(self._S[:, k])) + / np.matmul(np.transpose(self._Y[:, k]), + self._S[:, k])) + + self._B = ((np.matmul(np.transpose(self._Y[:, m]), + self._S[:, m]) + / (norm(self._Y[:, m], ord=2)**2)) * I) + + # Move to proposed point + self._current = self._proposed + self._current_f = np.asarray(proposed_f) + self._current_dfdx = np.asarray(proposed_dfdx) + + # storing the accepted value of alpha + self.__current_alpha = self._proposed_alpha + + # if self.__k == 0: + # print('\nThe first accepted value of alpha was: ', + # self.__current_alpha) + # print('set initial alpha to this in subsequent runs' + + # 'to speed up computation') + + # Update newton direction + self._px = - np.matmul(self._B, self._current_dfdx) + + # incrementing the number of accepted steps + self.__k += 1 + + # Resetting the number of steps in the current line search + # iteration as we have accepted a value and completed this + # line search. + self.__j = 0 + + # resetting line search logic + self.__1st_wolfe_check_needed = False + self.__1st_wolfe_check_done = False + self.__2nd_wolfe_check_needed = False + self.__2nd_wolfe_check_done = False + self.__need_update = False + + else: + # wolfe conditions haven't been meet so we continue the line search + if self.__1st_wolfe_check_needed: + self.__1st_wolfe_check_needed = False + self.__1st_wolfe_check_done = True + if self.__2nd_wolfe_check_needed: + self.__2nd_wolfe_check_needed = False + self.__2nd_wolfe_check_done = True + + # Checking if all gradients ~ 0, + # therefore the classical convergence test of a quasi-newton + # or conjugate gradient method has been meet. + if self.__convergence is not True: + + if norm(proposed_dfdx, ord=np.inf) <= 1e-6: + + self.__convergence = True + print('') + print(20 * '*' + ' Convergence after ', + self.__k, ' accepted steps!' + 20 * '*') + print('||df/dx_i||inf <= 1e-6 with parameters:') + print(self._proposed) + print('error function evaluation: ', proposed_f) + print('\nInverse Hessian matrix:\n', self._B) + + # Update xbest and fbest + if self._fbest > proposed_f: + self._fbest = proposed_f + self._xbest = self._current + + print('') + print('Hessian updates: ', self.__k, ' line steps: ', self.__j, + ' propose alpha: ', self._proposed_alpha, ' propose points: ', + self._proposed) + + def __update(self, a, b, c): + ''' + This function is part of the Hager-Zhang line search method [1]. + + Updates the bracketing boundary values of alpha. Ensuring the opposite + slope conditions are obeyed. + + The opposite slope conditions are: + φ(a) ≤ φ(0) + epsilon, + φ'(a) < 0 (w.r.t every parameter), + φ'(b) ≥ 0 (w.r.t every parameter). + where φ(α) = f(x+α*px), f() is the function binging minimised, x is + the current mparameter set, px is the newton direction, and alpha is + the step size. + + In the first condition, epsilon is a small positive constant. The + condition demands that the function at the left end point be not much + bigger than the starting point (i.e. alpha = 0). This is an easy to + satisfy condition because by assumption, we are in a direction where + the function value is decreasing. The second and third conditions + together demand that there is at least one zero of the derivative in + between a and b. In addition to the interval, the update algorithm + requires a third point to be supplied. Usually, this point would lie + within the interval [a, b]. If the point is outside this interval, + the current interval is returned. If the point lies within the + interval, the behaviour of the function and derivative value at this + point is used to squeeze the original interval in a manner that + preserves the opposite slope conditions. + + :param a: lower bound of alpha + :param b: upper bound of alpha + :param c: proposed value of alpha + :param dfdx_c: vector of gradients at the proposed point i.e alpha = c + :param f_c: function evaluation at the proposed point i.e alpha = c + :param f_0: function evaluation at the previous point i.e alpha = 0 + ''' + + # Check c is within the bracket conditions (steps U0 from [1]). + if c < a or c > b: + # if the proposed point is not in range we return + # the old brackets unmodified + return a, b + else: + + # evaluate function for alpha = c i.e at the proposed boundary + # point_c = self._current + c * self._px + # fs_c = self._evaluator.evaluate([point_c]) + f_c, dfdx_c = self.obj_and_grad_func(c) + + # Checking if the opposite slope condition at the upper bound is + # obeyed by the proposed point + # (steps U1 from [1]). + if dfdx_c >= 0.0: + # Updating the upper bound. + return a, c + + # Checking if the opposite slope condition at the lower bound is + # obeyed by the proposed point, if so it is a valid lower bound + # (steps U2 from [1]). + elif dfdx_c < 0.0 and f_c <= self._current_f + self.epsilon: + # Updating the lower bound. + return c, b + + # The proposed point doesn't obey the opposite slope condition + # i.e. dfdx_c < 0.0 and f_c > self._current_f + self.epsilon. + # Checking this is unnecessary as it is the opposite to the above + # conditions. A secant/bisect can narrow down an interval between + # the current lower bound and the trial point c. + else: + b = c + return self.__bisect_or_secant(a, b) + + def __bisect_or_secant(self, a: float, b: float): + ''' + This function is part of the Hager-Zhang line search method [1]. + + Actual implementation of secant (or bisetc if `self.theta` = 0.5) + given a bracketing intervale [a, b] used in `__update()` and + `__initial_bracketing_interval()`. (steps U3a-c from [1]) + + + :param a: lower bound of alpha + :param b: upper bound of alpha + :param c: proposed value of alpha + ''' + + secant = True + + # FIXME: + # problem is in this while loop a and b merge but don't satisfy + # opposite slope rule for upper, + # probably should return when difference between a and b almost + # nothing??? + + # The interval needs updating if the upper bracket has a negative + # slope and the value of the function at that point is too high. + # It is not a valid lower bracket but along with the current + # lower bracket, it encloses another minima. The below function + # is a loop which tries to narrow the interval so that it + # satisfies the opposite slope conditions. + + # (steps U3 from [1]) + + while secant is True: + # according to [1]] this while loop is guaranteed to terminate + # as the intervale between a and b tends to zero + + # Preforming secant (if theta = 0.5 as is default this is a + # bisection) to propose a new point which hopeful obeys the + # opposite slope conditions. + # (steps U3a from [1]) + d = (1.0 - self.theta) * a + self.theta * b + + # Evaluating function for alpha = d i.e at the proposed boundary. + # point_d = self._current + d * self._px + # fs = self._evaluator.evaluate([point_d]) + # f_d, dfdx_d = fs[0] + + f_d, dfdd = self.obj_and_grad_func(d) + + # Checking if the opposite slope condition at the upper bound is + # obeyed by the proposed point. + # If the proposed point has a positive slope, then we have found a + # suitable upper bound to bracket a minima within opposite slopes. + # (still steps U3a from [1]) + converged = self.__very_close(d, b) or self.__very_close(d, a) + if dfdd >= 0.0 or converged: + secant = False + # Updating the upper bound. + return a, d + + # Checking if the opposite slope condition at the lower bound is + # obeyed by the proposed point. + # If the proposed point has a negative slope and the function value + # at that point is small enough, we can use it as a new lower bound + # to narrow down the interval. + # (steps U3b from [1]) + elif dfdd < 0.0 and f_d <= self._current_f + self.epsilon: + # Updating the lower bound. + a = d + + # The proposed point doesn't obey the opposite slope condition + # i.e. dfdx_c < 0.0 and f_c > self._current_f + self.epsilon + # Checking this is unnecessary as it is the opposite to the above + # conditions. We are therefore in the same situation as when we + # started the loop so we update the upper bracket and continue. + # (steps U3c from [1]) + else: + b = d + + def __secant_for_alpha(self, a, b): + ''' + This function is part of the Hager-Zhang line search method [1]. + + Preforms a secant step to propose a value of alpha. This is the same as + the secant routine described in [1]. + + :param a: lower bound of alpha + :param b: upper bound of alpha + ''' + + # Evaluating function for alpha = a to obtain gradients. + f_a, dfda = self.obj_and_grad_func(a) + + # Evaluating function for alpha = b to obtain gradients. + f_b, dfdb = self.obj_and_grad_func(b) + + # Preforming secant. + numerator = a * dfdb - b * dfda + denominator = dfdb - dfda + + return float(numerator / denominator) + + def __initial_bracket(self, c, rho=5): + ''' + This function is part of the Hager-Zhang line search method [1]. + + This function is used to generate an initial interval [a, b] for alpha + satisfying the opposite slope conditions and therefore bracketing + the minimum, beginning with the initial guess [0, c]. + The opposite slope conditions: + φ(a) ≤ φ(0) + epsilon, + φ'(a) < 0 (w.r.t every parameter), + φ'(b) ≥ 0 (w.r.t every parameter). + where φ(α) = f(x+α*px), f() is the function binging minimised, x is + the current parameter set, px is the newton direction, and alpha is + the step size. + + This is the same as the bracket routine described in [1] as steps B0-3 + + :param c: initial guess for maximum value of alpha + :param row: range (1, ∞), expansion factor used in the bracket rule to + increase the upper limit c (c_j+1 = row*c_j) until a + suitable interval is found that contains the minimum. + ''' + + # (steps B0 from [1]) + # Initiating a list of proposed boundary values of alpha. + c = [c] + # Initiating a list of error function evaluations at + # the proposed boundary values of alpha. + f_c = [] + # Initiating lower bound. + a = 0 + + # Initiating an iteration counter for the below while loop. + j = 0 + bracketing = True + + while bracketing: + + # Evaluating function for alpha = c[j] + # i.e at the proposed boundary. + f_cj, dfdc_j = self.obj_and_grad_func(c[j]) + + # Appending the error function evaluations at proposed boundary + # values of alpha. + f_c.append(f_cj) + + # Checking if the opposite slope condition at the upper bound is + # obeyed by the proposed point. If the slope at the propsed point + # is positive, then the given points already bracket a minimum. + # (steps B1 from [1]) + if dfdc_j >= 0.0: + # Setting the upper bound. + b = c[j] + + bracketing = False + + # Checking if the non derivative opposite slope condition at + # the lower bound is obeyed by any of the previously evaluated + # points and returning the value for which the boundary + # conditions are as close together as possible. + for i in range(1, j + 1): + if f_c[j - i] <= self._current_f + self.epsilon: + a = c[j - i] + return a, b + return a, b + + # Checking if the proposed point doesn't obey the opposite slope + # condition. This means the upper bracket limit almost works as a + # new lower limit but the objective function(f_cj) is too large. + # We therefore need to preform a secant/bisect but the minimum is + # not yet bracketed. + # (steps B2 from [1]) + elif dfdc_j < 0.0 and f_cj > self._current_f + self.epsilon: + # The interval needs updating if the upper bracket has a + # negative slope and the value of the function at that point + # is too high. It is not a valid lower bracket but along with + # the current lower bracket, it encloses another minima. The + # below function tries to narrow the interval so that it + # satisfies the opposite slope conditions. + bracketing = False + return self.__bisect_or_secant(0.0, c[j]) + + # The proposed point obeys the opposite slope condition + # at the lower bound + # i.e. dfdx_d < 0.0 and f_d <= self._current_f + self.epsilon. + # Checking this is unnecessary as it is the opposite to + # the above conditions. This means the bracket interval needs + # expanding to ensure a minimum is bracketed. + # (steps B3 from [1]) + else: + # Increasing the proposed point by a factor of row to attempt + # to bracket minimum and trying again. + c.append(rho * c[j]) + + # Increamenting the iteration counter. + j += 1 + + def __initialising(self, k, alpha_k0): + ''' + This function is part of the Hager-Zhang line search method [1]. + + Generate the starting guess of alpha, 'c', used by + ``__initial_bracket()``. This is the same as the routine + called initial and described as I0-2 in [1]. + + :param k: number of accepted steps/newton direction updates that + have taken place. + :param alpha_k0: the alpha value used by the previously accepted + steps/newton direction + update. If k = 0 this is the initial alpha the user wants to be used + ''' + QuadStep = False + + # Small factor used in initial guess of step size, range (0, 1). + # As the initial guess is very crude this is a very small factor + # to keep the initial step short and close the starting parameters + # self.x_0. + ps_0 = self.__ps_0 + + # Small factor used in subsequent guesses of step size + # if Quadstep is true, range (0, 1). + # TODO: this hasn't been implement yet + ps_1 = self.__ps_1 + + # Sacling factor used in subsequent guesses of step size, + # range (1, inf). + ps_2 = self.__ps_2 + + # For the first line search do the following + # (step I0 in [1]) + if k == 0: + + if alpha_k0 is not None: + # returning user specified initial alpha + return alpha_k0 + + # (step I0a in [1]) + elif np.all(self._x0 != 0.0): + # Crude step size estimate + # :math: \alpha = ps_0*||x_0||_\inf / ||dfdx||_\inf + return ((ps_0 * norm(self._x0, ord=np.inf)) + / (norm(self._current_dfdx, ord=np.inf))) + + # If self._x0 = 0.0 the above statement would give alpha = 0, + # hence we use the following estimation. + # (step I0b in [1]) + elif self._current_f != 0: + # Crude step size estimate + # :math: \alpha = ps_0*|f(x_0)| / ||dfdx||^2 + return ((ps_0 * abs(self._current_f)) + / (pow(norm(self._current_dfdx, ord=2), 2.0))) + + # Otherwise self._current_f = 0.0 and we are already at the + # minimum therefore return alpha = 1.0 it should not matter what we + # return in this case as if self._current_f = 0.0 the gradients + # will also equal zero. + # (step I0c in [1]) + else: + return 1.0 + + # TODO: implement the below option using a quadratic interpolant ??? + # everything will work without this + # (step I1 in [1]) + elif QuadStep: + + # point_current_scaled = self._current + ps_1*alpha_k0* self._px + # fs = self._evaluator.evaluate([point_current_scaled]) + f, df = self.obj_and_grad_func(ps_1 * alpha_k0) + + if f <= self._current_f: + pass + #TODO: add quad step option + pass + + # For the subsequent line search do the following + # (step I2 in [1]) + else: + # Increases the step size of the previous accepted step as it + # is only decreased in subsequent boundary manipulation. + return ps_2 * alpha_k0 + + def __very_close(self, x, y): + ''' Returns true if x is very close in value to y. ''' + return np.nextafter(x, y) >= y + + def obj_and_grad_func(self, alpha: float): + ''' + For a given alpha this returns the values of the objective function + and it's derivative. + ''' + point_alpha = self._current + alpha * self._px + fs_alpha = self._evaluator.evaluate([point_alpha]) + f_alpha, dfdx_alpha = fs_alpha[0] + + dfdalpha = np.matmul(np.transpose(dfdx_alpha), self._px) + + return f_alpha, dfdalpha + + # def __secant2(self, a, b): + # ''' + # This function is part of the Hager-Zhang line search method [1]. + + # This function is referred to as secant^2 and described as steps + # S1-4 in [1]. + + # Preforms a secant step to update the bracketing interval of alpha. + # Given an interval that brackets a root, this procedure performs an + # update of both end points using two intermediate points generated + # using the secant interpolation `self.__secant_for_alpha()`. + # Assuming the interval [a, b] satisfy the opposite slope conditions. + + # The opposite slope conditions: + # φ(a) ≤ φ(0) + epsilon , + # φ'(a) < 0 (w.r.t every parameter), + # φ'(b) ≥ 0 (w.r.t every parameter). + # where φ(α) = f(x+α*px), f() is the function binging minimised, + # x is the current parameter set, px is the newton direction, + # and alpha is the step size. + + # :param a: power bound of alpha + # :param b: upper bound of alpha + # ''' + + # # Preforming a secant to propose a value of alpha. + # # (step S1 in [1]) + # c = self.__secant_for_alpha(a,b) + # # CHECK IF c SATISFY THE WOLFE CONDITIONS i.e pass to tell!!!!!!! + # # IF IT HAS STOP SEARCHING THIS DIRECTION!!!! + + # # IF WOLFE CONDITIONS AREAN'T MEET DO THIS thing below + + # # (typically) updating one side of the bracketing interval + # print('first update secant') + # A, B = self.__update(a, b, c) + + # # (typically) updating the otherside side of the bracketing interval + # # (step S2 in [1]) + # if c == A: + # # Preforming a secant to propose a value of alpha. + # C = self.__secant_for_alpha (a, A) + + # # (step S3 in [1]) + # if c == B: + # # Preforming a secant to propose a value of alpha. + # C = self.__secant_for_alpha (b, B) + + # # (step S4 in [1]) + # if c == A or c == B: + # # CHECK IF C SATISFY THE WOLFE CONDITIONS i.e pass to tell!!!!!!! + # # IF IT HAS STOP SEARCHING THIS DIRECTION!!!! + + # # IF WOLFE CONDITIONS AREAN'T MEET DO THIS thing below + + # print('second update secant') + # A, B = self.__update(A, B, C) + + # return A, B + diff --git a/pints/_optimisers/_bfgs_scipy.py b/pints/_optimisers/_bfgs_scipy.py new file mode 100644 index 000000000..21a21c447 --- /dev/null +++ b/pints/_optimisers/_bfgs_scipy.py @@ -0,0 +1,399 @@ +# +# Broyden–Fletcher–Goldfarb–Shanno algorithm +# +# This file is part of PINTS (https://github.com/pints-team/pints/) which is +# released under the BSD 3-clause license. See accompanying LICENSE.md for +# copyright notice and full license details. +# +from __future__ import absolute_import, division +from __future__ import print_function, unicode_literals +import numpy as np +from numpy.linalg import norm +from scipy.optimize.linesearch import line_search_wolfe2 +#from scipy.optimize.linesearch import line_search_wolfe1 +#from scipy.optimize.linesearch import line_search_BFGS +import pints + + +# TODO: move line search to abstract class +class BFGS_scipy(pints.LineSearchBasedOptimiser): + """ + Broyden–Fletcher–Goldfarb–Shanno algorithm [1] + + [1] Liu, D. C.; Nocedal, J. + On the Limited Memory BFGS Method for Large Scale Optimization. + Mathematical Programming 1989, 45 (1), + 503–528. https://doi.org/10.1007/BF01589116. + + [2] Nocedal, J. Updating Quasi-Newton Matrices with Limited Storage. + Math. Comp. 1980, 35 (151), 773–782. + https://doi.org/10.1090/S0025-5718-1980-0572855-7. + + [3] Nash, S. G.; Nocedal, J. A Numerical Study of the Limited Memory + BFGS Method and the Truncated-Newton Method for Large Scale Optimization. + SIAM J. Optim. 1991, 1 (3), 358–372. https://doi.org/10.1137/0801023. + + """ + + def __init__(self, x0, sigma0=None, boundaries=None): + super(BFGS_scipy, self).__init__(x0, sigma0, boundaries) + + # Set optimiser state + self._running = False + self._ready_for_tell = False + + # Best solution found + self._xbest = self._x0 + self._fbest = float('inf') + + # approximate inverse hessian + # initial the identity is used + self._B = np.identity(self._n_parameters) + + # newton direction + self._px = None + + # maximum number of correction matrices stored + self._m = 5 + + # Storage for vectors constructing the correction matrix + # this is the advised way of storing them. + self._S = np.zeros(shape=(self._n_parameters, self._m)) + self._Y = np.zeros(shape=(self._n_parameters, self._m)) + + # number of accepted steps/ newton direction updates + self.__k = 0 + + # Current point, score, and gradient + self._current = self._x0 + self._current_f = None + self._current_dfdx = None + + # Proposed next point (read-only, so can be passed to user) + self._proposed = self._x0 + self._proposed.setflags(write=False) + + self._previous_f = None + + # parameters for wolfe conditions on line search + + # As c1 approaches 0 and c2 approaches 1, the line search + # terminates more quickly. + self._c1 = 1E-4 # Parameter for Armijo condition rule, 0 < c1 < 0.5 + self._c2 = 0.9 # Parameter for curvature condition rule, c1 < c2 < 1.0 + + # boundary values of alpha + self._minimum_alpha = 0.0 + self._maximum_alpha = float("inf") + self._proposed_alpha = 0.001 # same default value as used in stan + + self.__current_alpha = 1.0 + + self.__convergence = False + + def fbest(self): + """ See :meth:`Optimiser.fbest()`. """ + return self._fbest + + def wolfe_line_search_parameters(self): + """ + Returns the wolfe line search parameters this optimiser is using + as a vector ``[c1, c2]``. + As c1 approaches 0 and c2 approaches 1, the line search terminates + more quickly. ``c1`` is the parameter for the Armijo condition + rule, ``0 < c1 < 0.5``. ``c2 `` is the parameter for the + curvature condition rule, ``c1 < c2 < 1.0``. + """ + return (self._c1, self._c2) + + def max_correction_matrice_storage(self): + """ + Returns ``m``, the maximum number of correction matrice for + calculating the inverse hessian used and stored + """ + return self._m + + def name(self): + """ See :meth:`Optimiser.name()`. """ + return 'Broyden–Fletcher–Goldfarb–Shanno (BFGS)' + + def needs_sensitivities(self): + """ See :meth:`Optimiser.needs_sensitivities()`. """ + return True + + def n_hyper_parameters(self): + """ See :meth:`pints.TunableMethod.n_hyper_parameters()`. """ + return 2 + + def running(self): + """ See :meth:`Optimiser.running()`. """ + return self._running + + def set_hyper_parameters(self, x): + """ + See :meth:`pints.TunableMethod.set_hyper_parameters()`. + + The hyper-parameter vector is ``[c1, c2, m]``. + ``c1`` is the parameter for the Armijo condition rule, + ``0 < c1 < 0.5``. + ``c2`` is the parameter for the curvature condition rule, + ``c1 < c2 < 1.0``. + ``m`` is the number of maximum number of correction matrices + that can be stored for the LM-BFGS update. + """ + + self.__set_wolfe_line_search_parameters(x[0], x[1]) + + def __set_wolfe_line_search_parameters(self, c1: float, c2: float): + """ + Sets the parameters for the wolfe conditions. + + Parameters + ---------- + c1: float + Parameter for the Armijo condition rule, ``0 < c1 < 0.5``. + + c2: float + Parameter for the curvature condition rule, ``c1 < c2 < 1.0``. + """ + if(0 < c1 < 0.5 and c1 < c2 < 1.0): + self._c1 = c1 + self._c2 = c2 + else: + cs = self.wolfe_line_search_parameters() + print('Invalid wolfe line search parameters!!!') + print('0 < c1 < 0.5 and c1 < c2 < 1.0') + print('using default parameters: c1 = ', cs[0], ' c2 = ', cs[1]) + + def __set_max_correction_matrice_storage(self, m: int): + """ + Sets the parameters for the wolfe conditions. + + Parameters + ---------- + m: int + The maximum number of correction matrice for calculating the + inverse hessian used and stored. + """ + if(m == int(m)): + self._m = m + self._S = np.zeros(self._n_parameters, m) + self._Y = np.zeros(self._n_parameters, m) + else: + print('Invalid value of m!!!\nm must be an integer') + print('using default parameters: m = ', self._m) + + def xbest(self): + """ See :meth:`Optimiser.xbest()`. """ + return self._xbest + + def __objective_function(self, point_alpha: float): + ''' + For a given alpha this returns the values of the objective function. + ''' + #point_alpha = self._current + alpha * self._px + fs_alpha = self._evaluator.evaluate([point_alpha]) + f_alpha, dfdx_alpha = fs_alpha[0] + + return f_alpha + + def __gradients_function(self, point_alpha: float): + ''' + For a given alpha this returns the values of the objective + functions derivative. + ''' + #point_alpha = self._current + alpha * self._px + fs_alpha = self._evaluator.evaluate([point_alpha]) + f_alpha, dfdx_alpha = fs_alpha[0] + + # dfdalpha = np.matmul(np.transpose(dfdx_alpha), self._px) + + # print('dfdalpha: ', dfdalpha) + # print('type dfdalpha: ', type(dfdalpha[0])) + + return dfdx_alpha + + def ask(self): + """ See :meth:`Optimiser.ask()`. """ + + # print('') + # print('in ask') + # print('') + + if not self._running: + self._proposed = np.asarray(self._x0) + else: + # line search using an algorithm from scipy to meet wolfe condtions + results = line_search_wolfe2(f=self.__objective_function, + myfprime=self.__gradients_function, + xk=self._current, pk=self._px, + gfk=self._current_dfdx, + old_fval=self._current_f, + old_old_fval=self._previous_f, + c1=self._c1, c2=self._c2, maxiter=50) + + # line_search_BFGS below only checks the Armijo rule, + # therefore it doesn't ensure the gradient is decreasing, + # this can cause problems with the BFGS algorithm as + # the Hessians from this approach may not be positive definite. + + # results = line_search_BFGS(f=self.__objective_function, + # xk=self._current, + # pk=self._px, gfk=self._current_dfdx, + # old_fval=self._current_f, c1=self._c1, + # alpha0=self.__current_alpha) + + # results = line_search_wolfe1(f=self.__objective_function, + # fprime=self.__gradients_function, + # xk=self._current, pk=self._px, + # gfk=self._current_dfdx, + # old_fval=self._current_f, + # old_old_fval=self._previous_f, + # c1=self._c1, c2=self._c2) + + self._proposed_alpha = results[0] + # print('alpha: ', self._proposed_alpha) + self._proposed = self._current + self._proposed_alpha * self._px + + # Running, and ready for tell now + self._ready_for_tell = True + self._running = True + return [self._proposed] + + def tell(self, reply): + """ See :meth:`Optimiser.tell()`. """ + + # Check ask-tell pattern + if not self._ready_for_tell: + raise Exception('ask() not called before tell()') + self._ready_for_tell = False + + # Unpack reply + proposed_f, proposed_dfdx = reply[0] + proposed_f = proposed_f + proposed_dfdx = np.asarray(proposed_dfdx) + + # We need the evaluation of the gradients before we can start the BFGS, + # the first tell gets these. + if self._current_f is None: + + # Move to proposed point + self._current = self._proposed + self._current_f = np.asarray(proposed_f) + self._current_dfdx = np.asarray(proposed_dfdx) + + # Update newton direction + # FIXME: is this right for inital newton direction??? + # if it isn't a desecnt direction the line searches will fail + # i.e return alpha = none + self._px = - np.matmul(self._B, self._current_dfdx) + + # If wolfe conditions meet the line search is stopped + # and the hessian matrix and newton direction are updated by the + # L-BFGS/BFGS approximation of the hessian described in reference [1] + # [2], and [3]. If the line search has converged we also accept the + # steps and update. + else: + + # identity matrix + I = np.identity(self._n_parameters) + + # We do this if we haven't exhausted existing memory yet, this is + # identical to the BFGS algorithm + if self.__k <= self._m - 1: + k = self.__k + # Defining the next column. + self._S[:, k] = self._proposed - self._current + self._Y[:, k] = proposed_dfdx - self._current_dfdx + + # Defining B_0. Scaling taken from [3]. + self._B = ((np.matmul(np.transpose(self._Y[:, k]), + self._S[:, k]) + / (norm(self._Y[:, k], ord=2) ** 2)) * I) + + # Updating inverse hessian. + for k in range(self.__k + 1): + + V = (I - np.matmul(self._Y[:, k], + np.transpose(self._S[:, k])) + / np.matmul(np.transpose(self._Y[:, k]), + self._S[:, k])) + + self._B = np.matmul(np.transpose(V), np.matmul(self._B, V)) + self._B += (np.matmul(self._S[:, k], + np.transpose(self._S[:, k])) + / np.matmul(np.transpose(self._Y[:, k]), + self._S[:, k])) + + # We have exhausted the limited memory and now enter + # the LM-BFGS algorithm + else: + + m = self._m - 1 + # Shifting everything one column to the left. + self._S[:, 0:m] = self._S[:, 1:self._m] + self._Y[:, 0:m] = self._Y[:, 1:self._m] + + # Renewing last column. + self._S[:, m] = self._proposed - self._current + self._Y[:, m] = proposed_dfdx - self._current_dfdx + + # Defining B_0. Scaling taken from [3]. + self._B = ((np.matmul(np.transpose(self._Y[:, m]), + self._S[:, m]) + / (norm(self._Y[:, m], ord=2) ** 2)) * I) + + # Updating inverse hessian. + for k in range(self._m): + + V = (I - np.matmul(self._Y[:, k], + np.transpose(self._S[:, k])) + / np.matmul(np.transpose(self._Y[:, k]), + self._S[:, k])) + + self._B = np.matmul(np.transpose(V), np.matmul(self._B, V)) + self._B += (np.matmul(self._S[:, k], + np.transpose(self._S[:, k])) + / np.matmul(np.transpose(self._Y[:, k]), + self._S[:, k])) + + self._B = ((np.matmul(np.transpose(self._Y[:, m]), + self._S[:, m]) + / (norm(self._Y[:, m], ord=2)**2)) * I) + + # Move to proposed point + self._previous_f = self._current_f + self._current = self._proposed + self._current_f = np.asarray(proposed_f) + self._current_dfdx = np.asarray(proposed_dfdx) + + # storing the accepted value of alpha + self.__current_alpha = self._proposed_alpha + + # Update newton direction + self._px = - np.matmul(self._B, self._current_dfdx) + + # incrementing the number of accepted steps + self.__k += 1 + + # Checking if all gradients ~ 0, + # therefore the classical convergence test of a quasi-newton + # or conjugate gradient method has been meet. + if self.__convergence is not True: + + if norm(proposed_dfdx, ord=np.inf) <= 1e-6: + + self.__convergence = True + print('') + print(20 * '*' + ' Convergence after ', + self.__k, ' accepted steps!' + 20 * '*') + print('||df/dx_i||inf <= 1e-6 with parameters:') + print(self._proposed) + print('error function evaluation: ', proposed_f) + print('\nInverse Hessian matrix:\n', self._B) + + # Update xbest and fbest + if self._fbest > proposed_f: + self._fbest = proposed_f + self._xbest = self._current + From a350cb6a6f46d61b0d88b3a34f69bca61017fa8a Mon Sep 17 00:00:00 2001 From: alisterde Date: Sun, 24 May 2020 18:49:15 +0100 Subject: [PATCH 03/16] Removing none ASCII characters --- pints/_optimisers/_bfgs_linesearch.py | 2 +- pints/_optimisers/_bfgs_scipy.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pints/_optimisers/_bfgs_linesearch.py b/pints/_optimisers/_bfgs_linesearch.py index 507dae302..1dbf70bf4 100644 --- a/pints/_optimisers/_bfgs_linesearch.py +++ b/pints/_optimisers/_bfgs_linesearch.py @@ -1,5 +1,5 @@ # -# Broyden–Fletcher–Goldfarb–Shanno algorithm +# Broyden-Fletcher-Goldfarb-Shanno algorithm # # This file is part of PINTS (https://github.com/pints-team/pints/) which is # released under the BSD 3-clause license. See accompanying LICENSE.md for diff --git a/pints/_optimisers/_bfgs_scipy.py b/pints/_optimisers/_bfgs_scipy.py index 21a21c447..4b39f6d74 100644 --- a/pints/_optimisers/_bfgs_scipy.py +++ b/pints/_optimisers/_bfgs_scipy.py @@ -1,5 +1,5 @@ # -# Broyden–Fletcher–Goldfarb–Shanno algorithm +# Broyden-Fletcher-Goldfarb-Shanno algorithm # # This file is part of PINTS (https://github.com/pints-team/pints/) which is # released under the BSD 3-clause license. See accompanying LICENSE.md for From 6d71df4f45224938ec2cdb7930c4b33544f2089c Mon Sep 17 00:00:00 2001 From: alisterde Date: Sun, 24 May 2020 18:50:39 +0100 Subject: [PATCH 04/16] :fire: --- examples/optimisation/bfgs_trial.ipynb | 827 ------------------------- 1 file changed, 827 deletions(-) delete mode 100644 examples/optimisation/bfgs_trial.ipynb diff --git a/examples/optimisation/bfgs_trial.ipynb b/examples/optimisation/bfgs_trial.ipynb deleted file mode 100644 index 10394190c..000000000 --- a/examples/optimisation/bfgs_trial.ipynb +++ /dev/null @@ -1,827 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "os.chdir(\"../..\")\n", - "import pints\n", - "import pints.toy" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using the attempted Hager-Zhang linesearch implimentation" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "tags": [ - "outputPrepend" - ] - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": "ps: 59.33333333333348 propose alpha: 5.447714322654104e-10 propose points: [ 4.50854165 450.00004005]\n60 61 1.9e+07 0:00.2\n\nHessian updates: 0 line steps: 60.333333333333485 propose alpha: 5.447684887856562e-10 propose points: [ 4.50851735 450.00004005]\n\nHessian updates: 0 line steps: 61.33333333333349 propose alpha: 5.447655453038777e-10 propose points: [ 4.50849304 450.00004005]\n\nHessian updates: 0 line steps: 62.3333333333335 propose alpha: 5.447626018200718e-10 propose points: [ 4.50846873 450.00004005]\n\nHessian updates: 0 line steps: 63.333333333333506 propose alpha: 5.447596583342355e-10 propose points: [ 4.50844443 450.00004004]\n\nHessian updates: 0 line steps: 64.3333333333335 propose alpha: 5.447567148463658e-10 propose points: [ 4.50842012 450.00004004]\n\nHessian updates: 0 line steps: 65.33333333333348 propose alpha: 5.447537713564598e-10 propose points: [ 4.50839582 450.00004004]\n\nHessian updates: 0 line steps: 66.33333333333347 propose alpha: 5.447508278645143e-10 propose points: [ 4.50837151 450.00004004]\n\nHessian updates: 0 line steps: 67.33333333333346 propose alpha: 5.447478843705264e-10 propose points: [ 4.5083472 450.00004004]\n\nHessian updates: 0 line steps: 68.33333333333344 propose alpha: 5.447449408744931e-10 propose points: [ 4.5083229 450.00004004]\n\nHessian updates: 0 line steps: 69.33333333333343 propose alpha: 5.447419973764114e-10 propose points: [ 4.50829859 450.00004004]\n\nHessian updates: 0 line steps: 70.33333333333341 propose alpha: 5.447390538762783e-10 propose points: [ 4.50827428 450.00004004]\n\nHessian updates: 0 line steps: 71.3333333333334 propose alpha: 5.447361103740907e-10 propose points: [ 4.50824998 450.00004004]\n\nHessian updates: 0 line steps: 72.33333333333339 propose alpha: 5.447331668698458e-10 propose points: [ 4.50822567 450.00004004]\n\nHessian updates: 0 line steps: 73.33333333333337 propose alpha: 5.447302233635402e-10 propose points: [ 4.50820136 450.00004004]\n\nHessian updates: 0 line steps: 74.33333333333336 propose alpha: 5.447272798551713e-10 propose points: [ 4.50817706 450.00004004]\n\nHessian updates: 0 line steps: 75.33333333333334 propose alpha: 5.447243363447359e-10 propose points: [ 4.50815275 450.00004004]\n\nHessian updates: 0 line steps: 76.33333333333333 propose alpha: 5.447213928322309e-10 propose points: [ 4.50812844 450.00004004]\n\nHessian updates: 0 line steps: 77.33333333333331 propose alpha: 5.447184493176534e-10 propose points: [ 4.50810414 450.00004004]\n\nHessian updates: 0 line steps: 78.3333333333333 propose alpha: 5.447155058010004e-10 propose points: [ 4.50807983 450.00004004]\n\nHessian updates: 0 line steps: 79.33333333333329 propose alpha: 5.44712562282269e-10 propose points: [ 4.50805552 450.00004004]\n80 81 1.9e+07 0:00.2\n\nHessian updates: 0 line steps: 80.33333333333327 propose alpha: 5.447096187614559e-10 propose points: [ 4.50803122 450.00004004]\n\nHessian updates: 0 line steps: 81.33333333333326 propose alpha: 5.447066752385582e-10 propose points: [ 4.50800691 450.00004004]\n\nHessian updates: 0 line steps: 82.33333333333324 propose alpha: 5.44703731713573e-10 propose points: [ 4.5079826 450.00004004]\n\nHessian updates: 0 line steps: 83.33333333333323 propose alpha: 5.447007881864974e-10 propose points: [ 4.5079583 450.00004004]\n\nHessian updates: 0 line steps: 84.33333333333321 propose alpha: 5.44697844657328e-10 propose points: [ 4.50793399 450.00004004]\n\nHessian updates: 0 line steps: 85.3333333333332 propose alpha: 5.446949011260622e-10 propose points: [ 4.50790968 450.00004004]\n\nHessian updates: 0 line steps: 86.33333333333319 propose alpha: 5.446919575926966e-10 propose points: [ 4.50788538 450.00004004]\n\nHessian updates: 0 line steps: 87.33333333333317 propose alpha: 5.446890140572285e-10 propose points: [ 4.50786107 450.00004004]\n\nHessian updates: 0 line steps: 88.33333333333316 propose alpha: 5.446860705196548e-10 propose points: [ 4.50783676 450.00004004]\n\nHessian updates: 0 line steps: 89.33333333333314 propose alpha: 5.446831269799725e-10 propose points: [ 4.50781246 450.00004004]\n\nHessian updates: 0 line steps: 90.33333333333313 propose alpha: 5.446801834381785e-10 propose points: [ 4.50778815 450.00004004]\n\nHessian updates: 0 line steps: 91.33333333333312 propose alpha: 5.446772398942698e-10 propose points: [ 4.50776384 450.00004004]\n\nHessian updates: 0 line steps: 92.3333333333331 propose alpha: 5.446742963482434e-10 propose points: [ 4.50773954 450.00004004]\n\nHessian updates: 0 line steps: 93.33333333333309 propose alpha: 5.446713528000964e-10 propose points: [ 4.50771523 450.00004004]\n\nHessian updates: 0 line steps: 94.33333333333307 propose alpha: 5.446684092498257e-10 propose points: [ 4.50769092 450.00004004]\n\nHessian updates: 0 line steps: 95.33333333333306 propose alpha: 5.446654656974282e-10 propose points: [ 4.50766662 450.00004004]\n\nHessian updates: 0 line steps: 96.33333333333304 propose alpha: 5.44662522142901e-10 propose points: [ 4.50764231 450.00004004]\n\nHessian updates: 0 line steps: 97.33333333333303 propose alpha: 5.446595785862412e-10 propose points: [ 4.507618 450.00004004]\n\nHessian updates: 0 line steps: 98.33333333333302 propose alpha: 5.446566350274456e-10 propose points: [ 4.5075937 450.00004004]\n\nHessian updates: 0 line steps: 99.333333333333 propose alpha: 5.446536914665113e-10 propose points: [ 4.50756939 450.00004004]\n100 101 1.9e+07 0:00.2\n\nHessian updates: 0 line steps: 100.33333333333299 propose alpha: 5.446507479034352e-10 propose points: [ 4.50754508 450.00004004]\n\nHessian updates: 0 line steps: 101.33333333333297 propose alpha: 5.446478043382142e-10 propose points: [ 4.50752077 450.00004004]\n\nHessian updates: 0 line steps: 102.33333333333296 propose alpha: 5.446448607708455e-10 propose points: [ 4.50749647 450.00004004]\n\nHessian updates: 0 line steps: 103.33333333333294 propose alpha: 5.44641917201326e-10 propose points: [ 4.50747216 450.00004004]\n\nHessian updates: 0 line steps: 104.33333333333293 propose alpha: 5.446389736296527e-10 propose points: [ 4.50744785 450.00004004]\n\nHessian updates: 0 line steps: 105.33333333333292 propose alpha: 5.446360300558226e-10 propose points: [ 4.50742355 450.00004004]\n\nHessian updates: 0 line steps: 106.3333333333329 propose alpha: 5.446330864798326e-10 propose points: [ 4.50739924 450.00004004]\n\nHessian updates: 0 line steps: 107.33333333333289 propose alpha: 5.446301429016798e-10 propose points: [ 4.50737493 450.00004004]\n\nHessian updates: 0 line steps: 108.33333333333287 propose alpha: 5.44627199321361e-10 propose points: [ 4.50735063 450.00004004]\n\nHessian updates: 0 line steps: 109.33333333333286 propose alpha: 5.446242557388735e-10 propose points: [ 4.50732632 450.00004003]\n\nHessian updates: 0 line steps: 110.33333333333285 propose alpha: 5.44621312154214e-10 propose points: [ 4.50730201 450.00004003]\n\nHessian updates: 0 line steps: 111.33333333333283 propose alpha: 5.446183685673796e-10 propose points: [ 4.5072777 450.00004003]\n\nHessian updates: 0 line steps: 112.33333333333282 propose alpha: 5.446154249783673e-10 propose points: [ 4.5072534 450.00004003]\n\nHessian updates: 0 line steps: 113.3333333333328 propose alpha: 5.446124813871741e-10 propose points: [ 4.50722909 450.00004003]\n\nHessian updates: 0 line steps: 114.33333333333279 propose alpha: 5.446095377937969e-10 propose points: [ 4.50720478 450.00004003]\n\nHessian updates: 0 line steps: 115.33333333333277 propose alpha: 5.446065941982327e-10 propose points: [ 4.50718047 450.00004003]\n\nHessian updates: 0 line steps: 116.33333333333276 propose alpha: 5.446036506004786e-10 propose points: [ 4.50715617 450.00004003]\n\nHessian updates: 0 line steps: 117.33333333333275 propose alpha: 5.446007070005316e-10 propose points: [ 4.50713186 450.00004003]\n\nHessian updates: 0 line steps: 118.33333333333273 propose alpha: 5.445977633983885e-10 propose points: [ 4.50710755 450.00004003]\n\nHessian updates: 0 line steps: 119.33333333333272 propose alpha: 5.445948197940464e-10 propose points: [ 4.50708325 450.00004003]\n120 121 1.9e+07 0:00.3\n\nHessian updates: 0 line steps: 120.3333333333327 propose alpha: 5.445918761875024e-10 propose points: [ 4.50705894 450.00004003]\n\nHessian updates: 0 line steps: 121.33333333333269 propose alpha: 5.445889325787533e-10 propose points: [ 4.50703463 450.00004003]\n\nHessian updates: 0 line steps: 122.33333333333267 propose alpha: 5.445859889677962e-10 propose points: [ 4.50701032 450.00004003]\n\nHessian updates: 0 line steps: 123.33333333333266 propose alpha: 5.44583045354628e-10 propose points: [ 4.50698602 450.00004003]\n\nHessian updates: 0 line steps: 124.33333333333265 propose alpha: 5.445801017392457e-10 propose points: [ 4.50696171 450.00004003]\n\nHessian updates: 0 line steps: 125.33333333333263 propose alpha: 5.445771581216464e-10 propose points: [ 4.5069374 450.00004003]\n\nHessian updates: 0 line steps: 126.33333333333262 propose alpha: 5.445742145018268e-10 propose points: [ 4.50691309 450.00004003]\n\nHessian updates: 0 line steps: 127.3333333333326 propose alpha: 5.445712708797842e-10 propose points: [ 4.50688879 450.00004003]\n\nHessian updates: 0 line steps: 128.3333333333326 propose alpha: 5.445683272555156e-10 propose points: [ 4.50686448 450.00004003]\n\nHessian updates: 0 line steps: 129.33333333333263 propose alpha: 5.445653836290176e-10 propose points: [ 4.50684017 450.00004003]\n\nHessian updates: 0 line steps: 130.33333333333266 propose alpha: 5.445624400002877e-10 propose points: [ 4.50681586 450.00004003]\n\nHessian updates: 0 line steps: 131.3333333333327 propose alpha: 5.445594963693226e-10 propose points: [ 4.50679156 450.00004003]\n\nHessian updates: 0 line steps: 132.33333333333272 propose alpha: 5.445565527361194e-10 propose points: [ 4.50676725 450.00004003]\n\nHessian updates: 0 line steps: 133.33333333333275 propose alpha: 5.445536091006749e-10 propose points: [ 4.50674294 450.00004003]\n\nHessian updates: 0 line steps: 134.33333333333277 propose alpha: 5.445506654629862e-10 propose points: [ 4.50671863 450.00004003]\n\nHessian updates: 0 line steps: 135.3333333333328 propose alpha: 5.445477218230503e-10 propose points: [ 4.50669433 450.00004003]\n\nHessian updates: 0 line steps: 136.33333333333283 propose alpha: 5.44544778180864e-10 propose points: [ 4.50667002 450.00004003]\n\nHessian updates: 0 line steps: 137.33333333333286 propose alpha: 5.445418345364246e-10 propose points: [ 4.50664571 450.00004003]\n\nHessian updates: 0 line steps: 138.3333333333329 propose alpha: 5.445388908897289e-10 propose points: [ 4.5066214 450.00004003]\n\nHessian updates: 0 line steps: 139.33333333333292 propose alpha: 5.44535947240774e-10 propose points: [ 4.5065971 450.00004003]\n140 141 1.9e+07 0:00.3\n\nHessian updates: 0 line steps: 140.33333333333294 propose alpha: 5.445330035895567e-10 propose points: [ 4.50657279 450.00004003]\n\nHessian updates: 0 line steps: 141.33333333333297 propose alpha: 5.445300599360741e-10 propose points: [ 4.50654848 450.00004003]\n\nHessian updates: 0 line steps: 142.333333333333 propose alpha: 5.445271162803231e-10 propose points: [ 4.50652417 450.00004003]\n\nHessian updates: 0 line steps: 143.33333333333303 propose alpha: 5.445241726223008e-10 propose points: [ 4.50649986 450.00004003]\n\nHessian updates: 0 line steps: 144.33333333333306 propose alpha: 5.445212289620042e-10 propose points: [ 4.50647556 450.00004003]\n\nHessian updates: 0 line steps: 145.3333333333331 propose alpha: 5.445182852994302e-10 propose points: [ 4.50645125 450.00004003]\n\nHessian updates: 0 line steps: 146.33333333333312 propose alpha: 5.445153416345758e-10 propose points: [ 4.50642694 450.00004003]\n\nHessian updates: 0 line steps: 147.33333333333314 propose alpha: 5.445123979674381e-10 propose points: [ 4.50640263 450.00004003]\n\nHessian updates: 0 line steps: 148.33333333333317 propose alpha: 5.44509454298014e-10 propose points: [ 4.50637833 450.00004003]\n\nHessian updates: 0 line steps: 149.3333333333332 propose alpha: 5.445065106263004e-10 propose points: [ 4.50635402 450.00004003]\n\nHessian updates: 0 line steps: 150.33333333333323 propose alpha: 5.445035669522944e-10 propose points: [ 4.50632971 450.00004003]\n\nHessian updates: 0 line steps: 151.33333333333326 propose alpha: 5.445006232759929e-10 propose points: [ 4.5063054 450.00004003]\n\nHessian updates: 0 line steps: 152.3333333333333 propose alpha: 5.444976795973929e-10 propose points: [ 4.50628109 450.00004003]\n\nHessian updates: 0 line steps: 153.33333333333331 propose alpha: 5.444947359164915e-10 propose points: [ 4.50625679 450.00004003]\n\nHessian updates: 0 line steps: 154.33333333333334 propose alpha: 5.444917922332856e-10 propose points: [ 4.50623248 450.00004003]\n\nHessian updates: 0 line steps: 155.33333333333337 propose alpha: 5.444888485477721e-10 propose points: [ 4.50620817 450.00004002]\n\nHessian updates: 0 line steps: 156.3333333333334 propose alpha: 5.444859048599481e-10 propose points: [ 4.50618386 450.00004002]\n\nHessian updates: 0 line steps: 157.33333333333343 propose alpha: 5.444829611698105e-10 propose points: [ 4.50615955 450.00004002]\n\nHessian updates: 0 line steps: 158.33333333333346 propose alpha: 5.444800174773563e-10 propose points: [ 4.50613525 450.00004002]\n\nHessian updates: 0 line steps: 159.33333333333348 propose alpha: 5.444770737825826e-10 propose points: [ 4.50611094 450.00004002]\n160 161 1.9e+07 0:00.3\n\nHessian updates: 0 line steps: 160.3333333333335 propose alpha: 5.444741300854863e-10 propose points: [ 4.50608663 450.00004002]\n\nHessian updates: 0 line steps: 161.33333333333354 propose alpha: 5.444711863860644e-10 propose points: [ 4.50606232 450.00004002]\n\nHessian updates: 0 line steps: 162.33333333333357 propose alpha: 5.444682426843138e-10 propose points: [ 4.50603801 450.00004002]\n\nHessian updates: 0 line steps: 163.3333333333336 propose alpha: 5.444652989802316e-10 propose points: [ 4.50601371 450.00004002]\n\nHessian updates: 0 line steps: 164.33333333333363 propose alpha: 5.444623552738148e-10 propose points: [ 4.5059894 450.00004002]\n\nHessian updates: 0 line steps: 165.33333333333366 propose alpha: 5.444594115650603e-10 propose points: [ 4.50596509 450.00004002]\n\nHessian updates: 0 line steps: 166.33333333333368 propose alpha: 5.444564678539651e-10 propose points: [ 4.50594078 450.00004002]\n\nHessian updates: 0 line steps: 167.3333333333337 propose alpha: 5.444535241405262e-10 propose points: [ 4.50591647 450.00004002]\n\nHessian updates: 0 line steps: 168.33333333333374 propose alpha: 5.444505804247405e-10 propose points: [ 4.50589216 450.00004002]\n\nHessian updates: 0 line steps: 169.33333333333377 propose alpha: 5.444476367066052e-10 propose points: [ 4.50586786 450.00004002]\n\nHessian updates: 0 line steps: 170.3333333333338 propose alpha: 5.444446929861171e-10 propose points: [ 4.50584355 450.00004002]\n\nHessian updates: 0 line steps: 171.33333333333383 propose alpha: 5.444417492632732e-10 propose points: [ 4.50581924 450.00004002]\n\nHessian updates: 0 line steps: 172.33333333333385 propose alpha: 5.444388055380704e-10 propose points: [ 4.50579493 450.00004002]\n\nHessian updates: 0 line steps: 173.33333333333388 propose alpha: 5.444358618105059e-10 propose points: [ 4.50577062 450.00004002]\n\nHessian updates: 0 line steps: 174.3333333333339 propose alpha: 5.444329180805766e-10 propose points: [ 4.50574632 450.00004002]\n\nHessian updates: 0 line steps: 175.33333333333394 propose alpha: 5.444299743482794e-10 propose points: [ 4.50572201 450.00004002]\n\nHessian updates: 0 line steps: 176.33333333333397 propose alpha: 5.444270306136114e-10 propose points: [ 4.5056977 450.00004002]\n\nHessian updates: 0 line steps: 177.333333333334 propose alpha: 5.444240868765695e-10 propose points: [ 4.50567339 450.00004002]\n\nHessian updates: 0 line steps: 178.33333333333402 propose alpha: 5.444211431371507e-10 propose points: [ 4.50564908 450.00004002]\n\nHessian updates: 0 line steps: 179.33333333333405 propose alpha: 5.444181993953519e-10 propose points: [ 4.50562477 450.00004002]\n180 181 1.9e+07 0:00.4\n\nHessian updates: 0 line steps: 180.33333333333408 propose alpha: 5.444152556511703e-10 propose points: [ 4.50560046 450.00004002]\n\nHessian updates: 0 line steps: 181.3333333333341 propose alpha: 5.444123119046026e-10 propose points: [ 4.50557616 450.00004002]\n\nHessian updates: 0 line steps: 182.33333333333414 propose alpha: 5.44409368155646e-10 propose points: [ 4.50555185 450.00004002]\n\nHessian updates: 0 line steps: 183.33333333333417 propose alpha: 5.444064244042975e-10 propose points: [ 4.50552754 450.00004002]\n\nHessian updates: 0 line steps: 184.3333333333342 propose alpha: 5.44403480650554e-10 propose points: [ 4.50550323 450.00004002]\n\nHessian updates: 0 line steps: 185.33333333333422 propose alpha: 5.444005368944125e-10 propose points: [ 4.50547892 450.00004002]\n\nHessian updates: 0 line steps: 186.33333333333425 propose alpha: 5.443975931358698e-10 propose points: [ 4.50545461 450.00004002]\n\nHessian updates: 0 line steps: 187.33333333333428 propose alpha: 5.443946493749232e-10 propose points: [ 4.5054303 450.00004002]\n\nHessian updates: 0 line steps: 188.3333333333343 propose alpha: 5.443917056115695e-10 propose points: [ 4.505406 450.00004002]\n\nHessian updates: 0 line steps: 189.33333333333434 propose alpha: 5.443887618458057e-10 propose points: [ 4.50538169 450.00004002]\n\nHessian updates: 0 line steps: 190.33333333333437 propose alpha: 5.443858180776289e-10 propose points: [ 4.50535738 450.00004002]\n\nHessian updates: 0 line steps: 191.3333333333344 propose alpha: 5.443828743070359e-10 propose points: [ 4.50533307 450.00004002]\n\nHessian updates: 0 line steps: 192.33333333333442 propose alpha: 5.443799305340237e-10 propose points: [ 4.50530876 450.00004002]\n\nHessian updates: 0 line steps: 193.33333333333445 propose alpha: 5.443769867585894e-10 propose points: [ 4.50528445 450.00004002]\n\nHessian updates: 0 line steps: 194.33333333333448 propose alpha: 5.4437404298073e-10 propose points: [ 4.50526014 450.00004002]\n\nHessian updates: 0 line steps: 195.3333333333345 propose alpha: 5.443710992004422e-10 propose points: [ 4.50523584 450.00004002]\n\nHessian updates: 0 line steps: 196.33333333333454 propose alpha: 5.443681554177234e-10 propose points: [ 4.50521153 450.00004002]\n\nHessian updates: 0 line steps: 197.33333333333456 propose alpha: 5.443652116325703e-10 propose points: [ 4.50518722 450.00004002]\n\nHessian updates: 0 line steps: 198.3333333333346 propose alpha: 5.4436226784498e-10 propose points: [ 4.50516291 450.00004002]\n\nHessian updates: 0 line steps: 199.33333333333462 propose alpha: 5.443593240549494e-10 propose points: [ 4.5051386 450.00004002]\n200 201 1.9e+07 0:00.4\n201 201 1.9e+07 0:00.4\nHalting: No significant change for 200 iterations.\nScore at true solution: \n106262.41486040733\nFound solution: True parameters:\n 1.00000000000000002e-02 1.49999999999999994e-02\n 4.50000000000000000e+02 5.00000000000000000e+02\n" - } - ], - "source": [ - "# Load a forward model\n", - "model = pints.toy.LogisticModel()\n", - "\n", - "# Create some toy data\n", - "real_parameters = [0.015, 500]\n", - "times = np.linspace(0, 1000, 1000)\n", - "values = model.simulate(real_parameters, times)\n", - "\n", - "# Add noise\n", - "values += np.random.normal(0, 10, values.shape)\n", - "\n", - "# Create an object with links to the model and time series\n", - "problem = pints.SingleOutputProblem(model, times, values)\n", - "\n", - "# Select a score function\n", - "score = pints.SumOfSquaresError(problem)\n", - "\n", - "# Perform an optimization\n", - "x0 = [0.01, 450]\n", - "opt = pints.OptimisationController(\n", - " score,\n", - " x0,\n", - " method=pints.BFGS\n", - ")\n", - "opt.set_max_unchanged_iterations(200)\n", - "opt.set_max_iterations(1000)\n", - "found_parameters, found_value = opt.run()\n", - "\n", - "# Show score of true solution\n", - "print('Score at true solution: ')\n", - "print(score(real_parameters))\n", - "\n", - "# Compare parameters with original\n", - "print('Found solution: True parameters:' )\n", - "for k, x in enumerate(found_parameters):\n", - " print(pints.strfloat(x) + ' ' + pints.strfloat(real_parameters[k]))" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "tags": [ - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend", - "outputPrepend" - ] - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": "ropose alpha: 0.9853004251050581 propose points: [ 0.05805812 -0.70685582 2.31663456]\nalpha_initial: 1.9706008502101162\nNumber of accepted steps: 928\nstep sized alpha: 41.19133925117548 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 929 line steps: 0 propose alpha: 41.19133925117548 propose points: [ 0.05804951 -0.70683788 2.31627516]\nalpha_initial: 82.38267850235096\nNumber of accepted steps: 929\nstep sized alpha: 0.9837835985315283 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 930 line steps: 0 propose alpha: 0.9837835985315283 propose points: [ 0.05807305 -0.70688811 2.31627209]\nalpha_initial: 1.9675671970630566\nNumber of accepted steps: 930\nstep sized alpha: 42.1802912347368 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 931 line steps: 0 propose alpha: 42.1802912347368 propose points: [ 0.05806474 -0.70687113 2.31591937]\nalpha_initial: 84.3605824694736\nNumber of accepted steps: 931\nstep sized alpha: 0.9822161226427419 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 932 line steps: 0 propose alpha: 0.9822161226427419 propose points: [ 0.05808755 -0.7069198 2.31591649]\nalpha_initial: 1.9644322452854839\nNumber of accepted steps: 932\nstep sized alpha: 43.19236875916908 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 933 line steps: 0 propose alpha: 43.19236875916908 propose points: [ 0.05807992 -0.70690358 2.31557068]\nalpha_initial: 86.38473751833816\nNumber of accepted steps: 933\nstep sized alpha: 0.9806273291958192 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 934 line steps: 0 propose alpha: 0.9806273291958192 propose points: [ 0.05810199 -0.70695071 2.31556798]\nalpha_initial: 1.9612546583916384\nNumber of accepted steps: 934\nstep sized alpha: 44.280002900249485 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 935 line steps: 0 propose alpha: 44.280002900249485 propose points: [ 0.05809443 -0.70693552 2.3152289 ]\nalpha_initial: 88.56000580049897\nNumber of accepted steps: 935\nstep sized alpha: 0.9789710383899488 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 936 line steps: 0 propose alpha: 0.9789710383899488 propose points: [ 0.0581158 -0.70698111 2.31522639]\n940 941 36718.74 3:40.2\nalpha_initial: 1.9579420767798976\nNumber of accepted steps: 936\nstep sized alpha: 45.38326240469792 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 937 line steps: 0 propose alpha: 45.38326240469792 propose points: [ 0.05810926 -0.70696648 2.31489437]\nalpha_initial: 90.76652480939585\nNumber of accepted steps: 937\nstep sized alpha: 0.9772736517059862 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 938 line steps: 0 propose alpha: 0.9772736517059862 propose points: [ 0.05812988 -0.70701055 2.31489202]\nalpha_initial: 1.9545473034119725\nNumber of accepted steps: 938\nstep sized alpha: 46.66076684351502 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 939 line steps: 0 propose alpha: 46.66076684351502 propose points: [ 0.05812278 -0.70699725 2.31456626]\nalpha_initial: 93.32153368703004\nNumber of accepted steps: 939\nstep sized alpha: 0.9754247000926022 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 940 line steps: 0 propose alpha: 0.9754247000926022 propose points: [ 0.05814275 -0.70703984 2.31456408]\nalpha_initial: 1.9508494001852044\nNumber of accepted steps: 940\nstep sized alpha: 47.77994076855586 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 941 line steps: 0 propose alpha: 47.77994076855586 propose points: [ 0.05813782 -0.7070265 2.31424617]\nalpha_initial: 95.55988153711172\nNumber of accepted steps: 941\nstep sized alpha: 0.9736747154152247 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 942 line steps: 0 propose alpha: 0.9736747154152247 propose points: [ 0.05815699 -0.70706757 2.31424415]\nalpha_initial: 1.9473494308304493\nNumber of accepted steps: 942\nstep sized alpha: 49.39344559848598 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 943 line steps: 0 propose alpha: 49.39344559848598 propose points: [ 0.05814922 -0.70705662 2.31393131]\nalpha_initial: 98.78689119697196\nNumber of accepted steps: 943\nstep sized alpha: 0.9714355070860714 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 944 line steps: 0 propose alpha: 0.9714355070860714 propose points: [ 0.05816788 -0.70709626 2.31392946]\nalpha_initial: 1.9428710141721428\nNumber of accepted steps: 944\nstep sized alpha: 50.178225674483606 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 945 line steps: 0 propose alpha: 50.178225674483606 propose points: [ 0.05816672 -0.70708308 2.3136274 ]\nalpha_initial: 100.35645134896721\nNumber of accepted steps: 945\nstep sized alpha: 0.9695061100904564 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 946 line steps: 0 propose alpha: 0.9695061100904564 propose points: [ 0.05818432 -0.70712114 2.31362567]\nalpha_initial: 1.9390122201809128\nNumber of accepted steps: 946\nstep sized alpha: 52.71079127859243 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 947 line steps: 0 propose alpha: 52.71079127859243 propose points: [ 0.05817109 -0.70711482 2.31332441]\nalpha_initial: 105.42158255718486\nNumber of accepted steps: 947\nstep sized alpha: 0.9645884702708936 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 948 line steps: 0 propose alpha: 0.9645884702708936 propose points: [ 0.05818871 -0.70715155 2.31332286]\nalpha_initial: 1.9291769405417871\nNumber of accepted steps: 948\nstep sized alpha: 50.553520962228234 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 949 line steps: 0 propose alpha: 50.553520962228234 propose points: [ 0.058201 -0.70713366 2.31304911]\nalpha_initial: 101.10704192445647\nNumber of accepted steps: 949\nstep sized alpha: 0.9558473081233407 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 950 line steps: 0 propose alpha: 0.9558473081233407 propose points: [ 0.05821625 -0.7071681 2.31304754]\nalpha_initial: 1.9116946162466815\nNumber of accepted steps: 950\nstep sized alpha: 52.84968946407165 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 951 line steps: 0 propose alpha: 52.84968946407165 propose points: [ 0.05817747 -0.70717588 2.31277594]\nalpha_initial: 105.6993789281433\nNumber of accepted steps: 951\nstep sized alpha: 0.9109547852317396 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 952 line steps: 0 propose alpha: 0.9109547852317396 propose points: [ 0.05819422 -0.70720776 2.31277447]\nalpha_initial: 1.8219095704634791\nNumber of accepted steps: 952\nstep sized alpha: 33.38194674986785 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 953 line steps: 0 propose alpha: 33.38194674986785 propose points: [ 0.05823593 -0.7071765 2.31261153]\nalpha_initial: 66.7638934997357\nNumber of accepted steps: 953\nstep sized alpha: 0.8284770353382566 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 954 line steps: 0 propose alpha: 0.8284770353382566 propose points: [ 0.05824577 -0.70720143 2.31260926]\nalpha_initial: 1.6569540706765131\nNumber of accepted steps: 954\nstep sized alpha: 26.556301410258673 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 955 line steps: 0 propose alpha: 26.556301410258673 propose points: [ 0.05820501 -0.70720799 2.31248365]\nalpha_initial: 53.112602820517345\nNumber of accepted steps: 955\nstep sized alpha: 0.8744500722765555 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 956 line steps: 0 propose alpha: 0.8744500722765555 propose points: [ 0.05821597 -0.70722912 2.3124812 ]\n960 961 36718.74 3:45.6\nalpha_initial: 1.748900144553111\nNumber of accepted steps: 956\nstep sized alpha: 23.439578485715582 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 957 line steps: 0 propose alpha: 23.439578485715582 propose points: [ 0.05822971 -0.70720982 2.31237319]\nalpha_initial: 46.879156971431165\nNumber of accepted steps: 957\nstep sized alpha: 0.9726783551307459 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 958 line steps: 0 propose alpha: 0.9726783551307459 propose points: [ 0.05823881 -0.70723022 2.3123707 ]\nalpha_initial: 1.9453567102614917\nNumber of accepted steps: 958\nstep sized alpha: 23.836450682939443 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 959 line steps: 0 propose alpha: 23.836450682939443 propose points: [ 0.05822621 -0.70722338 2.3122635 ]\nalpha_initial: 47.672901365878886\nNumber of accepted steps: 959\nstep sized alpha: 1.0100609012192157 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 960 line steps: 0 propose alpha: 1.0100609012192157 propose points: [ 0.05823573 -0.70724333 2.31226111]\nalpha_initial: 2.0201218024384313\nNumber of accepted steps: 960\nstep sized alpha: 24.603027303527533 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 961 line steps: 0 propose alpha: 24.603027303527533 propose points: [ 0.05823442 -0.70723149 2.3121532 ]\nalpha_initial: 49.206054607055066\nNumber of accepted steps: 961\nstep sized alpha: 1.0146568855352263 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 962 line steps: 0 propose alpha: 1.0146568855352263 propose points: [ 0.05824362 -0.70725135 2.31215091]\nalpha_initial: 2.0293137710704525\nNumber of accepted steps: 962\nstep sized alpha: 25.681859551539237 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 963 line steps: 0 propose alpha: 25.681859551539237 propose points: [ 0.0582373 -0.70724218 2.31204115]\nalpha_initial: 51.36371910307847\nNumber of accepted steps: 963\nstep sized alpha: 1.0130834315484123 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 964 line steps: 0 propose alpha: 1.0130834315484123 propose points: [ 0.05824656 -0.70726188 2.31203897]\nalpha_initial: 2.0261668630968246\nNumber of accepted steps: 964\nstep sized alpha: 26.840743781642743 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 965 line steps: 0 propose alpha: 26.840743781642743 propose points: [ 0.05824303 -0.70725174 2.31192732]\nalpha_initial: 53.681487563285486\nNumber of accepted steps: 965\nstep sized alpha: 1.0103905622892726 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 966 line steps: 0 propose alpha: 1.0103905622892726 propose points: [ 0.05825216 -0.70727134 2.31192525]\nalpha_initial: 2.020781124578545\nNumber of accepted steps: 966\nstep sized alpha: 28.191241579177273 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 967 line steps: 0 propose alpha: 28.191241579177273 propose points: [ 0.05824721 -0.70726222 2.31181124]\nalpha_initial: 56.382483158354546\nNumber of accepted steps: 967\nstep sized alpha: 1.007211291311263 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 968 line steps: 0 propose alpha: 1.007211291311263 propose points: [ 0.05825634 -0.70728171 2.31180928]\nalpha_initial: 2.014422582622526\nNumber of accepted steps: 968\nstep sized alpha: 29.72480561652972 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 969 line steps: 0 propose alpha: 29.72480561652972 propose points: [ 0.05825258 -0.70727241 2.31169257]\nalpha_initial: 59.44961123305944\nNumber of accepted steps: 969\nstep sized alpha: 1.0037472948306263 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 970 line steps: 0 propose alpha: 1.0037472948306263 propose points: [ 0.05826164 -0.70729182 2.31169073]\nalpha_initial: 2.0074945896612526\nNumber of accepted steps: 970\nstep sized alpha: 31.466011061357214 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 971 line steps: 0 propose alpha: 31.466011061357214 propose points: [ 0.05825725 -0.70728322 2.31157098]\nalpha_initial: 62.93202212271443\nNumber of accepted steps: 971\nstep sized alpha: 1.000087711218419 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 972 line steps: 0 propose alpha: 1.000087711218419 propose points: [ 0.05826629 -0.70730253 2.31156927]\nalpha_initial: 2.000175422436838\nNumber of accepted steps: 972\nstep sized alpha: 33.446108038120684 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 973 line steps: 0 propose alpha: 33.446108038120684 propose points: [ 0.05826276 -0.70729399 2.3114461 ]\nalpha_initial: 66.89221607624137\nNumber of accepted steps: 973\nstep sized alpha: 0.9961264224882628 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 974 line steps: 0 propose alpha: 0.9961264224882628 propose points: [ 0.05827174 -0.70731322 2.31144451]\nalpha_initial: 1.9922528449765255\nNumber of accepted steps: 974\nstep sized alpha: 35.78301193570644 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 975 line steps: 0 propose alpha: 35.78301193570644 propose points: [ 0.05826774 -0.70730539 2.31131728]\nalpha_initial: 71.56602387141288\nNumber of accepted steps: 975\nstep sized alpha: 0.9917693742743835 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 976 line steps: 0 propose alpha: 0.9917693742743835 propose points: [ 0.0582767 -0.70732454 2.31131582]\n980 981 36718.74 3:50.4\nalpha_initial: 1.983538748548767\nNumber of accepted steps: 976\nstep sized alpha: 38.49840024184752 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 977 line steps: 0 propose alpha: 38.49840024184752 propose points: [ 0.05827366 -0.70731681 2.31118395]\nalpha_initial: 76.99680048369504\nNumber of accepted steps: 977\nstep sized alpha: 0.9869979906592825 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 978 line steps: 0 propose alpha: 0.9869979906592825 propose points: [ 0.05828258 -0.70733591 2.31118263]\nalpha_initial: 1.973995981318565\nNumber of accepted steps: 978\nstep sized alpha: 41.797048018246215 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 979 line steps: 0 propose alpha: 41.797048018246215 propose points: [ 0.05827881 -0.70732914 2.3110451 ]\nalpha_initial: 83.59409603649243\nNumber of accepted steps: 979\nstep sized alpha: 0.9816152808443387 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 980 line steps: 0 propose alpha: 0.9816152808443387 propose points: [ 0.05828774 -0.70734819 2.31104392]\nalpha_initial: 1.9632305616886774\nNumber of accepted steps: 980\nstep sized alpha: 45.67253981776184 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 981 line steps: 0 propose alpha: 45.67253981776184 propose points: [ 0.05828572 -0.70734131 2.31090006]\nalpha_initial: 91.34507963552367\nNumber of accepted steps: 981\nstep sized alpha: 0.9755848179245643 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 982 line steps: 0 propose alpha: 0.9755848179245643 propose points: [ 0.05829457 -0.70736034 2.31089902]\nalpha_initial: 1.9511696358491286\nNumber of accepted steps: 982\nstep sized alpha: 50.613261706813304 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 983 line steps: 0 propose alpha: 50.613261706813304 propose points: [ 0.05829031 -0.70735534 2.31074701]\nalpha_initial: 101.22652341362661\nNumber of accepted steps: 983\nstep sized alpha: 0.9683582998152838 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 984 line steps: 0 propose alpha: 0.9683582998152838 propose points: [ 0.05829927 -0.70737433 2.31074613]\nalpha_initial: 1.9367165996305675\nNumber of accepted steps: 984\nstep sized alpha: 55.96523971136157 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 985 line steps: 0 propose alpha: 55.96523971136157 propose points: [ 0.05830066 -0.70736761 2.31058668]\nalpha_initial: 111.93047942272314\nNumber of accepted steps: 985\nstep sized alpha: 0.9594635667765983 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 986 line steps: 0 propose alpha: 0.9594635667765983 propose points: [ 0.05830932 -0.70738657 2.31058596]\nalpha_initial: 1.9189271335531966\nNumber of accepted steps: 986\nstep sized alpha: 63.235663883354555 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 987 line steps: 0 propose alpha: 63.235663883354555 propose points: [ 0.0582983 -0.70738689 2.310416 ]\nalpha_initial: 126.47132776670911\nNumber of accepted steps: 987\nstep sized alpha: 0.9404807259871993 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 988 line steps: 0 propose alpha: 0.9404807259871993 propose points: [ 0.05830751 -0.70740548 2.31041544]\nalpha_initial: 1.8809614519743987\nNumber of accepted steps: 988\nstep sized alpha: 57.2908705844076 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 989 line steps: 0 propose alpha: 57.2908705844076 propose points: [ 0.0583285 -0.70739124 2.31027116]\nalpha_initial: 114.5817411688152\nNumber of accepted steps: 989\nstep sized alpha: 0.8678577511508032 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 990 line steps: 0 propose alpha: 0.8678577511508032 propose points: [ 0.0583351 -0.70740799 2.31027047]\nalpha_initial: 1.7357155023016064\nNumber of accepted steps: 990\nstep sized alpha: 36.64845187727479 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 991 line steps: 0 propose alpha: 36.64845187727479 propose points: [ 0.05829955 -0.7074201 2.31018323]\nalpha_initial: 73.29690375454958\nNumber of accepted steps: 991\nstep sized alpha: 0.6739339523784038 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 992 line steps: 0 propose alpha: 0.6739339523784038 propose points: [ 0.05830615 -0.7074311 2.31018207]\nalpha_initial: 1.3478679047568076\nNumber of accepted steps: 992\nstep sized alpha: 17.2163699544149 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 993 line steps: 0 propose alpha: 17.2163699544149 propose points: [ 0.05832221 -0.70741722 2.31014225]\nalpha_initial: 34.4327399088298\nNumber of accepted steps: 993\nstep sized alpha: 0.8335795547119987 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 994 line steps: 0 propose alpha: 0.8335795547119987 propose points: [ 0.05832627 -0.70742607 2.3101408 ]\nalpha_initial: 1.6671591094239975\nNumber of accepted steps: 994\nstep sized alpha: 17.215946503275195 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 995 line steps: 0 propose alpha: 17.215946503275195 propose points: [ 0.05832296 -0.70742128 2.31010169]\n1000 1000 36718.74 3:55.2\nHalting: Maximum number of iterations (1000) reached.\nScore at true solution: \n36894.34103855895\nFound solution: True parameters:\n 5.83229645034749195e-02 1.00000000000000006e-01\n-7.07421276762396634e-01 5.00000000000000000e-01\n 2.31010169210899896e+00 3.00000000000000000e+00\n" - } - ], - "source": [ - "# Load a forward model\n", - "\n", - "f = pints.toy.FitzhughNagumoModel()\n", - "real_parameters = f.suggested_parameters()\n", - "# [0.1, 0.5, 3. ]\n", - "times =f.suggested_times()\n", - "values = f.simulate(real_parameters, times)\n", - "\n", - "# Add noise\n", - "values += np.random.normal(0, 10, values.shape)\n", - "\n", - "# Create an object with links to the model and time series\n", - "problem = pints.MultiOutputProblem(f, times, values)\n", - "\n", - "# Select a score function\n", - "score = pints.SumOfSquaresError(problem)\n", - "\n", - "# Perform an optimization\n", - "x0 = [0.2, 0.3, 2.5]\n", - "opt = pints.OptimisationController(\n", - " score,\n", - " x0,\n", - " method=pints.BFGS\n", - ")\n", - "opt.set_max_unchanged_iterations(200)\n", - "opt.set_max_iterations(1000)\n", - "found_parameters, found_value = opt.run()\n", - "\n", - "# Show score of true solution\n", - "print('Score at true solution: ')\n", - "print(score(real_parameters))\n", - "\n", - "# Compare parameters with original\n", - "print('Found solution: True parameters:' )\n", - "for k, x in enumerate(found_parameters):\n", - " print(pints.strfloat(x) + ' ' + pints.strfloat(real_parameters[k]))\n", - "\n", - "# [0.1, 0.5, 3. ] real\n", - "# [0.2, 0.3, 2.5] starting" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## now using scipy line search" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": "Minimising error measure\nUsing Broyden–Fletcher–Goldfarb–Shanno (BFGS)\nRunning in sequential mode.\nIter. Eval. Best Time m:s\n0 1 1.91e+07 0:00.0\n\n----------------------------------------\nUnexpected termination.\nCurrent best score: 19146011.016347833\nCurrent best position:\n 1.00000000000000002e-02\n 4.50000000000000000e+02\n----------------------------------------\n" - }, - { - "output_type": "error", - "ename": "TypeError", - "evalue": "unsupported operand type(s) for *: 'NoneType' and 'float'", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_max_unchanged_iterations\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m200\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_max_iterations\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m \u001b[0mfound_parameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfound_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0;31m# Show score of true solution\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/pints/pints-clone/pints/pints/_optimisers/__init__.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 540\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0mrunning\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 541\u001b[0m \u001b[0;31m# Get points\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 542\u001b[0;31m \u001b[0mxs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_optimiser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 543\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 544\u001b[0m \u001b[0;31m# Calculate scores\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/pints/pints-clone/pints/pints/_optimisers/_bfgs_scipy.py\u001b[0m in \u001b[0;36mask\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 245\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_proposed_alpha\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 246\u001b[0m \u001b[0;31m# print('alpha: ', self._proposed_alpha)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 247\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_proposed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_current\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_proposed_alpha\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_px\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 248\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 249\u001b[0m \u001b[0;31m# Running, and ready for tell now\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mTypeError\u001b[0m: unsupported operand type(s) for *: 'NoneType' and 'float'" - ] - } - ], - "source": [ - "# Load a forward model\n", - "model = pints.toy.LogisticModel()\n", - "\n", - "# Create some toy data\n", - "real_parameters = [0.015, 500]\n", - "times = np.linspace(0, 1000, 1000)\n", - "values = model.simulate(real_parameters, times)\n", - "\n", - "# Add noise\n", - "values += np.random.normal(0, 10, values.shape)\n", - "\n", - "# Create an object with links to the model and time series\n", - "problem = pints.SingleOutputProblem(model, times, values)\n", - "\n", - "# Select a score function\n", - "score = pints.SumOfSquaresError(problem)\n", - "\n", - "# Perform an optimization\n", - "x0 = [0.01, 450]\n", - "opt = pints.OptimisationController(\n", - " score,\n", - " x0,\n", - " method=pints.BFGS_scipy\n", - ")\n", - "opt.set_max_unchanged_iterations(200)\n", - "opt.set_max_iterations(1000)\n", - "found_parameters, found_value = opt.run()\n", - "\n", - "# Show score of true solution\n", - "print('Score at true solution: ')\n", - "print(score(real_parameters))\n", - "\n", - "# Compare parameters with original\n", - "print('Found solution: True parameters:' )\n", - "for k, x in enumerate(found_parameters):\n", - " print(pints.strfloat(x) + ' ' + pints.strfloat(real_parameters[k]))" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": "Minimising error measure\nUsing Broyden–Fletcher–Goldfarb–Shanno (BFGS)\nRunning in sequential mode.\nIter. Eval. Best Time m:s\n0 1 34916.69 0:00.1\n1 2 34498.91 0:00.7\n2 3 34481.78 0:00.9\n3 4 34478.34 0:01.1\n20 21 34233.43 0:06.2\n40 41 34172.38 0:11.5\n\n----------------------------------------\nUnexpected termination.\nCurrent best score: 34161.946320079565\nCurrent best position:\n-5.44201477518764937e-02\n-5.50556538864840572e-01\n 2.83944303459274172e+00\n----------------------------------------\n" - }, - { - "output_type": "error", - "ename": "KeyboardInterrupt", - "evalue": "", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 23\u001b[0m \u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_max_unchanged_iterations\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m200\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_max_iterations\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m \u001b[0mfound_parameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfound_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[0;31m# Show score of true solution\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/pints/pints-clone/pints/pints/_optimisers/__init__.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 540\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0mrunning\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 541\u001b[0m \u001b[0;31m# Get points\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 542\u001b[0;31m \u001b[0mxs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_optimiser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 543\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 544\u001b[0m \u001b[0;31m# Calculate scores\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/pints/pints-clone/pints/pints/_optimisers/_bfgs_scipy.py\u001b[0m in \u001b[0;36mask\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 216\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 217\u001b[0m \u001b[0;31m# line search using an algorithm from scipy to meet wolfe condtions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 218\u001b[0;31m results = line_search_wolfe2(f=self.__objective_function,\n\u001b[0m\u001b[1;32m 219\u001b[0m \u001b[0mmyfprime\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__gradients_function\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 220\u001b[0m \u001b[0mxk\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_current\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpk\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_px\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/envs/pints/lib/python3.8/site-packages/scipy/optimize/linesearch.py\u001b[0m in \u001b[0;36mline_search_wolfe2\u001b[0;34m(f, myfprime, xk, pk, gfk, old_fval, old_old_fval, args, c1, c2, amax, extra_condition, maxiter)\u001b[0m\n\u001b[1;32m 307\u001b[0m \u001b[0mextra_condition2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 308\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 309\u001b[0;31m alpha_star, phi_star, old_fval, derphi_star = scalar_search_wolfe2(\n\u001b[0m\u001b[1;32m 310\u001b[0m \u001b[0mphi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mderphi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mold_fval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mold_old_fval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mderphi0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mc1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mc2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mamax\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 311\u001b[0m extra_condition2, maxiter=maxiter)\n", - "\u001b[0;32m~/anaconda3/envs/pints/lib/python3.8/site-packages/scipy/optimize/linesearch.py\u001b[0m in \u001b[0;36mscalar_search_wolfe2\u001b[0;34m(phi, derphi, phi0, old_phi0, derphi0, c1, c2, amax, extra_condition, maxiter)\u001b[0m\n\u001b[1;32m 435\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 436\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 437\u001b[0;31m \u001b[0mderphi_a1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mderphi\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0malpha1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 438\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mabs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mderphi_a1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0mc2\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mderphi0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 439\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mextra_condition\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0malpha1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mphi_a1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/envs/pints/lib/python3.8/site-packages/scipy/optimize/linesearch.py\u001b[0m in \u001b[0;36mderphi\u001b[0;34m(alpha)\u001b[0m\n\u001b[1;32m 288\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mderphi\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0malpha\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 289\u001b[0m \u001b[0mgc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 290\u001b[0;31m \u001b[0mgval\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfprime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxk\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0malpha\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mpk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# store for later use\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 291\u001b[0m \u001b[0mgval_alpha\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0malpha\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 292\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgval\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/pints/pints-clone/pints/pints/_optimisers/_bfgs_scipy.py\u001b[0m in \u001b[0;36m__gradients_function\u001b[0;34m(self, point_alpha)\u001b[0m\n\u001b[1;32m 195\u001b[0m '''\n\u001b[1;32m 196\u001b[0m \u001b[0;31m#point_alpha = self._current + alpha * self._px\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 197\u001b[0;31m \u001b[0mfs_alpha\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_evaluator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpoint_alpha\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 198\u001b[0m \u001b[0mf_alpha\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdfdx_alpha\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfs_alpha\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 199\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/pints/pints-clone/pints/pints/_evaluation.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(self, positions)\u001b[0m\n\u001b[1;32m 104\u001b[0m \u001b[0;34m'The argument `positions` must be a sequence of input values'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 105\u001b[0m ' to the evaluator\\'s function.')\n\u001b[0;32m--> 106\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_evaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpositions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 107\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 108\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_evaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpositions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/pints/pints-clone/pints/pints/_evaluation.py\u001b[0m in \u001b[0;36m_evaluate\u001b[0;34m(self, positions)\u001b[0m\n\u001b[1;32m 396\u001b[0m \u001b[0mscores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpositions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 397\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpositions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 398\u001b[0;31m \u001b[0mscores\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 399\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mscores\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 400\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/pints/pints-clone/pints/pints/_error_measures.py\u001b[0m in \u001b[0;36mevaluateS1\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m 333\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mevaluateS1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 334\u001b[0m \u001b[0;34m\"\"\" See :meth:`ErrorMeasure.evaluateS1()`. \"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 335\u001b[0;31m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_problem\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluateS1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 336\u001b[0m \u001b[0mdy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_times\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_outputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_parameters\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 337\u001b[0m \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_values\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/pints/pints-clone/pints/pints/_core.py\u001b[0m in \u001b[0;36mevaluateS1\u001b[0;34m(self, parameters)\u001b[0m\n\u001b[1;32m 277\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;32mclass\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mForwardModelS1\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0minterface\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 278\u001b[0m \"\"\"\n\u001b[0;32m--> 279\u001b[0;31m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msimulateS1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_times\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 280\u001b[0m return (\n\u001b[1;32m 281\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_times\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_outputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/pints/pints-clone/pints/pints/toy/_toy_classes.py\u001b[0m in \u001b[0;36msimulateS1\u001b[0;34m(self, parameters, times)\u001b[0m\n\u001b[1;32m 227\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0msimulateS1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 228\u001b[0m \u001b[0;34m\"\"\" See :meth:`pints.ForwardModelS1.simulateS1()`. \"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 229\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_simulate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/Documents/pints/pints-clone/pints/pints/toy/_toy_classes.py\u001b[0m in \u001b[0;36m_simulate\u001b[0;34m(self, parameters, times, sensitivities)\u001b[0m\n\u001b[1;32m 214\u001b[0m \u001b[0my0\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_params\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mn_outputs\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mn_outputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 215\u001b[0m \u001b[0my0\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mn_outputs\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_y0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 216\u001b[0;31m result = scipy.integrate.odeint(\n\u001b[0m\u001b[1;32m 217\u001b[0m self._rhs_S1, y0, times, (parameters,))\n\u001b[1;32m 218\u001b[0m \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mn_outputs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/envs/pints/lib/python3.8/site-packages/scipy/integrate/odepack.py\u001b[0m in \u001b[0;36modeint\u001b[0;34m(func, y0, t, args, Dfun, col_deriv, full_output, ml, mu, rtol, atol, tcrit, h0, hmax, hmin, ixpr, mxstep, mxhnil, mxordn, mxords, printmessg, tfirst)\u001b[0m\n\u001b[1;32m 240\u001b[0m \u001b[0mt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 241\u001b[0m \u001b[0my0\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 242\u001b[0;31m output = _odepack.odeint(func, y0, t, args, Dfun, col_deriv, ml, mu,\n\u001b[0m\u001b[1;32m 243\u001b[0m \u001b[0mfull_output\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrtol\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0matol\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtcrit\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhmax\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhmin\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 244\u001b[0m \u001b[0mixpr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmxstep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmxhnil\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmxordn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmxords\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/pints/pints-clone/pints/pints/toy/_toy_classes.py\u001b[0m in \u001b[0;36m_rhs_S1\u001b[0;34m(self, y_and_dydp, t, p)\u001b[0m\n\u001b[1;32m 173\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdydp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjacobian\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 174\u001b[0m np.transpose(self._dfdp(y, t, p)))\n\u001b[0;32m--> 175\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdydt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0md_dydp_dt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 176\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 177\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0msimulate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m<__array_function__ internals>\u001b[0m in \u001b[0;36mconcatenate\u001b[0;34m(*args, **kwargs)\u001b[0m\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " - ] - } - ], - "source": [ - "f = pints.toy.FitzhughNagumoModel()\n", - "real_parameters = f.suggested_parameters()\n", - "# [0.1, 0.5, 3. ]\n", - "times =f.suggested_times()\n", - "values = f.simulate(real_parameters, times)\n", - "\n", - "# Add noise\n", - "values += np.random.normal(0, 10, values.shape)\n", - "\n", - "# Create an object with links to the model and time series\n", - "problem = pints.MultiOutputProblem(f, times, values)\n", - "\n", - "# Select a score function\n", - "score = pints.SumOfSquaresError(problem)\n", - "\n", - "# Perform an optimization\n", - "x0 = [0.2, 0.3, 2.5]\n", - "opt = pints.OptimisationController(\n", - " score,\n", - " x0,\n", - " method=pints.BFGS_scipy\n", - ")\n", - "opt.set_max_unchanged_iterations(200)\n", - "opt.set_max_iterations(1000)\n", - "found_parameters, found_value = opt.run()\n", - "\n", - "# Show score of true solution\n", - "print('Score at true solution: ')\n", - "print(score(real_parameters))\n", - "\n", - "# Compare parameters with original\n", - "print('Found solution: True parameters:' )\n", - "for k, x in enumerate(found_parameters):\n", - " print(pints.strfloat(x) + ' ' + pints.strfloat(real_parameters[k]))\n", - "\n", - "# [0.1, 0.5, 3. ] real\n", - "# [0.2, 0.3, 2.5] starting" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": "False\n" - } - ], - "source": [ - "print( 5 < 4 | 0 > 1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3.8.2 64-bit ('pints': conda)", - "language": "python", - "name": "python38264bitpintsconda8d0b754a30424fdd8045d82b54ea71e7" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.2-final" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} \ No newline at end of file From 635acc3fc47b321281acebeff6553faf4e0a65c6 Mon Sep 17 00:00:00 2001 From: alisterde Date: Sun, 24 May 2020 18:59:07 +0100 Subject: [PATCH 05/16] removing non ASCII characters --- pints/_optimisers/_bfgs_linesearch.py | 12 ++++++------ pints/_optimisers/_bfgs_scipy.py | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pints/_optimisers/_bfgs_linesearch.py b/pints/_optimisers/_bfgs_linesearch.py index 1dbf70bf4..bedc8df45 100644 --- a/pints/_optimisers/_bfgs_linesearch.py +++ b/pints/_optimisers/_bfgs_linesearch.py @@ -14,28 +14,28 @@ class BFGS(pints.LineSearchBasedOptimiser): """ - Broyden–Fletcher–Goldfarb–Shanno algorithm [2], [3], [4] + Broyden-Fletcher-Goldfarb-Shanno algorithm [2], [3], [4] The Hager-Zhang line search algorithm [1] is implemented in this class # TODO: when this is working move everything to an abstract class [1] Hager, W. W.; Zhang, H. Algorithm 851: CG_DESCENT, a Conjugate Gradient Method with Guaranteed Descent. - ACM Trans. Math. Softw. 2006, 32 (1), 113–137. + ACM Trans. Math. Softw. 2006, 32 (1), 113-137. https://doi.org/10.1145/1132973.1132979. [2] Liu, D. C.; Nocedal, J. On the Limited Memory BFGS Method for Large Scale Optimization. Mathematical Programming 1989, 45 (1), - 503–528. https://doi.org/10.1007/BF01589116. + 503-528. https://doi.org/10.1007/BF01589116. [3] Nocedal, J. Updating Quasi-Newton Matrices with Limited Storage. - Math. Comp. 1980, 35 (151), 773–782. + Math. Comp. 1980, 35 (151), 773-782. https://doi.org/10.1090/S0025-5718-1980-0572855-7. [4] Nash, S. G.; Nocedal, J. A Numerical Study of the Limited Memory BFGS Method and the Truncated-Newton Method for Large Scale Optimization. - SIAM J. Optim. 1991, 1 (3), 358–372. https://doi.org/10.1137/0801023. + SIAM J. Optim. 1991, 1 (3), 358-372. https://doi.org/10.1137/0801023. """ @@ -157,7 +157,7 @@ def max_correction_matrice_storage(self): def name(self): """ See :meth:`Optimiser.name()`. """ - return 'Broyden–Fletcher–Goldfarb–Shanno (BFGS)' + return 'Broyden-Fletcher-Goldfarb-Shanno (BFGS)' def needs_sensitivities(self): """ See :meth:`Optimiser.needs_sensitivities()`. """ diff --git a/pints/_optimisers/_bfgs_scipy.py b/pints/_optimisers/_bfgs_scipy.py index 4b39f6d74..a39d37999 100644 --- a/pints/_optimisers/_bfgs_scipy.py +++ b/pints/_optimisers/_bfgs_scipy.py @@ -18,20 +18,20 @@ # TODO: move line search to abstract class class BFGS_scipy(pints.LineSearchBasedOptimiser): """ - Broyden–Fletcher–Goldfarb–Shanno algorithm [1] + Broyden-Fletcher-Goldfarb-Shanno algorithm [1] [1] Liu, D. C.; Nocedal, J. On the Limited Memory BFGS Method for Large Scale Optimization. Mathematical Programming 1989, 45 (1), - 503–528. https://doi.org/10.1007/BF01589116. + 503-528. https://doi.org/10.1007/BF01589116. [2] Nocedal, J. Updating Quasi-Newton Matrices with Limited Storage. - Math. Comp. 1980, 35 (151), 773–782. + Math. Comp. 1980, 35 (151), 773-782. https://doi.org/10.1090/S0025-5718-1980-0572855-7. [3] Nash, S. G.; Nocedal, J. A Numerical Study of the Limited Memory BFGS Method and the Truncated-Newton Method for Large Scale Optimization. - SIAM J. Optim. 1991, 1 (3), 358–372. https://doi.org/10.1137/0801023. + SIAM J. Optim. 1991, 1 (3), 358-372. https://doi.org/10.1137/0801023. """ @@ -115,7 +115,7 @@ def max_correction_matrice_storage(self): def name(self): """ See :meth:`Optimiser.name()`. """ - return 'Broyden–Fletcher–Goldfarb–Shanno (BFGS)' + return 'Broyden-Fletcher-Goldfarb-Shanno (BFGS)' def needs_sensitivities(self): """ See :meth:`Optimiser.needs_sensitivities()`. """ From 7ea50f4ac522bd042742103b10a5e8e2104903f9 Mon Sep 17 00:00:00 2001 From: alisterde Date: Thu, 16 Jul 2020 12:31:01 +0100 Subject: [PATCH 06/16] LBFGS ready for sepration into abstract class --- pints/_optimisers/__init__.py | 8 ++ pints/_optimisers/_bfgs_linesearch.py | 154 +++++++++++++------------- 2 files changed, 88 insertions(+), 74 deletions(-) diff --git a/pints/_optimisers/__init__.py b/pints/_optimisers/__init__.py index 57a1c02db..1fa59ed26 100644 --- a/pints/_optimisers/__init__.py +++ b/pints/_optimisers/__init__.py @@ -284,6 +284,14 @@ class LineSearchBasedOptimiser(Optimiser): Base class for optimisers that incorporate a line search within their algorithm. + The Hager-Zhang line search algorithm [1] is implemented + in this class. + + [1] Hager, W. W.; Zhang, H. Algorithm 851: CG_DESCENT, + a Conjugate Gradient Method with Guaranteed Descent. + ACM Trans. Math. Softw. 2006, 32 (1), 113-137. + https://doi.org/10.1145/1132973.1132979. + Extends :class:`Optimiser`. """ diff --git a/pints/_optimisers/_bfgs_linesearch.py b/pints/_optimisers/_bfgs_linesearch.py index bedc8df45..a99766678 100644 --- a/pints/_optimisers/_bfgs_linesearch.py +++ b/pints/_optimisers/_bfgs_linesearch.py @@ -17,8 +17,7 @@ class BFGS(pints.LineSearchBasedOptimiser): Broyden-Fletcher-Goldfarb-Shanno algorithm [2], [3], [4] The Hager-Zhang line search algorithm [1] is implemented in this class - # TODO: when this is working move everything to an abstract class - + [1] Hager, W. W.; Zhang, H. Algorithm 851: CG_DESCENT, a Conjugate Gradient Method with Guaranteed Descent. ACM Trans. Math. Softw. 2006, 32 (1), 113-137. @@ -186,7 +185,7 @@ def set_hyper_parameters(self, x): self.__set_wolfe_line_search_parameters(x[0], x[1]) - def __set_wolfe_line_search_parameters(self, c1: float, c2: float): + def __set_wolfe_line_search_parameters(self, c1, c2): """ Sets the parameters for the wolfe conditions. @@ -326,7 +325,7 @@ def ask(self): self._proposed_alpha = self.__secant_for_alpha(a, A) # checking the proposed point is in range - if self._proposed_alpha < A | self._proposed_alpha > B: + if self._proposed_alpha < A or self._proposed_alpha > B: # If the point is out of range there is no need to # check wolfe conditions. self.__2nd_wolfe_check_needed = False @@ -473,9 +472,8 @@ def tell(self, reply): and approx_wolfe_applies) # If wolfe conditions meet the line search is stopped - # and the hessian matrix and newton direction are updated by the - # L-BFGS/BFGS approximation of the hessian described in reference [2] - # [3], and [4]. If the line search has converged we also accept the + # and the inverse hessian matrix and newton direction are updated. + # If the line search has converged we also accept the # steps and update. if exact_wolfe or approximate_wolfe or self.__converged_ls: @@ -484,72 +482,7 @@ def tell(self, reply): print('updating Hessian and changing newton direction') # Updating inverse hessian. - - # identity matrix - I = np.identity(self._n_parameters) - - # We do this if we haven't exhausted existing memory yet, this is - # identical to the BFGS algorithm - if self.__k <= self._m - 1: - k = self.__k - # Defining the next column. - self._S[:, k] = self._proposed - self._current - self._Y[:, k] = proposed_dfdx - self._current_dfdx - - # Defining B_0. Scaling taken from [4]. - self._B = ((np.matmul(np.transpose(self._Y[:, k]), - self._S[:, k]) - / (norm(self._Y[:, k], ord=2) ** 2)) * I) - - # Updating inverse hessian. - for k in range(self.__k + 1): - - V = (I - np.matmul(self._Y[:, k], - np.transpose(self._S[:, k])) - / np.matmul(np.transpose(self._Y[:, k]), - self._S[:, k])) - - self._B = np.matmul(np.transpose(V), np.matmul(self._B, V)) - self._B += (np.matmul(self._S[:, k], - np.transpose(self._S[:, k])) - / np.matmul(np.transpose(self._Y[:, k]), - self._S[:, k])) - - # We have exhausted the limited memory and now enter - # the LM-BFGS algorithm - else: - - m = self._m - 1 - # Shifting everything one column to the left. - self._S[:, 0:m] = self._S[:, 1:self._m] - self._Y[:, 0:m] = self._Y[:, 1:self._m] - - # Renewing last column. - self._S[:, m] = self._proposed - self._current - self._Y[:, m] = proposed_dfdx - self._current_dfdx - - # Defining B_0. Scaling taken from [4]. - self._B = ((np.matmul(np.transpose(self._Y[:, m]), - self._S[:, m]) - / (norm(self._Y[:, m], ord=2) ** 2)) * I) - - # Updating inverse hessian. - for k in range(self._m): - - V = (I - np.matmul(self._Y[:, k], - np.transpose(self._S[:, k])) - / np.matmul(np.transpose(self._Y[:, k]), - self._S[:, k])) - - self._B = np.matmul(np.transpose(V), np.matmul(self._B, V)) - self._B += (np.matmul(self._S[:, k], - np.transpose(self._S[:, k])) - / np.matmul(np.transpose(self._Y[:, k]), - self._S[:, k])) - - self._B = ((np.matmul(np.transpose(self._Y[:, m]), - self._S[:, m]) - / (norm(self._Y[:, m], ord=2)**2)) * I) + self._B = self.inverse_hessian_update(proposed_f, proposed_dfdx) # Move to proposed point self._current = self._proposed @@ -690,7 +623,7 @@ def __update(self, a, b, c): b = c return self.__bisect_or_secant(a, b) - def __bisect_or_secant(self, a: float, b: float): + def __bisect_or_secant(self, a, b): ''' This function is part of the Hager-Zhang line search method [1]. @@ -1049,3 +982,76 @@ def obj_and_grad_func(self, alpha: float): # return A, B + def inverse_hessian_update(self, proposed_f, proposed_dfdx): + '''the inverse hessian matrix and newton direction are updated by the + L-BFGS/BFGS approximation of the hessian described in reference [2] + [3], and [4]. + ''' + + # identity matrix + I = np.identity(self._n_parameters) + + # We do this if we haven't exhausted existing memory yet, this is + # identical to the BFGS algorithm + if self.__k <= self._m - 1: + k = self.__k + # Defining the next column. + self._S[:, k] = self._proposed - self._current + self._Y[:, k] = proposed_dfdx - self._current_dfdx + + # Defining B_0. Scaling taken from [4]. + B = ((np.matmul(np.transpose(self._Y[:, k]), + self._S[:, k]) + / (norm(self._Y[:, k], ord=2) ** 2)) * I) + + # Updating inverse hessian. + for k in range(self.__k + 1): + + V = (I - np.matmul(self._Y[:, k], + np.transpose(self._S[:, k])) + / np.matmul(np.transpose(self._Y[:, k]), + self._S[:, k])) + + B = np.matmul(np.transpose(V), np.matmul(self._B, V)) + B += (np.matmul(self._S[:, k], + np.transpose(self._S[:, k])) + / np.matmul(np.transpose(self._Y[:, k]), + self._S[:, k])) + + # We have exhausted the limited memory and now enter + # the LM-BFGS algorithm + else: + + m = self._m - 1 + # Shifting everything one column to the left. + self._S[:, 0:m] = self._S[:, 1:self._m] + self._Y[:, 0:m] = self._Y[:, 1:self._m] + + # Renewing last column. + self._S[:, m] = self._proposed - self._current + self._Y[:, m] = proposed_dfdx - self._current_dfdx + + # Defining B_0. Scaling taken from [4]. + B = ((np.matmul(np.transpose(self._Y[:, m]), + self._S[:, m]) + / (norm(self._Y[:, m], ord=2) ** 2)) * I) + + # Updating inverse hessian. + for k in range(self._m): + + V = (I - np.matmul(self._Y[:, k], + np.transpose(self._S[:, k])) + / np.matmul(np.transpose(self._Y[:, k]), + self._S[:, k])) + + B = np.matmul(np.transpose(V), np.matmul(self._B, V)) + B += (np.matmul(self._S[:, k], + np.transpose(self._S[:, k])) + / np.matmul(np.transpose(self._Y[:, k]), + self._S[:, k])) + + B = ((np.matmul(np.transpose(self._Y[:, m]), + self._S[:, m]) + / (norm(self._Y[:, m], ord=2)**2)) * I) + + return B \ No newline at end of file From 3985888d4e3d9298262f288fe284efcf842de523 Mon Sep 17 00:00:00 2001 From: alisterde Date: Tue, 21 Jul 2020 10:58:31 +0100 Subject: [PATCH 07/16] deleting bfgs_scipy --- pints/_optimisers/_bfgs_scipy.py | 399 ------------------------------- 1 file changed, 399 deletions(-) delete mode 100644 pints/_optimisers/_bfgs_scipy.py diff --git a/pints/_optimisers/_bfgs_scipy.py b/pints/_optimisers/_bfgs_scipy.py deleted file mode 100644 index a39d37999..000000000 --- a/pints/_optimisers/_bfgs_scipy.py +++ /dev/null @@ -1,399 +0,0 @@ -# -# Broyden-Fletcher-Goldfarb-Shanno algorithm -# -# This file is part of PINTS (https://github.com/pints-team/pints/) which is -# released under the BSD 3-clause license. See accompanying LICENSE.md for -# copyright notice and full license details. -# -from __future__ import absolute_import, division -from __future__ import print_function, unicode_literals -import numpy as np -from numpy.linalg import norm -from scipy.optimize.linesearch import line_search_wolfe2 -#from scipy.optimize.linesearch import line_search_wolfe1 -#from scipy.optimize.linesearch import line_search_BFGS -import pints - - -# TODO: move line search to abstract class -class BFGS_scipy(pints.LineSearchBasedOptimiser): - """ - Broyden-Fletcher-Goldfarb-Shanno algorithm [1] - - [1] Liu, D. C.; Nocedal, J. - On the Limited Memory BFGS Method for Large Scale Optimization. - Mathematical Programming 1989, 45 (1), - 503-528. https://doi.org/10.1007/BF01589116. - - [2] Nocedal, J. Updating Quasi-Newton Matrices with Limited Storage. - Math. Comp. 1980, 35 (151), 773-782. - https://doi.org/10.1090/S0025-5718-1980-0572855-7. - - [3] Nash, S. G.; Nocedal, J. A Numerical Study of the Limited Memory - BFGS Method and the Truncated-Newton Method for Large Scale Optimization. - SIAM J. Optim. 1991, 1 (3), 358-372. https://doi.org/10.1137/0801023. - - """ - - def __init__(self, x0, sigma0=None, boundaries=None): - super(BFGS_scipy, self).__init__(x0, sigma0, boundaries) - - # Set optimiser state - self._running = False - self._ready_for_tell = False - - # Best solution found - self._xbest = self._x0 - self._fbest = float('inf') - - # approximate inverse hessian - # initial the identity is used - self._B = np.identity(self._n_parameters) - - # newton direction - self._px = None - - # maximum number of correction matrices stored - self._m = 5 - - # Storage for vectors constructing the correction matrix - # this is the advised way of storing them. - self._S = np.zeros(shape=(self._n_parameters, self._m)) - self._Y = np.zeros(shape=(self._n_parameters, self._m)) - - # number of accepted steps/ newton direction updates - self.__k = 0 - - # Current point, score, and gradient - self._current = self._x0 - self._current_f = None - self._current_dfdx = None - - # Proposed next point (read-only, so can be passed to user) - self._proposed = self._x0 - self._proposed.setflags(write=False) - - self._previous_f = None - - # parameters for wolfe conditions on line search - - # As c1 approaches 0 and c2 approaches 1, the line search - # terminates more quickly. - self._c1 = 1E-4 # Parameter for Armijo condition rule, 0 < c1 < 0.5 - self._c2 = 0.9 # Parameter for curvature condition rule, c1 < c2 < 1.0 - - # boundary values of alpha - self._minimum_alpha = 0.0 - self._maximum_alpha = float("inf") - self._proposed_alpha = 0.001 # same default value as used in stan - - self.__current_alpha = 1.0 - - self.__convergence = False - - def fbest(self): - """ See :meth:`Optimiser.fbest()`. """ - return self._fbest - - def wolfe_line_search_parameters(self): - """ - Returns the wolfe line search parameters this optimiser is using - as a vector ``[c1, c2]``. - As c1 approaches 0 and c2 approaches 1, the line search terminates - more quickly. ``c1`` is the parameter for the Armijo condition - rule, ``0 < c1 < 0.5``. ``c2 `` is the parameter for the - curvature condition rule, ``c1 < c2 < 1.0``. - """ - return (self._c1, self._c2) - - def max_correction_matrice_storage(self): - """ - Returns ``m``, the maximum number of correction matrice for - calculating the inverse hessian used and stored - """ - return self._m - - def name(self): - """ See :meth:`Optimiser.name()`. """ - return 'Broyden-Fletcher-Goldfarb-Shanno (BFGS)' - - def needs_sensitivities(self): - """ See :meth:`Optimiser.needs_sensitivities()`. """ - return True - - def n_hyper_parameters(self): - """ See :meth:`pints.TunableMethod.n_hyper_parameters()`. """ - return 2 - - def running(self): - """ See :meth:`Optimiser.running()`. """ - return self._running - - def set_hyper_parameters(self, x): - """ - See :meth:`pints.TunableMethod.set_hyper_parameters()`. - - The hyper-parameter vector is ``[c1, c2, m]``. - ``c1`` is the parameter for the Armijo condition rule, - ``0 < c1 < 0.5``. - ``c2`` is the parameter for the curvature condition rule, - ``c1 < c2 < 1.0``. - ``m`` is the number of maximum number of correction matrices - that can be stored for the LM-BFGS update. - """ - - self.__set_wolfe_line_search_parameters(x[0], x[1]) - - def __set_wolfe_line_search_parameters(self, c1: float, c2: float): - """ - Sets the parameters for the wolfe conditions. - - Parameters - ---------- - c1: float - Parameter for the Armijo condition rule, ``0 < c1 < 0.5``. - - c2: float - Parameter for the curvature condition rule, ``c1 < c2 < 1.0``. - """ - if(0 < c1 < 0.5 and c1 < c2 < 1.0): - self._c1 = c1 - self._c2 = c2 - else: - cs = self.wolfe_line_search_parameters() - print('Invalid wolfe line search parameters!!!') - print('0 < c1 < 0.5 and c1 < c2 < 1.0') - print('using default parameters: c1 = ', cs[0], ' c2 = ', cs[1]) - - def __set_max_correction_matrice_storage(self, m: int): - """ - Sets the parameters for the wolfe conditions. - - Parameters - ---------- - m: int - The maximum number of correction matrice for calculating the - inverse hessian used and stored. - """ - if(m == int(m)): - self._m = m - self._S = np.zeros(self._n_parameters, m) - self._Y = np.zeros(self._n_parameters, m) - else: - print('Invalid value of m!!!\nm must be an integer') - print('using default parameters: m = ', self._m) - - def xbest(self): - """ See :meth:`Optimiser.xbest()`. """ - return self._xbest - - def __objective_function(self, point_alpha: float): - ''' - For a given alpha this returns the values of the objective function. - ''' - #point_alpha = self._current + alpha * self._px - fs_alpha = self._evaluator.evaluate([point_alpha]) - f_alpha, dfdx_alpha = fs_alpha[0] - - return f_alpha - - def __gradients_function(self, point_alpha: float): - ''' - For a given alpha this returns the values of the objective - functions derivative. - ''' - #point_alpha = self._current + alpha * self._px - fs_alpha = self._evaluator.evaluate([point_alpha]) - f_alpha, dfdx_alpha = fs_alpha[0] - - # dfdalpha = np.matmul(np.transpose(dfdx_alpha), self._px) - - # print('dfdalpha: ', dfdalpha) - # print('type dfdalpha: ', type(dfdalpha[0])) - - return dfdx_alpha - - def ask(self): - """ See :meth:`Optimiser.ask()`. """ - - # print('') - # print('in ask') - # print('') - - if not self._running: - self._proposed = np.asarray(self._x0) - else: - # line search using an algorithm from scipy to meet wolfe condtions - results = line_search_wolfe2(f=self.__objective_function, - myfprime=self.__gradients_function, - xk=self._current, pk=self._px, - gfk=self._current_dfdx, - old_fval=self._current_f, - old_old_fval=self._previous_f, - c1=self._c1, c2=self._c2, maxiter=50) - - # line_search_BFGS below only checks the Armijo rule, - # therefore it doesn't ensure the gradient is decreasing, - # this can cause problems with the BFGS algorithm as - # the Hessians from this approach may not be positive definite. - - # results = line_search_BFGS(f=self.__objective_function, - # xk=self._current, - # pk=self._px, gfk=self._current_dfdx, - # old_fval=self._current_f, c1=self._c1, - # alpha0=self.__current_alpha) - - # results = line_search_wolfe1(f=self.__objective_function, - # fprime=self.__gradients_function, - # xk=self._current, pk=self._px, - # gfk=self._current_dfdx, - # old_fval=self._current_f, - # old_old_fval=self._previous_f, - # c1=self._c1, c2=self._c2) - - self._proposed_alpha = results[0] - # print('alpha: ', self._proposed_alpha) - self._proposed = self._current + self._proposed_alpha * self._px - - # Running, and ready for tell now - self._ready_for_tell = True - self._running = True - return [self._proposed] - - def tell(self, reply): - """ See :meth:`Optimiser.tell()`. """ - - # Check ask-tell pattern - if not self._ready_for_tell: - raise Exception('ask() not called before tell()') - self._ready_for_tell = False - - # Unpack reply - proposed_f, proposed_dfdx = reply[0] - proposed_f = proposed_f - proposed_dfdx = np.asarray(proposed_dfdx) - - # We need the evaluation of the gradients before we can start the BFGS, - # the first tell gets these. - if self._current_f is None: - - # Move to proposed point - self._current = self._proposed - self._current_f = np.asarray(proposed_f) - self._current_dfdx = np.asarray(proposed_dfdx) - - # Update newton direction - # FIXME: is this right for inital newton direction??? - # if it isn't a desecnt direction the line searches will fail - # i.e return alpha = none - self._px = - np.matmul(self._B, self._current_dfdx) - - # If wolfe conditions meet the line search is stopped - # and the hessian matrix and newton direction are updated by the - # L-BFGS/BFGS approximation of the hessian described in reference [1] - # [2], and [3]. If the line search has converged we also accept the - # steps and update. - else: - - # identity matrix - I = np.identity(self._n_parameters) - - # We do this if we haven't exhausted existing memory yet, this is - # identical to the BFGS algorithm - if self.__k <= self._m - 1: - k = self.__k - # Defining the next column. - self._S[:, k] = self._proposed - self._current - self._Y[:, k] = proposed_dfdx - self._current_dfdx - - # Defining B_0. Scaling taken from [3]. - self._B = ((np.matmul(np.transpose(self._Y[:, k]), - self._S[:, k]) - / (norm(self._Y[:, k], ord=2) ** 2)) * I) - - # Updating inverse hessian. - for k in range(self.__k + 1): - - V = (I - np.matmul(self._Y[:, k], - np.transpose(self._S[:, k])) - / np.matmul(np.transpose(self._Y[:, k]), - self._S[:, k])) - - self._B = np.matmul(np.transpose(V), np.matmul(self._B, V)) - self._B += (np.matmul(self._S[:, k], - np.transpose(self._S[:, k])) - / np.matmul(np.transpose(self._Y[:, k]), - self._S[:, k])) - - # We have exhausted the limited memory and now enter - # the LM-BFGS algorithm - else: - - m = self._m - 1 - # Shifting everything one column to the left. - self._S[:, 0:m] = self._S[:, 1:self._m] - self._Y[:, 0:m] = self._Y[:, 1:self._m] - - # Renewing last column. - self._S[:, m] = self._proposed - self._current - self._Y[:, m] = proposed_dfdx - self._current_dfdx - - # Defining B_0. Scaling taken from [3]. - self._B = ((np.matmul(np.transpose(self._Y[:, m]), - self._S[:, m]) - / (norm(self._Y[:, m], ord=2) ** 2)) * I) - - # Updating inverse hessian. - for k in range(self._m): - - V = (I - np.matmul(self._Y[:, k], - np.transpose(self._S[:, k])) - / np.matmul(np.transpose(self._Y[:, k]), - self._S[:, k])) - - self._B = np.matmul(np.transpose(V), np.matmul(self._B, V)) - self._B += (np.matmul(self._S[:, k], - np.transpose(self._S[:, k])) - / np.matmul(np.transpose(self._Y[:, k]), - self._S[:, k])) - - self._B = ((np.matmul(np.transpose(self._Y[:, m]), - self._S[:, m]) - / (norm(self._Y[:, m], ord=2)**2)) * I) - - # Move to proposed point - self._previous_f = self._current_f - self._current = self._proposed - self._current_f = np.asarray(proposed_f) - self._current_dfdx = np.asarray(proposed_dfdx) - - # storing the accepted value of alpha - self.__current_alpha = self._proposed_alpha - - # Update newton direction - self._px = - np.matmul(self._B, self._current_dfdx) - - # incrementing the number of accepted steps - self.__k += 1 - - # Checking if all gradients ~ 0, - # therefore the classical convergence test of a quasi-newton - # or conjugate gradient method has been meet. - if self.__convergence is not True: - - if norm(proposed_dfdx, ord=np.inf) <= 1e-6: - - self.__convergence = True - print('') - print(20 * '*' + ' Convergence after ', - self.__k, ' accepted steps!' + 20 * '*') - print('||df/dx_i||inf <= 1e-6 with parameters:') - print(self._proposed) - print('error function evaluation: ', proposed_f) - print('\nInverse Hessian matrix:\n', self._B) - - # Update xbest and fbest - if self._fbest > proposed_f: - self._fbest = proposed_f - self._xbest = self._current - From ec2b7c709a9b47882b45cb4e566284b5a911cc82 Mon Sep 17 00:00:00 2001 From: alisterde Date: Wed, 12 Aug 2020 14:40:45 +0100 Subject: [PATCH 08/16] seperating linsearch --- examples/optimisation/bfgs_trial.ipynb | 368 ++++++++++ pints/_optimisers/__init__.py | 903 ++++++++++++++++++++++++ pints/_optimisers/_bfgs_linesearch.py | 938 +------------------------ 3 files changed, 1288 insertions(+), 921 deletions(-) create mode 100644 examples/optimisation/bfgs_trial.ipynb diff --git a/examples/optimisation/bfgs_trial.ipynb b/examples/optimisation/bfgs_trial.ipynb new file mode 100644 index 000000000..954e4db29 --- /dev/null +++ b/examples/optimisation/bfgs_trial.ipynb @@ -0,0 +1,368 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "os.chdir(\"../..\")\n", + "import pints\n", + "import pints.toy" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": "con1 : False\ncon2 : False\ncon3 : False\ncon1 changed to true\ncon1 : True\n\n******************************\ncon2 changed to true\ncon2 : True\n\n******************************\ncon3 changed to true\ncon3 : True\n\n******************************\nfinsihed while\ncon1 : True\ncon2 : True\ncon3 : True\n" + } + ], + "source": [ + "con1 = False\n", + "con2 = False\n", + "con3 = False\n", + "counter = 0\n", + "\n", + "while (con1 is not True and\n", + " con2 is not True and\n", + " con3 is not True):\n", + "\n", + " print('con1 :', con1)\n", + " print('con2 :', con2)\n", + " print('con3 :', con3)\n", + "\n", + " if counter == 0 and con1 is not True:\n", + " print('con1 changed to true')\n", + " con1 = True\n", + " print('con1 :', con1)\n", + " print('')\n", + " print('*'*30)\n", + " counter += 1\n", + " if counter == 1 and con2 is not True:\n", + " print('con2 changed to true')\n", + " con2 = True\n", + " print('con2 :', con2)\n", + " print('')\n", + " print('*'*30)\n", + " counter += 1\n", + " if counter == 2 and con3 is not True:\n", + " print('con3 changed to true')\n", + " con3 = True\n", + " print('con3 :', con3)\n", + " print('')\n", + " print('*'*30)\n", + " counter += 1\n", + "\n", + "print('finsihed while')\n", + "print('con1 :', con1)\n", + "print('con2 :', con2)\n", + "print('con3 :', con3)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "#specifing a simple model for BFGS line search trial\n", + "\n", + "# dzdt = ax^2 +by^2\n", + "# dz/dxdt = 2ax\n", + "# dz/dydt = 2by\n", + "# starting conditions z=0, y=0, x=0\n", + "# x and y increase by 1 in each step?\n", + "\n", + "class easyProblem(pints.ForwardModelS1):\n", + " \"\"\"\n", + " A model with parameters of a similar magantuide\n", + " \"\"\"\n", + "\n", + " def __init__(self):\n", + " super(easyProblem, self).__init__()\n", + "\n", + " def n_parameters(self):\n", + " return 2\n", + " def n_outputs(self):\n", + " return 1\n", + "\n", + " def _simulate(self, parameters, times, senstivities = False):\n", + "\n", + " # unpacking parameters\n", + " a = parameters[0]\n", + " b = parameters[1]\n", + " #ax^2 +by^2 where x= sin(times) and y=cos(time)\n", + " time_dim = len(times)\n", + " time_step = float(times[1])\n", + " x = np.sin(times)\n", + " y = np.cos(times)\n", + " dzdt = a*np.square(x) + b*np.square(y)\n", + " if senstivities is True:\n", + "\n", + " # creating senetivity matrix\n", + " sen = np.zeros((time_dim,2,3))\n", + " for i, xi in enumerate(x):\n", + " old_sen = sen[i,:,:]\n", + "\n", + " if i < time_dim:\n", + " # df/dtheta\n", + " dfdtheta = np.asarray([[0,0],\n", + " [0,0],\n", + " [xi**2, y[i]**2]])\n", + " # jacobian df/du\n", + " Jac = np.array([[0,1,0],\n", + " [-1,0,0],\n", + " [2*a*xi, 2*b*y[i], 0]])\n", + " temp = np.matmul(old_sen, np.transpose(Jac)) + np.transpose(dfdtheta)\n", + " new_sen = temp*time_step + old_sen\n", + "\n", + " # assigning the new sentivity to the right place in the senstivity array\n", + " sen[i,:,:] = new_sen\n", + "\n", + " return (dzdt, sen[:,:,-1])\n", + "\n", + " else:\n", + " return dzdt\n", + " \n", + " def simulate(self, parameters, times):\n", + " return self._simulate(parameters, times, senstivities = False)\n", + "\n", + " def simulateS1(self, parameters, times):\n", + " return self._simulate(parameters, times, senstivities = True)\n", + "\n", + " def suggested_parameters(self):\n", + " \"\"\"suggested parameters for the model [a, b]\"\"\"\n", + " return [3, 9]\n", + " def suggested_times(self):\n", + " \"\"\"suggested times for the model range 0 to 10\"\"\"\n", + " return np.linspace(0, 10, num=100000)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# making some toy data\n", + "model = easyProblem()\n", + "times = model.suggested_times()\n", + "real_parameters = model.suggested_parameters()\n", + "output = model.simulateS1(real_parameters, times)#\n", + "values = output[0]\n", + "sensetivities = output[1]" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": "Minimising error measure\nUsing Covariance Matrix Adaptation Evolution Strategy (CMA-ES)\nRunning in sequential mode.\nPopulation size: 6\nIter. Eval. Best Time m:s\n0 6 1.03e+07 0:00.1\n1 12 1.01e+07 0:00.1\n2 18 1.01e+07 0:00.1\n3 24 1.01e+07 0:00.2\n20 126 1e+07 0:00.7\n40 246 1e+07 0:01.5\n60 366 1e+07 0:02.2\n80 486 1e+07 0:02.8\n100 606 1e+07 0:03.5\n120 726 1e+07 0:04.1\n140 846 1e+07 0:04.7\n160 966 1e+07 0:05.3\n180 1086 1e+07 0:06.0\n200 1206 1e+07 0:06.7\n220 1326 1e+07 0:07.5\n240 1446 1e+07 0:08.1\n260 1566 1e+07 0:08.9\n280 1686 1e+07 0:09.6\n300 1806 1e+07 0:10.2\n320 1926 1e+07 0:10.9\n340 2046 1e+07 0:11.5\n360 2166 1e+07 0:12.1\n380 2286 1e+07 0:12.7\n393 2358 1e+07 0:13.1\nHalting: No significant change for 200 iterations.\nScore at true solution: \n10040056.240084708\nFound solution: True parameters:\n 3.02438339552395208e+00 3.00000000000000000e+00\n 8.94910569555093005e+00 9.00000000000000000e+00\n" + }, + { + "output_type": "display_data", + "data": { + "text/plain": "
", + "image/svg+xml": "\n\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEGCAYAAACO8lkDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd5xU1fn48c8zs42lL6wgLLB0pAtLkSYIIgiR2IiKJRpDEjUaY2LQfI0latSfJdEkGk1sCYoNFUVA6SDNpYkU6b0tIGWBhd2Z8/tjZpbZnbIzszNzZ3ee9+vFi9k7d+6cafc55TnnijEGpZRSypvN6gIopZRKPBoclFJK+dDgoJRSyocGB6WUUj40OCillPKRYnUBoqFhw4YmNzfX6mIopVSVsnz58kPGmGx/91WL4JCbm0t+fr7VxVBKqSpFRHYEuk+7lZRSSvnQ4KCUUsqHBgellFI+NDgopZTyocFBKaWUDw0OSimlfGhwUEop5UODg1JJoPBMCZ+u2mN1MVQVosEhzrYUFHLyTInVxVBJ5oHJa7hn0iq+23PM6qKoKkKDQ5wNfW4et735jdXFUElm/7HTAJw667C4JKqq0OBggaXbjlhdBKWUCkqDg1JKKR8aHJRSSvnQ4KAS0tPTN9D3yVlWF0OppFUtluxW1c/Lc7dYXQSlkpq2HJRKIsYYq4ugvBQ7nBw5edbqYvilwUHF3OaDhXyQv8vqYiQ1QawugvLjDx9+S48/f4XDmXhBW7uVVMxd9tf5OJyGa/OaWV0UpRLKZ9/uBcDhNNhtiRXAteWgYi6UWtHmg4WVnr17psTB8aLiSh1DKeWiwcEi6/Yep1CX0Sg17Pl5jH5pYaWOccNrS+n6yJdRKlH1Yki8bgt1TiJ+PhocLHL5iwsY/3a+1cWoVpbv+MHqIsTdiaJi9rmXxgiFSGJ1XSS7RB4L0uBgofztyXcyq4xjp4qrbLbNu8t2smTr4agf90cvLeSiv8yO+nGjZfWuo5wp0fWcKpKIX2sNDhEyxrB4y+Eqe7KqavYcPU23x77k1flbrS5KRB6YvIbrXl0S9eNuP3wq6seMlh2HTzLmH1/z6GfrrC5KWE6fdcQke8jhNMzZcLDsOSNxGw4aHCI1Y+1+rn9tCf9dssPqolQpB08UcbbEWWZbKIPIe35wdZ3MXH8gJuWKpy0FhazeddSS545nZeboKdfnWtWWCb/gT9P5zXuron7c/yzcyq1vfsOMtfujfuxY0OAQod3uk9X2Q5HX3BJxECqWjDH0fmIWv3lvJcYYZq47gMNpGPrcvJg/99kSJw9/+h0/JMCEo6HPzWPMP76O63N6922v33c8YSdeJYrPVu+N+jF3HXGdMw6eOFO6zZmA8xs8kj44fLfnGFNW741Jf7Dy74s1+5m6Zh+3v53P6wu3UeD1Y4mVz7/dy1uLd/DkF+tj/lyJbuTfFjD6xQVRO55m3UWuxB0cFm46ZHFJfCV1cHA4DaNfWsjd764M2h/87IzvGfrc3PgVLEYWbT7Ehv3HK3WMmesOMPXbfZUuy8HjroCw52jomTaV4elDTuCKWlztPVbks+3U2RIe/Wwtp8O4INBnq/fS+eEZfruOquJbXexwVrxTDBwqPFdBcjhNQgRcy4ODiNhFZKWIfO7+u6WILBWRzSLynoikxeq5Q+1//fuczWwpOBmrYlRascMZ0mu54d9LGfHXytUYb387nzvfWRHy/l9vTrwakfLv1flbeePr7bz+9bYy23MnTOWJqf4HledvLABc83YCSeAxVx9WXSnP+9f7x4/X0PnhGZYvqWF5cADuAbzb+k8DLxhj2gA/AD+zpFRxEOh8frbE6TNoG4jTaWj7x2kJmxEy7t9L/W6virXKRPNlBAOb6/cFPol7Tkb+TkqvLdjms02dY4xh1voDZSpp8zcW8KdPvwvp8YVFJex1t6I/XL4bgAPHfVt38WRpcBCRHGAU8G/33wJcAnzo3uUt4MfWlC4yxhjmbyyo1EBTh4em0efJmSHt6+mznLg0tllTx04XR7XJfcKdoaSXTI3Moi2HGP/f5WE/7pEAlYjNB0+UVla+3X2Mo6fCG7BOtuSK8lo+8AU/eyufd5edW2Dy5teX8fZi39+lv0rhE1+sp99Trvkqnt/0z96ydpKs1S2HvwL3A56zTgPgqDHG0+G2G2jq74EiMl5E8kUkv6CgIKInDzZbNHfCVB6ZstZn+3d7jvlNRVu96ygbD5zgq3UHuPn1ZfxnYeQ1LaeBH04l1hpB3R79krvfXRm143myZ4LVZENRVOwoDTTJ5GiQ78dbi7aTO2EqRcXnukj8nbx3/3AKp9OwcucPDHt+fml30sz1B7j2lcWux1XQXRlswnV1nAN0xd8X8o85mwPe72+2+uwNrvTrcCenxyNRIxjLgoOIjAYOGmPCr/4AxphXjTF5xpi87OzsKJfO5c1F2322jX5pIb/wU2Mb84+vGf7CfCZ946o57DgSuzEKh9OVBhrqj+/U2RLe+Lry3QLTvqtcfvas9Qcjfmyg1zr0uXl00fWUyvCcvI6dDh40Bzw9hzsmrmDnEVc6tnd/+6aDhfR/ajZbCgorX6BqsmTHd3uO8e3uY/y/Gd+H9bjb3oxNC+DU2RKenfF9yF3Q4bKy5dAfuEJEtgOTcHUn/Q2oJyKepcRzgD3WFC8yszdEfgIM1V9nbuT2t/P5Ys3+kJrzT03bkBBjEvleax9FWtsvvxZNvLKdqhJ/PZqB1vCZHmTcYs/R03y0IrSfn5WNhM0HT5A7YWrM09ErszDkloJCdrmDcLHDGZXVg/8+ezN/n7OZSd/srPSx/LEsOBhjHjDG5BhjcoHrgNnGmHHAHOAa9263AJ/Gqgzlfy6bD56I2rELi0oqrLlF+nt6abarZlhw4tyAVbAFvIJ1QcTTK/POXfoz2I8j2Pt21qJUw0QT6NOev7GgTFpk7MtRuVbB/I0FPDB5TaWO8fVmV1D4Yk3lU6xjZehz85jzvav7+/Gp66OyevAZd4uhOrYcAvkD8FsR2YxrDOI/8XriYc/PZ9uh8LuD/KXxfbJqL90e1e6OcD3/1Ubue3916d9vlevaWxXishPLth2J2Y8m3tbvO84/557r5/5+/wl+NdE3nXjZtiNM++7cCfLqlxfxo0oug+6vRXCmxHftIX8VnVAqPze/vox3l8Wm5ltU7Ahr3CNQSviJomJmhblsS7RWCHY6DZ3+NN0n4WTZtiOVGtcMRUIEB2PMXGPMaPftrcaY3saYNsaYa40xcR2VCTYI5J2ts/3wuSByeRRnm0bqrMPJ8BfmsTMBFmI7euosuROm8nqQL+/eo/7T9F6ctYk1e84FgIenBJ6UFeiz2rD/OGP/tThgbr6VVu4M/6Txo5cW8sz0c/3c4//r24f91boDjP3X4jLZMrt/OM2aPccqbBEHO3+W+Gmptf+/6dwx0TXuFspwghUjDrt/OEWHh6bzk38t4b0Qu13a/nEad73jm3Rx73urw8ocWrv3GFe/vCjk/QMzlDgNJ886fJJj/jIt9jP9EyI4JBrv2Yn7vWaStv3jtNLb8comCqf2s/FAoc8EJojveGCHh6bR/bGvAHjs88An571BxgoOHC970r/gT9P5ZFXZvu+iYgefrvLfH36k0JWGufFA+IOpxhg+Wr67TKZPqA4eLyrtV/b4fv+JMp/flf/0f9LYWlAY8HMuCSEtOtich28qsTT8vwME+BlrE3sBxK3uSavLth/hDx+F3m210M+kzR2HfXsTPlu9l5tfX+b3GIcKY7tu1X8WbmPlztgv3KjBoRxjTJmJK7e//U3Q/f2djP255Lm53PBa2SU6jDEYY3hg8hq/q3QWnDhDh4em89qCwMtUx3ogsMTh9Ltcxv99sobLXpjPI1PWllmvv6g4tK6ccIvtXSMGeHzqOh6fGrz2dKrYEfa1BOZvOsR9H6zmqWkb/N6/8cCJgEsb9H5yFgOfmVP69+Ith7nsr/P5XwUr967adZRLnptHywe+4Kt1rpPu7h9O8fO38/mFn1aCPx+4J04Fst3PCS5cwVYgNib0ReQ2BXkPrWCMCXvNrV+/u7J0dnikwlmttthhePjT7zDG8Odyla5YnQOSOjj4q1G/vWQHh70i/4ki/1/icNcX2lpwkkVbfLMpjp0u5t1lO0trId41T0/O9Ger4z/QtrWgkEemrKXNH6f5neX5vyU7+f7ACd5ctJ1PVoafUFbZHPidRyrOUlq96yjDX5jP59/uZcP+0JINPFlUgbqshr8wn5+9GbzC4OGpcX63x/9cjlNnS9h88ESZmulT7u6CAU/P4at1B8rU0PccPc2psyWUOMJ/7w4G6S5dui20LJ+HPvH9Hnh+Qw9+vIZWD35R5vsb6CO+9IX53PZGaO9hKPx9l8I5cR88cSbgdUIKz5SEVZE5fPIsf/jw25D2DTf76a3FO0pXg46HpA4O/izecrjMYNuOOPfhhzMLOdQvbSSTtUf+bUHpPI/DFSzvHMl5vjIXqTl91hHyj3/H4VPc9c7K0sG7j1bsrnRgitas7vFvL2fY8/NxhlieXUdO0fFPM6Kevlu+VVYZmw/6duX5q4Qt2x76e/j45+vInTC14gl5XrcDdYf5E6hys/HACTo/PMPva/IoX4l4Z+lO9ldi2YvyrQkrZ55rcCjnyMmzfvsdrVD+t7Dn6OmIcrkjWZv+TASZPvfG4AIp/ny43Pdk5m/gNJBQzsXx+FEu2uL6nm0LcVHHeyZFb4Z69IQ2oPXfxdvZeCCyVHHPiT5Wc4j+EqALcfgL8yt8bK8nQlvmJlSVmUsRbRocYix3wlQe/cx3GQ5w1fxDOVF5al5Dnp3rs7R4IqxQ4Jl78HEE3UuR8Be42nglC4Rj04ETZfLjg+XtB6u5hppi68+LswMvx+Ct/EB9qA5FcRkG75Tag8eLfFoFr3+9jdwJUzl1tmx37EOfrmXEXys+2QYT6nhWdXGo8CzvfRO9Vl24NDjEwRtfb69wn2AZRYVFJfx38faEzduPpA+8MioaiA7HpS/M5w4/cwaMcXXxFRU72HXkFLkTprIiSIbIxhDHNCqypeBk1CexPffVxqgdyzultveTszhebsLiAvdFa1xXmiv7vQjWvRmteQGB9PvLrNKWWlXyp0/LVizjWRnU4JDAPN+DrYdO8tCnvq2P9fuO+1y8JxqLnflbcDBZeAfpy/46nw4PTWeee3zDX3eWx14/C655BOqiCnSyzHs8ul0VsfR5CIkZgeo93vMv3lla8VyEP36yJuKJX3uPFfH09PDWREpEnkX8vMWqC1SDg8Uq87G+n787YN58ZfhbcLAqqeg9XbTlMEOfm1vhXIat5cYCPHHXXyvvrzM3ld72pGlWk/XmYmbexnM1+cIzZVsg3+055rNawdFTxfz583V8t+cYuROmctc7K/jHnM0h/4ZW7zrKocIzll8noTICLbkeCxocLOap6R89VUxRsYMbXlvqc1+4vt58KKKFvQ4XnmHk3yKb7Z0fRvaJ1W78z1K2FJwsc/Ivn6Mf7K0vf84vn7HS+eEZPrnoypf3ezRj7YHSSY2ey/cOeXau38d5kjI+/3ZfmRVSgy3B75H3+Ez6PDmrEqX2lTthalSPlyg0OFio/Anom+1HKpUGB64+33H/Xkq/v8wOuI/3tSq2FhTywlcbMcbw4MdrIrq+wmOfr+Ma9/r/VYn3ejWtHvwC8N8FUlGI9pexEut1bxLdsOfnlc7PWek1WL8iyPIh90xyZbt5TxqM5Tjbp6v2+J39rFxSKt5FxZL3GkObIljuobyt7qZ4RTNQ31y0nTcXbadBzTQOnzxL/zYNky4b5HSIS2R41ubxrpj+9v1VNKlbg99d1j7g48JZtqG6KSp2lnZPeleCrgqhG9R7Xs3fZ28KsqeLZzn6L9fu55ErOoVcxnsmrSIzzR7y/olKZ0jHQDxnGwbyo7+fy2suvxZRPBITPD/Esf+qejX/aPNeR8t7kM8zw/ncmIMwecUe/h7kimAqct4TQY+EcbnSvceKKDhxJqw5L6cCLOpYlcTqPJHUwSHQ0hgqOUwudyGbvn+ZVcFlL13/hzvOnAhzUaqSUxW0eoO9n72emMnDSZZtF+oM+3AldXBIdPHuD51XyYXEqoNf/s93zoNHpNlHFS2Kp8ryPtVFcjGhUNJrq5NX5m6peKcIaHBIYPe+t7rinZSPcf9eWvFOFdDavnX2HatcUkZFV2Csbo7HqAdEg0M1V13T7KwUyvUVVOQ8y5YDODRKW0aDQzXm7/KlKnL+zlMafGNrjp/F9qIxI/iqf35d6WNUdxocqrFEuHxpVeXv9PNevnWLoCWL8pd2rWwXUyDB1slSLhoclPLjaBgplCp6XltQ8eTBSC7/qsKnwUEpPypz3WUVWx9q9ldcaHBQSinlQ4ODUkopHxoclFJK+dDgoJRSyocGB6WUUj40OCillPKhwUEppZSPpA4Ov31/ldVFUEqphJTUwWHD/hNWF0EppRJSUgcHpZRS/mlwUEop5UODg1JKKR8aHJRSSvmwLDiISDMRmSMi60RkrYjc496eJSJficgm9//1rSqjUkolKytbDiXAfcaYjkBf4E4R6QhMAGYZY9oCs9x/K6WUiiPLgoMxZp8xZoX79glgPdAUGAO85d7tLeDH1pRQKaWSV0KMOYhILnAhsBRoZIzZ575rP9AowGPGi0i+iOQXFBTEpZxKKZUsLA8OIlIL+Aj4jTHmuPd9xhiD/8v5Yox51RiTZ4zJy87OjkNJlVIqeVgaHEQkFVdgmGiMmezefEBEznfffz5w0KryKaVUsrIyW0mA/wDrjTHPe901BbjFffsW4NN4l00ppZJdioXP3R+4CVgjIp4V8B4EngLeF5GfATuAsRaVTymlkpZlwcEYsxCQAHcPjWdZlFJKlWX5gLRSSqnEo8FBKaWUDw0OSimlfGhwUEop5UODg1JKKR8aHJRSSvnQ4KCUUsqHBgellFI+NDgopZTyocFBKaWUDw0OSimlfGhwUEop5UODg1JKKR8aHJRSSvnQ4KCUUsqHBgellFI+kjw4GKsLoJRSCSnkK8GJSKYx5lQsCxNvI2zf8FLqS5wmjSLSOWFqsNc0YK9pyA5zHutMC9Y6cymgvtVFrZLSKKad7KKLbRutZS9N5RA5UkB9KSSTImpyBidCEWkUkUaBqct+k8Ve04CNphnrnC343uRwmgyrX0qVVJ/jdLFto5PsoJkcIEcO0UQOU1tOUYMzpFNMMSkUkcZJk8F+sthvsthpzmO9sznrTAt2mEaYZK9DRkBw0lL200W20s62mxz3dz+bo2TKGTI5gx0nRaRyhjSOmNrsN1nsM1lsMU1YZ1qw3tmcH6hj2WuoMDiISD/g30AtoLmIdAN+YYy5I9aFi7VtpjH/coymBmepwRnqSiFN5AiDbatoJEdL99trsljs7MRCR2e+dnbmoAYLv1IoobtsZqB9DYNsa+gk20gTBwCnTDp7TEN2m4ZsNM0oNBmcJgPBSTrFZHKGbDlKMznIRbZ11JbTADiMsMa0YpGzE4ucnfjG2Z4zpFn5MhNWfY7T37aWAbY19LetpZmtoPS+AlOHPaYhG00Ox5w1OU06Z0gllRLSKaa2nOJ8OUJ32cwo2xJSUpwAHDeZLHFewCJnJxY6O7PZNCXw1X2TmaGt7GGQ7VsG2tbQ07ax9DtcbOzsM1nsNtkspx0nnRmcJAODjXTOks5ZGsgJGssROtm2cb3MKT3qFuf5LHZ2ZJGzE187O3OMWnF7RWJM8K4VEVkKXANMMcZc6N72nTGmcxzKF5K8vDyTn58f9uNyJ0wNeF9tTnGB7KCTbTs9bRvpZ1tLlhQCsMLZhqmOPkx39GYP2RGXuzpIpYQBtjWMti/mUtty6shpHEZYZdqwzNmBNc6WrDEt2WXOI/STiiFHDnGB7KCLbSsX2dbRXbaQKg5OmBrMcXbnC0cf5jm7Jn2rIpujjLQvZbR9CXmyEZsYjplMFjs7scLZhjWmFWuduRynZsjHTOcsbWQPHW076CGb6GdbSwvbQQC2Ohsz3dmbLxy9+c60JLkDhaGbbGG0fQmX25fSVA4DsMnZlMXOjqwxLVnjbMVm04SS0DtpyOI4F9h20Em208e2gT629dSSIoqNncXOjkx39uZLRx6HqFv6mO1PjYroFYjIcmNMnt/7QgkOxpg+IrLSKzisNsZ0i6g0MRCL4FCe4OQC2clg2ypG2pfRxbYdgHxnO953XMxUR19OUiPsMlRNhgtlM2PtcxlpX0Y9Ockxk8kMRy9mOS9ksbNTWCejUGRSRB/beobb8hluz6eBnOCkSecLRx/edwzmG9OeZDlRZXCG0fYlXGVbQB/beuxi2OBsxjRHb+Y5u7HGtMSBParPmSMFXGxbzQjbMi6yrSNFnGxzNuIDx2A+cgzkAFlRfb5E1kwO8BP7XK6wLaK5rYCzxs48Zze+cvZkgaMr+2gQ1edLoYSuspVh9hWMtC2lpe0ADiMsdHbhI8dAvnTmseGpqyI6dmWDw4fA88DfgT7APUCeMea6iEoTA/EIDuU1lwNcblvK1fYFtLXt4aRJZ6qjL+87Lia/mp6oanKaH9u/Zpx9Fh1tOzhp0pnh7MVnjotY6OxCcRi1o8qw46C3bQNX2BYx2r6E2nLafaK6mA8dF1fbbr+2spsb7LO4yr6AunKKbc5GTHH243PHRWwyOXErRz1OcKl9OVfbF9DXth6HEeY7u/K+YzAznT3j9j2IpxRKGGpbwTj7LAbZ15SenD9zXsSXjryoV4YCM3SQXYyyL+FK+0Jy5BD/LRnGTY9/FNHRKhscGgJ/A4bhOuN9CdxjjDkcUWliwIrgcI6hh2ziWvs8fmRfTC0pYpOzKe86LmGyYwBHqR2F57BWJ9nGOPssxti/pqacYa2zBRMdw/jU0c/y1lINihhpW8a19vlcZF9HibExx3kh7zqGMNfZHWcVH0xN5yyX25ZyQ8osetk2csakMN3Zm3dKhrLUdMDqSkhzOcA19nlcY59PEznCIVOHDx2DeM8xhG3mfEvLFg1NKeC6lDmMtc+lkRxlr8nivZIhvOcYzP4otxDCJTjpY9tAganLrL+Mj+wYlQkOVYG1weGcTIoYZV/C9fbZ9LBt5oxJZZqzF+8myA85HBmc4Uf2xYyzz6S7bSunTRqfOS5iomMoq01rEvG1tJD9/MQ+l2vt88iWY+w1WXzgGMz7JRdXubGhXNnHOPssrrXPo56cZIvzfN51XMJHjoGWZrAEYsPJINu3XGefwzDbclLEyRLnBbxTcgkznL2qVBKBDSeDbau40T6TwbbVGGCuszvvOC5hrrN71LvsosGqMYc38DMhwBhzW0SliYFECQ7eOshOrrPP5ir7QurIqYT/cXu0kd2Ms8/iavsC6sgpNjqbMtExjI8dA+LYdK4cVxfASq63z2aQ7VsA5ju78q5jCLOcPcIaHIwnOw6G2VYwzj6TQfY1FBs7M5y9mOgYymJnRxIxIPuTzQ9ca5/PT+xzaGE7yA+mFh87BvCu45K4dn+FqwHH+Il9LjekzCJHDnHA1GOSYwjvlQxhLw2tLl5QVgWHq73+zACuBPYaY+6OqDQxkIjBwSODM4yyLeX6lNnkubsFZjh78a7jkoT5wadSwgjbMsalzKKvbT1njZ1pzj5MLBnKsirW4imvKQWMTZnHtfa5NJEjFJi6fOC4mEmOIew0jawuHgCNOcxY+zyuT5nN+XKEPaYB75ZcwnuOIRRQz+riRUxw0s+2luvtcxhu+4Y0cZDvbMckxxA+d/SliHSriwgYessGxqXMYqRtKWni4GtHJ/7nGMZXzp4JW5Eoz5Lg4OdgNmChMaZfRKWJgUQODt7aym6ut8/mavv80gHFSY5L+NAxiMNeaWnx4RorudK+kNH2JdSXQnY6s3nHMZQPHBdbUJ7YsuHkYttqbrDPZohtJSni5GtHJyY5hjDD2YuzpMa1PDUoYoTtG66yL6C/bS02Mcx3dOG/jkuZ7bwwIbsuKiOL41xlX8D19tm0tu3juKnBJ44BTHIMYZ3JjXt5mssBrrYv4ErbAprbCjhuMvnQMYiJjqFsMU3jXp7KSpTg0B6YaoxpE1FpYqCqBAePdM4y0raM61Nm08e2gbPGzixnD2Y4ejHX2S1mg9g2nHSXzQy1r2CUbSm5tgOcNml86cxjsmMg851dkmI27Hn8wLX2eVxnn0MzWwFHTC0+dgxkuqMXK0zbmJ2Ya3GKwbbVDLfnc4ltJbWkiJ3ObCY7BzLZMTBhWjKx5aqpX5cyh1G2paRLMaudrZjsGMgs54XsNufF7JlbyV4utS1nuD2fnrZNOI3wtbMTkx0Dme7sVaXnzFjVrXQC15iDuP/fDzxgjIksdyoGqlpw8NZa9nC9fTZj7IvIlmM4jJBv2rPQ0Znlph2rna0rkRFkaC17ybNtpLdtAxfbVtNQjlNibCx2duQTxwBmOPMoJDOqr6mqEJz0t63levssLrUtJ00cHDOZLHB2ZaGzMyudbdhkciLOeErnLF1lK71tG+hrW0cf23rSxMEhU4evHD352DGAb0z7pAjI/tShkCvtX3OdfTYX2HYBrglkc5zd+cbZnlXONpXqVsvmKL1sG+ht28BA2xpa2/YB8J0zl88dffnE0d/yjKNoSYiWQyKqysHBQ3DSRbYx1L6CobaVdJQd2MTgMMIW04QtpglbzfnsNtn8YGpzlFqcMamIO1egjpwkixM0lGO0lP20tu2lneymnpwE4IipxQJnV2Y5ejDX2ZXjcZyGXxXU5hT9bd8xxLaKIfZVnOdePqXQZLDOtGC7szHbTSP2mQYcoybHTSbFpGDHSQoO6kkhDeU4jeQHWste2spuWsq+0uVD1jubMd/ZlS8deaw0bat8im205co+13tvW0Uf23rSpQTAtdyKM4ftxvX+HzZ1OUZNCo2rwmTHQboU05DjNJRjNJVDtJXdtLXt4Xw5AsBJk85yZztmOnsw09Ez4QeXIxHX4CAiPYId1BizIqLSxEB1CA7l1eEk3W2b6WnbSEfZSSvZS3M5SKr7ZBPMYVObLaYJm51NWGXasNzZji2mCVV5YDm+DLmyn+6yhQttm+hg20Wu7C+z3lYgTiPsMtlsNDlsNDmscLYl31Qx1DUAAB/fSURBVNk+rmviVHXpnKWTbOdC2yYutG2hleyjheynppyp8LGnTRqbTFM2maasd7bgG2d71prcKjOwHKlYBIdg79hzQe4zwCURlUaF5Dg1me/sxnznuVVKUiihAcfJkhPUk0LSKca42w4nTCZHqM0RU4cTSdpNFD3CdnM+2835fOIcULo1kyIayQ/U5hR15SQpOHBgowQ7x0xNDpm6HKZOtT8RxdoZ0lhh2rHC0Q5K60KGbI5RTwqpSyG15TQGwYGNsyaVQ9ThkLtVoZWg6Aj4LTbGDIlnQVTFSkjhAFkcMFl6KQoLnCLj3Kxfff/jTCigHgXGPQah73/MhVTFEZHOQEc4N5xvjHk7VoVSSillrQpHxUTkYeAl978hwDPAFTEuFyIyQkS+F5HNIjIh1s+nlFLqnFBSJq4BhgL7jTG3At0gtjOkRMQO/AMYiavFcr2IdIzlcyqllDonlOBQZIxxAiUiUgc4CDSLbbHoDWw2xmw1xpwFJgFjYvycSiml3AIGBxH5h4gMAJaJSD3gNWA5sAJYHONyNQV2ef29273Nu3zjRSRfRPILCgpQSikVPcEGpDcC/w9oApwE3gUuBeoYY76NQ9mCMsa8CrwKrnkOFhdHKaWqlYAtB2PM34wxFwGDgMPA68B04EoRaRvjcu2hbNdVjnubUkqpOKhwzMEYs8MY87T7+tHXAz8GNsS4XN8AbUWkpYikAdcBU2L8nEoppdxCSWVNEZEfichEYBrwPRDZ1axDZIwpAe4CZgDrgfeNMWtj+ZxKKaXOCTjmICKX4mopXA4sw5UxNN4YczIeBTPGfAF8EY/nUkopVVawAekHgHeA+4wxP8SpPEoppRJAsLWVdGE9pZRKUrqovFJKKR9JHRxqZ+jSykop5U9SBwellFL+aXBQSinlQ4ODUkopHxoclFJK+dDgoJRSykdSBwe9DLlSSvmX3MFBNDwopZQ/SR0cLu3YyOoiKKVUQkrq4NC/TQOri6CUUgkpqYOD6KiDUkr5ldTBQSmllH9JHRx0PFoppfxL8uCg0UEppfxJ6uBg09iglFJ+JXVw0AFppZTyL6mDg1JKKf+SOjjokINSSvmX1MGhd8ssq4uglFIJKamDQ4qOSCullF9JHRyUUkr5l9TBQbOVlFLKv6QODkoppfzT4KCUUsqHBgellFI+NDgopZTyocFBJaT0FP1qquTx84EtrS6Cj6T+BdbKSLG6CNVS/czUSh+je7N6USiJUlVD/ZppVhfBR1IHB7tOglNKKb+SOjgAdMupa3URqh29Tkb1FI0WofIvEedcJX1wGNC2odVFUH4M9PO51EpP4esJl1hQmqqhWVaNiB/7m2FtK9yneVZmxMdXwSVifcqS4CAi/09ENojItyLysYjU87rvARHZLCLfi8hlsS7LvcPa8cEvL4r106gwjeraxGebAE3r1eCh0R3jX6Aq4IbeLawugrJAZSoFwVjVcvgK6GyM6QpsBB4AEJGOwHVAJ2AE8E8RsceyICl2G71y/a/OelGrBhU+ftgF50W7SEnhnZ/3KfP3yocuDfmxt/XPpXeAzyyZ+WttheLpq7vE5LgqNhrUTOO1m/NK/25YKz0mz2NJcDDGfGmMKXH/uQTIcd8eA0wyxpwxxmwDNgO9rSgjgMFY9dRVzpu39iq9fU3PnCB7uvRrfe6Ec9+l7cLK1hARsmvH5gdRlVzRzbd1FYlOTXzH3bJrp2s3UgQGtAk9kP7tuu4h71sz7Vwd+feXtY/LJY4TYczhNmCa+3ZTYJfXfbvd23yIyHgRyReR/IKCgqgW6O6hbbnqQr9Pq0IwNq/i4ACuMQQVnlYNa8bkuPVqlB1s/uaPw0qDfKcmdRnZuTHPXNMtJs9dnXjX6CsypntTFtw/hM/uGhD2cHQ8xihiFhxEZKaIfOfn3xivff4IlAATwz2+MeZVY0yeMSYvOzs7auW+a0gbfntpO57/SfcytdvAQvuUWmfH5kdtBX8TdtJT7IzNy2FI+9A/i1v6ufrIPV/0Gqmh9yCO6R6dWnNVUy9AxlBlW1I39m3BE1d2LrPNuBvOqXbh5Rt70r5x7TL3b39qVKWeszrIqV+2v79Gmp2m9VzbFtw/pMLHN8vKpEuCZkzGLDgYY4YZYzr7+fcpgIj8FBgNjDPG8zVkD9DM6zA57m0x9+L1F/LM1V353WXtS7fdNaQN7RrVquCRoXU9Vaf0zvPrnvtBrH54OA+N7kjfVlk8c0033rg19F5AU+6t+/LeQSE/dninxmx/alSZ5nYyqFsjNumkKXYb4/r4H9D2982ddd/FZf5+5uquMSiVtZpnZTKyc+Og+7TKruj8UJZ3pcaU/wF4eefnfXjmGv/vaYMYjTGUZ1W20gjgfuAKY8wpr7umANeJSLqItATaAsviUaYrujVhbK9mZbbZbFJaC7i6h/+uEn+fb5rdxrUh9LtXFQ9e3qH0dqq97Kmibo1UfjagZaWCn+exzbIyyW0QXj/3p3f1j/h5q6Lnx/rvpw713Z9694ColKN1uZNioBZNVZbbsCaDK2gJ10r3rZx0a+ZqCdTwU3F5fmx3Hry8Axv+PKLMb8b75/PRr/rRr3VDxuY183k8xC8Jxqoxh78DtYGvRGSViLwCYIxZC7wPrAOmA3caYxwWlRE41y7wfHieYFH+fm82G7Qt1+Jo36i2nz0De3lcj7D29ydQzaMyQqu5hnaqisZwf5vzwntfq7La6SllBu7Lv38VZa0MbNvQ7+BzIKO6umrNo0MY+M4KklCw8qFLq2TL4qXrL6xwH3+Vw+eu7c4nd/anYa10n646u00YP6g1GeW6UD2T4G4f0JKeLer7HLNsIIlPL4RV2UptjDHNjDHd3f9+6XXfE8aY1saY9saYacGOE0+ej6NrTt2IBlLvHto2rPkUI7uc77NtXJ/mYT1noJqHP0sfHBrWsVX82SpIUWlcN4NFEy7h7zdUfFILRZvzarP9qVG0C6FikxtkoLx+zTRahNkiTASRduHVSLOXrg0WqKuuKkiEbKUqZ87vBpfeDjWd0G4TeuVmVWpgullWJq/c2DPEfcObGNOoTga9W4Y2d6B2mAsW3tjXf1AL0uXqV9P6wV/TR78KbzLjf3/WO6RBw0QRrI/ao0m9GozyU7HwtuHPI1jzyPBoFQuo+LP01HZ75frWigMJ57cSSvr0gvuHMPu+i8NKfOgbwlynaAr1J5HbIPYJLhocIuCdGeIvSyTYOimp9tDfcn/ptGkp0W9SXtcr9BZG6+xaAcdfPMq/J7XSg9fAQmkl/3NcD97+WfDB7p4twpsYN7BtNs2inMv/2JhOUT1eJESE0V0DB4iMVDu1M1LpeH6dmJajsskCs+4bHPK+oVQ0mmVlhj2A3KJBzahlZf0kSEs+3J6iVtm1eP2noafNRkKDQyWl+Gnq3z+ivZ89XcIJDunuGk5tr26saC3QVT4FLxQdGtfm1ZvyKuzeCLU5Huokw1n3XczlXc7nvNoZIe0PrlmkVshIscdsxqrHn8d0YuLtffhpv9yYPk84/H2Wn989sNLjXufXPfeZd4vxMu6htsoj9XSUxwDr1ojtd1yDQwV+fUkb6mWm0sPPIJE/qx8ezq39fecBeE5W/wxjoNlTA68Tg/TFKXcN4IZyYxjeTfNJ4/vSPCuT4R0blfY5/2pwa+pGkJVSURAIFPAmje/Li9df6JMZU55N4MpyrayL3VkmmdUo1dXzLt50US792zSkZ4v6LHnA/1hRIqROt2xY02fcq3wN/6f9crmkQ2jZN5N+3jfg64XKLyORF0KX16NXlG0Zer+eF0MYwI5U6/PCa/FEgwaHCvRskcWqPw2nfqbr5F5RrdnD86W5fUBL1jwyvDTLJJxujLuGtOG5a7sxKkgXQSBpFbRQsmqm0aVp2cwV7x9y31YNmH//EF69OY/B7c9jzu8GM6Z7lGeNV9Bw6NuqQUhjOlv/MooXfuJK8RzZuTEjOzfmySu78J47wCWDSGJBB/ektnDHkPxyf5aBTtCByvfIFZ1KP7tgpt0zkBppdhrXzeCqC5tyYfOyrQiD4f1f9PV5nL+urQkjO/hsA1fZvwow1+axMZ24rFMjbumXW2YsxLviU5nlTDznl0BZX2/+tJff7bGkwSFEwy44j9v6tyxTc7ioVYMKO0ZsNqF2RvDa9n2XtvO7PS3FxtU9c8hzt1q6Nq1bYZbo578ewC8ubsXrYXyZQumvbRmlZRva+KkBRbOS+/KNPXn5xp5kpNrp06pBxDXoX1zcKuIyeJ7yx92b0KN5FLtCorzU15NXuQJoizAGN5+8skvQMadoN1g8h/MOYM//pDsf3+E7v8Xf+J+/5SxuCdId1zZAZtbNF+Xyr5tcx3rz1t40qRt6F2corrywKc9d241fDPL/vfO39lj7xrWpnZ7CbwOcPypLg0OIUuw2/vSjjqU1o+8fH8H/bu/jc2KN5MfRooIT7/BOjcn/v2H0a9OwzPiDP52b1uWBkReE9IOPW8eD13s05a7+fPPHYeU3l3rmmm7ktagf0ZiIP0PLdVmUr3EGEs7Yjveqpd5dEw9efgG/HlrxdRJCHViM9jKQngAajhv6NOe5sbFfY8mTRRZOcPf3/qSnhnaKC+f71iwrk18Obu16zih9KDabcHXPHFLCGJOslZ7CmkcvY2Db6C0fVKZMMTlqEkhPsUftMqOhDOB6glJegixV/etL2vDXIN0Bfxjhv+memZbiU8Pzfhd7t8ziw1/1C2vgPpjfXtquNBjBucysaK5q6ekSgPCXUwC4pEOj6BXGLdKX9974vjGdsObvXFp+1j34dr9WeBIOeL9U2Hq7Y3BrPrkzvJn2nt9sg1qJd+3naNHgEGX+spcqMqhtw7CW740Wz0zMEeXWjwmlyXzf8Pb8OMjKtb9y16yCaVTH9TyxXCvGZvNd3vu1m/PKzFWprD+P6RzwPuuHhcPTp1UDn2VkQlHRuTvY+5CZlsKnYZ6cw/H2z/oE/bxvvii3zFjJn3/cucJu1B91bcIz13TljsFtolXMhKPBIcoy08If3BOR6A/2hqBtI9cM2CFeXS9f3D2Qz+8eGNXnCdQ18NN+ufzjhh5c3SO+r/3Sjo0q7HYrX+Q7ggS78hlc/7qpJ6O6nh9wcLaieSKB1vMJZRKcR4af7hRbDDOYPEUL5Rl+2i+XDo1rl2nRRZKmWv7674GSMGqlpwQ82c/7/WAal6sM3dS3RYWVB5tNGJvXjJrVeNn56vvKLPDRr/qV3vb8EP39WJpl1WDXkdNxKlVZvxvejtF+LsHp0bFJ9CdG3TmkNa/M2+Kz3W6TiDKxKiPYWMJTV3Xh4SlrOVPi9Lnv/hEd+Odc39cwopPvqp09mtenxw3+0yKvvLApf7mqC05j+Hil/wWHXbNfQ7tGSZ0arp/wtT3L1vb/OKojBSfOMOf7Ato1qkW/1g355cUVt+bC9dtL29HV6yQdSvx55IrQJwqmpwSuv3561wDmbyzg5teX0Swrk4xU13LZe45W/Nv6/NcD2HesKKzBeH+CrSlV1WnLIUr6tMwqs2DWjX1bcEOf5tx5iW+zc+rdAyuVDRNoyYdQVsZsVCcj6Do4sVBRtlY8BRug9D6xhVrHfuWm4BOnPBP3PJO5bCKkpdhoUs9/192fRncMOF7jr92QmZbC94+P4L7hZTNW6tZILV1+3m6z8cgVnXxqyNFw99C2DG4fu1VCX/9pL+4a0ibggPGgdtm8eWsvfu3+nVW0xIpH56Z1ubRj9Md6qhNtOcRIjTQ7T17p/9q8dTJS+cNlHfjXvK0RHTvQQHgoJ7R4Xvg01S4UOxLjUquts2uypeBkpbOg8lrUJ3/HDyHv37FJHT65sz/tG9Xm6ekbuCdA9tIbt/aiVcOaQWuygXqV0lP8T/Rr7B7TufLC2F8YKdTZ7sG6xnq3zGLZtiNltuU2rFnmGiv+xDI4heLjO/px9HSxpWWIBQ0OFgl1Mp0/gZqy/vr2X7mxJ7/83/KIn6sypt0zkMVbDlvy3OUFywhrUjeDvceKKjzGwj8MIatmGh3/NCOs5/as0BmsO2VIDE5wDWqls/HxkX6zgWIlULddRd1NxcXFPD28MZv2pSEI69evj+j57+9Tix+6nBvrSTm+l/Xr90d0rFBlAI2B9ev9f9dfu8LVdRrJa/J+bGWOk5GRQU5ODqmpobfiNThYLJIZvBmpdrJqpnHk5NkK9y2fiRRPbc6rXXq9hQcv7xD3FS69BavXDmjbkPfzd1d4jJz68Z9tvfrh4ZwpcdD7iVkh1869pQXps08ku3fvpnbt2jRKqYeIcEFO5SYPfrv7KOBaKNLqQeNid1kieU3ej430OMYYDh8+zO7du2nZ0ndpn0CqxjcngXmuq3vbgNDfdI/vHr0srEtjVmXjB7WmayV/8JVxrjfDtwrr3dNx80Wu9fejNc8ikJ8PDG3MqW6N1IiuHxJvnhaDv6ufhaKoqIgGDSKf0Z4MWjasGVEFRURo0KABRUUVt469aXCopKyaaWx/ahSX+claqUit9BSfK0Kp2Ap27hGEBy+/gM1PjIxovko46mWmUTPNHlK2S7RW4o2lRnXS+f1l7XkrwDXEs2q6unrKr+flrboGhvNqp0ecFZVTP7M0Jbp2RmrE2VGRvLeJXyVRQeX/3zDyHp8JBB6Qrp2ewokzJfErVBUmIqSE2Ef/50peu2H1w4EvuPPe+L6lg7Oe33WqLXHrciLCnUMCTwhr2bAmn901oLSlHS/hXNgnVhrXjTwJwspU2cT9tiWh9Y+NCPsxAgEX6/KYfu8gzvOzKJkKzc8CdBnedFFu6e23busd9sVXUuy2gGvp9GnVoHRdpoxUO7+9tB0f3dHP775VRZecunEbA+l4fh06N60bVuKHiHDfffeV/v3ss8/yyCOPhP3c+fn53H333WE/ziM3N5dDhw4F3efJJ5+M+Pih0uCQQMLpr+3knqyWmmLj5xUEh6b1asRsca7qwN8wb2d398crN/bgodEdfe7vU+6Sqhe3y47JGkkedw9tG9K1nJVLit0W9ozw9PR0Jk+eXOGJuSJ5eXm8+OKLlTpGReIRHLRbqYp6+caefL//OHUyUjlUeMbq4iS8ni3qs2rXURrWDNKC8jqXDOlwHgv/MMTvAGC0Lhup/HttwVYOHo/ud7pjkzo8/KPg3YApKSmMHz+eF154gSeeeKLMfdu3b+e2227j0KFDZGdn88Ybb9C8eXM++OADHn30Uex2O3Xr1mX+/PnMnTuXZ599lilTptC+fXsWLVpEdnY2TqeTdu3asXjxYrKzz1XWDh8+zPXXX8+ePXu46KKLyswF+fGPf8yuXbsoKirinnvuYfz48UyYMIHTp0/TvXt3OnXqxMSJE/3uV1nacqiiaqWnhHXN5Go61heyCSM78OW9g2jeIPRsDytSV5W17rzzTiZOnMixY8fKbP/1r3/NLbfcwrfffsu4ceNKu40ee+wxZsyYwerVq5kyZUqZx9hsNm688UYmTpwIwMyZM+nWrVuZwADw6KOPMmDAANauXcuVV17Jzp07S+97/fXXWb58Ofn5+bz44oscPnyYp556iho1arBq1arSY/vbr7K05aCSQqrdpt0ycTLv94M5eiryGcM/H9jKsrTnOnXqcPPNN/Piiy9So8a5geTFixczefJkAG666Sbuv/9+APr3789Pf/pTxo4dy1VXXeVzvNtuu40xY8bwm9/8htdff51bb73VZ5/58+eXHnvUqFHUr39uGZ4XX3yRjz/+GIBdu3axadMmGjTwnS8U6n7h0OCglIqqFg1q0iLC81Kb82px0uLMut/85jf06NHD74m8vFdeeYWlS5cydepUevbsyfLlZVcjaNasGY0aNWL27NksW7astKYfirlz5zJz5kwWL15MZmYmgwcP9jtXIdT9wqXdSgmmXaNa9ArhQucqeqJ1NS9Vea6LQUV/gcBwZGVlMXbsWP7zn/+UbuvXrx+TJk0CYOLEiQwc6FrWfsuWLfTp04fHHnuM7Oxsdu3a5XO822+/nRtvvJFrr70Wu9036WTQoEG88847AEybNo0ffnCt3XXs2DHq169PZmYmGzZsYMmSJaWPSU1Npbi4uML9KkODQ4L58t6L+eCXVTtlsapK8mEZ5eW+++4rk7X00ksv8cYbb9C1a1f++9//8re//Q2A3//+93Tp0oXOnTvTr18/unXzvYTqFVdcQWFhYcCWyMMPP8z8+fPp1KkTkydPpnnz5gCMGDGCkpISLrjgAiZMmEDfvn1LHzN+/Hi6du3KuHHjgu5XGdqtVA1ozVepyissLCy93ahRI06dOlX6d4sWLZg9e7bPYzxjBd4GDx7M4MGDS/9evXo13bp1o0MH/0uxN2jQgC+//NLvfdOmTfO7/emnn+bpp5+ucL/K0OBQjSR7RpJSieapp57i5ZdfDmusIVFot1KS8CwjEM/lm6uKXHd663l1rO3rVtXPhAkT2LFjBwMGDLC6KGHTlkOSuH9Ee+pnpvKjIJcITVZ3DGlDt2b1GNROZ5Er5aHBIUnUzkjlt8ODX1ErWdltooFBqXK0W6kaiOQiMEopFYwGh2pFxxOUUtGhwUEppQC73U737t1L/23fvp1+/VxzjrZv3146US1ZWBocROQ+ETEi0tD9t4jIiyKyWUS+FZEeVpZPKZU8PIvZef7l5uayaNEiIDmDg2UD0iLSDBgO7PTaPBJo6/7XB3jZ/b9SKllMmwD710T3mI27wMinwn5YrVq1KCwsZMKECaxfv57u3btzyy23cO+990a3fAnIymylF4D7gU+9to0B3jauBc2XiEg9ETnfGLPPkhJWFToerVSlea6RANCyZcvSVU7BNZnt2Wef5fPPP7eqeHFnSXAQkTHAHmPM6nIXvm4KeK9ctdu9zSc4iMh4YDxQuhZJstMZ0qpaiKCGHw2ebiXlErPgICIzgcZ+7voj8CCuLqWIGWNeBV4FyMvL07qzUkpFUcyCgzFmmL/tItIFaAl4Wg05wAoR6Q3sAZp57Z7j3qaUUpapXbs2J06csLoYcRX3bCVjzBpjzHnGmFxjTC6urqMexpj9wBTgZnfWUl/gmI43VMzTNZeRqpnJSsVC165dsdvtdOvWjRdeeMHq4sRFoi2f8QVwObAZOAVUfCkmRXbtdH5/WXtGdz3f6qIoVWV5L9ldfltqaqrfJburM8uDg7v14LltgDutK03VdeeQNlYXQSlVjWg/hFJKKR8aHJRSCcHoJQ1jJpL3VoODUspyGRkZHD58WANEDBhjOHz4MBkZ4V3MyvIxB6WUysnJYffu3RQUFFhdlGopIyODnJycsB6jwUEpZbnU1FRatmxpdTGUF+1WUkop5UODg1JKKR8aHJRSSvmQ6pAdICIFwI4IH94QOBTF4lQF+pqTg77m5FCZ19zCGJPt745qERwqQ0TyjTF5VpcjnvQ1Jwd9zckhVq9Zu5WUUkr50OCglFLKhwYH9wWDkoy+5uSgrzk5xOQ1J/2Yg1JKKV/aclBKKeVDg4NSSikfSR0cRGSEiHwvIptFZILV5Yk1EWkmInNEZJ2IrBWRe6wuUzyIiF1EVorI51aXJV5EpJ6IfCgiG0RkvYhcZHWZYklE7nV/p78TkXdFJLwlSKsIEXldRA6KyHde27JE5CsR2eT+v340nitpg4OI2IF/ACOBjsD1ItLR2lLFXAlwnzGmI9AXuDMJXjPAPcB6qwsRZ38DphtjOgDdqMavX0SaAncDecaYzoAduM7aUsXMm8CIctsmALOMMW2BWe6/Ky1pgwPQG9hsjNlqjDkLTALGWFymmDLG7DPGrHDfPoHrhNHU2lLFlojkAKOAf1tdlngRkbrAIOA/AMaYs8aYo9aWKuZSgBoikgJkAnstLk9MGGPmA0fKbR4DvOW+/Rbw42g8VzIHh6bALq+/d1PNT5TeRCQXuBBYam1JYu6vwP2A0+qCxFFLoAB4w92d9m8RqWl1oWLFGLMHeBbYCewDjhljvrS2VHHVyBizz317P9AoGgdN5uCQtESkFvAR8BtjzHGryxMrIjIaOGiMWW51WeIsBegBvGyMuRA4SZS6GhKRu499DK6g2ASoKSI3WlsqaxjX3ISozE9I5uCwB2jm9XeOe1u1JiKpuALDRGPMZKvLE2P9gStEZDuubsNLROR/1hYpLnYDu40xnlbhh7iCRXU1DNhmjCkwxhQDk4F+Fpcpng6IyPkA7v8PRuOgyRwcvgHaikhLEUnDNYA1xeIyxZSICK5+6PXGmOetLk+sGWMeMMbkGGNycX2+s40x1b5GaYzZD+wSkfbuTUOBdRYWKdZ2An1FJNP9HR9KNR6A92MKcIv79i3Ap9E4aNJeJtQYUyIidwEzcGU3vG6MWWtxsWKtP3ATsEZEVrm3PWiM+cLCMqnY+DUw0V3x2QrcanF5YsYYs1REPgRW4MrIW0k1XUZDRN4FBgMNRWQ38DDwFPC+iPwM16ULxkbluXT5DKWUUuUlc7eSUkqpADQ4KKWU8qHBQSmllA8NDkoppXxocFBKKeVDg4NSYRCRBiKyyv1vv4jscd8uFJF/Wl0+paJFU1mVipCIPAIUGmOetbosSkWbthyUigIRGey5XoSIPCIib4nIAhHZISJXicgzIrJGRKa7lzBBRHqKyDwRWS4iMzxLICiVCDQ4KBUbrYFLgCuA/wFzjDFdgNPAKHeAeAm4xhjTE3gdeMKqwipVXtIun6FUjE0zxhSLyBpcy7NMd29fA+QC7YHOwFeu5YCw41puWqmEoMFBqdg4A2CMcYpIsTk3uOfE9bsTYK0xplpfvlNVXdqtpJQ1vgeyPdd2FpFUEelkcZmUKqXBQSkLuC9New3wtIisBlaRXNcgUAlOU1mVUkr50JaDUkopHxoclFJK+dDgoJRSyocGB6WUUj40OCillPKhwUEppZQPDQ5KKaV8/H9YGy66hJ7MyAAAAABJRU5ErkJggg==\n" + }, + "metadata": { + "needs_background": "light" + } + } + ], + "source": [ + "# fiiting using cmaes to the toy data\n", + "\n", + "# Add noise\n", + "values += np.random.normal(0, 10, values.shape)\n", + "\n", + "# Create an object with links to the model and time series\n", + "problem = pints.SingleOutputProblem(model, times, values)\n", + "\n", + "# Select a score function\n", + "score = pints.SumOfSquaresError(problem)\n", + "\n", + "# Select some boundaries\n", + "boundaries = pints.RectangularBoundaries([0, 0], [20, 20])\n", + "\n", + "# Perform an optimization with boundaries and hints\n", + "x0 = 0.01, 0.01\n", + "#sigma0 = [0.01, 100]\n", + "found_parameters, found_value = pints.optimise(\n", + " score,\n", + " x0,\n", + " boundaries = boundaries,\n", + " method=pints.CMAES\n", + " )\n", + "\n", + "# Show score of true solution\n", + "print('Score at true solution: ')\n", + "print(score(real_parameters))\n", + "\n", + "# Compare parameters with original\n", + "print('Found solution: True parameters:' )\n", + "for k, x in enumerate(found_parameters):\n", + " print(pints.strfloat(x) + ' ' + pints.strfloat(real_parameters[k]))\n", + "\n", + "# Show quality of fit\n", + "plt.figure()\n", + "plt.xlabel('Time')\n", + "plt.ylabel('Value')\n", + "plt.plot(times, values, label='Nosiy data')\n", + "plt.plot(times, problem.evaluate(found_parameters), label='Fit')\n", + "plt.legend()\n", + "plt.show()\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using the attempted Hager-Zhang linesearch implimentation" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": "Minimising error measure\nUsing Broyden-Fletcher-Goldfarb-Shanno (BFGS)\nRunning in sequential mode.\n\nHessian updates: 0 line steps: 0 propose alpha: 0.001 propose points: [0.01 0.01]\nIter. Eval. Best Time m:s\n0 1 1.42e+07 0:02.0\nalpha_initial: 1.266832724547819e-06\nNumber of accepted steps: 0\nstep sized alpha: 0.10140829994739889 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 1 line steps: 0 propose alpha: 0.10140829994739889 propose points: [4.40138067 8.01486899]\n1 2 1.01e+07 0:22.2\nalpha_initial: 0.20281659989479778\nNumber of accepted steps: 1\nstep sized alpha: 0.18362729523646867 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 2 line steps: 0 propose alpha: 0.18362729523646867 propose points: [2.96671004 8.70466184]\n2 3 1e+07 0:28.6\nalpha_initial: 0.36725459047293735\nNumber of accepted steps: 2\nstep sized alpha: 0.06538164147441025 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 3 line steps: 0 propose alpha: 0.06538164147441025 propose points: [3.09125057 8.90041531]\n3 4 1e+07 0:35.6\nalpha_initial: 0.1307632829488205\nNumber of accepted steps: 3\nstep sized alpha: 0.172828183643652 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 4 line steps: 0 propose alpha: 0.172828183643652 propose points: [3.02128895 8.9307388 ]\nalpha_initial: 0.345656367287304\nNumber of accepted steps: 4\nstep sized alpha: 0.04836245024157738 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 5 line steps: 0 propose alpha: 0.04836245024157738 propose points: [3.03066513 8.94435074]\nalpha_initial: 0.09672490048315475\nNumber of accepted steps: 5\nstep sized alpha: 0.16443120896872018 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 6 line steps: 0 propose alpha: 0.16443120896872018 propose points: [3.02412874 8.94694275]\nalpha_initial: 0.32886241793744037\nNumber of accepted steps: 6\nstep sized alpha: 0.619769876204881 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 7 line steps: 0 propose alpha: 0.619769876204881 propose points: [3.02487266 8.94881872]\nalpha_initial: 1.239539752409762\nNumber of accepted steps: 7\nstep sized alpha: 1.7690537545373526 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 8 line steps: 0 propose alpha: 1.7690537545373526 propose points: [3.02437294 8.94901689]\nalpha_initial: 3.538107509074705\nNumber of accepted steps: 8\nstep sized alpha: 0.6197698762049499 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 9 line steps: 0 propose alpha: 0.6197698762049499 propose points: [3.02440349 8.94909392]\nalpha_initial: 1.2395397524098999\nNumber of accepted steps: 9\nstep sized alpha: 1.7690537545447989 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 10 line steps: 0 propose alpha: 1.7690537545447989 propose points: [3.02438297 8.94910206]\nalpha_initial: 3.5381075090895977\nNumber of accepted steps: 10\nstep sized alpha: 0.6197698762511816 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 11 line steps: 0 propose alpha: 0.6197698762511816 propose points: [3.02438422 8.94910522]\nalpha_initial: 1.2395397525023633\nNumber of accepted steps: 11\nstep sized alpha: 1.7690537551010947 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 12 line steps: 0 propose alpha: 1.7690537551010947 propose points: [3.02438338 8.94910555]\nalpha_initial: 3.5381075102021895\nNumber of accepted steps: 12\nstep sized alpha: 0.6197698765290248 accepted\nupdating Hessian and changing newton direction\n\n******************** Convergence after 13 accepted steps!********************\n||df/dx_i||inf <= 1e-6 with parameters:\n[3.02438343 8.94910568]\nerror function evaluation: 10039961.960134586\n\nInverse Hessian matrix:\n [[0.10216867 0. ]\n [0. 0.10216867]]\n\nHessian updates: 13 line steps: 0 propose alpha: 0.6197698765290248 propose points: [3.02438343 8.94910568]\nalpha_initial: 1.2395397530580496\nNumber of accepted steps: 13\nstep sized alpha: 1.7690537478879769 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 14 line steps: 0 propose alpha: 1.7690537478879769 propose points: [3.0243834 8.9491057]\nalpha_initial: 3.5381074957759537\nNumber of accepted steps: 14\nstep sized alpha: 0.6197698655560488 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 15 line steps: 0 propose alpha: 0.6197698655560488 propose points: [3.0243834 8.9491057]\nalpha_initial: 1.2395397311120977\nNumber of accepted steps: 15\nstep sized alpha: 1.7690541518228553 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 16 line steps: 0 propose alpha: 1.7690541518228553 propose points: [3.0243834 8.9491057]\nalpha_initial: 3.5381083036457106\nNumber of accepted steps: 16\nstep sized alpha: 0.6197695047582213 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 17 line steps: 0 propose alpha: 0.6197695047582213 propose points: [3.0243834 8.9491057]\nalpha_initial: 1.2395390095164427\nNumber of accepted steps: 17\nstep sized alpha: 1.7690649194144203 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 18 line steps: 0 propose alpha: 1.7690649194144203 propose points: [3.0243834 8.9491057]\nalpha_initial: 3.5381298388288407\nNumber of accepted steps: 18\nstep sized alpha: 0.6197554232665176 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 19 line steps: 0 propose alpha: 0.6197554232665176 propose points: [3.0243834 8.9491057]\nalpha_initial: 1.2395108465330351\nNumber of accepted steps: 19\nstep sized alpha: 1.769066827970866 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 20 line steps: 0 propose alpha: 1.769066827970866 propose points: [3.0243834 8.9491057]\n20 21 1e+07 2:44.5\nalpha_initial: 3.538133655941732\nNumber of accepted steps: 20\nstep sized alpha: 0.6197901482068541 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 21 line steps: 0 propose alpha: 0.6197901482068541 propose points: [3.0243834 8.9491057]\nalpha_initial: 1.2395802964137081\nNumber of accepted steps: 21\nstep sized alpha: 1.7782084058779175 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 22 line steps: 0 propose alpha: 1.7782084058779175 propose points: [3.0243834 8.9491057]\nalpha_initial: 3.556416811755835\nNumber of accepted steps: 22\nstep sized alpha: 0.6265448885525534 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 23 line steps: 0 propose alpha: 0.6265448885525534 propose points: [3.0243834 8.9491057]\nalpha_initial: 1.2530897771051068\nNumber of accepted steps: 23\nstep sized alpha: 1.7775552079681158 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 24 line steps: 0 propose alpha: 1.7775552079681158 propose points: [3.0243834 8.9491057]\nalpha_initial: 3.5551104159362317\nNumber of accepted steps: 24\nstep sized alpha: 0.7438191314882705 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 25 line steps: 0 propose alpha: 0.7438191314882705 propose points: [3.0243834 8.9491057]\nalpha_initial: 1.487638262976541\n\nHessian updates: 25 line steps: 0.3333333333333333 propose alpha: 1.6810429221572856 propose points: [3.0243834 8.9491057]\nNumber of accepted steps: 25\nstep sized alpha: 1.511137650925769 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 26 line steps: 0 propose alpha: 1.511137650925769 propose points: [3.0243834 8.9491057]\n28 28 1e+07 3:40.0\nHalting: No significant change for 10 iterations.\nScore at true solution: \n10040056.240084708\nFound solution: True parameters:\n 3.02438339959221913e+00 3.00000000000000000e+00\n 8.94910570291299479e+00 9.00000000000000000e+00\n" + } + ], + "source": [ + "# Create an object with links to the model and time series\n", + "problem = pints.SingleOutputProblem(model, times, values)\n", + "\n", + "# Select a score function\n", + "score = pints.SumOfSquaresError(problem)\n", + "\n", + "# Select some boundaries\n", + "boundaries = pints.RectangularBoundaries([0, 0], [20, 20])\n", + "\n", + "# Perform an optimization with boundaries and hints\n", + "x0 = 0.01, 0.01\n", + "opt = pints.OptimisationController(\n", + " score,\n", + " x0,\n", + " method=pints.BFGS\n", + ")\n", + "opt.set_max_unchanged_iterations(10)\n", + "opt.set_max_iterations(200)\n", + "found_parameters, found_value = opt.run()\n", + "\n", + "# Show score of true solution\n", + "print('Score at true solution: ')\n", + "print(score(real_parameters))\n", + "\n", + "# Compare parameters with original\n", + "print('Found solution: True parameters:' )\n", + "for k, x in enumerate(found_parameters):\n", + " print(pints.strfloat(x) + ' ' + pints.strfloat(real_parameters[k]))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## now using scipy line search" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": "Minimising error measure\nUsing Broyden-Fletcher-Goldfarb-Shanno (BFGS)\nRunning in sequential mode.\nIter. Eval. Best Time m:s\n0 1 1.43e+07 0:01.6\n1 2 1.02e+07 0:11.4\n2 3 1.01e+07 0:18.3\n3 4 1.01e+07 0:24.9\n" + } + ], + "source": [ + "# Create an object with links to the model and time series\n", + "problem = pints.SingleOutputProblem(model, times, values)\n", + "\n", + "# Select a score function\n", + "score = pints.SumOfSquaresError(problem)\n", + "\n", + "# Select some boundaries\n", + "boundaries = pints.RectangularBoundaries([0, 0], [20, 20])\n", + "\n", + "# Perform an optimization with boundaries and hints\n", + "x0 = 0.01, 0.01\n", + "opt = pints.OptimisationController(\n", + " score,\n", + " x0,\n", + " method=pints.BFGS_scipy\n", + ")\n", + "opt.set_max_unchanged_iterations(10)\n", + "opt.set_max_iterations(200)\n", + "found_parameters, found_value = opt.run()\n", + "\n", + "# Show score of true solution\n", + "print('Score at true solution: ')\n", + "print(score(real_parameters))\n", + "\n", + "# Compare parameters with original\n", + "print('Found solution: True parameters:' )\n", + "for k, x in enumerate(found_parameters):\n", + " print(pints.strfloat(x) + ' ' + pints.strfloat(real_parameters[k]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.8.2 64-bit ('pints': conda)", + "language": "python", + "name": "python38264bitpintsconda8d0b754a30424fdd8045d82b54ea71e7" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.2-final" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/pints/_optimisers/__init__.py b/pints/_optimisers/__init__.py index 1fa59ed26..6dc48426e 100644 --- a/pints/_optimisers/__init__.py +++ b/pints/_optimisers/__init__.py @@ -9,6 +9,7 @@ from __future__ import print_function, unicode_literals import pints import numpy as np +from numpy.linalg import norm class Optimiser(pints.Loggable, pints.TunableMethod): @@ -300,6 +301,89 @@ def __init__(self, x0, sigma0=None, boundaries=None): self._evaluator = None + # Set optimiser state + self._running = False + self._ready_for_tell = False + + # Best solution found + self._xbest = self._x0 + self._fbest = float('inf') + + # Number of iterations run + self._iterations = 0 + + # Parameters for wolfe conditions on line search + + # As c1 approaches 0 and c2 approaches 1, the line search + # terminates more quickly. + self._c1 = 1E-4 # Parameter for Armijo condition rule, 0 < c1 < 0.5 + self._c2 = 0.9 # Parameter for curvature condition rule, c1 < c2 < 1.0 + + # boundary values of alpha + self._minimum_alpha = 0.0 + self._maximum_alpha = float("inf") + self._proposed_alpha = 0.001 # same default value as used in stan + + self.__first_update_step_not_completed = True + self.__update_step_not_completed = True + self.__performing_line_search = False + + # Increase allowed between accepted positions when using approximate + # wolfe conditions, this takes into acccount machine error and + # insures decreasing. + self.epsilon = 1E-6 + + # range (0, 1), used in the ``self.__update()`` and + # ``self.__initial_bracket()`` when the potential intervals violate + # the opposite slope condition (see function definition) + self.theta = 0.5 + + self.__gamma = 0.66 + + # range (0, 1) small factor used in initial guess of step size + self.__ps_0 = 0.01 + # range (0, 1) small factor used in subsequent guesses of step size + self.__ps_1 = 0.1 + # range (1, inf) factor used in subsequent guesses of step size + self.__ps_2 = 2.0 + + self.__current_alpha = None + + # approximate inverse hessian + # initial the identity is used + self._B = np.identity(self._n_parameters) + + # newton direction + self._px = None + + # number of accepted steps/ newton direction updates + self._k = 0 + + # number of steps in the current line search iteration + self.__j = 0 + self.__logic_steps_left = 0 + + # maximum number of line search steps before a successful point + + # Current point, score, and gradient + self._current = self._x0 + self._current_f = None + self._current_dfdx = None + + # Proposed next point (read-only, so can be passed to user) + self._proposed = self._x0 + self._proposed.setflags(write=False) + + self.__convergence = False + + # logic for passing to tell at the right moments + self.__1st_wolfe_check_needed = False + self.__1st_wolfe_check_done = False + self.__2nd_wolfe_check_needed = False + self.__2nd_wolfe_check_done = False + self.__need_update = False + self.__converged_ls = False + def _set_function_evaluator(self, function): f = function @@ -309,6 +393,825 @@ def _set_function_evaluator(self, function): # Create evaluator object self._evaluator = pints.SequentialEvaluator(f) + def fbest(self): + """ See :meth:`Optimiser.fbest()`. """ + return self._fbest + + def wolfe_line_search_parameters(self): + """ + Returns the wolfe line search parameters this optimiser is using + as a vector ``[c1, c2]``. + As c1 approaches 0 and c2 approaches 1, the line search terminates + more quickly. ``c1`` is the parameter for the Armijo condition + rule, ``0 < c1 < 0.5``. ``c2 `` is the parameter for the + curvature condition rule, ``c1 < c2 < 1.0``. + """ + return (self._c1, self._c2) + + def needs_sensitivities(self): + """ See :meth:`Optimiser.needs_sensitivities()`. """ + return True + + def n_hyper_parameters(self): + """ See :meth:`pints.TunableMethod.n_hyper_parameters()`. """ + return 2 + + def running(self): + """ See :meth:`Optimiser.running()`. """ + return self._running + + def set_hyper_parameters(self, x): + """ + See :meth:`pints.TunableMethod.set_hyper_parameters()`. + + The hyper-parameter vector is ``[c1, c2, m]``. + ``c1`` is the parameter for the Armijo condition rule, + ``0 < c1 < 0.5``. + ``c2`` is the parameter for the curvature condition rule, + ``c1 < c2 < 1.0``. + ``m`` is the number of maximum number of correction matrices + that can be stored for the LM-BFGS update. + """ + + self.__set_wolfe_line_search_parameters(x[0], x[1]) + + def __set_wolfe_line_search_parameters(self, c1, c2): + """ + Sets the parameters for the wolfe conditions. + + Parameters + ---------- + c1: float + Parameter for the Armijo condition rule, ``0 < c1 < 0.5``. + + c2: float + Parameter for the curvature condition rule, ``c1 < c2 < 1.0``. + """ + if(0 < c1 < 0.5 and c1 < c2 < 1.0): + self._c1 = c1 + self._c2 = c2 + else: + cs = self.wolfe_line_search_parameters() + print('Invalid wolfe line search parameters!!!') + print('0 < c1 < 0.5 and c1 < c2 < 1.0') + print('using default parameters: c1 = ', cs[0], ' c2 = ', cs[1]) + + def xbest(self): + """ See :meth:`Optimiser.xbest()`. """ + return self._xbest + + def ask(self): + """ See :meth:`Optimiser.ask()`. """ + + # print('') + # print('in ask') + # print('') + + if not self._running: + self._proposed = np.asarray(self._x0) + else: + + if self.__j == 0: + # working out an initial stepsize value alpha + alpha_0 = self.__initialising(k=self._k, + alpha_k0=self.__current_alpha) + print('alpha_initial: ', alpha_0) + # Creating an initial bracketing interval of alpha values + # satisfying the opposite slope condition (see function + # docstring) beginning with initial guess [0, alpha_initial]. + bracket = (self.__initial_bracket(c=alpha_0)) + self._minimum_alpha = bracket[0] + self._maximum_alpha = bracket[1] + self._updated_minimum_alpha = self._minimum_alpha + self._updated_maximum_alpha = self._maximum_alpha + + # Looping while wolfe conditions don't need to be checked + # and the line search hasn't converged. + while (self.__1st_wolfe_check_needed is not True and + self.__2nd_wolfe_check_needed is not True and + self.__converged_ls is not True): + + # secant squared step of the line search + + # *************************************************************** + # a, b = self.__secant2(self._minimum_alpha, + # self._maximum_alpha) + a = self._updated_minimum_alpha + b = self._updated_maximum_alpha + c = self._proposed_alpha + + # checking if the bracketing interval has converged + self.__converged_ls = self.__very_close(a, b) + self.__logic_steps_left = 'not started' + if self.__converged_ls: + self.__logic_steps_left = ' converged in ls ' + # if converged is True don't do anything more. + + # ************ beginning of secant squared see [1] ************ + + # Preforming a secant to propose a value of alpha. + if (self.__1st_wolfe_check_done is not True and + self.__2nd_wolfe_check_done is not True and + self.__converged_ls is not True): + + # step S1 in [1] + self._proposed_alpha = self.__secant_for_alpha(a, b) + # passing to tell to check wolfe conditions + self.__1st_wolfe_check_needed = True + + # checking the proposed point is in range + if self._proposed_alpha < a or self._proposed_alpha > b: + # If the point is out of range there is no need to + # check wolfe conditions. + self.__1st_wolfe_check_needed = False + self.__1st_wolfe_check_done = True + + self.__logic_steps_left = 2 + self.__j += 1 / 3 + + elif (self.__1st_wolfe_check_done and + self.__2nd_wolfe_check_done is not True and + self.__converged_ls is not True): + + # (typically) updating one side of the + # bracketing interval + A, B = self.__update(a, b, c) + # end of step S1 in [1] + + # (typically) updating the otherside side of + # the bracketing interval + if c == B: + # S2 in [1] + # Preforming a secant to propose a value of alpha. + self._proposed_alpha = self.__secant_for_alpha(b, B) + + # checking the proposed point is in range + if (self._proposed_alpha < A or + self._proposed_alpha > B): + # If the point is out of range there is no need to + # check wolfe conditions. + self.__2nd_wolfe_check_needed = False + self.__2nd_wolfe_check_done = True + else: + self.__2nd_wolfe_check_needed = True + self.__need_update = True + elif c == A: + # S3 in [1] + # Preforming a secant to propose a value of alpha. + self._proposed_alpha = self.__secant_for_alpha(a, A) + + # checking the proposed point is in range + if (self._proposed_alpha < A or + self._proposed_alpha > B): + # If the point is out of range there is no need to + # check wolfe conditions. + self.__2nd_wolfe_check_needed = False + self.__2nd_wolfe_check_done = True + else: + self.__2nd_wolfe_check_needed = True + self.__need_update = True + else: + # No new point has been proposed therefore there + # is no need to check the wolfe conditions. + self.__2nd_wolfe_check_needed = False + self.__2nd_wolfe_check_done = True + + self._updated_minimum_alpha = A + self._updated_maximum_alpha = B + + self.__logic_steps_left = 1 + self.__j += 1 / 3 + + elif (self.__1st_wolfe_check_done and + self.__2nd_wolfe_check_done and + self.__converged_ls is not True): + + # S4 in [1], this is only preformed if S2 or S3 was done + # and the propsed point was in range. + if self.__need_update: + a, b = self.__update(a, b, c) + self.__need_update = False + + # ***************** end of secant squared ***************** + + # preforming steps L2 from [1] + # determing whether a bisection step should be preformed + # i.e if self.__secant_for_interval() didn't shrink the + # bracketing interval by the propotion self.__gamma + new_width = b - a + old_width = (self._maximum_alpha - self._minimum_alpha) + + if new_width > (self.__gamma * old_width): + # preforming bisection + c = (a + b) / 2.0 + a, b = self.__update(a=a, b=b, c=c) + + # preforming steps L3 from [1] + # updating bracketing interval + self._minimum_alpha, self._maximum_alpha = a, b + self.__j += 1 / 3 + self.__logic_steps_left = 0 + + # reset logic + self.__1st_wolfe_check_needed = False + self.__1st_wolfe_check_done = False + self.__2nd_wolfe_check_needed = False + self.__2nd_wolfe_check_done = False + + # *********************** CONTINUE LOOP *********************** + + if self.__converged_ls: + self._proposed = (self._current + + self._updated_maximum_alpha * self._px) + else: + self._proposed = (self._current + + self._proposed_alpha * self._px) + + # Running, and ready for tell now + self._ready_for_tell = True + self._running = True + + # print('') + # print('finished ask') + # print('') + # Return proposed points (just the one) in the search space to evaluate + return [self._proposed] + + def tell(self, reply): + """ See :meth:`Optimiser.tell()`. """ + + # Check ask-tell pattern + if not self._ready_for_tell: + raise Exception('ask() not called before tell()') + self._ready_for_tell = False + + # Unpack reply + proposed_f, proposed_dfdx = reply[0] + proposed_f = proposed_f + proposed_dfdx = np.asarray(proposed_dfdx) + + # We need the evaluation of the gradients before we can start the BFGS, + # the first tell gets these. + if self._current_f is None: + + # Move to proposed point + self._current = self._proposed + self._current_f = np.asarray(proposed_f) + self._current_dfdx = np.asarray(proposed_dfdx) + + # Update newton direction + # if it isn't a descent direction the line searches will fail + # i.e return nonsense + self._px = - np.matmul(self._B, self._current_dfdx) + + # resetting the number of steps in the current + # line search iteration. + self.__j = 0 + + # Checking if exact wolfe conditions. + proposed_grad = np.matmul(np.transpose(proposed_dfdx), self._px) + + wolfe_curvature = (proposed_grad >= + self._c2 * + np.matmul(np.transpose(self._current_dfdx), + self._px)) + + exact_wolfe_suff_dec = (self._c1 * + np.matmul(np.transpose(self._current_dfdx), + self._px) + >= proposed_f - self._current_f) + + exact_wolfe = (exact_wolfe_suff_dec and wolfe_curvature) + + # Checking if approximate wolfe conditions are meet. + approx_wolfe_suff_dec = ((2.0 * self._c1 - 1.0) * + np.matmul(np.transpose(self._current_dfdx), + self._px) >= proposed_grad) + + approx_wolfe_applies = proposed_f <= self._current_f + self.epsilon + + approximate_wolfe = (approx_wolfe_suff_dec and wolfe_curvature + and approx_wolfe_applies) + + # If wolfe conditions meet the line search is stopped + # and the inverse hessian matrix and newton direction are updated. + # If the line search has converged we also accept the + # steps and update. + if exact_wolfe or approximate_wolfe or self.__converged_ls: + + print('Number of accepted steps: ', self._k) + print('step size of alpha accepted: ', self._proposed_alpha) + print('updating Hessian and changing newton direction') + + # Updating inverse hessian. + self._B = self.inverse_hessian_update(proposed_f, proposed_dfdx) + + # Move to proposed point + self._current = self._proposed + self._current_f = np.asarray(proposed_f) + self._current_dfdx = np.asarray(proposed_dfdx) + + # storing the accepted value of alpha + self.__current_alpha = self._proposed_alpha + + # if self.__k == 0: + # print('\nThe first accepted value of alpha was: ', + # self.__current_alpha) + # print('set initial alpha to this in subsequent runs' + + # 'to speed up computation') + + # Update newton direction + self._px = - np.matmul(self._B, self._current_dfdx) + + # incrementing the number of accepted steps + self._k += 1 + + # Resetting the number of steps in the current line search + # iteration as we have accepted a value and completed this + # line search. + self.__j = 0 + + # resetting line search logic + self.__1st_wolfe_check_needed = False + self.__1st_wolfe_check_done = False + self.__2nd_wolfe_check_needed = False + self.__2nd_wolfe_check_done = False + self.__need_update = False + + else: + # wolfe conditions haven't been meet so we continue the line search + if self.__1st_wolfe_check_needed: + self.__1st_wolfe_check_needed = False + self.__1st_wolfe_check_done = True + if self.__2nd_wolfe_check_needed: + self.__2nd_wolfe_check_needed = False + self.__2nd_wolfe_check_done = True + + # Checking if all gradients ~ 0, + # therefore the classical convergence test of a quasi-newton + # or conjugate gradient method has been meet. + if self.__convergence is not True: + + if norm(proposed_dfdx, ord=np.inf) <= 1e-6: + + self.__convergence = True + print('') + print(20 * '*' + ' Convergence after ', + self._k, ' accepted steps!' + 20 * '*') + print('||df/dx_i||inf <= 1e-6 with parameters:') + print(self._proposed) + print('error function evaluation: ', proposed_f) + print('\nInverse Hessian matrix:\n', self._B) + + # Update xbest and fbest + if self._fbest > proposed_f: + self._fbest = proposed_f + self._xbest = self._current + + print('') + print('Hessian updates: ', self._k, ' line steps: ', self.__j, + ' propose alpha: ', self._proposed_alpha, ' propose points: ', + self._proposed) + + def __update(self, a, b, c): + ''' + This function is part of the Hager-Zhang line search method [1]. + + Updates the bracketing boundary values of alpha. Ensuring the opposite + slope conditions are obeyed. + + The opposite slope conditions are: + φ(a) ≤ φ(0) + epsilon, + φ'(a) < 0 (w.r.t every parameter), + φ'(b) ≥ 0 (w.r.t every parameter). + where φ(α) = f(x+α*px), f() is the function binging minimised, x is + the current mparameter set, px is the newton direction, and alpha is + the step size. + + In the first condition, epsilon is a small positive constant. The + condition demands that the function at the left end point be not much + bigger than the starting point (i.e. alpha = 0). This is an easy to + satisfy condition because by assumption, we are in a direction where + the function value is decreasing. The second and third conditions + together demand that there is at least one zero of the derivative in + between a and b. In addition to the interval, the update algorithm + requires a third point to be supplied. Usually, this point would lie + within the interval [a, b]. If the point is outside this interval, + the current interval is returned. If the point lies within the + interval, the behaviour of the function and derivative value at this + point is used to squeeze the original interval in a manner that + preserves the opposite slope conditions. + + :param a: lower bound of alpha + :param b: upper bound of alpha + :param c: proposed value of alpha + :param dfdx_c: vector of gradients at the proposed point i.e alpha = c + :param f_c: function evaluation at the proposed point i.e alpha = c + :param f_0: function evaluation at the previous point i.e alpha = 0 + ''' + + # Check c is within the bracket conditions (steps U0 from [1]). + if c < a or c > b: + # if the proposed point is not in range we return + # the old brackets unmodified + return a, b + else: + + # evaluate function for alpha = c i.e at the proposed boundary + # point_c = self._current + c * self._px + # fs_c = self._evaluator.evaluate([point_c]) + f_c, dfdx_c = self.obj_and_grad_func(c) + + # Checking if the opposite slope condition at the upper bound is + # obeyed by the proposed point + # (steps U1 from [1]). + if dfdx_c >= 0.0: + # Updating the upper bound. + return a, c + + # Checking if the opposite slope condition at the lower bound is + # obeyed by the proposed point, if so it is a valid lower bound + # (steps U2 from [1]). + elif dfdx_c < 0.0 and f_c <= self._current_f + self.epsilon: + # Updating the lower bound. + return c, b + + # The proposed point doesn't obey the opposite slope condition + # i.e. dfdx_c < 0.0 and f_c > self._current_f + self.epsilon. + # Checking this is unnecessary as it is the opposite to the above + # conditions. A secant/bisect can narrow down an interval between + # the current lower bound and the trial point c. + else: + b = c + return self.__bisect_or_secant(a, b) + + def __bisect_or_secant(self, a, b): + ''' + This function is part of the Hager-Zhang line search method [1]. + + Actual implementation of secant (or bisetc if `self.theta` = 0.5) + given a bracketing intervale [a, b] used in `__update()` and + `__initial_bracketing_interval()`. (steps U3a-c from [1]) + + + :param a: lower bound of alpha + :param b: upper bound of alpha + :param c: proposed value of alpha + ''' + + secant = True + + # FIXME: + # problem is in this while loop a and b merge but don't satisfy + # opposite slope rule for upper, + # probably should return when difference between a and b almost + # nothing??? + + # The interval needs updating if the upper bracket has a negative + # slope and the value of the function at that point is too high. + # It is not a valid lower bracket but along with the current + # lower bracket, it encloses another minima. The below function + # is a loop which tries to narrow the interval so that it + # satisfies the opposite slope conditions. + + # (steps U3 from [1]) + + while secant is True: + # according to [1]] this while loop is guaranteed to terminate + # as the intervale between a and b tends to zero + + # Preforming secant (if theta = 0.5 as is default this is a + # bisection) to propose a new point which hopeful obeys the + # opposite slope conditions. + # (steps U3a from [1]) + d = (1.0 - self.theta) * a + self.theta * b + + # Evaluating function for alpha = d i.e at the proposed boundary. + # point_d = self._current + d * self._px + # fs = self._evaluator.evaluate([point_d]) + # f_d, dfdx_d = fs[0] + + f_d, dfdd = self.obj_and_grad_func(d) + + # Checking if the opposite slope condition at the upper bound is + # obeyed by the proposed point. + # If the proposed point has a positive slope, then we have found a + # suitable upper bound to bracket a minima within opposite slopes. + # (still steps U3a from [1]) + converged = self.__very_close(d, b) or self.__very_close(d, a) + if dfdd >= 0.0 or converged: + secant = False + # Updating the upper bound. + return a, d + + # Checking if the opposite slope condition at the lower bound is + # obeyed by the proposed point. + # If the proposed point has a negative slope and the function value + # at that point is small enough, we can use it as a new lower bound + # to narrow down the interval. + # (steps U3b from [1]) + elif dfdd < 0.0 and f_d <= self._current_f + self.epsilon: + # Updating the lower bound. + a = d + + # The proposed point doesn't obey the opposite slope condition + # i.e. dfdx_c < 0.0 and f_c > self._current_f + self.epsilon + # Checking this is unnecessary as it is the opposite to the above + # conditions. We are therefore in the same situation as when we + # started the loop so we update the upper bracket and continue. + # (steps U3c from [1]) + else: + b = d + + def __secant_for_alpha(self, a, b): + ''' + This function is part of the Hager-Zhang line search method [1]. + + Preforms a secant step to propose a value of alpha. This is the same as + the secant routine described in [1]. + + :param a: lower bound of alpha + :param b: upper bound of alpha + ''' + + # Evaluating function for alpha = a to obtain gradients. + f_a, dfda = self.obj_and_grad_func(a) + + # Evaluating function for alpha = b to obtain gradients. + f_b, dfdb = self.obj_and_grad_func(b) + + # Preforming secant. + numerator = a * dfdb - b * dfda + denominator = dfdb - dfda + + return float(numerator / denominator) + + def __initial_bracket(self, c, rho=5): + ''' + This function is part of the Hager-Zhang line search method [1]. + + This function is used to generate an initial interval [a, b] for alpha + satisfying the opposite slope conditions and therefore bracketing + the minimum, beginning with the initial guess [0, c]. + The opposite slope conditions: + φ(a) ≤ φ(0) + epsilon, + φ'(a) < 0 (w.r.t every parameter), + φ'(b) ≥ 0 (w.r.t every parameter). + where φ(α) = f(x+α*px), f() is the function binging minimised, x is + the current parameter set, px is the newton direction, and alpha is + the step size. + + This is the same as the bracket routine described in [1] as steps B0-3 + + :param c: initial guess for maximum value of alpha + :param row: range (1, ∞), expansion factor used in the bracket rule to + increase the upper limit c (c_j+1 = row*c_j) until a + suitable interval is found that contains the minimum. + ''' + + # (steps B0 from [1]) + # Initiating a list of proposed boundary values of alpha. + c = [c] + # Initiating a list of error function evaluations at + # the proposed boundary values of alpha. + f_c = [] + # Initiating lower bound. + a = 0 + + # Initiating an iteration counter for the below while loop. + j = 0 + bracketing = True + + while bracketing: + + # Evaluating function for alpha = c[j] + # i.e at the proposed boundary. + f_cj, dfdc_j = self.obj_and_grad_func(c[j]) + + # Appending the error function evaluations at proposed boundary + # values of alpha. + f_c.append(f_cj) + + # Checking if the opposite slope condition at the upper bound is + # obeyed by the proposed point. If the slope at the proposed point + # is positive, then the given points already bracket a minimum. + # (steps B1 from [1]) + if dfdc_j >= 0.0: + # Setting the upper bound. + b = c[j] + + bracketing = False + + # Checking if the non derivative opposite slope condition at + # the lower bound is obeyed by any of the previously evaluated + # points and returning the value for which the boundary + # conditions are as close together as possible. + for i in range(1, j + 1): + if f_c[j - i] <= self._current_f + self.epsilon: + a = c[j - i] + return a, b + return a, b + + # Checking if the proposed point doesn't obey the opposite slope + # condition. This means the upper bracket limit almost works as a + # new lower limit but the objective function(f_cj) is too large. + # We therefore need to preform a secant/bisect but the minimum is + # not yet bracketed. + # (steps B2 from [1]) + elif dfdc_j < 0.0 and f_cj > self._current_f + self.epsilon: + # The interval needs updating if the upper bracket has a + # negative slope and the value of the function at that point + # is too high. It is not a valid lower bracket but along with + # the current lower bracket, it encloses another minima. The + # below function tries to narrow the interval so that it + # satisfies the opposite slope conditions. + bracketing = False + return self.__bisect_or_secant(0.0, c[j]) + + # The proposed point obeys the opposite slope condition + # at the lower bound + # i.e. dfdx_d < 0.0 and f_d <= self._current_f + self.epsilon. + # Checking this is unnecessary as it is the opposite to + # the above conditions. This means the bracket interval needs + # expanding to ensure a minimum is bracketed. + # (steps B3 from [1]) + else: + # Increasing the proposed point by a factor of row to attempt + # to bracket minimum and trying again. + c.append(rho * c[j]) + + # Increamenting the iteration counter. + j += 1 + + def __initialising(self, k, alpha_k0): + ''' + This function is part of the Hager-Zhang line search method [1]. + + Generate the starting guess of alpha, 'c', used by + ``__initial_bracket()``. This is the same as the routine + called initial and described as I0-2 in [1]. + + :param k: number of accepted steps/newton direction updates that + have taken place. + :param alpha_k0: the alpha value used by the previously accepted + steps/newton direction + update. If k = 0 this is the initial alpha the user wants to be used + ''' + QuadStep = False + + # Small factor used in initial guess of step size, range (0, 1). + # As the initial guess is very crude this is a very small factor + # to keep the initial step short and close the starting parameters + # self.x_0. + ps_0 = self.__ps_0 + + # Small factor used in subsequent guesses of step size + # if Quadstep is true, range (0, 1). + # TODO: this hasn't been implement yet + ps_1 = self.__ps_1 + + # Sacling factor used in subsequent guesses of step size, + # range (1, inf). + ps_2 = self.__ps_2 + + # For the first line search do the following + # (step I0 in [1]) + if k == 0: + + if alpha_k0 is not None: + # returning user specified initial alpha + return alpha_k0 + + # (step I0a in [1]) + elif np.all(self._x0 != 0.0): + # Crude step size estimate + # :math: \alpha = ps_0*||x_0||_\inf / ||dfdx||_\inf + return ((ps_0 * norm(self._x0, ord=np.inf)) + / (norm(self._current_dfdx, ord=np.inf))) + + # If self._x0 = 0.0 the above statement would give alpha = 0, + # hence we use the following estimation. + # (step I0b in [1]) + elif self._current_f != 0: + # Crude step size estimate + # :math: \alpha = ps_0*|f(x_0)| / ||dfdx||^2 + return ((ps_0 * abs(self._current_f)) + / (pow(norm(self._current_dfdx, ord=2), 2.0))) + + # Otherwise self._current_f = 0.0 and we are already at the + # minimum therefore return alpha = 1.0 it should not matter what we + # return in this case as if self._current_f = 0.0 the gradients + # will also equal zero. + # (step I0c in [1]) + else: + return 1.0 + + # TODO: implement the below option using a quadratic interpolant ??? + # everything will work without this + # (step I1 in [1]) + elif QuadStep: + + # point_current_scaled = self._current + ps_1*alpha_k0* self._px + # fs = self._evaluator.evaluate([point_current_scaled]) + f, df = self.obj_and_grad_func(ps_1 * alpha_k0) + + if f <= self._current_f: + pass + #TODO: add quad step option + pass + + # For the subsequent line search do the following + # (step I2 in [1]) + else: + # Increases the step size of the previous accepted step as it + # is only decreased in subsequent boundary manipulation. + return ps_2 * alpha_k0 + + def __very_close(self, x, y): + ''' Returns true if x is very close in value to y. ''' + return np.nextafter(x, y) >= y + + def obj_and_grad_func(self, alpha: float): + ''' + For a given alpha this returns the values of the objective function + and it's derivative. + ''' + point_alpha = self._current + alpha * self._px + fs_alpha = self._evaluator.evaluate([point_alpha]) + f_alpha, dfdx_alpha = fs_alpha[0] + + dfdalpha = np.matmul(np.transpose(dfdx_alpha), self._px) + + return f_alpha, dfdalpha + + def inverse_hessian_update(self, proposed_f, proposed_dfdx): + """ + Returns the newly calculated/updated inverse hessian matrix + by whichever quasi-Newton/ linesearch based optimiser is used + by the inherited class. + """ + raise NotImplementedError + + # def __secant2(self, a, b): + # ''' + # This function is part of the Hager-Zhang line search method [1]. + + # This function is referred to as secant^2 and described as steps + # S1-4 in [1]. + + # Preforms a secant step to update the bracketing interval of alpha. + # Given an interval that brackets a root, this procedure performs an + # update of both end points using two intermediate points generated + # using the secant interpolation `self.__secant_for_alpha()`. + # Assuming the interval [a, b] satisfy the opposite slope conditions. + + # The opposite slope conditions: + # φ(a) ≤ φ(0) + epsilon , + # φ'(a) < 0 (w.r.t every parameter), + # φ'(b) ≥ 0 (w.r.t every parameter). + # where φ(α) = f(x+α*px), f() is the function binging minimised, + # x is the current parameter set, px is the newton direction, + # and alpha is the step size. + + # :param a: power bound of alpha + # :param b: upper bound of alpha + # ''' + + # # Preforming a secant to propose a value of alpha. + # # (step S1 in [1]) + # c = self.__secant_for_alpha(a,b) + # # CHECK IF c SATISFY THE WOLFE CONDITIONS i.e pass to tell!!!!!!! + # # IF IT HAS STOP SEARCHING THIS DIRECTION!!!! + + # # IF WOLFE CONDITIONS AREAN'T MEET DO THIS thing below + + # # (typically) updating one side of the bracketing interval + # print('first update secant') + # A, B = self.__update(a, b, c) + + # # (typically) updating the otherside side of the bracketing interval + # # (step S2 in [1]) + # if c == A: + # # Preforming a secant to propose a value of alpha. + # C = self.__secant_for_alpha (a, A) + + # # (step S3 in [1]) + # if c == B: + # # Preforming a secant to propose a value of alpha. + # C = self.__secant_for_alpha (b, B) + + # # (step S4 in [1]) + # if c == A or c == B: + # # CHECK IF C SATISFY THE WOLFE CONDITIONS i.e pass to tell!!!!!!! + # # IF IT HAS STOP SEARCHING THIS DIRECTION!!!! + + # # IF WOLFE CONDITIONS AREAN'T MEET DO THIS thing below + + # print('second update secant') + # A, B = self.__update(A, B, C) + + # return A, B + class OptimisationController(object): """ diff --git a/pints/_optimisers/_bfgs_linesearch.py b/pints/_optimisers/_bfgs_linesearch.py index a99766678..e37b6ed21 100644 --- a/pints/_optimisers/_bfgs_linesearch.py +++ b/pints/_optimisers/_bfgs_linesearch.py @@ -17,7 +17,7 @@ class BFGS(pints.LineSearchBasedOptimiser): Broyden-Fletcher-Goldfarb-Shanno algorithm [2], [3], [4] The Hager-Zhang line search algorithm [1] is implemented in this class - + [1] Hager, W. W.; Zhang, H. Algorithm 851: CG_DESCENT, a Conjugate Gradient Method with Guaranteed Descent. ACM Trans. Math. Softw. 2006, 32 (1), 113-137. @@ -41,61 +41,6 @@ class BFGS(pints.LineSearchBasedOptimiser): def __init__(self, x0, sigma0=None, boundaries=None): super(BFGS, self).__init__(x0, sigma0, boundaries) - # Set optimiser state - self._running = False - self._ready_for_tell = False - - # Best solution found - self._xbest = self._x0 - self._fbest = float('inf') - - # Number of iterations run - self._iterations = 0 - - # Parameters for wolfe conditions on line search - - # As c1 approaches 0 and c2 approaches 1, the line search - # terminates more quickly. - self._c1 = 1E-4 # Parameter for Armijo condition rule, 0 < c1 < 0.5 - self._c2 = 0.9 # Parameter for curvature condition rule, c1 < c2 < 1.0 - - # boundary values of alpha - self._minimum_alpha = 0.0 - self._maximum_alpha = float("inf") - self._proposed_alpha = 0.001 # same default value as used in stan - - self.__first_update_step_not_completed = True - self.__update_step_not_completed = True - self.__performing_line_search = False - - # Increase allowed between accepted positions when using approximate - # wolfe conditions, this takes into acccount machine error and - # insures decreasing. - self.epsilon = 1E-6 - - # range (0, 1), used in the ``self.__update()`` and - # ``self.__initial_bracket()`` when the potential intervals violate - # the opposite slope condition (see function definition) - self.theta = 0.5 - - self.__gamma = 0.66 - - # range (0, 1) small factor used in initial guess of step size - self.__ps_0 = 0.01 - # range (0, 1) small factor used in subsequent guesses of step size - self.__ps_1 = 0.1 - # range (1, inf) factor used in subsequent guesses of step size - self.__ps_2 = 2.0 - - self.__current_alpha = None - - # approximate inverse hessian - # initial the identity is used - self._B = np.identity(self._n_parameters) - - # newton direction - self._px = None - # maximum number of correction matrices stored self._m = 5 @@ -104,49 +49,6 @@ def __init__(self, x0, sigma0=None, boundaries=None): self._S = np.zeros(shape=(self._n_parameters, self._m)) self._Y = np.zeros(shape=(self._n_parameters, self._m)) - # number of accepted steps/ newton direction updates - self.__k = 0 - - # number of steps in the current line search iteration - self.__j = 0 - self.__logic_steps_left = 0 - - # maximum number of line search steps before a successful point - - # Current point, score, and gradient - self._current = self._x0 - self._current_f = None - self._current_dfdx = None - - # Proposed next point (read-only, so can be passed to user) - self._proposed = self._x0 - self._proposed.setflags(write=False) - - self.__convergence = False - - # logic for passing to tell at the right moments - self.__1st_wolfe_check_needed = False - self.__1st_wolfe_check_done = False - self.__2nd_wolfe_check_needed = False - self.__2nd_wolfe_check_done = False - self.__need_update = False - self.__converged_ls = False - - def fbest(self): - """ See :meth:`Optimiser.fbest()`. """ - return self._fbest - - def wolfe_line_search_parameters(self): - """ - Returns the wolfe line search parameters this optimiser is using - as a vector ``[c1, c2]``. - As c1 approaches 0 and c2 approaches 1, the line search terminates - more quickly. ``c1`` is the parameter for the Armijo condition - rule, ``0 < c1 < 0.5``. ``c2 `` is the parameter for the - curvature condition rule, ``c1 < c2 < 1.0``. - """ - return (self._c1, self._c2) - def max_correction_matrice_storage(self): """ Returns ``m``, the maximum number of correction matrice for @@ -158,54 +60,6 @@ def name(self): """ See :meth:`Optimiser.name()`. """ return 'Broyden-Fletcher-Goldfarb-Shanno (BFGS)' - def needs_sensitivities(self): - """ See :meth:`Optimiser.needs_sensitivities()`. """ - return True - - def n_hyper_parameters(self): - """ See :meth:`pints.TunableMethod.n_hyper_parameters()`. """ - return 2 - - def running(self): - """ See :meth:`Optimiser.running()`. """ - return self._running - - def set_hyper_parameters(self, x): - """ - See :meth:`pints.TunableMethod.set_hyper_parameters()`. - - The hyper-parameter vector is ``[c1, c2, m]``. - ``c1`` is the parameter for the Armijo condition rule, - ``0 < c1 < 0.5``. - ``c2`` is the parameter for the curvature condition rule, - ``c1 < c2 < 1.0``. - ``m`` is the number of maximum number of correction matrices - that can be stored for the LM-BFGS update. - """ - - self.__set_wolfe_line_search_parameters(x[0], x[1]) - - def __set_wolfe_line_search_parameters(self, c1, c2): - """ - Sets the parameters for the wolfe conditions. - - Parameters - ---------- - c1: float - Parameter for the Armijo condition rule, ``0 < c1 < 0.5``. - - c2: float - Parameter for the curvature condition rule, ``c1 < c2 < 1.0``. - """ - if(0 < c1 < 0.5 and c1 < c2 < 1.0): - self._c1 = c1 - self._c2 = c2 - else: - cs = self.wolfe_line_search_parameters() - print('Invalid wolfe line search parameters!!!') - print('0 < c1 < 0.5 and c1 < c2 < 1.0') - print('using default parameters: c1 = ', cs[0], ' c2 = ', cs[1]) - def __set_max_correction_matrice_storage(self, m: int): """ Sets the parameters for the wolfe conditions. @@ -224,764 +78,6 @@ def __set_max_correction_matrice_storage(self, m: int): print('Invalid value of m!!!\nm must be an integer') print('using default parameters: m = ', self._m) - def xbest(self): - """ See :meth:`Optimiser.xbest()`. """ - return self._xbest - - def ask(self): - """ See :meth:`Optimiser.ask()`. """ - - # print('') - # print('in ask') - # print('') - - if not self._running: - self._proposed = np.asarray(self._x0) - else: - - if self.__j == 0: - # working out an initial stepsize value alpha - alpha_0 = self.__initialising(k=self.__k, - alpha_k0=self.__current_alpha) - print('alpha_initial: ', alpha_0) - # Creating an initial bracketing interval of alpha values - # satisfying the opposite slope condition (see function - # docstring) beginning with initial guess [0, alpha_initial]. - bracket = (self.__initial_bracket(c=alpha_0)) - self._minimum_alpha = bracket[0] - self._maximum_alpha = bracket[1] - self._updated_minimum_alpha = self._minimum_alpha - self._updated_maximum_alpha = self._maximum_alpha - - # Looping while wolfe conditions don't need to be checked - # and the line search hasn't converged. - while (~(self.__1st_wolfe_check_needed) & - ~(self.__2nd_wolfe_check_needed) & ~(self.__converged_ls)): - - # secant squared step of the line search - - # *************************************************************** - # a, b = self.__secant2(self._minimum_alpha, - # self._maximum_alpha) - a = self._updated_minimum_alpha - b = self._updated_maximum_alpha - c = self._proposed_alpha - - # checking if the bracketing interval has converged - self.__converged_ls = self.__very_close(a, b) - self.__logic_steps_left = 'not started' - if self.__converged_ls: - self.__logic_steps_left = ' converged in ls ' - # if converged is True don't do anything more. - - # ************ beginning of secant squared see [1] ************ - - # Preforming a secant to propose a value of alpha. - if (~(self.__1st_wolfe_check_done) & - ~(self.__2nd_wolfe_check_done) & ~(self.__converged_ls)): - - # step S1 in [1] - self._proposed_alpha = self.__secant_for_alpha(a, b) - # passing to tell to check wolfe conditions - self.__1st_wolfe_check_needed = True - - # checking the proposed point is in range - if self._proposed_alpha < a or self._proposed_alpha > b: - # If the point is out of range there is no need to - # check wolfe conditions. - self.__1st_wolfe_check_needed = False - self.__1st_wolfe_check_done = True - - self.__logic_steps_left = 2 - self.__j += 1 / 3 - - elif (self.__1st_wolfe_check_done & - ~(self.__2nd_wolfe_check_done) & ~(self.__converged_ls)): - - # (typically) updating one side of the - # bracketing interval - A, B = self.__update(a, b, c) - # end of step S1 in [1] - - # (typically) updating the otherside side of - # the bracketing interval - if c == B: - # S2 in [1] - # Preforming a secant to propose a value of alpha. - self._proposed_alpha = self.__secant_for_alpha(b, B) - - # checking the proposed point is in range - if self._proposed_alpha < A | self._proposed_alpha > B: - # If the point is out of range there is no need to - # check wolfe conditions. - self.__2nd_wolfe_check_needed = False - self.__2nd_wolfe_check_done = True - else: - self.__2nd_wolfe_check_needed = True - self.__need_update = True - elif c == A: - # S3 in [1] - # Preforming a secant to propose a value of alpha. - self._proposed_alpha = self.__secant_for_alpha(a, A) - - # checking the proposed point is in range - if self._proposed_alpha < A or self._proposed_alpha > B: - # If the point is out of range there is no need to - # check wolfe conditions. - self.__2nd_wolfe_check_needed = False - self.__2nd_wolfe_check_done = True - else: - self.__2nd_wolfe_check_needed = True - self.__need_update = True - else: - # No new point has been proposed therefore there - # is no need to check the wolfe conditions. - self.__2nd_wolfe_check_needed = False - self.__2nd_wolfe_check_done = True - - self._updated_minimum_alpha = A - self._updated_maximum_alpha = B - - self.__logic_steps_left = 1 - self.__j += 1 / 3 - - elif (self.__1st_wolfe_check_done & - self.__2nd_wolfe_check_done & ~(self.__converged_ls)): - - # S4 in [1], this is only preformed if S2 or S3 was done - # and the propsed point was in range. - if self.__need_update: - a, b = self.__update(a, b, c) - self.__need_update = False - - # ***************** end of secant squared ***************** - - # preforming steps L2 from [1] - # determing whether a bisection step should be preformed - # i.e if self.__secant_for_interval() didn't shrink the - # bracketing interval by the propotion self.__gamma - new_width = b - a - old_width = (self._maximum_alpha - self._minimum_alpha) - # print('lower_alpha: ', self._updated_minimum_alpha, - # 'updated_upper_alpha: ', self._updated_maximum_alpha) - # print('_maximum_alpha: ', self._minimum_alpha, - # ' maximum alpha: ', self._maximum_alpha) - if new_width > self.__gamma * old_width: - # preforming bisection - #print('preforming bisection') - c = (a + b) / 2.0 - a, b = self.__update(a=a, b=b, c=c) - #print('bisected_lower: ',a,' bisected upper: ', b) - #print('finished bisection') - - # preforming steps L3 from [1] - # updating bracketing interval - self._minimum_alpha, self._maximum_alpha = a, b - self.__j += 1 / 3 - self.__logic_steps_left = 0 - - # reset logic - self.__1st_wolfe_check_needed = False - self.__1st_wolfe_check_done = False - self.__2nd_wolfe_check_needed = False - self.__2nd_wolfe_check_done = False - - # print('line step loops: ', self.__j, - # ' logic steps left: ', self.__logic_steps_left) - # print('lower_alpha: ', self._updated_minimum_alpha, - # 'updated_upper_alpha: ', self._updated_maximum_alpha) - # print('_maximum_alpha: ', self._minimum_alpha, - # ' maximum alpha: ', self._maximum_alpha) - # print('converged: ', - # self.__very_close(self._updated_minimum_alpha, - # self._updated_maximum_alpha)) - # *********************** CONTINUE LOOP *********************** - - if self.__converged_ls: - self._proposed = (self._current + - self._updated_maximum_alpha * self._px) - else: - self._proposed = (self._current + - self._proposed_alpha * self._px) - - # Running, and ready for tell now - self._ready_for_tell = True - self._running = True - - # print('') - # print('finished ask') - # print('') - # Return proposed points (just the one) in the search space to evaluate - return [self._proposed] - - def tell(self, reply): - """ See :meth:`Optimiser.tell()`. """ - - # Check ask-tell pattern - if not self._ready_for_tell: - raise Exception('ask() not called before tell()') - self._ready_for_tell = False - - # Unpack reply - proposed_f, proposed_dfdx = reply[0] - proposed_f = proposed_f - proposed_dfdx = np.asarray(proposed_dfdx) - - # We need the evaluation of the gradients before we can start the BFGS, - # the first tell gets these. - if self._current_f is None: - - # Move to proposed point - self._current = self._proposed - self._current_f = np.asarray(proposed_f) - self._current_dfdx = np.asarray(proposed_dfdx) - - # Update newton direction - # FIXME: is this right for inital newton direction??? - # if it isn't a desecnt direction the line searches will fail - # i.e return alpha = none - self._px = - np.matmul(self._B, self._current_dfdx) - - # resetting the number of steps in the current - # line search iteration. - self.__j = 0 - - # Checking if exact wolfe conditions. - proposed_grad = np.matmul(np.transpose(proposed_dfdx), self._px) - - wolfe_curvature = (proposed_grad >= - self._c2 * - np.matmul(np.transpose(self._current_dfdx), - self._px)) - - exact_wolfe_suff_dec = (self._c1 * - np.matmul(np.transpose(self._current_dfdx), - self._px) - >= proposed_f - self._current_f) - - exact_wolfe = (exact_wolfe_suff_dec and wolfe_curvature) - - # Checking if approximate wolfe conditions are meet. - approx_wolfe_suff_dec = ((2.0 * self._c1 - 1.0) * - np.matmul(np.transpose(self._current_dfdx), - self._px) >= proposed_grad) - - approx_wolfe_applies = proposed_f <= self._current_f + self.epsilon - - approximate_wolfe = (approx_wolfe_suff_dec and wolfe_curvature - and approx_wolfe_applies) - - # If wolfe conditions meet the line search is stopped - # and the inverse hessian matrix and newton direction are updated. - # If the line search has converged we also accept the - # steps and update. - if exact_wolfe or approximate_wolfe or self.__converged_ls: - - print('Number of accepted steps: ', self.__k) - print('step sized alpha: ', self._proposed_alpha, ' accepted') - print('updating Hessian and changing newton direction') - - # Updating inverse hessian. - self._B = self.inverse_hessian_update(proposed_f, proposed_dfdx) - - # Move to proposed point - self._current = self._proposed - self._current_f = np.asarray(proposed_f) - self._current_dfdx = np.asarray(proposed_dfdx) - - # storing the accepted value of alpha - self.__current_alpha = self._proposed_alpha - - # if self.__k == 0: - # print('\nThe first accepted value of alpha was: ', - # self.__current_alpha) - # print('set initial alpha to this in subsequent runs' + - # 'to speed up computation') - - # Update newton direction - self._px = - np.matmul(self._B, self._current_dfdx) - - # incrementing the number of accepted steps - self.__k += 1 - - # Resetting the number of steps in the current line search - # iteration as we have accepted a value and completed this - # line search. - self.__j = 0 - - # resetting line search logic - self.__1st_wolfe_check_needed = False - self.__1st_wolfe_check_done = False - self.__2nd_wolfe_check_needed = False - self.__2nd_wolfe_check_done = False - self.__need_update = False - - else: - # wolfe conditions haven't been meet so we continue the line search - if self.__1st_wolfe_check_needed: - self.__1st_wolfe_check_needed = False - self.__1st_wolfe_check_done = True - if self.__2nd_wolfe_check_needed: - self.__2nd_wolfe_check_needed = False - self.__2nd_wolfe_check_done = True - - # Checking if all gradients ~ 0, - # therefore the classical convergence test of a quasi-newton - # or conjugate gradient method has been meet. - if self.__convergence is not True: - - if norm(proposed_dfdx, ord=np.inf) <= 1e-6: - - self.__convergence = True - print('') - print(20 * '*' + ' Convergence after ', - self.__k, ' accepted steps!' + 20 * '*') - print('||df/dx_i||inf <= 1e-6 with parameters:') - print(self._proposed) - print('error function evaluation: ', proposed_f) - print('\nInverse Hessian matrix:\n', self._B) - - # Update xbest and fbest - if self._fbest > proposed_f: - self._fbest = proposed_f - self._xbest = self._current - - print('') - print('Hessian updates: ', self.__k, ' line steps: ', self.__j, - ' propose alpha: ', self._proposed_alpha, ' propose points: ', - self._proposed) - - def __update(self, a, b, c): - ''' - This function is part of the Hager-Zhang line search method [1]. - - Updates the bracketing boundary values of alpha. Ensuring the opposite - slope conditions are obeyed. - - The opposite slope conditions are: - φ(a) ≤ φ(0) + epsilon, - φ'(a) < 0 (w.r.t every parameter), - φ'(b) ≥ 0 (w.r.t every parameter). - where φ(α) = f(x+α*px), f() is the function binging minimised, x is - the current mparameter set, px is the newton direction, and alpha is - the step size. - - In the first condition, epsilon is a small positive constant. The - condition demands that the function at the left end point be not much - bigger than the starting point (i.e. alpha = 0). This is an easy to - satisfy condition because by assumption, we are in a direction where - the function value is decreasing. The second and third conditions - together demand that there is at least one zero of the derivative in - between a and b. In addition to the interval, the update algorithm - requires a third point to be supplied. Usually, this point would lie - within the interval [a, b]. If the point is outside this interval, - the current interval is returned. If the point lies within the - interval, the behaviour of the function and derivative value at this - point is used to squeeze the original interval in a manner that - preserves the opposite slope conditions. - - :param a: lower bound of alpha - :param b: upper bound of alpha - :param c: proposed value of alpha - :param dfdx_c: vector of gradients at the proposed point i.e alpha = c - :param f_c: function evaluation at the proposed point i.e alpha = c - :param f_0: function evaluation at the previous point i.e alpha = 0 - ''' - - # Check c is within the bracket conditions (steps U0 from [1]). - if c < a or c > b: - # if the proposed point is not in range we return - # the old brackets unmodified - return a, b - else: - - # evaluate function for alpha = c i.e at the proposed boundary - # point_c = self._current + c * self._px - # fs_c = self._evaluator.evaluate([point_c]) - f_c, dfdx_c = self.obj_and_grad_func(c) - - # Checking if the opposite slope condition at the upper bound is - # obeyed by the proposed point - # (steps U1 from [1]). - if dfdx_c >= 0.0: - # Updating the upper bound. - return a, c - - # Checking if the opposite slope condition at the lower bound is - # obeyed by the proposed point, if so it is a valid lower bound - # (steps U2 from [1]). - elif dfdx_c < 0.0 and f_c <= self._current_f + self.epsilon: - # Updating the lower bound. - return c, b - - # The proposed point doesn't obey the opposite slope condition - # i.e. dfdx_c < 0.0 and f_c > self._current_f + self.epsilon. - # Checking this is unnecessary as it is the opposite to the above - # conditions. A secant/bisect can narrow down an interval between - # the current lower bound and the trial point c. - else: - b = c - return self.__bisect_or_secant(a, b) - - def __bisect_or_secant(self, a, b): - ''' - This function is part of the Hager-Zhang line search method [1]. - - Actual implementation of secant (or bisetc if `self.theta` = 0.5) - given a bracketing intervale [a, b] used in `__update()` and - `__initial_bracketing_interval()`. (steps U3a-c from [1]) - - - :param a: lower bound of alpha - :param b: upper bound of alpha - :param c: proposed value of alpha - ''' - - secant = True - - # FIXME: - # problem is in this while loop a and b merge but don't satisfy - # opposite slope rule for upper, - # probably should return when difference between a and b almost - # nothing??? - - # The interval needs updating if the upper bracket has a negative - # slope and the value of the function at that point is too high. - # It is not a valid lower bracket but along with the current - # lower bracket, it encloses another minima. The below function - # is a loop which tries to narrow the interval so that it - # satisfies the opposite slope conditions. - - # (steps U3 from [1]) - - while secant is True: - # according to [1]] this while loop is guaranteed to terminate - # as the intervale between a and b tends to zero - - # Preforming secant (if theta = 0.5 as is default this is a - # bisection) to propose a new point which hopeful obeys the - # opposite slope conditions. - # (steps U3a from [1]) - d = (1.0 - self.theta) * a + self.theta * b - - # Evaluating function for alpha = d i.e at the proposed boundary. - # point_d = self._current + d * self._px - # fs = self._evaluator.evaluate([point_d]) - # f_d, dfdx_d = fs[0] - - f_d, dfdd = self.obj_and_grad_func(d) - - # Checking if the opposite slope condition at the upper bound is - # obeyed by the proposed point. - # If the proposed point has a positive slope, then we have found a - # suitable upper bound to bracket a minima within opposite slopes. - # (still steps U3a from [1]) - converged = self.__very_close(d, b) or self.__very_close(d, a) - if dfdd >= 0.0 or converged: - secant = False - # Updating the upper bound. - return a, d - - # Checking if the opposite slope condition at the lower bound is - # obeyed by the proposed point. - # If the proposed point has a negative slope and the function value - # at that point is small enough, we can use it as a new lower bound - # to narrow down the interval. - # (steps U3b from [1]) - elif dfdd < 0.0 and f_d <= self._current_f + self.epsilon: - # Updating the lower bound. - a = d - - # The proposed point doesn't obey the opposite slope condition - # i.e. dfdx_c < 0.0 and f_c > self._current_f + self.epsilon - # Checking this is unnecessary as it is the opposite to the above - # conditions. We are therefore in the same situation as when we - # started the loop so we update the upper bracket and continue. - # (steps U3c from [1]) - else: - b = d - - def __secant_for_alpha(self, a, b): - ''' - This function is part of the Hager-Zhang line search method [1]. - - Preforms a secant step to propose a value of alpha. This is the same as - the secant routine described in [1]. - - :param a: lower bound of alpha - :param b: upper bound of alpha - ''' - - # Evaluating function for alpha = a to obtain gradients. - f_a, dfda = self.obj_and_grad_func(a) - - # Evaluating function for alpha = b to obtain gradients. - f_b, dfdb = self.obj_and_grad_func(b) - - # Preforming secant. - numerator = a * dfdb - b * dfda - denominator = dfdb - dfda - - return float(numerator / denominator) - - def __initial_bracket(self, c, rho=5): - ''' - This function is part of the Hager-Zhang line search method [1]. - - This function is used to generate an initial interval [a, b] for alpha - satisfying the opposite slope conditions and therefore bracketing - the minimum, beginning with the initial guess [0, c]. - The opposite slope conditions: - φ(a) ≤ φ(0) + epsilon, - φ'(a) < 0 (w.r.t every parameter), - φ'(b) ≥ 0 (w.r.t every parameter). - where φ(α) = f(x+α*px), f() is the function binging minimised, x is - the current parameter set, px is the newton direction, and alpha is - the step size. - - This is the same as the bracket routine described in [1] as steps B0-3 - - :param c: initial guess for maximum value of alpha - :param row: range (1, ∞), expansion factor used in the bracket rule to - increase the upper limit c (c_j+1 = row*c_j) until a - suitable interval is found that contains the minimum. - ''' - - # (steps B0 from [1]) - # Initiating a list of proposed boundary values of alpha. - c = [c] - # Initiating a list of error function evaluations at - # the proposed boundary values of alpha. - f_c = [] - # Initiating lower bound. - a = 0 - - # Initiating an iteration counter for the below while loop. - j = 0 - bracketing = True - - while bracketing: - - # Evaluating function for alpha = c[j] - # i.e at the proposed boundary. - f_cj, dfdc_j = self.obj_and_grad_func(c[j]) - - # Appending the error function evaluations at proposed boundary - # values of alpha. - f_c.append(f_cj) - - # Checking if the opposite slope condition at the upper bound is - # obeyed by the proposed point. If the slope at the propsed point - # is positive, then the given points already bracket a minimum. - # (steps B1 from [1]) - if dfdc_j >= 0.0: - # Setting the upper bound. - b = c[j] - - bracketing = False - - # Checking if the non derivative opposite slope condition at - # the lower bound is obeyed by any of the previously evaluated - # points and returning the value for which the boundary - # conditions are as close together as possible. - for i in range(1, j + 1): - if f_c[j - i] <= self._current_f + self.epsilon: - a = c[j - i] - return a, b - return a, b - - # Checking if the proposed point doesn't obey the opposite slope - # condition. This means the upper bracket limit almost works as a - # new lower limit but the objective function(f_cj) is too large. - # We therefore need to preform a secant/bisect but the minimum is - # not yet bracketed. - # (steps B2 from [1]) - elif dfdc_j < 0.0 and f_cj > self._current_f + self.epsilon: - # The interval needs updating if the upper bracket has a - # negative slope and the value of the function at that point - # is too high. It is not a valid lower bracket but along with - # the current lower bracket, it encloses another minima. The - # below function tries to narrow the interval so that it - # satisfies the opposite slope conditions. - bracketing = False - return self.__bisect_or_secant(0.0, c[j]) - - # The proposed point obeys the opposite slope condition - # at the lower bound - # i.e. dfdx_d < 0.0 and f_d <= self._current_f + self.epsilon. - # Checking this is unnecessary as it is the opposite to - # the above conditions. This means the bracket interval needs - # expanding to ensure a minimum is bracketed. - # (steps B3 from [1]) - else: - # Increasing the proposed point by a factor of row to attempt - # to bracket minimum and trying again. - c.append(rho * c[j]) - - # Increamenting the iteration counter. - j += 1 - - def __initialising(self, k, alpha_k0): - ''' - This function is part of the Hager-Zhang line search method [1]. - - Generate the starting guess of alpha, 'c', used by - ``__initial_bracket()``. This is the same as the routine - called initial and described as I0-2 in [1]. - - :param k: number of accepted steps/newton direction updates that - have taken place. - :param alpha_k0: the alpha value used by the previously accepted - steps/newton direction - update. If k = 0 this is the initial alpha the user wants to be used - ''' - QuadStep = False - - # Small factor used in initial guess of step size, range (0, 1). - # As the initial guess is very crude this is a very small factor - # to keep the initial step short and close the starting parameters - # self.x_0. - ps_0 = self.__ps_0 - - # Small factor used in subsequent guesses of step size - # if Quadstep is true, range (0, 1). - # TODO: this hasn't been implement yet - ps_1 = self.__ps_1 - - # Sacling factor used in subsequent guesses of step size, - # range (1, inf). - ps_2 = self.__ps_2 - - # For the first line search do the following - # (step I0 in [1]) - if k == 0: - - if alpha_k0 is not None: - # returning user specified initial alpha - return alpha_k0 - - # (step I0a in [1]) - elif np.all(self._x0 != 0.0): - # Crude step size estimate - # :math: \alpha = ps_0*||x_0||_\inf / ||dfdx||_\inf - return ((ps_0 * norm(self._x0, ord=np.inf)) - / (norm(self._current_dfdx, ord=np.inf))) - - # If self._x0 = 0.0 the above statement would give alpha = 0, - # hence we use the following estimation. - # (step I0b in [1]) - elif self._current_f != 0: - # Crude step size estimate - # :math: \alpha = ps_0*|f(x_0)| / ||dfdx||^2 - return ((ps_0 * abs(self._current_f)) - / (pow(norm(self._current_dfdx, ord=2), 2.0))) - - # Otherwise self._current_f = 0.0 and we are already at the - # minimum therefore return alpha = 1.0 it should not matter what we - # return in this case as if self._current_f = 0.0 the gradients - # will also equal zero. - # (step I0c in [1]) - else: - return 1.0 - - # TODO: implement the below option using a quadratic interpolant ??? - # everything will work without this - # (step I1 in [1]) - elif QuadStep: - - # point_current_scaled = self._current + ps_1*alpha_k0* self._px - # fs = self._evaluator.evaluate([point_current_scaled]) - f, df = self.obj_and_grad_func(ps_1 * alpha_k0) - - if f <= self._current_f: - pass - #TODO: add quad step option - pass - - # For the subsequent line search do the following - # (step I2 in [1]) - else: - # Increases the step size of the previous accepted step as it - # is only decreased in subsequent boundary manipulation. - return ps_2 * alpha_k0 - - def __very_close(self, x, y): - ''' Returns true if x is very close in value to y. ''' - return np.nextafter(x, y) >= y - - def obj_and_grad_func(self, alpha: float): - ''' - For a given alpha this returns the values of the objective function - and it's derivative. - ''' - point_alpha = self._current + alpha * self._px - fs_alpha = self._evaluator.evaluate([point_alpha]) - f_alpha, dfdx_alpha = fs_alpha[0] - - dfdalpha = np.matmul(np.transpose(dfdx_alpha), self._px) - - return f_alpha, dfdalpha - - # def __secant2(self, a, b): - # ''' - # This function is part of the Hager-Zhang line search method [1]. - - # This function is referred to as secant^2 and described as steps - # S1-4 in [1]. - - # Preforms a secant step to update the bracketing interval of alpha. - # Given an interval that brackets a root, this procedure performs an - # update of both end points using two intermediate points generated - # using the secant interpolation `self.__secant_for_alpha()`. - # Assuming the interval [a, b] satisfy the opposite slope conditions. - - # The opposite slope conditions: - # φ(a) ≤ φ(0) + epsilon , - # φ'(a) < 0 (w.r.t every parameter), - # φ'(b) ≥ 0 (w.r.t every parameter). - # where φ(α) = f(x+α*px), f() is the function binging minimised, - # x is the current parameter set, px is the newton direction, - # and alpha is the step size. - - # :param a: power bound of alpha - # :param b: upper bound of alpha - # ''' - - # # Preforming a secant to propose a value of alpha. - # # (step S1 in [1]) - # c = self.__secant_for_alpha(a,b) - # # CHECK IF c SATISFY THE WOLFE CONDITIONS i.e pass to tell!!!!!!! - # # IF IT HAS STOP SEARCHING THIS DIRECTION!!!! - - # # IF WOLFE CONDITIONS AREAN'T MEET DO THIS thing below - - # # (typically) updating one side of the bracketing interval - # print('first update secant') - # A, B = self.__update(a, b, c) - - # # (typically) updating the otherside side of the bracketing interval - # # (step S2 in [1]) - # if c == A: - # # Preforming a secant to propose a value of alpha. - # C = self.__secant_for_alpha (a, A) - - # # (step S3 in [1]) - # if c == B: - # # Preforming a secant to propose a value of alpha. - # C = self.__secant_for_alpha (b, B) - - # # (step S4 in [1]) - # if c == A or c == B: - # # CHECK IF C SATISFY THE WOLFE CONDITIONS i.e pass to tell!!!!!!! - # # IF IT HAS STOP SEARCHING THIS DIRECTION!!!! - - # # IF WOLFE CONDITIONS AREAN'T MEET DO THIS thing below - - # print('second update secant') - # A, B = self.__update(A, B, C) - - # return A, B - def inverse_hessian_update(self, proposed_f, proposed_dfdx): '''the inverse hessian matrix and newton direction are updated by the L-BFGS/BFGS approximation of the hessian described in reference [2] @@ -993,19 +89,19 @@ def inverse_hessian_update(self, proposed_f, proposed_dfdx): # We do this if we haven't exhausted existing memory yet, this is # identical to the BFGS algorithm - if self.__k <= self._m - 1: - k = self.__k + if self._k <= self._m - 1: + k = self._k # Defining the next column. self._S[:, k] = self._proposed - self._current self._Y[:, k] = proposed_dfdx - self._current_dfdx # Defining B_0. Scaling taken from [4]. B = ((np.matmul(np.transpose(self._Y[:, k]), - self._S[:, k]) - / (norm(self._Y[:, k], ord=2) ** 2)) * I) + self._S[:, k]) + / (norm(self._Y[:, k], ord=2) ** 2)) * I) # Updating inverse hessian. - for k in range(self.__k + 1): + for k in range(self._k + 1): V = (I - np.matmul(self._Y[:, k], np.transpose(self._S[:, k])) @@ -1014,9 +110,9 @@ def inverse_hessian_update(self, proposed_f, proposed_dfdx): B = np.matmul(np.transpose(V), np.matmul(self._B, V)) B += (np.matmul(self._S[:, k], - np.transpose(self._S[:, k])) - / np.matmul(np.transpose(self._Y[:, k]), - self._S[:, k])) + np.transpose(self._S[:, k])) + / np.matmul(np.transpose(self._Y[:, k]), + self._S[:, k])) # We have exhausted the limited memory and now enter # the LM-BFGS algorithm @@ -1033,8 +129,8 @@ def inverse_hessian_update(self, proposed_f, proposed_dfdx): # Defining B_0. Scaling taken from [4]. B = ((np.matmul(np.transpose(self._Y[:, m]), - self._S[:, m]) - / (norm(self._Y[:, m], ord=2) ** 2)) * I) + self._S[:, m]) + / (norm(self._Y[:, m], ord=2) ** 2)) * I) # Updating inverse hessian. for k in range(self._m): @@ -1046,12 +142,12 @@ def inverse_hessian_update(self, proposed_f, proposed_dfdx): B = np.matmul(np.transpose(V), np.matmul(self._B, V)) B += (np.matmul(self._S[:, k], - np.transpose(self._S[:, k])) - / np.matmul(np.transpose(self._Y[:, k]), - self._S[:, k])) + np.transpose(self._S[:, k])) + / np.matmul(np.transpose(self._Y[:, k]), + self._S[:, k])) B = ((np.matmul(np.transpose(self._Y[:, m]), - self._S[:, m]) - / (norm(self._Y[:, m], ord=2)**2)) * I) + self._S[:, m]) + / (norm(self._Y[:, m], ord=2)**2)) * I) - return B \ No newline at end of file + return B From ef4de154049a7eb17fc7e00c147a8bc487ef1cca Mon Sep 17 00:00:00 2001 From: alisterde Date: Sun, 8 Nov 2020 15:30:26 +0000 Subject: [PATCH 09/16] renaming bfgs to lbfgs --- pints/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pints/__init__.py b/pints/__init__.py index a3fe61c2e..0519eca9a 100644 --- a/pints/__init__.py +++ b/pints/__init__.py @@ -176,8 +176,7 @@ def version(formatted=False): from ._optimisers._pso import PSO from ._optimisers._snes import SNES from ._optimisers._xnes import XNES -from ._optimisers._bfgs_scipy import BFGS_scipy -from ._optimisers._bfgs_linesearch import BFGS +from ._optimisers._lbfgs import LBFGS # From e1b3585978d1ce217cd5099d0d385831d1fb4813 Mon Sep 17 00:00:00 2001 From: alisterde Date: Sun, 8 Nov 2020 15:34:57 +0000 Subject: [PATCH 10/16] splitting optimser and linesearch --- pints/_optimisers/__init__.py | 7 ++++++- .../_optimisers/{_bfgs_linesearch.py => _lbfgs.py} | 13 ++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) rename pints/_optimisers/{_bfgs_linesearch.py => _lbfgs.py} (91%) diff --git a/pints/_optimisers/__init__.py b/pints/_optimisers/__init__.py index 6dc48426e..6c5847b6e 100644 --- a/pints/_optimisers/__init__.py +++ b/pints/_optimisers/__init__.py @@ -748,8 +748,11 @@ def tell(self, reply): self.__2nd_wolfe_check_done = True # Checking if all gradients ~ 0, - # therefore the classical convergence test of a quasi-newton + # Therefore the classical convergence test of a quasi-newton # or conjugate gradient method has been meet. + # TODO: Implement a means of stopping the optimiser is this + # condition is meet (apparently something similar is done + # in CMAES) if self.__convergence is not True: if norm(proposed_dfdx, ord=np.inf) <= 1e-6: @@ -1057,6 +1060,8 @@ def __initialising(self, k, alpha_k0): steps/newton direction update. If k = 0 this is the initial alpha the user wants to be used ''' + # TODO: The Quadstep option has not been implemented yet. + # However, the linesearch is functional without is QuadStep = False # Small factor used in initial guess of step size, range (0, 1). diff --git a/pints/_optimisers/_bfgs_linesearch.py b/pints/_optimisers/_lbfgs.py similarity index 91% rename from pints/_optimisers/_bfgs_linesearch.py rename to pints/_optimisers/_lbfgs.py index e37b6ed21..f5c3f854d 100644 --- a/pints/_optimisers/_bfgs_linesearch.py +++ b/pints/_optimisers/_lbfgs.py @@ -12,7 +12,7 @@ import pints -class BFGS(pints.LineSearchBasedOptimiser): +class LBFGS(pints.LineSearchBasedOptimiser): """ Broyden-Fletcher-Goldfarb-Shanno algorithm [2], [3], [4] @@ -39,7 +39,7 @@ class BFGS(pints.LineSearchBasedOptimiser): """ def __init__(self, x0, sigma0=None, boundaries=None): - super(BFGS, self).__init__(x0, sigma0, boundaries) + super(LBFGS, self).__init__(x0, sigma0, boundaries) # maximum number of correction matrices stored self._m = 5 @@ -60,9 +60,11 @@ def name(self): """ See :meth:`Optimiser.name()`. """ return 'Broyden-Fletcher-Goldfarb-Shanno (BFGS)' - def __set_max_correction_matrice_storage(self, m: int): + def __set_max_correction_matrice_storage(self, m): """ - Sets the parameters for the wolfe conditions. + Sets the maximum number of correction matrice to be stored and used, + in subsequent inverse hessian updates, if ``m`` is set large enough + for the problem this method becomes the BFGS rather than the L-BFGS. Parameters ---------- @@ -79,7 +81,8 @@ def __set_max_correction_matrice_storage(self, m: int): print('using default parameters: m = ', self._m) def inverse_hessian_update(self, proposed_f, proposed_dfdx): - '''the inverse hessian matrix and newton direction are updated by the + ''' + The inverse hessian matrix and newton direction are updated by the L-BFGS/BFGS approximation of the hessian described in reference [2] [3], and [4]. ''' From 3c0244bd3a1fc32622555e0bba133d6c79e45cff Mon Sep 17 00:00:00 2001 From: alisterde Date: Sun, 8 Nov 2020 15:50:38 +0000 Subject: [PATCH 11/16] documentation related to LBFGS and linesearch --- docs/source/optimisers/base_classes.rst | 1 + docs/source/optimisers/index.rst | 1 + docs/source/optimisers/lbfgs.rst | 7 +++++++ 3 files changed, 9 insertions(+) create mode 100644 docs/source/optimisers/lbfgs.rst diff --git a/docs/source/optimisers/base_classes.rst b/docs/source/optimisers/base_classes.rst index 82b5cf7e7..9fce056ca 100644 --- a/docs/source/optimisers/base_classes.rst +++ b/docs/source/optimisers/base_classes.rst @@ -8,3 +8,4 @@ Optimiser base classes .. autoclass:: PopulationBasedOptimiser +.. autoclass:: LineSearchBasedOptimiser diff --git a/docs/source/optimisers/index.rst b/docs/source/optimisers/index.rst index 3a812e597..eef64719d 100644 --- a/docs/source/optimisers/index.rst +++ b/docs/source/optimisers/index.rst @@ -20,6 +20,7 @@ or the :class:`OptimisationController` class. cmaes_bare cmaes gradient_descent + lbfgs nelder_mead pso snes diff --git a/docs/source/optimisers/lbfgs.rst b/docs/source/optimisers/lbfgs.rst new file mode 100644 index 000000000..977712194 --- /dev/null +++ b/docs/source/optimisers/lbfgs.rst @@ -0,0 +1,7 @@ +****** +L-BFGS +****** + +.. currentmodule:: pints + +.. autoclass:: LBFGS From 009e7be1ff5cb2be477f3d5c5ef3f9ed35596acf Mon Sep 17 00:00:00 2001 From: alisterde <56870042+alisterde@users.noreply.github.com> Date: Sun, 8 Nov 2020 15:52:10 +0000 Subject: [PATCH 12/16] Delete bfgs_trial.ipynb --- examples/optimisation/bfgs_trial.ipynb | 368 ------------------------- 1 file changed, 368 deletions(-) delete mode 100644 examples/optimisation/bfgs_trial.ipynb diff --git a/examples/optimisation/bfgs_trial.ipynb b/examples/optimisation/bfgs_trial.ipynb deleted file mode 100644 index 954e4db29..000000000 --- a/examples/optimisation/bfgs_trial.ipynb +++ /dev/null @@ -1,368 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "os.chdir(\"../..\")\n", - "import pints\n", - "import pints.toy" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "tags": [] - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": "con1 : False\ncon2 : False\ncon3 : False\ncon1 changed to true\ncon1 : True\n\n******************************\ncon2 changed to true\ncon2 : True\n\n******************************\ncon3 changed to true\ncon3 : True\n\n******************************\nfinsihed while\ncon1 : True\ncon2 : True\ncon3 : True\n" - } - ], - "source": [ - "con1 = False\n", - "con2 = False\n", - "con3 = False\n", - "counter = 0\n", - "\n", - "while (con1 is not True and\n", - " con2 is not True and\n", - " con3 is not True):\n", - "\n", - " print('con1 :', con1)\n", - " print('con2 :', con2)\n", - " print('con3 :', con3)\n", - "\n", - " if counter == 0 and con1 is not True:\n", - " print('con1 changed to true')\n", - " con1 = True\n", - " print('con1 :', con1)\n", - " print('')\n", - " print('*'*30)\n", - " counter += 1\n", - " if counter == 1 and con2 is not True:\n", - " print('con2 changed to true')\n", - " con2 = True\n", - " print('con2 :', con2)\n", - " print('')\n", - " print('*'*30)\n", - " counter += 1\n", - " if counter == 2 and con3 is not True:\n", - " print('con3 changed to true')\n", - " con3 = True\n", - " print('con3 :', con3)\n", - " print('')\n", - " print('*'*30)\n", - " counter += 1\n", - "\n", - "print('finsihed while')\n", - "print('con1 :', con1)\n", - "print('con2 :', con2)\n", - "print('con3 :', con3)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "#specifing a simple model for BFGS line search trial\n", - "\n", - "# dzdt = ax^2 +by^2\n", - "# dz/dxdt = 2ax\n", - "# dz/dydt = 2by\n", - "# starting conditions z=0, y=0, x=0\n", - "# x and y increase by 1 in each step?\n", - "\n", - "class easyProblem(pints.ForwardModelS1):\n", - " \"\"\"\n", - " A model with parameters of a similar magantuide\n", - " \"\"\"\n", - "\n", - " def __init__(self):\n", - " super(easyProblem, self).__init__()\n", - "\n", - " def n_parameters(self):\n", - " return 2\n", - " def n_outputs(self):\n", - " return 1\n", - "\n", - " def _simulate(self, parameters, times, senstivities = False):\n", - "\n", - " # unpacking parameters\n", - " a = parameters[0]\n", - " b = parameters[1]\n", - " #ax^2 +by^2 where x= sin(times) and y=cos(time)\n", - " time_dim = len(times)\n", - " time_step = float(times[1])\n", - " x = np.sin(times)\n", - " y = np.cos(times)\n", - " dzdt = a*np.square(x) + b*np.square(y)\n", - " if senstivities is True:\n", - "\n", - " # creating senetivity matrix\n", - " sen = np.zeros((time_dim,2,3))\n", - " for i, xi in enumerate(x):\n", - " old_sen = sen[i,:,:]\n", - "\n", - " if i < time_dim:\n", - " # df/dtheta\n", - " dfdtheta = np.asarray([[0,0],\n", - " [0,0],\n", - " [xi**2, y[i]**2]])\n", - " # jacobian df/du\n", - " Jac = np.array([[0,1,0],\n", - " [-1,0,0],\n", - " [2*a*xi, 2*b*y[i], 0]])\n", - " temp = np.matmul(old_sen, np.transpose(Jac)) + np.transpose(dfdtheta)\n", - " new_sen = temp*time_step + old_sen\n", - "\n", - " # assigning the new sentivity to the right place in the senstivity array\n", - " sen[i,:,:] = new_sen\n", - "\n", - " return (dzdt, sen[:,:,-1])\n", - "\n", - " else:\n", - " return dzdt\n", - " \n", - " def simulate(self, parameters, times):\n", - " return self._simulate(parameters, times, senstivities = False)\n", - "\n", - " def simulateS1(self, parameters, times):\n", - " return self._simulate(parameters, times, senstivities = True)\n", - "\n", - " def suggested_parameters(self):\n", - " \"\"\"suggested parameters for the model [a, b]\"\"\"\n", - " return [3, 9]\n", - " def suggested_times(self):\n", - " \"\"\"suggested times for the model range 0 to 10\"\"\"\n", - " return np.linspace(0, 10, num=100000)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# making some toy data\n", - "model = easyProblem()\n", - "times = model.suggested_times()\n", - "real_parameters = model.suggested_parameters()\n", - "output = model.simulateS1(real_parameters, times)#\n", - "values = output[0]\n", - "sensetivities = output[1]" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "tags": [] - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": "Minimising error measure\nUsing Covariance Matrix Adaptation Evolution Strategy (CMA-ES)\nRunning in sequential mode.\nPopulation size: 6\nIter. Eval. Best Time m:s\n0 6 1.03e+07 0:00.1\n1 12 1.01e+07 0:00.1\n2 18 1.01e+07 0:00.1\n3 24 1.01e+07 0:00.2\n20 126 1e+07 0:00.7\n40 246 1e+07 0:01.5\n60 366 1e+07 0:02.2\n80 486 1e+07 0:02.8\n100 606 1e+07 0:03.5\n120 726 1e+07 0:04.1\n140 846 1e+07 0:04.7\n160 966 1e+07 0:05.3\n180 1086 1e+07 0:06.0\n200 1206 1e+07 0:06.7\n220 1326 1e+07 0:07.5\n240 1446 1e+07 0:08.1\n260 1566 1e+07 0:08.9\n280 1686 1e+07 0:09.6\n300 1806 1e+07 0:10.2\n320 1926 1e+07 0:10.9\n340 2046 1e+07 0:11.5\n360 2166 1e+07 0:12.1\n380 2286 1e+07 0:12.7\n393 2358 1e+07 0:13.1\nHalting: No significant change for 200 iterations.\nScore at true solution: \n10040056.240084708\nFound solution: True parameters:\n 3.02438339552395208e+00 3.00000000000000000e+00\n 8.94910569555093005e+00 9.00000000000000000e+00\n" - }, - { - "output_type": "display_data", - "data": { - "text/plain": "
", - "image/svg+xml": "\n\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEGCAYAAACO8lkDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd5xU1fn48c8zs42lL6wgLLB0pAtLkSYIIgiR2IiKJRpDEjUaY2LQfI0latSfJdEkGk1sCYoNFUVA6SDNpYkU6b0tIGWBhd2Z8/tjZpbZnbIzszNzZ3ee9+vFi9k7d+6cafc55TnnijEGpZRSypvN6gIopZRKPBoclFJK+dDgoJRSyocGB6WUUj40OCillPKRYnUBoqFhw4YmNzfX6mIopVSVsnz58kPGmGx/91WL4JCbm0t+fr7VxVBKqSpFRHYEuk+7lZRSSvnQ4KCUUsqHBgellFI+NDgopZTyocFBKaWUDw0OSimlfGhwUEop5UODg1JJoPBMCZ+u2mN1MVQVosEhzrYUFHLyTInVxVBJ5oHJa7hn0iq+23PM6qKoKkKDQ5wNfW4et735jdXFUElm/7HTAJw667C4JKqq0OBggaXbjlhdBKWUCkqDg1JKKR8aHJRSSvnQ4KAS0tPTN9D3yVlWF0OppFUtluxW1c/Lc7dYXQSlkpq2HJRKIsYYq4ugvBQ7nBw5edbqYvilwUHF3OaDhXyQv8vqYiQ1QawugvLjDx9+S48/f4XDmXhBW7uVVMxd9tf5OJyGa/OaWV0UpRLKZ9/uBcDhNNhtiRXAteWgYi6UWtHmg4WVnr17psTB8aLiSh1DKeWiwcEi6/Yep1CX0Sg17Pl5jH5pYaWOccNrS+n6yJdRKlH1Yki8bgt1TiJ+PhocLHL5iwsY/3a+1cWoVpbv+MHqIsTdiaJi9rmXxgiFSGJ1XSS7RB4L0uBgofztyXcyq4xjp4qrbLbNu8t2smTr4agf90cvLeSiv8yO+nGjZfWuo5wp0fWcKpKIX2sNDhEyxrB4y+Eqe7KqavYcPU23x77k1flbrS5KRB6YvIbrXl0S9eNuP3wq6seMlh2HTzLmH1/z6GfrrC5KWE6fdcQke8jhNMzZcLDsOSNxGw4aHCI1Y+1+rn9tCf9dssPqolQpB08UcbbEWWZbKIPIe35wdZ3MXH8gJuWKpy0FhazeddSS545nZeboKdfnWtWWCb/gT9P5zXuron7c/yzcyq1vfsOMtfujfuxY0OAQod3uk9X2Q5HX3BJxECqWjDH0fmIWv3lvJcYYZq47gMNpGPrcvJg/99kSJw9/+h0/JMCEo6HPzWPMP76O63N6922v33c8YSdeJYrPVu+N+jF3HXGdMw6eOFO6zZmA8xs8kj44fLfnGFNW741Jf7Dy74s1+5m6Zh+3v53P6wu3UeD1Y4mVz7/dy1uLd/DkF+tj/lyJbuTfFjD6xQVRO55m3UWuxB0cFm46ZHFJfCV1cHA4DaNfWsjd764M2h/87IzvGfrc3PgVLEYWbT7Ehv3HK3WMmesOMPXbfZUuy8HjroCw52jomTaV4elDTuCKWlztPVbks+3U2RIe/Wwtp8O4INBnq/fS+eEZfruOquJbXexwVrxTDBwqPFdBcjhNQgRcy4ODiNhFZKWIfO7+u6WILBWRzSLynoikxeq5Q+1//fuczWwpOBmrYlRascMZ0mu54d9LGfHXytUYb387nzvfWRHy/l9vTrwakfLv1flbeePr7bz+9bYy23MnTOWJqf4HledvLABc83YCSeAxVx9WXSnP+9f7x4/X0PnhGZYvqWF5cADuAbzb+k8DLxhj2gA/AD+zpFRxEOh8frbE6TNoG4jTaWj7x2kJmxEy7t9L/W6virXKRPNlBAOb6/cFPol7Tkb+TkqvLdjms02dY4xh1voDZSpp8zcW8KdPvwvp8YVFJex1t6I/XL4bgAPHfVt38WRpcBCRHGAU8G/33wJcAnzo3uUt4MfWlC4yxhjmbyyo1EBTh4em0efJmSHt6+mznLg0tllTx04XR7XJfcKdoaSXTI3Moi2HGP/f5WE/7pEAlYjNB0+UVla+3X2Mo6fCG7BOtuSK8lo+8AU/eyufd5edW2Dy5teX8fZi39+lv0rhE1+sp99Trvkqnt/0z96ydpKs1S2HvwL3A56zTgPgqDHG0+G2G2jq74EiMl5E8kUkv6CgIKInDzZbNHfCVB6ZstZn+3d7jvlNRVu96ygbD5zgq3UHuPn1ZfxnYeQ1LaeBH04l1hpB3R79krvfXRm143myZ4LVZENRVOwoDTTJ5GiQ78dbi7aTO2EqRcXnukj8nbx3/3AKp9OwcucPDHt+fml30sz1B7j2lcWux1XQXRlswnV1nAN0xd8X8o85mwPe72+2+uwNrvTrcCenxyNRIxjLgoOIjAYOGmPCr/4AxphXjTF5xpi87OzsKJfO5c1F2322jX5pIb/wU2Mb84+vGf7CfCZ946o57DgSuzEKh9OVBhrqj+/U2RLe+Lry3QLTvqtcfvas9Qcjfmyg1zr0uXl00fWUyvCcvI6dDh40Bzw9hzsmrmDnEVc6tnd/+6aDhfR/ajZbCgorX6BqsmTHd3uO8e3uY/y/Gd+H9bjb3oxNC+DU2RKenfF9yF3Q4bKy5dAfuEJEtgOTcHUn/Q2oJyKepcRzgD3WFC8yszdEfgIM1V9nbuT2t/P5Ys3+kJrzT03bkBBjEvleax9FWtsvvxZNvLKdqhJ/PZqB1vCZHmTcYs/R03y0IrSfn5WNhM0HT5A7YWrM09ErszDkloJCdrmDcLHDGZXVg/8+ezN/n7OZSd/srPSx/LEsOBhjHjDG5BhjcoHrgNnGmHHAHOAa9263AJ/Gqgzlfy6bD56I2rELi0oqrLlF+nt6abarZlhw4tyAVbAFvIJ1QcTTK/POXfoz2I8j2Pt21qJUw0QT6NOev7GgTFpk7MtRuVbB/I0FPDB5TaWO8fVmV1D4Yk3lU6xjZehz85jzvav7+/Gp66OyevAZd4uhOrYcAvkD8FsR2YxrDOI/8XriYc/PZ9uh8LuD/KXxfbJqL90e1e6OcD3/1Ubue3916d9vlevaWxXishPLth2J2Y8m3tbvO84/557r5/5+/wl+NdE3nXjZtiNM++7cCfLqlxfxo0oug+6vRXCmxHftIX8VnVAqPze/vox3l8Wm5ltU7Ahr3CNQSviJomJmhblsS7RWCHY6DZ3+NN0n4WTZtiOVGtcMRUIEB2PMXGPMaPftrcaY3saYNsaYa40xcR2VCTYI5J2ts/3wuSByeRRnm0bqrMPJ8BfmsTMBFmI7euosuROm8nqQL+/eo/7T9F6ctYk1e84FgIenBJ6UFeiz2rD/OGP/tThgbr6VVu4M/6Txo5cW8sz0c/3c4//r24f91boDjP3X4jLZMrt/OM2aPccqbBEHO3+W+Gmptf+/6dwx0TXuFspwghUjDrt/OEWHh6bzk38t4b0Qu13a/nEad73jm3Rx73urw8ocWrv3GFe/vCjk/QMzlDgNJ886fJJj/jIt9jP9EyI4JBrv2Yn7vWaStv3jtNLb8comCqf2s/FAoc8EJojveGCHh6bR/bGvAHjs88An571BxgoOHC970r/gT9P5ZFXZvu+iYgefrvLfH36k0JWGufFA+IOpxhg+Wr67TKZPqA4eLyrtV/b4fv+JMp/flf/0f9LYWlAY8HMuCSEtOtich28qsTT8vwME+BlrE3sBxK3uSavLth/hDx+F3m210M+kzR2HfXsTPlu9l5tfX+b3GIcKY7tu1X8WbmPlztgv3KjBoRxjTJmJK7e//U3Q/f2djP255Lm53PBa2SU6jDEYY3hg8hq/q3QWnDhDh4em89qCwMtUx3ogsMTh9Ltcxv99sobLXpjPI1PWllmvv6g4tK6ccIvtXSMGeHzqOh6fGrz2dKrYEfa1BOZvOsR9H6zmqWkb/N6/8cCJgEsb9H5yFgOfmVP69+Ith7nsr/P5XwUr967adZRLnptHywe+4Kt1rpPu7h9O8fO38/mFn1aCPx+4J04Fst3PCS5cwVYgNib0ReQ2BXkPrWCMCXvNrV+/u7J0dnikwlmttthhePjT7zDG8Odyla5YnQOSOjj4q1G/vWQHh70i/4ki/1/icNcX2lpwkkVbfLMpjp0u5t1lO0trId41T0/O9Ger4z/QtrWgkEemrKXNH6f5neX5vyU7+f7ACd5ctJ1PVoafUFbZHPidRyrOUlq96yjDX5jP59/uZcP+0JINPFlUgbqshr8wn5+9GbzC4OGpcX63x/9cjlNnS9h88ESZmulT7u6CAU/P4at1B8rU0PccPc2psyWUOMJ/7w4G6S5dui20LJ+HPvH9Hnh+Qw9+vIZWD35R5vsb6CO+9IX53PZGaO9hKPx9l8I5cR88cSbgdUIKz5SEVZE5fPIsf/jw25D2DTf76a3FO0pXg46HpA4O/izecrjMYNuOOPfhhzMLOdQvbSSTtUf+bUHpPI/DFSzvHMl5vjIXqTl91hHyj3/H4VPc9c7K0sG7j1bsrnRgitas7vFvL2fY8/NxhlieXUdO0fFPM6Kevlu+VVYZmw/6duX5q4Qt2x76e/j45+vInTC14gl5XrcDdYf5E6hys/HACTo/PMPva/IoX4l4Z+lO9ldi2YvyrQkrZ55rcCjnyMmzfvsdrVD+t7Dn6OmIcrkjWZv+TASZPvfG4AIp/ny43Pdk5m/gNJBQzsXx+FEu2uL6nm0LcVHHeyZFb4Z69IQ2oPXfxdvZeCCyVHHPiT5Wc4j+EqALcfgL8yt8bK8nQlvmJlSVmUsRbRocYix3wlQe/cx3GQ5w1fxDOVF5al5Dnp3rs7R4IqxQ4Jl78HEE3UuR8Be42nglC4Rj04ETZfLjg+XtB6u5hppi68+LswMvx+Ct/EB9qA5FcRkG75Tag8eLfFoFr3+9jdwJUzl1tmx37EOfrmXEXys+2QYT6nhWdXGo8CzvfRO9Vl24NDjEwRtfb69wn2AZRYVFJfx38faEzduPpA+8MioaiA7HpS/M5w4/cwaMcXXxFRU72HXkFLkTprIiSIbIxhDHNCqypeBk1CexPffVxqgdyzultveTszhebsLiAvdFa1xXmiv7vQjWvRmteQGB9PvLrNKWWlXyp0/LVizjWRnU4JDAPN+DrYdO8tCnvq2P9fuO+1y8JxqLnflbcDBZeAfpy/46nw4PTWeee3zDX3eWx14/C655BOqiCnSyzHs8ul0VsfR5CIkZgeo93vMv3lla8VyEP36yJuKJX3uPFfH09PDWREpEnkX8vMWqC1SDg8Uq87G+n787YN58ZfhbcLAqqeg9XbTlMEOfm1vhXIat5cYCPHHXXyvvrzM3ld72pGlWk/XmYmbexnM1+cIzZVsg3+055rNawdFTxfz583V8t+cYuROmctc7K/jHnM0h/4ZW7zrKocIzll8noTICLbkeCxocLOap6R89VUxRsYMbXlvqc1+4vt58KKKFvQ4XnmHk3yKb7Z0fRvaJ1W78z1K2FJwsc/Ivn6Mf7K0vf84vn7HS+eEZPrnoypf3ezRj7YHSSY2ey/cOeXau38d5kjI+/3ZfmRVSgy3B75H3+Ez6PDmrEqX2lTthalSPlyg0OFio/Anom+1HKpUGB64+33H/Xkq/v8wOuI/3tSq2FhTywlcbMcbw4MdrIrq+wmOfr+Ma9/r/VYn3ejWtHvwC8N8FUlGI9pexEut1bxLdsOfnlc7PWek1WL8iyPIh90xyZbt5TxqM5Tjbp6v2+J39rFxSKt5FxZL3GkObIljuobyt7qZ4RTNQ31y0nTcXbadBzTQOnzxL/zYNky4b5HSIS2R41ubxrpj+9v1VNKlbg99d1j7g48JZtqG6KSp2lnZPeleCrgqhG9R7Xs3fZ28KsqeLZzn6L9fu55ErOoVcxnsmrSIzzR7y/olKZ0jHQDxnGwbyo7+fy2suvxZRPBITPD/Esf+qejX/aPNeR8t7kM8zw/ncmIMwecUe/h7kimAqct4TQY+EcbnSvceKKDhxJqw5L6cCLOpYlcTqPJHUwSHQ0hgqOUwudyGbvn+ZVcFlL13/hzvOnAhzUaqSUxW0eoO9n72emMnDSZZtF+oM+3AldXBIdPHuD51XyYXEqoNf/s93zoNHpNlHFS2Kp8ryPtVFcjGhUNJrq5NX5m6peKcIaHBIYPe+t7rinZSPcf9eWvFOFdDavnX2HatcUkZFV2Csbo7HqAdEg0M1V13T7KwUyvUVVOQ8y5YDODRKW0aDQzXm7/KlKnL+zlMafGNrjp/F9qIxI/iqf35d6WNUdxocqrFEuHxpVeXv9PNevnWLoCWL8pd2rWwXUyDB1slSLhoclPLjaBgplCp6XltQ8eTBSC7/qsKnwUEpPypz3WUVWx9q9ldcaHBQSinlQ4ODUkopHxoclFJK+dDgoJRSyocGB6WUUj40OCillPKhwUEppZSPpA4Ov31/ldVFUEqphJTUwWHD/hNWF0EppRJSUgcHpZRS/mlwUEop5UODg1JKKR8aHJRSSvmwLDiISDMRmSMi60RkrYjc496eJSJficgm9//1rSqjUkolKytbDiXAfcaYjkBf4E4R6QhMAGYZY9oCs9x/K6WUiiPLgoMxZp8xZoX79glgPdAUGAO85d7tLeDH1pRQKaWSV0KMOYhILnAhsBRoZIzZ575rP9AowGPGi0i+iOQXFBTEpZxKKZUsLA8OIlIL+Aj4jTHmuPd9xhiD/8v5Yox51RiTZ4zJy87OjkNJlVIqeVgaHEQkFVdgmGiMmezefEBEznfffz5w0KryKaVUsrIyW0mA/wDrjTHPe901BbjFffsW4NN4l00ppZJdioXP3R+4CVgjIp4V8B4EngLeF5GfATuAsRaVTymlkpZlwcEYsxCQAHcPjWdZlFJKlWX5gLRSSqnEo8FBKaWUDw0OSimlfGhwUEop5UODg1JKKR8aHJRSSvnQ4KCUUsqHBgellFI+NDgopZTyocFBKaWUDw0OSimlfGhwUEop5UODg1JKKR8aHJRSSvnQ4KCUUsqHBgellFI+kjw4GKsLoJRSCSnkK8GJSKYx5lQsCxNvI2zf8FLqS5wmjSLSOWFqsNc0YK9pyA5zHutMC9Y6cymgvtVFrZLSKKad7KKLbRutZS9N5RA5UkB9KSSTImpyBidCEWkUkUaBqct+k8Ve04CNphnrnC343uRwmgyrX0qVVJ/jdLFto5PsoJkcIEcO0UQOU1tOUYMzpFNMMSkUkcZJk8F+sthvsthpzmO9sznrTAt2mEaYZK9DRkBw0lL200W20s62mxz3dz+bo2TKGTI5gx0nRaRyhjSOmNrsN1nsM1lsMU1YZ1qw3tmcH6hj2WuoMDiISD/g30AtoLmIdAN+YYy5I9aFi7VtpjH/coymBmepwRnqSiFN5AiDbatoJEdL99trsljs7MRCR2e+dnbmoAYLv1IoobtsZqB9DYNsa+gk20gTBwCnTDp7TEN2m4ZsNM0oNBmcJgPBSTrFZHKGbDlKMznIRbZ11JbTADiMsMa0YpGzE4ucnfjG2Z4zpFn5MhNWfY7T37aWAbY19LetpZmtoPS+AlOHPaYhG00Ox5w1OU06Z0gllRLSKaa2nOJ8OUJ32cwo2xJSUpwAHDeZLHFewCJnJxY6O7PZNCXw1X2TmaGt7GGQ7VsG2tbQ07ax9DtcbOzsM1nsNtkspx0nnRmcJAODjXTOks5ZGsgJGssROtm2cb3MKT3qFuf5LHZ2ZJGzE187O3OMWnF7RWJM8K4VEVkKXANMMcZc6N72nTGmcxzKF5K8vDyTn58f9uNyJ0wNeF9tTnGB7KCTbTs9bRvpZ1tLlhQCsMLZhqmOPkx39GYP2RGXuzpIpYQBtjWMti/mUtty6shpHEZYZdqwzNmBNc6WrDEt2WXOI/STiiFHDnGB7KCLbSsX2dbRXbaQKg5OmBrMcXbnC0cf5jm7Jn2rIpujjLQvZbR9CXmyEZsYjplMFjs7scLZhjWmFWuduRynZsjHTOcsbWQPHW076CGb6GdbSwvbQQC2Ohsz3dmbLxy9+c60JLkDhaGbbGG0fQmX25fSVA4DsMnZlMXOjqwxLVnjbMVm04SS0DtpyOI4F9h20Em208e2gT629dSSIoqNncXOjkx39uZLRx6HqFv6mO1PjYroFYjIcmNMnt/7QgkOxpg+IrLSKzisNsZ0i6g0MRCL4FCe4OQC2clg2ypG2pfRxbYdgHxnO953XMxUR19OUiPsMlRNhgtlM2PtcxlpX0Y9Ockxk8kMRy9mOS9ksbNTWCejUGRSRB/beobb8hluz6eBnOCkSecLRx/edwzmG9OeZDlRZXCG0fYlXGVbQB/beuxi2OBsxjRHb+Y5u7HGtMSBParPmSMFXGxbzQjbMi6yrSNFnGxzNuIDx2A+cgzkAFlRfb5E1kwO8BP7XK6wLaK5rYCzxs48Zze+cvZkgaMr+2gQ1edLoYSuspVh9hWMtC2lpe0ADiMsdHbhI8dAvnTmseGpqyI6dmWDw4fA88DfgT7APUCeMea6iEoTA/EIDuU1lwNcblvK1fYFtLXt4aRJZ6qjL+87Lia/mp6oanKaH9u/Zpx9Fh1tOzhp0pnh7MVnjotY6OxCcRi1o8qw46C3bQNX2BYx2r6E2nLafaK6mA8dF1fbbr+2spsb7LO4yr6AunKKbc5GTHH243PHRWwyOXErRz1OcKl9OVfbF9DXth6HEeY7u/K+YzAznT3j9j2IpxRKGGpbwTj7LAbZ15SenD9zXsSXjryoV4YCM3SQXYyyL+FK+0Jy5BD/LRnGTY9/FNHRKhscGgJ/A4bhOuN9CdxjjDkcUWliwIrgcI6hh2ziWvs8fmRfTC0pYpOzKe86LmGyYwBHqR2F57BWJ9nGOPssxti/pqacYa2zBRMdw/jU0c/y1lINihhpW8a19vlcZF9HibExx3kh7zqGMNfZHWcVH0xN5yyX25ZyQ8osetk2csakMN3Zm3dKhrLUdMDqSkhzOcA19nlcY59PEznCIVOHDx2DeM8xhG3mfEvLFg1NKeC6lDmMtc+lkRxlr8nivZIhvOcYzP4otxDCJTjpY9tAganLrL+Mj+wYlQkOVYG1weGcTIoYZV/C9fbZ9LBt5oxJZZqzF+8myA85HBmc4Uf2xYyzz6S7bSunTRqfOS5iomMoq01rEvG1tJD9/MQ+l2vt88iWY+w1WXzgGMz7JRdXubGhXNnHOPssrrXPo56cZIvzfN51XMJHjoGWZrAEYsPJINu3XGefwzDbclLEyRLnBbxTcgkznL2qVBKBDSeDbau40T6TwbbVGGCuszvvOC5hrrN71LvsosGqMYc38DMhwBhzW0SliYFECQ7eOshOrrPP5ir7QurIqYT/cXu0kd2Ms8/iavsC6sgpNjqbMtExjI8dA+LYdK4cVxfASq63z2aQ7VsA5ju78q5jCLOcPcIaHIwnOw6G2VYwzj6TQfY1FBs7M5y9mOgYymJnRxIxIPuTzQ9ca5/PT+xzaGE7yA+mFh87BvCu45K4dn+FqwHH+Il9LjekzCJHDnHA1GOSYwjvlQxhLw2tLl5QVgWHq73+zACuBPYaY+6OqDQxkIjBwSODM4yyLeX6lNnkubsFZjh78a7jkoT5wadSwgjbMsalzKKvbT1njZ1pzj5MLBnKsirW4imvKQWMTZnHtfa5NJEjFJi6fOC4mEmOIew0jawuHgCNOcxY+zyuT5nN+XKEPaYB75ZcwnuOIRRQz+riRUxw0s+2luvtcxhu+4Y0cZDvbMckxxA+d/SliHSriwgYessGxqXMYqRtKWni4GtHJ/7nGMZXzp4JW5Eoz5Lg4OdgNmChMaZfRKWJgUQODt7aym6ut8/mavv80gHFSY5L+NAxiMNeaWnx4RorudK+kNH2JdSXQnY6s3nHMZQPHBdbUJ7YsuHkYttqbrDPZohtJSni5GtHJyY5hjDD2YuzpMa1PDUoYoTtG66yL6C/bS02Mcx3dOG/jkuZ7bwwIbsuKiOL41xlX8D19tm0tu3juKnBJ44BTHIMYZ3JjXt5mssBrrYv4ErbAprbCjhuMvnQMYiJjqFsMU3jXp7KSpTg0B6YaoxpE1FpYqCqBAePdM4y0raM61Nm08e2gbPGzixnD2Y4ejHX2S1mg9g2nHSXzQy1r2CUbSm5tgOcNml86cxjsmMg851dkmI27Hn8wLX2eVxnn0MzWwFHTC0+dgxkuqMXK0zbmJ2Ya3GKwbbVDLfnc4ltJbWkiJ3ObCY7BzLZMTBhWjKx5aqpX5cyh1G2paRLMaudrZjsGMgs54XsNufF7JlbyV4utS1nuD2fnrZNOI3wtbMTkx0Dme7sVaXnzFjVrXQC15iDuP/fDzxgjIksdyoGqlpw8NZa9nC9fTZj7IvIlmM4jJBv2rPQ0Znlph2rna0rkRFkaC17ybNtpLdtAxfbVtNQjlNibCx2duQTxwBmOPMoJDOqr6mqEJz0t63levssLrUtJ00cHDOZLHB2ZaGzMyudbdhkciLOeErnLF1lK71tG+hrW0cf23rSxMEhU4evHD352DGAb0z7pAjI/tShkCvtX3OdfTYX2HYBrglkc5zd+cbZnlXONpXqVsvmKL1sG+ht28BA2xpa2/YB8J0zl88dffnE0d/yjKNoSYiWQyKqysHBQ3DSRbYx1L6CobaVdJQd2MTgMMIW04QtpglbzfnsNtn8YGpzlFqcMamIO1egjpwkixM0lGO0lP20tu2lneymnpwE4IipxQJnV2Y5ejDX2ZXjcZyGXxXU5hT9bd8xxLaKIfZVnOdePqXQZLDOtGC7szHbTSP2mQYcoybHTSbFpGDHSQoO6kkhDeU4jeQHWste2spuWsq+0uVD1jubMd/ZlS8deaw0bat8im205co+13tvW0Uf23rSpQTAtdyKM4ftxvX+HzZ1OUZNCo2rwmTHQboU05DjNJRjNJVDtJXdtLXt4Xw5AsBJk85yZztmOnsw09Ez4QeXIxHX4CAiPYId1BizIqLSxEB1CA7l1eEk3W2b6WnbSEfZSSvZS3M5SKr7ZBPMYVObLaYJm51NWGXasNzZji2mCVV5YDm+DLmyn+6yhQttm+hg20Wu7C+z3lYgTiPsMtlsNDlsNDmscLYl31Qx1DUAAB/fSURBVNk+rmviVHXpnKWTbOdC2yYutG2hleyjheynppyp8LGnTRqbTFM2maasd7bgG2d71prcKjOwHKlYBIdg79hzQe4zwCURlUaF5Dg1me/sxnznuVVKUiihAcfJkhPUk0LSKca42w4nTCZHqM0RU4cTSdpNFD3CdnM+2835fOIcULo1kyIayQ/U5hR15SQpOHBgowQ7x0xNDpm6HKZOtT8RxdoZ0lhh2rHC0Q5K60KGbI5RTwqpSyG15TQGwYGNsyaVQ9ThkLtVoZWg6Aj4LTbGDIlnQVTFSkjhAFkcMFl6KQoLnCLj3Kxfff/jTCigHgXGPQah73/MhVTFEZHOQEc4N5xvjHk7VoVSSillrQpHxUTkYeAl978hwDPAFTEuFyIyQkS+F5HNIjIh1s+nlFLqnFBSJq4BhgL7jTG3At0gtjOkRMQO/AMYiavFcr2IdIzlcyqllDonlOBQZIxxAiUiUgc4CDSLbbHoDWw2xmw1xpwFJgFjYvycSiml3AIGBxH5h4gMAJaJSD3gNWA5sAJYHONyNQV2ef29273Nu3zjRSRfRPILCgpQSikVPcEGpDcC/w9oApwE3gUuBeoYY76NQ9mCMsa8CrwKrnkOFhdHKaWqlYAtB2PM34wxFwGDgMPA68B04EoRaRvjcu2hbNdVjnubUkqpOKhwzMEYs8MY87T7+tHXAz8GNsS4XN8AbUWkpYikAdcBU2L8nEoppdxCSWVNEZEfichEYBrwPRDZ1axDZIwpAe4CZgDrgfeNMWtj+ZxKKaXOCTjmICKX4mopXA4sw5UxNN4YczIeBTPGfAF8EY/nUkopVVawAekHgHeA+4wxP8SpPEoppRJAsLWVdGE9pZRKUrqovFJKKR9JHRxqZ+jSykop5U9SBwellFL+aXBQSinlQ4ODUkopHxoclFJK+dDgoJRSykdSBwe9DLlSSvmX3MFBNDwopZQ/SR0cLu3YyOoiKKVUQkrq4NC/TQOri6CUUgkpqYOD6KiDUkr5ldTBQSmllH9JHRx0PFoppfxL8uCg0UEppfxJ6uBg09iglFJ+JXVw0AFppZTyL6mDg1JKKf+SOjjokINSSvmX1MGhd8ssq4uglFIJKamDQ4qOSCullF9JHRyUUkr5l9TBQbOVlFLKv6QODkoppfzT4KCUUsqHBgellFI+NDgopZTyocFBJaT0FP1qquTx84EtrS6Cj6T+BdbKSLG6CNVS/czUSh+je7N6USiJUlVD/ZppVhfBR1IHB7tOglNKKb+SOjgAdMupa3URqh29Tkb1FI0WofIvEedcJX1wGNC2odVFUH4M9PO51EpP4esJl1hQmqqhWVaNiB/7m2FtK9yneVZmxMdXwSVifcqS4CAi/09ENojItyLysYjU87rvARHZLCLfi8hlsS7LvcPa8cEvL4r106gwjeraxGebAE3r1eCh0R3jX6Aq4IbeLawugrJAZSoFwVjVcvgK6GyM6QpsBB4AEJGOwHVAJ2AE8E8RsceyICl2G71y/a/OelGrBhU+ftgF50W7SEnhnZ/3KfP3yocuDfmxt/XPpXeAzyyZ+WttheLpq7vE5LgqNhrUTOO1m/NK/25YKz0mz2NJcDDGfGmMKXH/uQTIcd8eA0wyxpwxxmwDNgO9rSgjgMFY9dRVzpu39iq9fU3PnCB7uvRrfe6Ec9+l7cLK1hARsmvH5gdRlVzRzbd1FYlOTXzH3bJrp2s3UgQGtAk9kP7tuu4h71sz7Vwd+feXtY/LJY4TYczhNmCa+3ZTYJfXfbvd23yIyHgRyReR/IKCgqgW6O6hbbnqQr9Pq0IwNq/i4ACuMQQVnlYNa8bkuPVqlB1s/uaPw0qDfKcmdRnZuTHPXNMtJs9dnXjX6CsypntTFtw/hM/uGhD2cHQ8xihiFhxEZKaIfOfn3xivff4IlAATwz2+MeZVY0yeMSYvOzs7auW+a0gbfntpO57/SfcytdvAQvuUWmfH5kdtBX8TdtJT7IzNy2FI+9A/i1v6ufrIPV/0Gqmh9yCO6R6dWnNVUy9AxlBlW1I39m3BE1d2LrPNuBvOqXbh5Rt70r5x7TL3b39qVKWeszrIqV+2v79Gmp2m9VzbFtw/pMLHN8vKpEuCZkzGLDgYY4YZYzr7+fcpgIj8FBgNjDPG8zVkD9DM6zA57m0x9+L1F/LM1V353WXtS7fdNaQN7RrVquCRoXU9Vaf0zvPrnvtBrH54OA+N7kjfVlk8c0033rg19F5AU+6t+/LeQSE/dninxmx/alSZ5nYyqFsjNumkKXYb4/r4H9D2982ddd/FZf5+5uquMSiVtZpnZTKyc+Og+7TKruj8UJZ3pcaU/wF4eefnfXjmGv/vaYMYjTGUZ1W20gjgfuAKY8wpr7umANeJSLqItATaAsviUaYrujVhbK9mZbbZbFJaC7i6h/+uEn+fb5rdxrUh9LtXFQ9e3qH0dqq97Kmibo1UfjagZaWCn+exzbIyyW0QXj/3p3f1j/h5q6Lnx/rvpw713Z9694ColKN1uZNioBZNVZbbsCaDK2gJ10r3rZx0a+ZqCdTwU3F5fmx3Hry8Axv+PKLMb8b75/PRr/rRr3VDxuY183k8xC8Jxqoxh78DtYGvRGSViLwCYIxZC7wPrAOmA3caYxwWlRE41y7wfHieYFH+fm82G7Qt1+Jo36i2nz0De3lcj7D29ydQzaMyQqu5hnaqisZwf5vzwntfq7La6SllBu7Lv38VZa0MbNvQ7+BzIKO6umrNo0MY+M4KklCw8qFLq2TL4qXrL6xwH3+Vw+eu7c4nd/anYa10n646u00YP6g1GeW6UD2T4G4f0JKeLer7HLNsIIlPL4RV2UptjDHNjDHd3f9+6XXfE8aY1saY9saYacGOE0+ej6NrTt2IBlLvHto2rPkUI7uc77NtXJ/mYT1noJqHP0sfHBrWsVX82SpIUWlcN4NFEy7h7zdUfFILRZvzarP9qVG0C6FikxtkoLx+zTRahNkiTASRduHVSLOXrg0WqKuuKkiEbKUqZ87vBpfeDjWd0G4TeuVmVWpgullWJq/c2DPEfcObGNOoTga9W4Y2d6B2mAsW3tjXf1AL0uXqV9P6wV/TR78KbzLjf3/WO6RBw0QRrI/ao0m9GozyU7HwtuHPI1jzyPBoFQuo+LP01HZ75frWigMJ57cSSvr0gvuHMPu+i8NKfOgbwlynaAr1J5HbIPYJLhocIuCdGeIvSyTYOimp9tDfcn/ptGkp0W9SXtcr9BZG6+xaAcdfPMq/J7XSg9fAQmkl/3NcD97+WfDB7p4twpsYN7BtNs2inMv/2JhOUT1eJESE0V0DB4iMVDu1M1LpeH6dmJajsskCs+4bHPK+oVQ0mmVlhj2A3KJBzahlZf0kSEs+3J6iVtm1eP2noafNRkKDQyWl+Gnq3z+ivZ89XcIJDunuGk5tr26saC3QVT4FLxQdGtfm1ZvyKuzeCLU5Huokw1n3XczlXc7nvNoZIe0PrlmkVshIscdsxqrHn8d0YuLtffhpv9yYPk84/H2Wn989sNLjXufXPfeZd4vxMu6htsoj9XSUxwDr1ojtd1yDQwV+fUkb6mWm0sPPIJE/qx8ezq39fecBeE5W/wxjoNlTA68Tg/TFKXcN4IZyYxjeTfNJ4/vSPCuT4R0blfY5/2pwa+pGkJVSURAIFPAmje/Li9df6JMZU55N4MpyrayL3VkmmdUo1dXzLt50US792zSkZ4v6LHnA/1hRIqROt2xY02fcq3wN/6f9crmkQ2jZN5N+3jfg64XKLyORF0KX16NXlG0Zer+eF0MYwI5U6/PCa/FEgwaHCvRskcWqPw2nfqbr5F5RrdnD86W5fUBL1jwyvDTLJJxujLuGtOG5a7sxKkgXQSBpFbRQsmqm0aVp2cwV7x9y31YNmH//EF69OY/B7c9jzu8GM6Z7lGeNV9Bw6NuqQUhjOlv/MooXfuJK8RzZuTEjOzfmySu78J47wCWDSGJBB/ektnDHkPxyf5aBTtCByvfIFZ1KP7tgpt0zkBppdhrXzeCqC5tyYfOyrQiD4f1f9PV5nL+urQkjO/hsA1fZvwow1+axMZ24rFMjbumXW2YsxLviU5nlTDznl0BZX2/+tJff7bGkwSFEwy44j9v6tyxTc7ioVYMKO0ZsNqF2RvDa9n2XtvO7PS3FxtU9c8hzt1q6Nq1bYZbo578ewC8ubsXrYXyZQumvbRmlZRva+KkBRbOS+/KNPXn5xp5kpNrp06pBxDXoX1zcKuIyeJ7yx92b0KN5FLtCorzU15NXuQJoizAGN5+8skvQMadoN1g8h/MOYM//pDsf3+E7v8Xf+J+/5SxuCdId1zZAZtbNF+Xyr5tcx3rz1t40qRt6F2corrywKc9d241fDPL/vfO39lj7xrWpnZ7CbwOcPypLg0OIUuw2/vSjjqU1o+8fH8H/bu/jc2KN5MfRooIT7/BOjcn/v2H0a9OwzPiDP52b1uWBkReE9IOPW8eD13s05a7+fPPHYeU3l3rmmm7ktagf0ZiIP0PLdVmUr3EGEs7Yjveqpd5dEw9efgG/HlrxdRJCHViM9jKQngAajhv6NOe5sbFfY8mTRRZOcPf3/qSnhnaKC+f71iwrk18Obu16zih9KDabcHXPHFLCGJOslZ7CmkcvY2Db6C0fVKZMMTlqEkhPsUftMqOhDOB6glJegixV/etL2vDXIN0Bfxjhv+memZbiU8Pzfhd7t8ziw1/1C2vgPpjfXtquNBjBucysaK5q6ekSgPCXUwC4pEOj6BXGLdKX9974vjGdsObvXFp+1j34dr9WeBIOeL9U2Hq7Y3BrPrkzvJn2nt9sg1qJd+3naNHgEGX+spcqMqhtw7CW740Wz0zMEeXWjwmlyXzf8Pb8OMjKtb9y16yCaVTH9TyxXCvGZvNd3vu1m/PKzFWprD+P6RzwPuuHhcPTp1UDn2VkQlHRuTvY+5CZlsKnYZ6cw/H2z/oE/bxvvii3zFjJn3/cucJu1B91bcIz13TljsFtolXMhKPBIcoy08If3BOR6A/2hqBtI9cM2CFeXS9f3D2Qz+8eGNXnCdQ18NN+ufzjhh5c3SO+r/3Sjo0q7HYrX+Q7ggS78hlc/7qpJ6O6nh9wcLaieSKB1vMJZRKcR4af7hRbDDOYPEUL5Rl+2i+XDo1rl2nRRZKmWv7674GSMGqlpwQ82c/7/WAal6sM3dS3RYWVB5tNGJvXjJrVeNn56vvKLPDRr/qV3vb8EP39WJpl1WDXkdNxKlVZvxvejtF+LsHp0bFJ9CdG3TmkNa/M2+Kz3W6TiDKxKiPYWMJTV3Xh4SlrOVPi9Lnv/hEd+Odc39cwopPvqp09mtenxw3+0yKvvLApf7mqC05j+Hil/wWHXbNfQ7tGSZ0arp/wtT3L1vb/OKojBSfOMOf7Ato1qkW/1g355cUVt+bC9dtL29HV6yQdSvx55IrQJwqmpwSuv3561wDmbyzg5teX0Swrk4xU13LZe45W/Nv6/NcD2HesKKzBeH+CrSlV1WnLIUr6tMwqs2DWjX1bcEOf5tx5iW+zc+rdAyuVDRNoyYdQVsZsVCcj6Do4sVBRtlY8BRug9D6xhVrHfuWm4BOnPBP3PJO5bCKkpdhoUs9/192fRncMOF7jr92QmZbC94+P4L7hZTNW6tZILV1+3m6z8cgVnXxqyNFw99C2DG4fu1VCX/9pL+4a0ibggPGgdtm8eWsvfu3+nVW0xIpH56Z1ubRj9Md6qhNtOcRIjTQ7T17p/9q8dTJS+cNlHfjXvK0RHTvQQHgoJ7R4Xvg01S4UOxLjUquts2uypeBkpbOg8lrUJ3/HDyHv37FJHT65sz/tG9Xm6ekbuCdA9tIbt/aiVcOaQWuygXqV0lP8T/Rr7B7TufLC2F8YKdTZ7sG6xnq3zGLZtiNltuU2rFnmGiv+xDI4heLjO/px9HSxpWWIBQ0OFgl1Mp0/gZqy/vr2X7mxJ7/83/KIn6sypt0zkMVbDlvy3OUFywhrUjeDvceKKjzGwj8MIatmGh3/NCOs5/as0BmsO2VIDE5wDWqls/HxkX6zgWIlULddRd1NxcXFPD28MZv2pSEI69evj+j57+9Tix+6nBvrSTm+l/Xr90d0rFBlAI2B9ev9f9dfu8LVdRrJa/J+bGWOk5GRQU5ODqmpobfiNThYLJIZvBmpdrJqpnHk5NkK9y2fiRRPbc6rXXq9hQcv7xD3FS69BavXDmjbkPfzd1d4jJz68Z9tvfrh4ZwpcdD7iVkh1869pQXps08ku3fvpnbt2jRKqYeIcEFO5SYPfrv7KOBaKNLqQeNid1kieU3ej430OMYYDh8+zO7du2nZ0ndpn0CqxjcngXmuq3vbgNDfdI/vHr0srEtjVmXjB7WmayV/8JVxrjfDtwrr3dNx80Wu9fejNc8ikJ8PDG3MqW6N1IiuHxJvnhaDv6ufhaKoqIgGDSKf0Z4MWjasGVEFRURo0KABRUUVt469aXCopKyaaWx/ahSX+claqUit9BSfK0Kp2Ap27hGEBy+/gM1PjIxovko46mWmUTPNHlK2S7RW4o2lRnXS+f1l7XkrwDXEs2q6unrKr+flrboGhvNqp0ecFZVTP7M0Jbp2RmrE2VGRvLeJXyVRQeX/3zDyHp8JBB6Qrp2ewokzJfErVBUmIqSE2Ef/50peu2H1w4EvuPPe+L6lg7Oe33WqLXHrciLCnUMCTwhr2bAmn901oLSlHS/hXNgnVhrXjTwJwspU2cT9tiWh9Y+NCPsxAgEX6/KYfu8gzvOzKJkKzc8CdBnedFFu6e23busd9sVXUuy2gGvp9GnVoHRdpoxUO7+9tB0f3dHP775VRZecunEbA+l4fh06N60bVuKHiHDfffeV/v3ss8/yyCOPhP3c+fn53H333WE/ziM3N5dDhw4F3efJJ5+M+Pih0uCQQMLpr+3knqyWmmLj5xUEh6b1asRsca7qwN8wb2d398crN/bgodEdfe7vU+6Sqhe3y47JGkkedw9tG9K1nJVLit0W9ozw9PR0Jk+eXOGJuSJ5eXm8+OKLlTpGReIRHLRbqYp6+caefL//OHUyUjlUeMbq4iS8ni3qs2rXURrWDNKC8jqXDOlwHgv/MMTvAGC0Lhup/HttwVYOHo/ud7pjkzo8/KPg3YApKSmMHz+eF154gSeeeKLMfdu3b+e2227j0KFDZGdn88Ybb9C8eXM++OADHn30Uex2O3Xr1mX+/PnMnTuXZ599lilTptC+fXsWLVpEdnY2TqeTdu3asXjxYrKzz1XWDh8+zPXXX8+ePXu46KKLyswF+fGPf8yuXbsoKirinnvuYfz48UyYMIHTp0/TvXt3OnXqxMSJE/3uV1nacqiiaqWnhHXN5Go61heyCSM78OW9g2jeIPRsDytSV5W17rzzTiZOnMixY8fKbP/1r3/NLbfcwrfffsu4ceNKu40ee+wxZsyYwerVq5kyZUqZx9hsNm688UYmTpwIwMyZM+nWrVuZwADw6KOPMmDAANauXcuVV17Jzp07S+97/fXXWb58Ofn5+bz44oscPnyYp556iho1arBq1arSY/vbr7K05aCSQqrdpt0ycTLv94M5eiryGcM/H9jKsrTnOnXqcPPNN/Piiy9So8a5geTFixczefJkAG666Sbuv/9+APr3789Pf/pTxo4dy1VXXeVzvNtuu40xY8bwm9/8htdff51bb73VZ5/58+eXHnvUqFHUr39uGZ4XX3yRjz/+GIBdu3axadMmGjTwnS8U6n7h0OCglIqqFg1q0iLC81Kb82px0uLMut/85jf06NHD74m8vFdeeYWlS5cydepUevbsyfLlZVcjaNasGY0aNWL27NksW7astKYfirlz5zJz5kwWL15MZmYmgwcP9jtXIdT9wqXdSgmmXaNa9ArhQucqeqJ1NS9Vea6LQUV/gcBwZGVlMXbsWP7zn/+UbuvXrx+TJk0CYOLEiQwc6FrWfsuWLfTp04fHHnuM7Oxsdu3a5XO822+/nRtvvJFrr70Wu9036WTQoEG88847AEybNo0ffnCt3XXs2DHq169PZmYmGzZsYMmSJaWPSU1Npbi4uML9KkODQ4L58t6L+eCXVTtlsapK8mEZ5eW+++4rk7X00ksv8cYbb9C1a1f++9//8re//Q2A3//+93Tp0oXOnTvTr18/unXzvYTqFVdcQWFhYcCWyMMPP8z8+fPp1KkTkydPpnnz5gCMGDGCkpISLrjgAiZMmEDfvn1LHzN+/Hi6du3KuHHjgu5XGdqtVA1ozVepyissLCy93ahRI06dOlX6d4sWLZg9e7bPYzxjBd4GDx7M4MGDS/9evXo13bp1o0MH/0uxN2jQgC+//NLvfdOmTfO7/emnn+bpp5+ucL/K0OBQjSR7RpJSieapp57i5ZdfDmusIVFot1KS8CwjEM/lm6uKXHd663l1rO3rVtXPhAkT2LFjBwMGDLC6KGHTlkOSuH9Ee+pnpvKjIJcITVZ3DGlDt2b1GNROZ5Er5aHBIUnUzkjlt8ODX1ErWdltooFBqXK0W6kaiOQiMEopFYwGh2pFxxOUUtGhwUEppQC73U737t1L/23fvp1+/VxzjrZv3146US1ZWBocROQ+ETEi0tD9t4jIiyKyWUS+FZEeVpZPKZU8PIvZef7l5uayaNEiIDmDg2UD0iLSDBgO7PTaPBJo6/7XB3jZ/b9SKllMmwD710T3mI27wMinwn5YrVq1KCwsZMKECaxfv57u3btzyy23cO+990a3fAnIymylF4D7gU+9to0B3jauBc2XiEg9ETnfGLPPkhJWFToerVSlea6RANCyZcvSVU7BNZnt2Wef5fPPP7eqeHFnSXAQkTHAHmPM6nIXvm4KeK9ctdu9zSc4iMh4YDxQuhZJstMZ0qpaiKCGHw2ebiXlErPgICIzgcZ+7voj8CCuLqWIGWNeBV4FyMvL07qzUkpFUcyCgzFmmL/tItIFaAl4Wg05wAoR6Q3sAZp57Z7j3qaUUpapXbs2J06csLoYcRX3bCVjzBpjzHnGmFxjTC6urqMexpj9wBTgZnfWUl/gmI43VMzTNZeRqpnJSsVC165dsdvtdOvWjRdeeMHq4sRFoi2f8QVwObAZOAVUfCkmRXbtdH5/WXtGdz3f6qIoVWV5L9ldfltqaqrfJburM8uDg7v14LltgDutK03VdeeQNlYXQSlVjWg/hFJKKR8aHJRSCcHoJQ1jJpL3VoODUspyGRkZHD58WANEDBhjOHz4MBkZ4V3MyvIxB6WUysnJYffu3RQUFFhdlGopIyODnJycsB6jwUEpZbnU1FRatmxpdTGUF+1WUkop5UODg1JKKR8aHJRSSvmQ6pAdICIFwI4IH94QOBTF4lQF+pqTg77m5FCZ19zCGJPt745qERwqQ0TyjTF5VpcjnvQ1Jwd9zckhVq9Zu5WUUkr50OCglFLKhwYH9wWDkoy+5uSgrzk5xOQ1J/2Yg1JKKV/aclBKKeVDg4NSSikfSR0cRGSEiHwvIptFZILV5Yk1EWkmInNEZJ2IrBWRe6wuUzyIiF1EVorI51aXJV5EpJ6IfCgiG0RkvYhcZHWZYklE7nV/p78TkXdFJLwlSKsIEXldRA6KyHde27JE5CsR2eT+v340nitpg4OI2IF/ACOBjsD1ItLR2lLFXAlwnzGmI9AXuDMJXjPAPcB6qwsRZ38DphtjOgDdqMavX0SaAncDecaYzoAduM7aUsXMm8CIctsmALOMMW2BWe6/Ky1pgwPQG9hsjNlqjDkLTALGWFymmDLG7DPGrHDfPoHrhNHU2lLFlojkAKOAf1tdlngRkbrAIOA/AMaYs8aYo9aWKuZSgBoikgJkAnstLk9MGGPmA0fKbR4DvOW+/Rbw42g8VzIHh6bALq+/d1PNT5TeRCQXuBBYam1JYu6vwP2A0+qCxFFLoAB4w92d9m8RqWl1oWLFGLMHeBbYCewDjhljvrS2VHHVyBizz317P9AoGgdN5uCQtESkFvAR8BtjzHGryxMrIjIaOGiMWW51WeIsBegBvGyMuRA4SZS6GhKRu499DK6g2ASoKSI3WlsqaxjX3ISozE9I5uCwB2jm9XeOe1u1JiKpuALDRGPMZKvLE2P9gStEZDuubsNLROR/1hYpLnYDu40xnlbhh7iCRXU1DNhmjCkwxhQDk4F+Fpcpng6IyPkA7v8PRuOgyRwcvgHaikhLEUnDNYA1xeIyxZSICK5+6PXGmOetLk+sGWMeMMbkGGNycX2+s40x1b5GaYzZD+wSkfbuTUOBdRYWKdZ2An1FJNP9HR9KNR6A92MKcIv79i3Ap9E4aNJeJtQYUyIidwEzcGU3vG6MWWtxsWKtP3ATsEZEVrm3PWiM+cLCMqnY+DUw0V3x2QrcanF5YsYYs1REPgRW4MrIW0k1XUZDRN4FBgMNRWQ38DDwFPC+iPwM16ULxkbluXT5DKWUUuUlc7eSUkqpADQ4KKWU8qHBQSmllA8NDkoppXxocFBKKeVDg4NSYRCRBiKyyv1vv4jscd8uFJF/Wl0+paJFU1mVipCIPAIUGmOetbosSkWbthyUigIRGey5XoSIPCIib4nIAhHZISJXicgzIrJGRKa7lzBBRHqKyDwRWS4iMzxLICiVCDQ4KBUbrYFLgCuA/wFzjDFdgNPAKHeAeAm4xhjTE3gdeMKqwipVXtIun6FUjE0zxhSLyBpcy7NMd29fA+QC7YHOwFeu5YCw41puWqmEoMFBqdg4A2CMcYpIsTk3uOfE9bsTYK0xplpfvlNVXdqtpJQ1vgeyPdd2FpFUEelkcZmUKqXBQSkLuC9New3wtIisBlaRXNcgUAlOU1mVUkr50JaDUkopHxoclFJK+dDgoJRSyocGB6WUUj40OCillPKhwUEppZQPDQ5KKaV8/H9YGy66hJ7MyAAAAABJRU5ErkJggg==\n" - }, - "metadata": { - "needs_background": "light" - } - } - ], - "source": [ - "# fiiting using cmaes to the toy data\n", - "\n", - "# Add noise\n", - "values += np.random.normal(0, 10, values.shape)\n", - "\n", - "# Create an object with links to the model and time series\n", - "problem = pints.SingleOutputProblem(model, times, values)\n", - "\n", - "# Select a score function\n", - "score = pints.SumOfSquaresError(problem)\n", - "\n", - "# Select some boundaries\n", - "boundaries = pints.RectangularBoundaries([0, 0], [20, 20])\n", - "\n", - "# Perform an optimization with boundaries and hints\n", - "x0 = 0.01, 0.01\n", - "#sigma0 = [0.01, 100]\n", - "found_parameters, found_value = pints.optimise(\n", - " score,\n", - " x0,\n", - " boundaries = boundaries,\n", - " method=pints.CMAES\n", - " )\n", - "\n", - "# Show score of true solution\n", - "print('Score at true solution: ')\n", - "print(score(real_parameters))\n", - "\n", - "# Compare parameters with original\n", - "print('Found solution: True parameters:' )\n", - "for k, x in enumerate(found_parameters):\n", - " print(pints.strfloat(x) + ' ' + pints.strfloat(real_parameters[k]))\n", - "\n", - "# Show quality of fit\n", - "plt.figure()\n", - "plt.xlabel('Time')\n", - "plt.ylabel('Value')\n", - "plt.plot(times, values, label='Nosiy data')\n", - "plt.plot(times, problem.evaluate(found_parameters), label='Fit')\n", - "plt.legend()\n", - "plt.show()\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using the attempted Hager-Zhang linesearch implimentation" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "tags": [] - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": "Minimising error measure\nUsing Broyden-Fletcher-Goldfarb-Shanno (BFGS)\nRunning in sequential mode.\n\nHessian updates: 0 line steps: 0 propose alpha: 0.001 propose points: [0.01 0.01]\nIter. Eval. Best Time m:s\n0 1 1.42e+07 0:02.0\nalpha_initial: 1.266832724547819e-06\nNumber of accepted steps: 0\nstep sized alpha: 0.10140829994739889 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 1 line steps: 0 propose alpha: 0.10140829994739889 propose points: [4.40138067 8.01486899]\n1 2 1.01e+07 0:22.2\nalpha_initial: 0.20281659989479778\nNumber of accepted steps: 1\nstep sized alpha: 0.18362729523646867 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 2 line steps: 0 propose alpha: 0.18362729523646867 propose points: [2.96671004 8.70466184]\n2 3 1e+07 0:28.6\nalpha_initial: 0.36725459047293735\nNumber of accepted steps: 2\nstep sized alpha: 0.06538164147441025 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 3 line steps: 0 propose alpha: 0.06538164147441025 propose points: [3.09125057 8.90041531]\n3 4 1e+07 0:35.6\nalpha_initial: 0.1307632829488205\nNumber of accepted steps: 3\nstep sized alpha: 0.172828183643652 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 4 line steps: 0 propose alpha: 0.172828183643652 propose points: [3.02128895 8.9307388 ]\nalpha_initial: 0.345656367287304\nNumber of accepted steps: 4\nstep sized alpha: 0.04836245024157738 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 5 line steps: 0 propose alpha: 0.04836245024157738 propose points: [3.03066513 8.94435074]\nalpha_initial: 0.09672490048315475\nNumber of accepted steps: 5\nstep sized alpha: 0.16443120896872018 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 6 line steps: 0 propose alpha: 0.16443120896872018 propose points: [3.02412874 8.94694275]\nalpha_initial: 0.32886241793744037\nNumber of accepted steps: 6\nstep sized alpha: 0.619769876204881 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 7 line steps: 0 propose alpha: 0.619769876204881 propose points: [3.02487266 8.94881872]\nalpha_initial: 1.239539752409762\nNumber of accepted steps: 7\nstep sized alpha: 1.7690537545373526 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 8 line steps: 0 propose alpha: 1.7690537545373526 propose points: [3.02437294 8.94901689]\nalpha_initial: 3.538107509074705\nNumber of accepted steps: 8\nstep sized alpha: 0.6197698762049499 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 9 line steps: 0 propose alpha: 0.6197698762049499 propose points: [3.02440349 8.94909392]\nalpha_initial: 1.2395397524098999\nNumber of accepted steps: 9\nstep sized alpha: 1.7690537545447989 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 10 line steps: 0 propose alpha: 1.7690537545447989 propose points: [3.02438297 8.94910206]\nalpha_initial: 3.5381075090895977\nNumber of accepted steps: 10\nstep sized alpha: 0.6197698762511816 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 11 line steps: 0 propose alpha: 0.6197698762511816 propose points: [3.02438422 8.94910522]\nalpha_initial: 1.2395397525023633\nNumber of accepted steps: 11\nstep sized alpha: 1.7690537551010947 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 12 line steps: 0 propose alpha: 1.7690537551010947 propose points: [3.02438338 8.94910555]\nalpha_initial: 3.5381075102021895\nNumber of accepted steps: 12\nstep sized alpha: 0.6197698765290248 accepted\nupdating Hessian and changing newton direction\n\n******************** Convergence after 13 accepted steps!********************\n||df/dx_i||inf <= 1e-6 with parameters:\n[3.02438343 8.94910568]\nerror function evaluation: 10039961.960134586\n\nInverse Hessian matrix:\n [[0.10216867 0. ]\n [0. 0.10216867]]\n\nHessian updates: 13 line steps: 0 propose alpha: 0.6197698765290248 propose points: [3.02438343 8.94910568]\nalpha_initial: 1.2395397530580496\nNumber of accepted steps: 13\nstep sized alpha: 1.7690537478879769 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 14 line steps: 0 propose alpha: 1.7690537478879769 propose points: [3.0243834 8.9491057]\nalpha_initial: 3.5381074957759537\nNumber of accepted steps: 14\nstep sized alpha: 0.6197698655560488 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 15 line steps: 0 propose alpha: 0.6197698655560488 propose points: [3.0243834 8.9491057]\nalpha_initial: 1.2395397311120977\nNumber of accepted steps: 15\nstep sized alpha: 1.7690541518228553 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 16 line steps: 0 propose alpha: 1.7690541518228553 propose points: [3.0243834 8.9491057]\nalpha_initial: 3.5381083036457106\nNumber of accepted steps: 16\nstep sized alpha: 0.6197695047582213 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 17 line steps: 0 propose alpha: 0.6197695047582213 propose points: [3.0243834 8.9491057]\nalpha_initial: 1.2395390095164427\nNumber of accepted steps: 17\nstep sized alpha: 1.7690649194144203 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 18 line steps: 0 propose alpha: 1.7690649194144203 propose points: [3.0243834 8.9491057]\nalpha_initial: 3.5381298388288407\nNumber of accepted steps: 18\nstep sized alpha: 0.6197554232665176 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 19 line steps: 0 propose alpha: 0.6197554232665176 propose points: [3.0243834 8.9491057]\nalpha_initial: 1.2395108465330351\nNumber of accepted steps: 19\nstep sized alpha: 1.769066827970866 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 20 line steps: 0 propose alpha: 1.769066827970866 propose points: [3.0243834 8.9491057]\n20 21 1e+07 2:44.5\nalpha_initial: 3.538133655941732\nNumber of accepted steps: 20\nstep sized alpha: 0.6197901482068541 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 21 line steps: 0 propose alpha: 0.6197901482068541 propose points: [3.0243834 8.9491057]\nalpha_initial: 1.2395802964137081\nNumber of accepted steps: 21\nstep sized alpha: 1.7782084058779175 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 22 line steps: 0 propose alpha: 1.7782084058779175 propose points: [3.0243834 8.9491057]\nalpha_initial: 3.556416811755835\nNumber of accepted steps: 22\nstep sized alpha: 0.6265448885525534 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 23 line steps: 0 propose alpha: 0.6265448885525534 propose points: [3.0243834 8.9491057]\nalpha_initial: 1.2530897771051068\nNumber of accepted steps: 23\nstep sized alpha: 1.7775552079681158 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 24 line steps: 0 propose alpha: 1.7775552079681158 propose points: [3.0243834 8.9491057]\nalpha_initial: 3.5551104159362317\nNumber of accepted steps: 24\nstep sized alpha: 0.7438191314882705 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 25 line steps: 0 propose alpha: 0.7438191314882705 propose points: [3.0243834 8.9491057]\nalpha_initial: 1.487638262976541\n\nHessian updates: 25 line steps: 0.3333333333333333 propose alpha: 1.6810429221572856 propose points: [3.0243834 8.9491057]\nNumber of accepted steps: 25\nstep sized alpha: 1.511137650925769 accepted\nupdating Hessian and changing newton direction\n\nHessian updates: 26 line steps: 0 propose alpha: 1.511137650925769 propose points: [3.0243834 8.9491057]\n28 28 1e+07 3:40.0\nHalting: No significant change for 10 iterations.\nScore at true solution: \n10040056.240084708\nFound solution: True parameters:\n 3.02438339959221913e+00 3.00000000000000000e+00\n 8.94910570291299479e+00 9.00000000000000000e+00\n" - } - ], - "source": [ - "# Create an object with links to the model and time series\n", - "problem = pints.SingleOutputProblem(model, times, values)\n", - "\n", - "# Select a score function\n", - "score = pints.SumOfSquaresError(problem)\n", - "\n", - "# Select some boundaries\n", - "boundaries = pints.RectangularBoundaries([0, 0], [20, 20])\n", - "\n", - "# Perform an optimization with boundaries and hints\n", - "x0 = 0.01, 0.01\n", - "opt = pints.OptimisationController(\n", - " score,\n", - " x0,\n", - " method=pints.BFGS\n", - ")\n", - "opt.set_max_unchanged_iterations(10)\n", - "opt.set_max_iterations(200)\n", - "found_parameters, found_value = opt.run()\n", - "\n", - "# Show score of true solution\n", - "print('Score at true solution: ')\n", - "print(score(real_parameters))\n", - "\n", - "# Compare parameters with original\n", - "print('Found solution: True parameters:' )\n", - "for k, x in enumerate(found_parameters):\n", - " print(pints.strfloat(x) + ' ' + pints.strfloat(real_parameters[k]))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## now using scipy line search" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "tags": [] - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": "Minimising error measure\nUsing Broyden-Fletcher-Goldfarb-Shanno (BFGS)\nRunning in sequential mode.\nIter. Eval. Best Time m:s\n0 1 1.43e+07 0:01.6\n1 2 1.02e+07 0:11.4\n2 3 1.01e+07 0:18.3\n3 4 1.01e+07 0:24.9\n" - } - ], - "source": [ - "# Create an object with links to the model and time series\n", - "problem = pints.SingleOutputProblem(model, times, values)\n", - "\n", - "# Select a score function\n", - "score = pints.SumOfSquaresError(problem)\n", - "\n", - "# Select some boundaries\n", - "boundaries = pints.RectangularBoundaries([0, 0], [20, 20])\n", - "\n", - "# Perform an optimization with boundaries and hints\n", - "x0 = 0.01, 0.01\n", - "opt = pints.OptimisationController(\n", - " score,\n", - " x0,\n", - " method=pints.BFGS_scipy\n", - ")\n", - "opt.set_max_unchanged_iterations(10)\n", - "opt.set_max_iterations(200)\n", - "found_parameters, found_value = opt.run()\n", - "\n", - "# Show score of true solution\n", - "print('Score at true solution: ')\n", - "print(score(real_parameters))\n", - "\n", - "# Compare parameters with original\n", - "print('Found solution: True parameters:' )\n", - "for k, x in enumerate(found_parameters):\n", - " print(pints.strfloat(x) + ' ' + pints.strfloat(real_parameters[k]))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3.8.2 64-bit ('pints': conda)", - "language": "python", - "name": "python38264bitpintsconda8d0b754a30424fdd8045d82b54ea71e7" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.2-final" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} \ No newline at end of file From 683a2a574fafcfa4b6477512d87d0b437fa55307 Mon Sep 17 00:00:00 2001 From: alisterde Date: Sun, 8 Nov 2020 16:17:29 +0000 Subject: [PATCH 13/16] Adding convergence stopping crtirea to lbfgs --- pints/_optimisers/__init__.py | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/pints/_optimisers/__init__.py b/pints/_optimisers/__init__.py index 6c5847b6e..ed620cdba 100644 --- a/pints/_optimisers/__init__.py +++ b/pints/_optimisers/__init__.py @@ -308,6 +308,7 @@ def __init__(self, x0, sigma0=None, boundaries=None): # Best solution found self._xbest = self._x0 self._fbest = float('inf') + self._dfdx_best = [float('inf'), float('inf')] # Number of iterations run self._iterations = 0 @@ -636,6 +637,17 @@ def ask(self): # print('') # Return proposed points (just the one) in the search space to evaluate return [self._proposed] + + def stop(self): + """ See :meth:`Optimiser.stop()`. """ + + # We use the condition number defined in the pycma code at + # cma/evolution_strategy.py#L2965. + + cond = norm(self._dfdx_best, ord=np.inf) + if cond <= 1e-6: + return 'Convergences i.e gradients ~ 0' + return False def tell(self, reply): """ See :meth:`Optimiser.tell()`. """ @@ -747,28 +759,10 @@ def tell(self, reply): self.__2nd_wolfe_check_needed = False self.__2nd_wolfe_check_done = True - # Checking if all gradients ~ 0, - # Therefore the classical convergence test of a quasi-newton - # or conjugate gradient method has been meet. - # TODO: Implement a means of stopping the optimiser is this - # condition is meet (apparently something similar is done - # in CMAES) - if self.__convergence is not True: - - if norm(proposed_dfdx, ord=np.inf) <= 1e-6: - - self.__convergence = True - print('') - print(20 * '*' + ' Convergence after ', - self._k, ' accepted steps!' + 20 * '*') - print('||df/dx_i||inf <= 1e-6 with parameters:') - print(self._proposed) - print('error function evaluation: ', proposed_f) - print('\nInverse Hessian matrix:\n', self._B) - # Update xbest and fbest if self._fbest > proposed_f: self._fbest = proposed_f + self._dfdx_best = proposed_dfdx self._xbest = self._current print('') From 858857f8a66bb7ea1ccf1a189d97cc7f4b55b809 Mon Sep 17 00:00:00 2001 From: alisterde Date: Sun, 8 Nov 2020 16:43:37 +0000 Subject: [PATCH 14/16] lbfgs example (needs improving) --- examples/optimisation/lbfgs.ipynb | 434 ++++++++++++++++++++++++++++++ 1 file changed, 434 insertions(+) create mode 100644 examples/optimisation/lbfgs.ipynb diff --git a/examples/optimisation/lbfgs.ipynb b/examples/optimisation/lbfgs.ipynb new file mode 100644 index 000000000..4bc6c259e --- /dev/null +++ b/examples/optimisation/lbfgs.ipynb @@ -0,0 +1,434 @@ +{ + "cells": [ + { + "source": [ + "# Optimisation: L-BFGS\n", + "\n", + "This example shows you how to run a global optimisation with [L-BFGS](http://pints.readthedocs.io/en/latest/optimisers/lbfgs.html).\n", + "\n", + "For a more elaborate example of an optimisation, see: [basic optimisation example](./first-example.ipynb)." + ], + "cell_type": "markdown", + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import print_function\n", + "import pints\n", + "import pints.toy as toy\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "#specifing a simple model for LBFGS line search trial\n", + "\n", + "# dzdt = ax^2 +by^2\n", + "# dz/dxdt = 2ax\n", + "# dz/dydt = 2by\n", + "# starting conditions z=0, y=0, x=0\n", + "# x and y increase by 1 in each step?\n", + "\n", + "class easyProblem(pints.ForwardModelS1):\n", + " \"\"\"\n", + " A model with parameters of a similar magantuide\n", + " \"\"\"\n", + "\n", + " def __init__(self):\n", + " super(easyProblem, self).__init__()\n", + "\n", + " def n_parameters(self):\n", + " return 2\n", + " def n_outputs(self):\n", + " return 1\n", + "\n", + " def _simulate(self, parameters, times, senstivities = False):\n", + "\n", + " # unpacking parameters\n", + " a = parameters[0]\n", + " b = parameters[1]\n", + " #ax^2 +by^2 where x= sin(times) and y=cos(time)\n", + " time_dim = len(times)\n", + " time_step = float(times[1])\n", + " x = np.sin(times)\n", + " y = np.cos(times)\n", + " dzdt = a*np.square(x) + b*np.square(y)\n", + " if senstivities is True:\n", + "\n", + " # creating senetivity matrix\n", + " sen = np.zeros((time_dim,2,3))\n", + " for i, xi in enumerate(x):\n", + " old_sen = sen[i,:,:]\n", + "\n", + " if i < time_dim:\n", + " # df/dtheta\n", + " dfdtheta = np.asarray([[0,0],\n", + " [0,0],\n", + " [xi**2, y[i]**2]])\n", + " # jacobian df/du\n", + " Jac = np.array([[0,1,0],\n", + " [-1,0,0],\n", + " [2*a*xi, 2*b*y[i], 0]])\n", + " temp = np.matmul(old_sen, np.transpose(Jac)) + np.transpose(dfdtheta)\n", + " new_sen = temp*time_step + old_sen\n", + "\n", + " # assigning the new sentivity to the right place in the senstivity array\n", + " sen[i,:,:] = new_sen\n", + "\n", + " return (dzdt, sen[:,:,-1])\n", + "\n", + " else:\n", + " return dzdt\n", + " \n", + " def simulate(self, parameters, times):\n", + " return self._simulate(parameters, times, senstivities = False)\n", + "\n", + " def simulateS1(self, parameters, times):\n", + " return self._simulate(parameters, times, senstivities = True)\n", + "\n", + " def suggested_parameters(self):\n", + " \"\"\"suggested parameters for the model [a, b]\"\"\"\n", + " return [3, 9]\n", + " def suggested_times(self):\n", + " \"\"\"suggested times for the model range 0 to 10\"\"\"\n", + " return np.linspace(0, 10, num=100000)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# making some toy data\n", + "model = easyProblem()\n", + "times = model.suggested_times()\n", + "real_parameters = model.suggested_parameters()\n", + "output = model.simulateS1(real_parameters, times)#\n", + "values = output[0]\n", + "sensetivities = output[1]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using the Hager-Zhang linesearch implimentation of the LBFGS optimser" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Minimising error measure\n", + "Using Broyden-Fletcher-Goldfarb-Shanno (BFGS)\n", + "Running in sequential mode.\n", + "\n", + "Hessian updates: 0 line steps: 0 propose alpha: 0.001 propose points: [0.01 0.01]\n", + "Iter. Eval. Best Time m:s\n", + "0 1 1.42e+07 0:01.9\n", + "alpha_initial: 1.2626003255377118e-06\n", + "Number of accepted steps: 0\n", + "step size of alpha accepted: 0.10154474127440917\n", + "updating Hessian and changing newton direction\n", + "\n", + "Hessian updates: 1 line steps: 0 propose alpha: 0.10154474127440917 propose points: [4.34929732 8.05250872]\n", + "1 2 1.01e+07 0:22.8\n", + "alpha_initial: 0.20308948254881834\n", + "Number of accepted steps: 1\n", + "step size of alpha accepted: 0.18279326950513297\n", + "updating Hessian and changing newton direction\n", + "\n", + "Hessian updates: 2 line steps: 0 propose alpha: 0.18279326950513297 propose points: [2.86351878 8.75197834]\n", + "2 3 9996514 0:29.6\n", + "alpha_initial: 0.36558653901026594\n", + "Number of accepted steps: 2\n", + "step size of alpha accepted: 0.06556133817314082\n", + "updating Hessian and changing newton direction\n", + "\n", + "Hessian updates: 3 line steps: 0 propose alpha: 0.06556133817314082 propose points: [2.99467023 8.96074361]\n", + "3 4 9993489 0:36.7\n", + "alpha_initial: 0.13112267634628164\n", + "Number of accepted steps: 3\n", + "step size of alpha accepted: 0.17156343822848208\n", + "updating Hessian and changing newton direction\n", + "\n", + "Hessian updates: 4 line steps: 0 propose alpha: 0.17156343822848208 propose points: [2.91829932 8.99303174]\n", + "alpha_initial: 0.34312687645696416\n", + "Number of accepted steps: 4\n", + "step size of alpha accepted: 0.0485337191905153\n", + "updating Hessian and changing newton direction\n", + "\n", + "Hessian updates: 5 line steps: 0 propose alpha: 0.0485337191905153 propose points: [2.92869329 9.00828149]\n", + "alpha_initial: 0.0970674383810306\n", + "Number of accepted steps: 5\n", + "step size of alpha accepted: 0.16286406909804865\n", + "updating Hessian and changing newton direction\n", + "\n", + "Hessian updates: 6 line steps: 0 propose alpha: 0.16286406909804865 propose points: [2.92121559 9.01116373]\n", + "alpha_initial: 0.3257281381960973\n", + "Number of accepted steps: 6\n", + "step size of alpha accepted: 0.626615636979039\n", + "updating Hessian and changing newton direction\n", + "\n", + "Hessian updates: 7 line steps: 0 propose alpha: 0.626615636979039 propose points: [2.92206729 9.01337339]\n", + "alpha_initial: 1.253231273958078\n", + "Number of accepted steps: 7\n", + "step size of alpha accepted: 1.7570238298625835\n", + "updating Hessian and changing newton direction\n", + "\n", + "Hessian updates: 8 line steps: 0 propose alpha: 1.7570238298625835 propose points: [2.92146643 9.01360499]\n", + "alpha_initial: 3.514047659725167\n", + "Number of accepted steps: 8\n", + "step size of alpha accepted: 0.6266156369792899\n", + "updating Hessian and changing newton direction\n", + "\n", + "Hessian updates: 9 line steps: 0 propose alpha: 0.6266156369792899 propose points: [2.92150318 9.01370032]\n", + "alpha_initial: 1.2532312739585798\n", + "Number of accepted steps: 9\n", + "step size of alpha accepted: 1.757023829805599\n", + "updating Hessian and changing newton direction\n", + "\n", + "Hessian updates: 10 line steps: 0 propose alpha: 1.757023829805599 propose points: [2.92147725 9.01371031]\n", + "alpha_initial: 3.514047659611198\n", + "Number of accepted steps: 10\n", + "step size of alpha accepted: 0.6266156369924621\n", + "updating Hessian and changing newton direction\n", + "\n", + "Hessian updates: 11 line steps: 0 propose alpha: 0.6266156369924621 propose points: [2.92147884 9.01371442]\n", + "alpha_initial: 1.2532312739849243\n", + "Number of accepted steps: 11\n", + "step size of alpha accepted: 1.7570238302294703\n", + "updating Hessian and changing newton direction\n", + "\n", + "Hessian updates: 12 line steps: 0 propose alpha: 1.7570238302294703 propose points: [2.92147772 9.01371486]\n", + "alpha_initial: 3.5140476604589406\n", + "Number of accepted steps: 12\n", + "step size of alpha accepted: 0.6266156369695461\n", + "updating Hessian and changing newton direction\n", + "\n", + "******************** Convergence after 13 accepted steps!********************\n", + "||df/dx_i||inf <= 1e-6 with parameters:\n", + "[2.92147779 9.01371503]\n", + "error function evaluation: 9993281.681014169\n", + "\n", + "Inverse Hessian matrix:\n", + " [[0.10233609 0. ]\n", + " [0. 0.10233609]]\n", + "\n", + "Hessian updates: 13 line steps: 0 propose alpha: 0.6266156369695461 propose points: [2.92147779 9.01371503]\n", + "14 14 9993282 2:05.3\n", + "Halting: Ill-conditioned covariance matrix\n", + "Score at true solution: \n", + "9993481.362921719\n", + "Found solution: True parameters:\n", + " 2.92147778828319726e+00 3.00000000000000000e+00\n", + " 9.01371503353561465e+00 9.00000000000000000e+00\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": "
", + "image/svg+xml": "\n\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEGCAYAAACO8lkDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd3yT17nA8d+R5MEw25iAAbMhDLP3XoGQkNWkEEhI0pTmNmmS3rS9JGmb0QzSpGkzm2YWEpK02YMwwoawwg57GjDTmGVj8JDO/UOSka0tS34l6/l+PnywpFevjmXpfc54zjlKa40QQgjhymR0AYQQQkQfCQ5CCCHcSHAQQgjhRoKDEEIINxIchBBCuLEYXYBwaNCggc7IyDC6GEIIEVPWr19/Smud6umxKhEcMjIyWLdundHFEEKImKKUOujtMelWEkII4UaCgxBCCDcSHIQQQripEmMOnhQXF5Odnc2lS5eMLkqVlJycTHp6OgkJCUYXRQgRAVU2OGRnZ5OSkkJGRgZKKaOLU6VorcnNzSU7O5sWLVoYXRwhRARU2W6lS5cuUb9+fQkMEaCUon79+tIqE6IKq7LBAZDAEEHy3gpRtVXp4CBEVbdqXy77cvKNLoaogiQ4RJBSioceeqj09gsvvMDjjz8e9HnWrVvH/fffH3I5MjIyOHXqlM9jnnnmmZDPL4wz8a3VjPjbUqOLIaogCQ4RlJSUxOeff+73wuxPz549efnll8NUKs8kOAghXElwiCCLxcLUqVP5+9//7vZYVlYWw4cPp0uXLowYMYJDhw4B8Mknn9CpUycyMzMZPHgwAEuWLOGaa67BZrPRpk0bcnJyALDZbLRu3br0tlNubi6jR4+mY8eO3H333bju9nf99dfTo0cPOnbsyJtvvgnAtGnTuHjxIl27dmXSpElejxNCxI8qm8rq6olvtrH96PmwnvPKxrV47NqOWG2a/Tn5pNetTrVEs9tx9957L126dOEPf/hDmft/85vfMGXKFKZMmcK7777L/fffz5dffsmTTz7JvHnzaNKkCWfPni3zHJPJxOTJk5k1axYPPvggCxYsIDMzk9TUsutmPfHEEwwcOJA///nPzJ49m3feeaf0sXfffZd69epx8eJFevXqxU033cT06dN59dVX2bRpk8/j6tevH463TggRA6TlUEEXCku4WGzlxHnPaZ21atXi9ttvd+sWWrVqFbfeeisAt912GytWrABgwIAB3HHHHbz11ltYrVa38911113MnDkTsF/A77zzTrdjli1bxuTJkwEYN24cdevWLX3s5ZdfJjMzk759+3L48GH27NnjsdyBHieEqJriouXw2LUdDX39Bx98kO7du3u8kJf3xhtvsGbNGmbPnk2PHj1Yv359mcebNm1KWloaixYtYu3atcyaNSvgcixZsoQFCxawatUqqlevztChQz3OVQj0OCFEcKw2zffbT3BVx7SoTweXlkMlqFevHrfcckuZ7p3+/fvz8ccfAzBr1iwGDRoEwL59++jTpw9PPvkkqampHD582O18d999N5MnT+bmm2/GbHbvyho8eDAffvghAHPmzOHMmTMAnDt3jrp161K9enV27tzJ6tWrS5+TkJBAcXGx3+OEEKF7Z8V+7vlgPd9uOWZ0UfyS4FBJHnrooTJZS6+88grvvfceXbp04f333+ell14C4Pe//z2dO3emU6dO9O/fn8zMTLdzjR8/nvz8fK8tkccee4xly5bRsWNHPv/8c5o1awbAmDFjKCkpoUOHDkybNo2+ffuWPmfq1Kl06dKFSZMm+Twu1hQUlWCzaf8HClEJjp61t8BP5RcaXBL/4qJbySj5+ZcnJ6WlpVFQUFB6u3nz5ixatMjtOZ9//rnbfUOHDmXo0KGltzdv3kxmZibt27f3+Lr169dn/vz5Hh+bM2eOx/ufe+45nnvuOb/HxZJzF4vJfGI+949ow/+Oamt0cYSIKdJyiDHTp0/npptu4tlnnzW6KFHvzIUiAL7adMTgkggReyQ4xJhp06Zx8OBBBg4caHRRhBBBcp1zFO0kOAghRCVz5inlXSrmtcV7o3JcTIKDEEIY5Klvd/D8vF3M337C6KK4keAghAhJYYmV7DMF/g8Upcq3D/KLSgAottoqvzB+SHAQVV4MdfPGlD98uoWBzy3mUrH7TH7hW7RPgAMJDhFlNpvp2rVr6b+srCz69+8P2Bfec05UE5ERA9+/mLZo50kACkuir9Yb7c5dLObVRXuieoBa5jlEULVq1cosZgewcuVK4HJwcK6vJISIHy9+vxuAmknRewmWlkMlq1mzJmBPSV2+fDldu3b1uKR3vNt1PI9RLy7l3MVio4si/Ineym/UKd9QcI41fLnxSNS1IqI3bIXTnGlw/KfwnrNRZxg73echzj0SAFq0aMEXX3xR+tj06dN54YUX+Pbbb8NbriripYW72XMynxV7TjGuyxUReY0Ve04x+Z01rPvjSBrUTIrIa1Rl0msXPF0ukjq7PhfuPMl3Px2P2Gc9FNJyiCBnt9KmTZvKBAZhd/h0AUt2nYz465T/Qjq9s2I/AFuyz3p8XAhX0+fsJGPa7Iid/+zFooidOxTx0XLwU8N3ZbVptNZYzBI3I23435ZQbNVkTR8XtnMePl3Agh0nuHNAC5TUbSPKmXHjLfhWNW8s3Wd0ESqVXAHL2XUij+3HvO8al32mgIO5F7w+rrV9Z7i8S777ylNSUsjLywu5nFVBsdX/RSXYC8+kt9fwxDfbOVsQXbWwaHDg1AXmbzsetvM5u0SirKu80hw7d5Fb3lhVuoZXIGLpvZLgUE6Jn8kopy8U+Rwk1RryC0s4mOt7clCXLl0wm81kZmbKgLQHodb6nUE5ClcjiKgF20+QMW02u094r3AMe2EJU99f7/Xx8tYfPMNP2efCUbwq6V9L97M26zRfbHRf2HHVvtyAKijR3LqNj24lg7gu2V3+voSEBI9LdsezuVuPcc8HG9j4p1EVPteJ85eokRg/H++5jhbBpsNnaZuWEpZz3vRPR9q1n26/OIvDfhWWWJn41moym9bhq3sH+Dw2mufixH3LocRqkxmeUeLt5QcA2JtzOahqDa8t3kvWKe9deZ6MfWl5WMsWq95cto9Hvghzpp5DFF/XDLM/J59Djl6DXcfdu6fLB9Jofg/jPjjsPpHvsykufPsx63TplyGsHN+a0xeKeH7eLia9vSbkU8VSP2+4PfPdTj5cc4iFO4Jb2O3YuYsBHxtt+fmV7clvt5f+PPxvSxn192WV8rrnLhazdHdOxM5veHBQSpmVUhuVUt86brdQSq1RSu1VSv1HKZUY6rkD+dCW2GJ76n9RiY1cA7YcdL63N7+xisHPL67QufaedA/Ox89dKvM6gbbuXNesieYme2X7xYx1QR3f71n/XZ6Xs5V8s9k0by/fz4XCkqDKEM0uFVtZFuCFWWsdkcmc//PBeqa8u5bTQQyIB8Pw4AA8AOxwuf0c8HetdWvgDPCLUE6anJxMbm5ula/VHDh1gSNnL/odSA+Gv/dMa01ubi7Jyclheb3tx+zBwfVV1x88E5Zzx4tQP+YZ02Zz34cbQnpuoLF34c6TPDV7B0/N3uH/4BjxxDfb2R9gV+eby/aT+cR8jp51b42FsgDfywv3MPyFJew9ae9+jdSKroaO2Cml0oFxwNPA/yr7OzUccC44NAN4HPhnsOdOT08nOzubnBzf0f3EGfsfbEdeNY+3/R1/qdjKqfwizieYuJiThNaaE2cvoRSYz3s+RzgdO3cRqw1M55IxmypeVS622jhxvpAGNRNJTjB7PS45OZn09HRgu9djQuX6W1RGaK9K9YdQPgHfbjnGfcPPk1G/hs+/uTdaQ+fH5vF/Y9szuW9zt8cvOlp9/tK7Y8n+HPdkE2/mOZIFjp276PZZy3dpTTkfs9nsCdzevs/OdZkapkR2Vr/R6Rz/AP4AONMr6gNntdbOdywbaOLpiUqpqcBUgGbNmrk9npCQQIsWLfwWYKxjxmPW9HHsPZnHL2csK73t73iAhTtO8Muv1zGsXSrv3dmVS8VWrv7TXBItJnY/Ndbna7+/+iBpKUmM7tjIbzm9ue2p7zmVX8SPj44kNQwflllrDvLo11lM7N2MZ2/sUOHzlXffhxvolVGPKf0zQnr+V5uOMKB1g7AsdxELyyZXljH/WM74zMa8PLGb22MHTl3gu5+OMaFXU+p7eN+11uQVlvDnr7Z6DA6VZdfxPJIsJjIa1OBikZUt2Wfp07J+2F9n4purOXza/zib5zRV7zWR1xfvZUKvpox7eQW7TuSFdXJoKAzrVlJKXQOc1FoHnnjtQmv9pta6p9a6Z2pqaljK9MbS/V4f23jojMdBOtfrS7B9/3/6cmtQeefR7Kfsc3y09hD/WrqP297xPnj87ZZjPPb1tjL3WcuN+yzZdbm1dzLP/p7mXigiJ6+QBz7exC9neu8/9/h1DLJl8OqiPXGZ378u67TH+29+YxXPz9tFj6cWeHw8WhpeV/1jGUNfWALAw59v4ecBXsSDtWp/LkcdY2LhdPTcJT7fcIRdHhJk9pzIc7v+RPp9N7LlMAAYr5S6GkgGagEvAXWUUhZH6yEdcJ9hYoAbXl/ps9tm7YHT9HhqAS9N6FqJpYoe1766IuTn/vY/m7mhW3rpWMeri/eWPrbx0OWxB2ff6vEIfDHBvnSKTWtemL+bF+bv9ltzyy8sIf9SCY1qh2fsJVoVFHkeSI7mGdI7j9svsPkxNgh+zMtn25kB9euhrdwei1T717CWg9b6Ya11utY6A5gALNJaTwIWAz9zHDYF+MqgIrqx+ph2e6HI3q+69oDn2pcvO4+f5+vNR0MuV1Xw0dpDbDjkvgDe2YLK66ee+OZq2jw6J+Djr31lBX2fXRiRsuQXlvDa4r0+P3OuTpyveMA8eu4S2WcKAn7N8pclb8+KlaSQH/aeCjmr6M1l+/jvusN+j/P3Vry/+qDPx19fUnnrO0VDtlJ5/4d9cHov9jGIdwwuj1fHz13ynuMfxPdhzD+Wc/9HG8NTqDD5aO0h9pzII2PabPY4mrlnLhRFrCb28OeeJ2o5a4Cujp27xPajnte/8vTlLrLaAsrmWluuW+WPX/qePHYgyIl5wZg+x77x/JytxwI6fsXeU0Bg4yjnLxWzLyefIx6yZ655ZQWtHvmuzH2uZ/zlzHVeL6D+LnzROMazYPsJPl2fzflLxUx6ew1TfXRZ+vLMdzv5w6db3O4vKrGVVnqyThXw8Y++A8ipILqmIx1zoyI4aK2XaK2vcfy8X2vdW2vdWmt9s9a68pP4y1m+x3PGU99nF/L4N2WzdTx9/u+e8SNzt4ZvwTNXwX5Aikps/Oilb7m8n7+5GrCPEwB0+8v3DHwusCU/bBFe3Ojql91nQH+6PpsSD6+bk1fIxLdWez3X8/N2ebz/g9WHPN7/w95TPPOd57TM0xeKfC7MGKgLhfaW6MHcArYdDe/4x+8/2cyIvy1lwHT3v6W/ltr320/wX8cFLpgLWbS6e+Y6fvfJZoocW50600PD5TWXLtKHPtkc1nOXilDMjYrgEA3Gv7qCT9dnl7nvvR8OsGpfLre9s9bteNc/uidFLjXVBTtOcs8HkR14DrRS9uycHdz8xiqPNe+cvEIe/WJr6W1Pk2tcLx6FJd4nprV85Dt2+FjdNhSefscLhSWlNdnf+fjy/Zjlfd6Ep9aJL5PeXsObyzwnLwx8bhFDnl/i8/k5eYUBt8Cen7eLcS+HPp7jybxtwc2WDrXGv+nwWU6GoburImxas/XIOWat8d1d48tvPtoY0j4OSsV2ADU6lTVqbPGQnfLEN95z+L3VNl1r8rtP5IVtETR/nK+7fE8OKckJdG1ax+NxuxwXwjMeVoz01qLw1gY456eWufHQWTpcUcvjY12fnO/zuYG4UFhCx8fmAZ5TjwPvOw+fgiL/M7l7Pb2AxrWTWfnwiDL3Pzd3J6k1k7hroP8UbG+01nyy7jDXdW1CoiU8db/yoeHcxWJy8i5f9Lwtq379az+Qkmzhqes7haUcoXANrJP6hJZm+02cjgdKyyGCXL9AkVK+UnfbO2u5/rUfwvoaufmFjHxxqdv9nrpwAhWOgWZ/XQDPzoneGbmeUiH/uWRfmXV6QjF363F+/+kWXlq4O2ILSr66eC+9nr6c1nq3j6U58i6Fd4zKZtO8sXRflVqKwx9v42WRbpVIcPAi1OyhSI65uU6TX7TzBKv25XIq3/O6Kt9vPxH0SqbefLvlmMcL8e8/9d2HGqnVQAMVj0twnHfMQs7NL+KaV8LbHeXNNi/JAeX5+mos2nmizDyhC4UlHrOc5m47zvQ5OyMa+PMLS8iYNpsWD0duS9BAvL18P3e8t9Zvt2ek9oSQ4ODFPysxZSwQ2WcKaPPoHP7zo32Q9K5/r/M5yPrLmetKJwQFKtB+8Nz8QjKmzeaHvbl+j918+GzYUhnLZwe5BktPe1GXD5wr950KSzkqwl+qYkU5x1b2nswP2+BqXhC19FAmnRUUlXDXv9dxx3s/AnAy7xIdH5vHG0v3u82xcLaGnAP2Tv66OF19tj6bxT72Ln91kX08UevAF3yMhKdm7ygzIbSyxXVw8JXeGI5mq7cJLaHYl2O/MDozhzypSEth5b5THlPxwD091NtMWU+ue+0HPlmXzcZDZ0prtaG69a2yM69dEwicFxZ/z3edDxBIfev+jzby0VrPWUuuDpy64Hfp8gOnLvCnL7f6PCYQi3ae4P1VWT6PWWdQq2nQXxezeOfJMsHaX93A2T3p/Pw6Jzk+N3cnV/55HusPXh4Lu+BlTOdAuQwxX6/50CebudPL5yX3QlGZvaLb/2mu78L7UFBkZdYa/58dfyqrBVheXA9IL9rpvfYQ6kqHH6+9nMf8u08287Me6SGdJ1jnLhYx8sXg15F/Y+k+xnRsxHof2TwV9YfP7EGnR/O6YT2vv5xxT37MOs19H27kjv4ZLPTx93f6evNRvt58lIm93dfvcjWsXCvt5PlLNKxVduZ0+c/U8j05DGrjeemX8uv0/3PJPn41uCUmk+Kuf9v7+G/rl+G3/Ea489+eL7zeulydmX95hSU89tVWembUK/P4xkNn6dHcfp9rcP1iYzY/Zp3hmRs6u53TFkBr1XWV1Cd9JJ/Eq7huOfj6AIXai+drkHbT4bOlOfA9PdS+LwaQ6eLN64s9d4P5ypE/W1DE9Dk7mfjWas5UwkzkaFiv6NvN9pbXv1dmRfR1nLW9cwXFPPnN9tI8eleeUqTBnlFWPo34ubk7+b7chj2RWDcoErxlMzn9y2VNsxmrDvKbACeE/vY/m/nQS818TwBdav1d5nnE8goFkRrnjOvg4Eu4kiBP5l3uxrj+tR9Kc+A9ZRp0+PPc0q6u5+bu5IuNl7tN/OWL/3TE84XXV478HMfEvIvF1oBqWlWBvwtVuDgXDPzrvJ28+8MBvtx0JKDuKcBr99vvP9lcJt9+0F8rtslSZVm4w95C83QN2xfE0te+xMoSHZEQqRyYuO5Wqgy9n3Zfe8fXINep/CJqJJlLB8SPnr1ETl6h35qur6/GG0v3cc8Q+4Jdzu/Qfpf+77MFxWUm7UVKZbxGqHxN6HPSWgfVwnp9yV5KrPY33GbTvPdDVqjFA+B8mNNCK4uncbJFO0+w7ch5erWo5+EZIhpIcPAinIPJ5fka5Cq/kJu3yXbBmD5nZ2lwcLpYLgvEW/O8qvFWwXTWbn1574esoOYh/HXuLn7es2nAx8cT57jJlH6hTUyL1O5n4jLpVqoi/DUtD58uu9pmHLfCPQpkwuKSCmzmHkxK5Ot+lmapSmasCiy19/yl4jLvYfnZ7/H8cZ79U2CLMwZLWg5xYtBfF3PfsNalt1ft9z9HIZ6U34AoXJxdaeUXaHRatS+Xw2cKymRyLTYwt70yuKamBqrL4/NpWi/y2+7Goj9/tY3bI5C5FufBIfqWEPbHW40/kIwF1010jJxcY6RQV/zwtAihk6/Jg/66P3xNZKxqlFKs2HOKyT52CvTl8Gn3JcY9+c5HTTrcq65WZXHdrRSFy8uLCFuwI7gVSZ12HjvP5sPumxGB7y6paNzDwCha65ADg/9zX/7517M2eH19T2uECc/ivOUQe5ybupQXqfVVhN2tb4d2UZO/ymXfhbinyVOzw7OOkoyzBSeuWw5GrptSEeU3GgdpBRmp/OxoV7E8uSrcPE0EDMe5zvjo8hOhi+vgEKv9j56+ZL76xIWoalyXPrGP6/hvFmSFYYe+eBLXwSFWvbvigNt9OTG845QQFRVIKvLwv8l4QzAkOMQgT7nh0p8q4tk9H3gehBahi+vgIN30QsS+YgO2g40HcR0cZBRXiNh3Q5i3xRV2cR0cJDQIEftOVsJe7fEoroODEEIIzyQ4CCGEcCPBQQghhJu4Dg4yHi2EEJ7FdXBw7tIlhBCirLgODq/G0aYqQggRjLgODkIIITyT4CCEEMKNBAchhBBuJDgIIYRwI8FBCCGEGwkOQggh3BgWHJRSTZVSi5VS25VS25RSDzjur6eU+l4ptcfxf12jyiiEEPHKyJZDCfCQ1vpKoC9wr1LqSmAasFBr3QZY6LgthBCiEhkWHLTWx7TWGxw/5wE7gCbAdcAMx2EzgOuNKaEQQsSvqBhzUEplAN2ANUCa1vqY46HjQJqX50xVSq1TSq3LycmplHIKIUS8MDw4KKVqAp8BD2qtz7s+prXWgMcFkLTWb2qte2qte6amplZCSYUQIn4YGhyUUgnYA8MsrfXnjrtPKKWucDx+BXDSqPIJIUS8MjJbSQHvADu01i+6PPQ1MMXx8xTgq8oumxBCxDuLga89ALgN+Ekptclx3yPAdOC/SqlfAAeBWwwqnxBCxC3DgoPWegXgbbudEZVZFiGEEGUZPiAthBAi+khwEEII4UaCgxBCCDcSHIQQQriR4CCEEMKNBAchhBBuJDgIIYRwI8FBCCGEGwkOQggh3AQcHJRS1SNZECGEENHDb3BQSvVXSm0HdjpuZyqlXo94yYQQQhgmkJbD34GrgFwArfVmYHAkCyWEEMJYAXUraa0Pl7vLGoGyCCGEiBKBrMp6WCnVH9COzXkewL7fsxBCiCoqkJbDPcC9QBPgCNDVcVsIIUQV5bfloLU+BUyqhLIIIYSIEn6Dg1LqPUCXv19rfVdESiSEEMJwgYw5fOvyczJwA3A0MsWpXD3VTu61fMVFkrhIIud1DY7q+hzVDTigG7FHp1Ns6E6qVYEmlXO0NR2mhTpOQ3WGVM5RW13AjA0TNoqwcE7X4Bw1OaIbkKUbccDWiCM0wPtmgSIQFkpork7QWh2hhTpOPZVHPZVHCgWlxxSSwGmdwmldi2zdgH26Mft0Y/KRqU0VVYsLtFHZNFcnaGY6SSrnqKEuUoNLABRhoZBETunanNB1OaIbsEs35aBOw2bwHOVAupU+c72tlPoIWBGxElWiJFVMPZVHNU5RjSLqmPJJURdLHy/UFnbqZmywtWG5rTNrbB24QDUDSxz9FDY6qEP0Ne2gr2k7PUy7qa/ySh+3akUutTmna1CCCRsmEimhtukCtcknSZWUHpurU9hoa80GW1uW2TqzTWegZVK/T9W4RF/TDnqbdtLLtIvOan+Z97RAJ3GaFPJ0NZyBN5lC6pnyqK0Kypxrn+0KfrS1Y51uxzJrF05StzJ/lZjUiFyGmLfQx7SDTLWPVqZjpY/ZtOIMNcnT1blAMhpFEsUkU0Sq6SzJqrj02AKdxHbdnNW2Dqy0dWS9rS2FJFbq76K0dusx8v0EpdoBs7XWrSNTpOD17NlTr1u3LujnZUyb7XZfCgU0Uadoo7LpZDpAZ3WAbqa9VFNFFGsza23tmW3ry1xrL05TKxzFj3lmrPQ27WSMaS1XmdfRSJ0BIMuWxlpbe7br5uzSTdlna8wpavuoEWkacpaWpmO0UkfJVPvobtpDa5O9oXpS12GxtStzbL1YYetMibTqAKjLeUaZ1zPKtJ5Bpp9IVsUUaTM/6Zass7Vlp60Ze3UTDuhGPlsDCZTQVJ2klTpKG5VNd9Meepp2U0ddAGCzrSULrN2ZY+vNXp1eWb9e1GurDjPevJIRpg10MNmz/k/qOmy0tWazrRXbdXMO6jSO6AYUkeDlLJo65JOucuhgOkQHdYiupr10UfuxKBuXdAJLbZl8Z+3NIlt38sr9HbOmjwup7Eqp9Vrrnh4f8xcclFJ52McclOP/48DD5VsURgpncPAkiSJ6mHYzyPQTo03raGU6Rok2sdLWkU+sQ5hn6+Xjj151XUEut1oWMsG8mFR1jos6kaW2TL639mClrSPHqB+W16nHeYaYNjPcvJEhpi3UUgWc0rX4xtqPL60D2KxbEX/dT5ruag+TLQsYZ1pNkiohWzfge2sPvrf1CFtNU2GjrcpmhGkjI83r6ar2YVKaLbYWfGYdzNfWfpyJw0pSGqe53vwD15t/oIPpECXaxFpbe5bYMlli68punU44PpM1uEhv006GmDYzxvwjjdQZCrWFRbZu/Nc6lGW2LlgxGxMcYkGkg0NZmg7qEOPMqxlvWkkzUw6ndU0+sw7mY+sw9ukmIZwzlmj6m7Zxu/l7Rpns7/kiW3c+sw5iqa0LF0mO6KsnUMIQ02auN69glGkDSaqYbbbmvG8dxVfW/hF/faNV4xLjzau4zfw9nUxZ5OlqfGYdxCfWoWzTzYl0kEzlLNeaV3GjeTmdTFkUazPzbD2ZWTKatbp9xF/fWJp+pu3cbp7PKNN6LMrGBltrvrQOYLa1L7nUjuirK2x0U3u5xrya8eaVNFDnOaHr8I+Sm3j26RdDO2cowUEp1d3XSbXWG0IqTQRUbnC4TGGjv2kbE82LuMq0jgRlZaX1St6zjmGhrbvhA0rhVJMCbjIv5zbz97Q2HSVXp/Af6zBmlYzgCKmGlCmFAsabVzLZvIAOpkOc19X4zDqYD6wjq1yQbqGOMdm8gJvNS6mlCthha8oH1lF8YR1IgUEBsZ06xM/My7jZvJQ66gI7bE2ZaR3Nl9YBVSpIl//sn9Y1+a91GB9Zh3FQNzKkTAmUMNy0kZvNS5hn68XzTz8f0nlCDQ6LfZxTa62Hh1SaCDAqOLhqwDluNi9lsuV7mqhcDtoaMsN6FR8gLZcAACAASURBVJ9Yh7j1D8aStuowt5vnc4N5BTVUIZtsrZhZMorZtr6VPkDmnaaH2s1tlu8Za1pLkiphhbUjM62jWWDrEbNB2oyVEaYNTDYvYLD5J4q0mbm23swsGcU63Y5oqaUnU8h480qmmOfT0XSQ87o6n1iHMNM6yrCLZzi0Uke4wzyPG83LHZ/9lrxfMppvo+qzbyfdSl5EQ3BwMmPlKtOP3GmZSy/TbvJ1Mp9YhzDDOposfUXYXy8SLJQw2rSOKZb59DHtpFAn8LW1HzOto/lJtzS6eD7V5xw/Ny9msmUBjdVpsnUDPigZyX+sQ2Omb7yB43e41bKQJiqXo7oeH5aM4D/WYeRQx+ji+WAfB7nDMo+xprWYsbHElslM62iW2rrERKaZwsYQ0xbuNM9liHkLhdrCN7b+zCwZxRbdyujieWVYcFBKdQKuhMttRa31zJBKEwHRFBxcdVL7udMyl2tNq7BgY7GtK+9Zx7DC1oloqfW5SuM0E8yLmWhZRCN1hkO2VD6wjuS/1qGcJcXo4gXFjJWRpvVMMc+nv3l7aYCbYR3N1qgMcJqeahe3WRYw1rSGRGVlubUTH1hHscDWHStmowsYlFTOcKt5EZMsC2mozrLf1oj3raP4NEpb0tW5xE3mZdxhnkcr0zFO6jq8XzKSD60jIj6WEA5GZSs9BgzFHhy+A8YCK7TWPwupNBEQrcHBKZWzTLIsYJJ5AanqPPttjfjYOozPrIMN/+ApbAw0bWWSeSEjHYNsS6yZzLCOZqktM2a7ZFy1Udncbp5f2j2wwdaaGSWjmWPrY3iWWS3yudG8gonmRbQzZXNeV+dTx7jJft3Y0LKFQwIljDWt5XbLfHqadnNBJ/G5dRAzrKOjIh22nTrEBPNibjIvp5YqYJOtJe+VjOE7W9+YmgBrVHD4CcgENmqtM5VSacAHWutRIZUmAqI9ODglUszVpjVMtCyij2knRdrMfFtP/mMdxkpbx0qtHTZVJxhvWsUt5iU0N50kV6fwiXUoH1qHc0inVVo5KlMKBdxkXsbt5vm0NB0nR9fiI+twPrcOqtQuPxM2+ph2cJN5OdeYVpGsitlka8lH1hF8be1XpQZzXXVUB7jDPI/x5lUkqWJWWq/kc9sg5ll7VWproiYFjDOvYaJ5EV1N+yjUFubaejOjZDQbdBuisVXvj1HB4UetdS+l1HpgGJAH7NBatw+pNBEQK8HBVSt1hInmRdxkXk5dlU+uTmGetRezbX1YbbsyIoEiXZ1kpGkD480r6W7aC8AaW3tmlYxkbhzN1XC2lm43z2eEaSMmpdlqy+Bba1++s/WJSHA0Y6Wb2sM48xrGmdfQUJ0lXyfzlXUAH1pHsE1nhP01o1VdzjPBvIQJ5kU0N52kUCew0NaNb6z9WGHrHJFAUYsLjDStZ6x5LYNNW0hSJeyypfOxdRhfWAfGXLdpeZUaHJRSrwEfAROBR4EJwENAPrBJa31nSKWJgFgMDk5JFDHUtIlx5jWMMG2ghirkvK7GGse0+bW2DuwOaY0nzRWcpotpH30ck2icU/l32JrxlbU/31j7GZaGGi0akcs48xquMa+mmyNgZtnS+MHWiR9sHdmiW5Gtg1/jyUIJ7VQ2maZ9DDJtYYBpG7VUAYU6gUW2rnxj7cciWzcukRSB3ypWaLqqfVxn/oFrzKtIVecp0SbW67Yss3Zhg27DVluLkIJFLfLpZMqin2k7A0xbS2caH9H1mWPtzbfWfmyqQpMnKzs4PIA9IDQG/oM9UJwBammtt4RUkgiJ5eDgyh4oNjPYtIX+pq20MJ0AoFib2asbs1c34biux3Fdl/PUoEhbKMZCEsXUVBcd0+9P0UydpJXpKA3VWQAKdQKrbR1YYstkqS2zSvRlR0K6ymGEaQMDTVvpa9peus7WWV2DHbbmZOsGHKU+OboOF3UShY6WVnV1iZpcoqE6S7o6STN1knYqmyTHWjlHdT2WWbuwzGb/JwvauXO2rIaaNzPEtJnOpqzSx/bZriBLN+KwTuWork8+1cnXyZRgJpESklUR9cjjCpVLY5VLO9Nh0tUpAEq0iS26JT/YOrHQ2r1KBQRXRnUrNcceJCYA1bAHiQ+11ntCKk0EVJXgUF4Tcuhm2ksH00GuVAfJUMdppM5QTRV5fc5JXYdDuiEHdRqbbS3ZbGvFDt08brqMwsVCCR1VFp1MWXRUWbQzHaaxyqUhZzArz9+ZQm3hiG5Atk5lp27GT7YWbNEtOajTqIoXpEiqQx5dTPvpovbTyZRFU3WSpiqHWuUWB3R1RtfkuK7HHt2E7bbmbNfN2WhrE5XZUeFm+DwHpVQ34F2gi9Y6anLrqmpw8ExTiwukcJFEVUICJVwikXxdjXyqSRCIMAsl1CWPJGVfTVMBF3QyBSRxjhoxkcsfy6pziZpcpIa6RAIlFJJAoU7gHDWq7EB+ICIRHALZ7MeCPX11AjACWAI8HlJJRBgozlOT89T0sAWTiLQSLORQV957gxSQbF8uRN7/iPMaHJRSo7APRl8NrAU+BqZqrS9UUtmEEEIYxFcb+GFgJdBBaz1ea/1hZQYGpdQYpdQupdRepdS0ynpdIYQQPloORi6sp5QyA68Bo4Bs4Eel1Nda6+1GlUkIIeJJtI6e9Qb2aq33a62LsHdpXWdwmYQQIm5Ea3BoAhx2uZ3tuK+UUmqqUmqdUmpdTk5OpRZOCCGqumgNDn5prd/UWvfUWvdMTY3vWb5CCBFu0RocjgBNXW6nO+4TQghRCaI1OPwItFFKtVBKJWKfY/G1wWUSQoi4EZULlmutS5RS9wHzADPwrtZ6m8HFEkKIuBGVwQFAa/0d9s2FhBBCVLJo7VYSQghhoLgODp2bRP/esEIIYYS4Dg4mWUVZCCE8iuvgkGCO619fCCG8iuurY9emdYwughBCRKW4Dg5KupWEEMKjuA4OQgghPJPgIIQQwk1cBwcl/UpCCOFRXAcHIYQQnklwEEII4Saug4N0KgkhhGdxHRyEEEJ4Ft/BQZoOQgjhUXwHByGEEB7FdXBonVrT6CIIIURUiuvgcE2XxkYXQQgholJcBwchhBCexXVwkAnSQgjhWVwHByGEEJ5JcBBCCOFGgoMQQgg3EhyEEEK4keAghBDCjQQHIYQQbiQ4CCGEcCPBQQghhBsJDkIIIdzEdXCQGdIVYzHJGyhEVRXXwUFUzA3dmhhdBCFEhEhwEEII4UaCgzDcO1N6Gl2EuPerIS0rfI7P/qdfGEoiooUEBxGyGkmWsJwnrVZyWM4jKkBX/BSt4mTzrD+O6xD2c9atngDA/SPahP3coYrr4KBkE+mQvTShK52a1PZ73LwHB5e5fc+QVjx5XUe349b/cST/N6Z92MpXlbVvlBL2c/qKDe3Swv96sSpr+jjuHtSSvU+PDet5qyfaK1qdGtcK+rnf/mZgWMviFNfBQYRuTKdGHu9/ZWK3MrfbNUrhrgEtSm9PG9veY0iuXzOJmsnhaYlUdXWrJ4b9nFp7Dw+/HtYqwHOEqzTx7TfDWwd1fCCVtFAYEhyUUs8rpXYqpbYopb5QStVxeexhpdRepdQupdRVRpQvHP2vVV2Sxex2X9b0cVyb6b71alJC2Y+Zv2tIiwY1KlI0UQFXd3YP+ibJ+Q5aWq0k7uifEfTzlFI8NLod/zM0sIAcSUa1HL4HOmmtuwC7gYcBlFJXAhOAjsAY4HWllPtVKMJSayb5PeYXA1twe7/mAZ/ztVu7V6RIMWXqYM/B9ZeDWni836lbU3sd4eGx0r3ki79rdVot75/fH6YNZ8nvhvLW7WWTAJy1/trVEtyeU79mIrf2aRZ0OeOZxWTi8fHu3acAk/vGxntpSHDQWs/XWpc4bq4G0h0/Xwd8rLUu1FofAPYCvSu7fP1a1S/9ef5vB/PKxG7c1D29zDFd0mtTLSHwuDWuyxVhK1+0e+RqzwN2dfx0h3RqUptdT41hdEfPXVZOzepVD7lssaJpvWohP3fmXX28PtakTjUyGtRg1JVpXo5QPH7tlW73TpLg4NfPeqT7PwhI9tDq9tWtZ5RoGHO4C5jj+LkJcNjlsWzHfW6UUlOVUuuUUutycnJCemFnDaz8TN/6New1r4YpSbRNS+HazMb87ZbMMsf0zKhX+vPUwS35YdrwoF47mmcXv3Bzpv+D8P2BHtoulfEeupjsz/N+Tk/dVeX1a1nf7zGx7oau3icY9mnh+/evVa1iYzd3DPDdwhOeubaYh7ZLDeq5zq9ENF0VIhYclFILlFJbPfy7zuWYR4ESYFaw59dav6m17qm17pmaGtwfwp9gu1jr1UikSZ3ganrROvj67I2d/daAAvng//vO3rxcbnDaqUEA3XbBqIpddr7qkT0z6rLrqTFe+7QDzcL78O7LLYxaju4kj5/9coVp3yiFtY+M8Di+JCAlyeK1S8mbZEcvhLdrj9mlMrnwoSEhly0YEQsOWuuRWutOHv59BaCUugO4BpikL1dBjwBNXU6T7rivUoW7hffenb3c7rvyiuBT1irDhF5N/R6TaA7uY+NsJTkHNntm1PX7HH8X/N+Nblv6c9+W9XwcGXsC6cJJsph55OoOTOxt/3sF8ncrr3/rBux9eiwrpw2nbg17l5+/sJJWK4kvfj2AhrWSecyl++mO/hlxu1ZZ+e9yo9rJJAT4Hdn859G8dmt3tySMmuXmELm+ta7zSbx3D1acUdlKY4A/AOO11gUuD30NTFBKJSmlWgBtgLVGlBF8tyBcH/IXTDwN8kXrF0l5KNjrkypWM//VkFZM6dc8qOwNf2M0yUGM90S7jX8aVfpz35b1ePqGzgE9L9Fiokt6Hf8H+mAxm2gcRKu3Xo0kqiXa33vXFmAwNeWqNGZkNim+uLc/m/48yv/BHtSunsC4Lle4ddHePahFQIkZ5YNIOBk15vAqkAJ8r5TapJR6A0BrvQ34L7AdmAvcq7W2RqoQFpNiZIc03rnjcs1+1t3eB/PclLuOrn1kRJhKdlmwfZeRUNG8+ppJFp64rlPpRaWidLl+jugbyguOs9Ye6wJtcX9zX2QmbRklyWKmTvVEUhxdxRUJ2M66WZLFzK+GeE9n/XhqX4CIprwala3UWmvdVGvd1fHvHpfHntZat9Jat9Naz/F1nopSSvH2lJ4MaWu/AE/q04wBrRuEfL6GtZJ59sbLtb6uTStWqwPjalmutfzGdWJ3eYtpAdS+9oR5tmukRSSZoRKzZWpXd29JxxpP79YVtavxzX0DefqGTm6Pmcv9zRLNJga3Db7i5+zS7duyPlnTx9E2grPXoyFbKSpkTR8XcHMevHcLTewdWMpfJJbuWPfHkWE71+PjO7L/matZ/fAImtf3PCnNYg7tdyh/HfL2XtZMstDZw+zPYe0aBvxagfTfB9o/XBl8XaPbNLT3NfvrUvOWrfTG5B5+X9/T36J7c/9jRJWpdcPoWcOp/NvVOb12QF2eu58ey8y7/GfpT+zdtEz24MqHg8uKrIjoTJkxWL0aidRMsvDoOPd870hpVq86h04X8MLNmSSYFWcuFPH4N9uDCiHhzgIymRSNantvNVzTpTE7j+fxr6X7w/q6TlufsE+Qz5g2u8z9qSlJHDl7sfR2oP2uax4ZwfqDZ/j1rA3hK2SEWEzuActft9xvR7alRpK5dJ2e8jqnB7/Mwu39mkfd+E6TOtXYezLf6GIEZGynRszZetzv9/hyKmvZI5+9sQsATetW4z/rDlO/Ersgo6fKFEUSLSa2PnGV1zz9UF2b2ZgrvFxsnZOe0molcV25HPfHPExKMtKkvvaZ4QlmEw+PDf8KlYFwXrDuHdbK58XLdYA9rVYyV3eOjcmIvxzsfa6Bt8bFAyPbcPegsrPTE82m0s9PPR9jRw0dK+M2rWvsYHHW9HF+j2lS1/MAeijLVYQq0F64f07uEdDvpP1MdOjTsj4v3tLVY8JIpEhwCILrchkKVTq3wdtyBT0dzXFntH9lYjf++rMuPl/DWXMItAf4qevt/ZspEZ43sfMvY1j7yAiypo8rHaOpbD0c76cGft6rKb8d2Zb7hoVnieOGKUmG/V6unH/36okWt4pEqJeFOwe0IGv6OJ8tj9FXpvHenb3cgkuUJtWVfhZapl7u8qzImlztG6XwxuQeLPjfwf4PdhGtWYfhIMEhCE9e14mGKZcDweQ+zXlnSk+v22VOG9ue+b8dXKbPPthxv0BrCuUnJHkqkzPDIRTJCebS2qVRXN+JBLOJB0a2CVsG1NpHRzLDpQ+4ef3ga9CBrgnls4Yb5nHh67oG1vpVSjGsXUO3gVMn59In3ZtVPMkiWPU8dKU4y3mVn6VWApVetxpjOjWidcMUEi1lL4uB1PyrIgkOIVLK3ic/okOa1wu4xWzymk1Q/im+gobPx7zc7xrEnPpGybIT5VNR/bmxW5NK3y3uFwODX0LCW+ph+0YpZS5wvuYEBPve9G5hnwDobbaya/ZcRTSpU425Dw7isWuDm/nrtCaINO+JvZuVSSRY+L/uM4LDVWGfeVdvpo1tz19/dnnQN5BksGD/Tv7PF30kOEQZZ9DwFhD+dVsPGrt0N6Q7urbK78KV5Kj9jCvXx/7hL/vE3LacL/68KyM6hDYT1NP6T66rw/73V5Hf2nLug4O5M5TlmwM4plVqTbKmj/Oagm0JYyZW+0a13GrVrqoneW/Fedvt70MP84qevbFzmczBujUSSXeMMyRZTEwd5HnV32C6eP44rgN/ua4jg9o04J4hrTy2Tl67tXuZDahapbp3W4WSdXi9l54G+/mihwSHIBkZ4e/on8FVHRux/P8up7MNa9+QT+7p53bxcbZmyqf99W/VgBEd0soswVzRZvP1XRvTK4AlMSrK2bXhL8//jcndmfPAIP5naCuPmUyuWWjOmrc3N/r4IgMkJ1TNr1DbNPvnpkMQy7wkWczsf+bqMuNfKUkWtj3hfVuW/gHOK5rUxz7et/5Po8gIYGzhg1/4nsxaI8nCbf0yPLb6nRf8Ye1TSyeZ/fjoSL6pwI5rzm7efc9cHbHNecJNUllDFM4IP7ZTI85dLA74+PL9wr0ygl9bKJzzqP4xwfMCe+H2ws1d+GLjEb+TCxvUTKLDFbV8XtjuHdaKRLOPLCfH/55qwylJFvIK7SvOL3xoKAOmLyp9bNOfR3Eqv5CRLy4re74A3+9wzkWryLLfHRvXZveJfJ8tBYC3bu/J9qPnS2+bTJfr0vN/O5hGtZM97jXer2X9gMdDwD4T2NNsYNf3y/m6k/o0Y2Cb0Cezejp3qodu2mA8e2Nn/njNlV7HdKKRBIcoUCPJwtmCssEhnC2UaB9Qy/Ayya68OtUTuTNMy0n//qrANxSqkWjmQpF9FZf5vx3M2YJibvnXKjpcUcttNd461RP97lvhyRPjO/LY19vK3PfPyT3417J9fPfT8aDPt+2JqyrlQjTqyjSvi7+lpSRTK9nzbOiPAkiOGNI2lQQvEy09BtswpQ4Fcppgg7jFbKJ2Ne+Btl1aTZbtzgn7XKWKqJpt4gj6/VXtgNCXAOjXqj4TezfjuZsup7S6psKW/1yG8nkP5KIwMsQ+/HCoU81+8Xz06g5kTR/nsWYZLF8rs740oSvv3lHxcZZ6NRJpm5ZC/Zr28nubsxIM55jQlR42ls9sWofXJ/XgvmGt+eLX/Uu7QALZGKZGkiXqJq8Fa8ZdvXl7ivuKxoHyVSkKZGXhgK7/YYq/fxjTnk/u6RdVXU7ScgjSLT2bcktP38sjv3lbD48DXGBPwSyfQfLAiLZMeTc8i88+cnV7BrdNZY6f2ubQIJagCLdqieawt2Y+ntqvdCZ1+S91+UmFoVr6+6GAfRD4pQldGdo2+Pfw1+W6Rl5zrHjb3LGG1nUexjh+56iQvDKxG28t31/hlVirov4uuzf6u6g/NKqtzy4tIzp+EsymkLqHI0mCQwT42+ayvESLqTQbw7kJUKjbBk4dbL/4+AsOALPvH8ieE7GxDEE0cB28dA04E3s35aO1hz09xeNzPWlYK5l9z1ztcyyoab3qPHmd+6Ju8eiZGzrz17m7SifDmU2B5w39ZkRgEyd9fQedf05f+3XHOgkOBpo2tj0DHdkaT17XiREdGrrVCkNdoG9K/ww2HT5bZlZ3eR0b16Zj4+hpxsaqZ2/sUroGTqB+N7otLRqUzSSLpcHKcHId4A9Uy9SavHFbD5bu9r1F8AMj2vDSwj1BnTuQiadJFjN//3mm3y1bY5mMORjoniGtSvsYqyWaGdMpfOv+1KuRyIy7elM/iga4YpHWlGa+BLNU9q8Gt6RJnWq8eqvnTK77hrfxu6FRvFj0u6ERO/dvR7VlRHt791/nJrVD2jHPmxu6pQe1UVKskZZDjPrsf/pzqTi8+yDVCNNSFLHsvTt6YdOao+culd730oRuHDt3KagB3oev7sDDVxuzKGGsqWiaqNO1mY35dstR7h3W2uPjD4xow8gAttX85+Tu/Gvpfmp4Wd02XsT3bx8DvLVwe4R5jf3nbuocdQNiwfry3gF8vPYQPZqF/t4Mc9Qy3199sPS+5ARzhRZ1cxWNyyRUFbWrJfDx1IrPeB/UJpVBbYxfhNFoEhwEAD/vFdgmRdGsa9M6Ydl9T1Qtf7m+E7Xn72JQ24pPjIsnEhyiVCXu2ihElda4TjVevKWr0cWIOTIgHeXiM39FiMA4F6GM9S7RaCQtByE8cAblcG/mIi1Cz9b/cWRIu5y1SUthye+G0qyesTvYVUUSHKJUuNeLF8H5WY90th09x+9GtwvL+aryjmHhUJGU60BWaRXBk+AQ5eSiYozkBHPQE9uEqEokOMSA7o601cGSbSEqiXOzqHDO2l4cwcluIvwkOEQp177prk3rsPMvY2J+lc145lwGJVa6Cx++ugN1qie67SRYEeGaKyIqhwSHKOccpJPAENtu79ecA6fy+fVQz7N3o03taglMGxv4nhei6pHgIEQlqJFkKbOJvRDRTuY5RKnY6HwQwp1zpV+Ll13cRGyQlkOUk6+XiDX/ur0Hu47nhWWHP2EcaTkIIcKqVnKCzFiuAiQ4RCnn3gEJAex1K4QQ4Sbtvig1uW9zcvIK+fWwVv4PFkKIMJPgEKWSE8yyWYwQwjDSZyGEEMKNBAchhBBuDA0OSqmHlFJaKdXAcVsppV5WSu1VSm1RSnU3snxCCBGvDAsOSqmmwGjgkMvdY4E2jn9TgX8aUDQhhIh7RrYc/g78gbKTga8DZmq71UAdpVT4Vv4SQggREEOCg1LqOuCI1npzuYeaAIddbmc77vN0jqlKqXVKqXU5OTkRKqkQQsSniKWyKqUWAI08PPQo8Aj2LqWQaa3fBN4E6NmzpyxFJIQQYRSx4KC1HunpfqVUZ6AFsNmxHHU6sEEp1Rs4AjR1OTzdcZ8QQohKpLTBO54rpbKAnlrrU0qpccB9wNVAH+BlrXXvAM6RAxwMsQgNgFMhPjdWye8cH+R3jg8V+Z2ba61TPT0QbTOkv8MeGPYCBcCdgTzJ2y8XCKXUOq11z1CfH4vkd44P8jvHh0j9zoYHB611hsvPGrjXuNIIIYQAmSEthBDCAwkOjoynOCO/c3yQ3zk+ROR3NnxAWgghRPSRloMQQgg3EhyEEEK4ievgoJQao5Ta5VgFdprR5Yk0pVRTpdRipdR2pdQ2pdQDRpepMiilzEqpjUqpb40uS2VRStVRSn2qlNqplNqhlOpndJkiSSn1W8dneqtS6iOlVLLRZYoEpdS7SqmTSqmtLvfVU0p9r5Ta4/i/bjheK26Dg1LKDLyGfSXYK4GJSqkrjS1VxJUAD2mtrwT6AvfGwe8M8ACww+hCVLKXgLla6/ZAJlX491dKNQHuxz6ZthNgBiYYW6qI+Tcwptx904CFWus2wELH7QqL2+AA9Ab2aq33a62LgI+xrwpbZWmtj2mtNzh+zsN+wfC4sGFVoZRKB8YBbxtdlsqilKoNDAbeAdBaF2mtzxpbqoizANWUUhagOnDU4PJEhNZ6GXC63N3XATMcP88Arg/Ha8VzcAh4BdiqSCmVAXQD1hhbkoj7B/al4W1GF6QStQBygPcc3WlvK6VqGF2oSNFaHwFewL43zDHgnNZ6vrGlqlRpWutjjp+PA2nhOGk8B4e4pZSqCXwGPKi1Pm90eSJFKXUNcFJrvd7oslQyC9Ad+KfWuhtwgTB1NUQjRx/7ddiDYmOghlJqsrGlMoZjlYmwzE+I5+AQlyvAKqUSsAeGWVrrz40uT4QNAMY7Fnf8GBiulPrA2CJVimwgW2vtbBV+ij1YVFUjgQNa6xytdTHwOdDf4DJVphPOTdEc/58Mx0njOTj8CLRRSrVQSiViH8D62uAyRZSyr5H+DrBDa/2i0eWJNK31w1rrdMf6XROARVrrKl+j1FofBw4rpdo57hoBbDewSJF2COirlKru+IyPoAoPwHvwNTDF8fMU4KtwnNTwhfeMorUuUUrdB8zDnt3wrtZ6m8HFirQBwG3AT0qpTY77HtFaf2dgmURk/AaY5aj47CfAFY5jkdZ6jVLqU2AD9oy8jVTRZTSUUh8BQ4EGSqls4DFgOvBfpdQvsG9dcEtYXkuWzxBCCFFePHcrCSGE8EKCgxBCCDcSHIQQQriR4CCEEMKNBAchhBBuJDgIEQSlVH2l1CbHv+NKqSOOn/OVUq8bXT4hwkVSWYUIkVLqcSBfa/2C0WURItyk5SBEGCilhjr3i1BKPa6UmqGUWq6UOqiUulEp9Vel1E9KqbmOJUxQSvVQSi1VSq1XSs1zLoEgRDSQ4CBEZLQChgPjgQ+AxVrrzsBFYJwjQLwC/Exr3QN4F3jaqMIKUV7cLp8hRITN0VoXK6V+wr48y1zH/T8BGUA7oBPwvX05IMzYl5sWIipIcBAiMgoBtNY2pVSxvjy4Z8P+vVPANq11ld6+U8Qu6VYSwhi7gFTn3s5KqQSlVEeDyyREKQkOQhjAsTXtz4DnlFKbgU3E1x4EIspJKqsQQgg30nIQQgjhTdZPNgAAAC1JREFURoKDEEIINxIchBBCuJHgIIQQwo0EByGEEG4kOAghhHAjwUEIIYSb/wefOv/flNO5twAAAABJRU5ErkJggg==\n" + }, + "metadata": { + "needs_background": "light" + } + } + ], + "source": [ + "# Create an object with links to the model and time series\n", + "problem = pints.SingleOutputProblem(model, times, values)\n", + "\n", + "# Select a score function\n", + "score = pints.SumOfSquaresError(problem)\n", + "\n", + "# Select some boundaries\n", + "boundaries = pints.RectangularBoundaries([0, 0], [20, 20])\n", + "\n", + "# Perform an optimization with boundaries and hints\n", + "x0 = 0.01, 0.01\n", + "opt = pints.OptimisationController(\n", + " score,\n", + " x0,\n", + " method=pints.LBFGS\n", + ")\n", + "opt.set_max_unchanged_iterations(10)\n", + "opt.set_max_iterations(200)\n", + "found_parameters, found_value = opt.run()\n", + "\n", + "# Show score of true solution\n", + "print('Score at true solution: ')\n", + "print(score(real_parameters))\n", + "\n", + "# Compare parameters with original\n", + "print('Found solution: True parameters:' )\n", + "for k, x in enumerate(found_parameters):\n", + " print(pints.strfloat(x) + ' ' + pints.strfloat(real_parameters[k]))\n", + "\n", + "# Show quality of fit\n", + "plt.figure()\n", + "plt.xlabel('Time')\n", + "plt.ylabel('Value')\n", + "plt.plot(times, values, label='Nosiy data')\n", + "plt.plot(times, problem.evaluate(found_parameters), label='Fit')\n", + "plt.legend()\n", + "plt.show()\n" + ] + }, + { + "source": [ + "## Using the CMA-ES to solve the same problem" + ], + "cell_type": "markdown", + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Minimising error measure\n", + "Using Covariance Matrix Adaptation Evolution Strategy (CMA-ES)\n", + "Running in sequential mode.\n", + "Population size: 6\n", + "Iter. Eval. Best Time m:s\n", + "0 6 1.12e+07 0:00.1\n", + "1 12 1.07e+07 0:00.1\n", + "2 18 1.04e+07 0:00.2\n", + "3 24 1.02e+07 0:00.2\n", + "20 126 9993283 0:00.7\n", + "40 246 9993282 0:01.2\n", + "60 366 9993282 0:01.8\n", + "80 486 9993282 0:02.3\n", + "100 606 9993282 0:02.8\n", + "120 726 9993282 0:03.3\n", + "140 846 9993282 0:03.9\n", + "160 966 9993282 0:04.4\n", + "180 1086 9993282 0:04.9\n", + "200 1206 9993282 0:05.4\n", + "220 1326 9993282 0:05.8\n", + "240 1446 9993282 0:06.3\n", + "260 1566 9993282 0:06.7\n", + "272 1632 9993282 0:06.9\n", + "Halting: No significant change for 200 iterations.\n", + "Score at true solution: \n", + "9993481.362921719\n", + "Found solution: True parameters:\n", + " 2.92147774048697517e+00 3.00000000000000000e+00\n", + " 9.01371505503933079e+00 9.00000000000000000e+00\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": "
", + "image/svg+xml": "\n\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEGCAYAAACO8lkDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd3yT17nA8d+R5MEw25iAAbMhDLP3XoGQkNWkEEhI0pTmNmmS3rS9JGmb0QzSpGkzm2YWEpK02YMwwoawwg57GjDTmGVj8JDO/UOSka0tS34l6/l+PnywpFevjmXpfc54zjlKa40QQgjhymR0AYQQQkQfCQ5CCCHcSHAQQgjhRoKDEEIINxIchBBCuLEYXYBwaNCggc7IyDC6GEIIEVPWr19/Smud6umxKhEcMjIyWLdundHFEEKImKKUOujtMelWEkII4UaCgxBCCDcSHIQQQripEmMOnhQXF5Odnc2lS5eMLkqVlJycTHp6OgkJCUYXRQgRAVU2OGRnZ5OSkkJGRgZKKaOLU6VorcnNzSU7O5sWLVoYXRwhRARU2W6lS5cuUb9+fQkMEaCUon79+tIqE6IKq7LBAZDAEEHy3gpRtVXp4CBEVbdqXy77cvKNLoaogiQ4RJBSioceeqj09gsvvMDjjz8e9HnWrVvH/fffH3I5MjIyOHXqlM9jnnnmmZDPL4wz8a3VjPjbUqOLIaogCQ4RlJSUxOeff+73wuxPz549efnll8NUKs8kOAghXElwiCCLxcLUqVP5+9//7vZYVlYWw4cPp0uXLowYMYJDhw4B8Mknn9CpUycyMzMZPHgwAEuWLOGaa67BZrPRpk0bcnJyALDZbLRu3br0tlNubi6jR4+mY8eO3H333bju9nf99dfTo0cPOnbsyJtvvgnAtGnTuHjxIl27dmXSpElejxNCxI8qm8rq6olvtrH96PmwnvPKxrV47NqOWG2a/Tn5pNetTrVEs9tx9957L126dOEPf/hDmft/85vfMGXKFKZMmcK7777L/fffz5dffsmTTz7JvHnzaNKkCWfPni3zHJPJxOTJk5k1axYPPvggCxYsIDMzk9TUsutmPfHEEwwcOJA///nPzJ49m3feeaf0sXfffZd69epx8eJFevXqxU033cT06dN59dVX2bRpk8/j6tevH463TggRA6TlUEEXCku4WGzlxHnPaZ21atXi9ttvd+sWWrVqFbfeeisAt912GytWrABgwIAB3HHHHbz11ltYrVa38911113MnDkTsF/A77zzTrdjli1bxuTJkwEYN24cdevWLX3s5ZdfJjMzk759+3L48GH27NnjsdyBHieEqJriouXw2LUdDX39Bx98kO7du3u8kJf3xhtvsGbNGmbPnk2PHj1Yv359mcebNm1KWloaixYtYu3atcyaNSvgcixZsoQFCxawatUqqlevztChQz3OVQj0OCFEcKw2zffbT3BVx7SoTweXlkMlqFevHrfcckuZ7p3+/fvz8ccfAzBr1iwGDRoEwL59++jTpw9PPvkkqampHD582O18d999N5MnT+bmm2/GbHbvyho8eDAffvghAHPmzOHMmTMAnDt3jrp161K9enV27tzJ6tWrS5+TkJBAcXGx3+OEEKF7Z8V+7vlgPd9uOWZ0UfyS4FBJHnrooTJZS6+88grvvfceXbp04f333+ell14C4Pe//z2dO3emU6dO9O/fn8zMTLdzjR8/nvz8fK8tkccee4xly5bRsWNHPv/8c5o1awbAmDFjKCkpoUOHDkybNo2+ffuWPmfq1Kl06dKFSZMm+Twu1hQUlWCzaf8HClEJjp61t8BP5RcaXBL/4qJbySj5+ZcnJ6WlpVFQUFB6u3nz5ixatMjtOZ9//rnbfUOHDmXo0KGltzdv3kxmZibt27f3+Lr169dn/vz5Hh+bM2eOx/ufe+45nnvuOb/HxZJzF4vJfGI+949ow/+Oamt0cYSIKdJyiDHTp0/npptu4tlnnzW6KFHvzIUiAL7adMTgkggReyQ4xJhp06Zx8OBBBg4caHRRhBBBcp1zFO0kOAghRCVz5inlXSrmtcV7o3JcTIKDEEIY5Klvd/D8vF3M337C6KK4keAghAhJYYmV7DMF/g8Upcq3D/KLSgAottoqvzB+SHAQVV4MdfPGlD98uoWBzy3mUrH7TH7hW7RPgAMJDhFlNpvp2rVr6b+srCz69+8P2Bfec05UE5ERA9+/mLZo50kACkuir9Yb7c5dLObVRXuieoBa5jlEULVq1cosZgewcuVK4HJwcK6vJISIHy9+vxuAmknRewmWlkMlq1mzJmBPSV2+fDldu3b1uKR3vNt1PI9RLy7l3MVio4si/Ineym/UKd9QcI41fLnxSNS1IqI3bIXTnGlw/KfwnrNRZxg73echzj0SAFq0aMEXX3xR+tj06dN54YUX+Pbbb8NbriripYW72XMynxV7TjGuyxUReY0Ve04x+Z01rPvjSBrUTIrIa1Rl0msXPF0ukjq7PhfuPMl3Px2P2Gc9FNJyiCBnt9KmTZvKBAZhd/h0AUt2nYz465T/Qjq9s2I/AFuyz3p8XAhX0+fsJGPa7Iid/+zFooidOxTx0XLwU8N3ZbVptNZYzBI3I23435ZQbNVkTR8XtnMePl3Agh0nuHNAC5TUbSPKmXHjLfhWNW8s3Wd0ESqVXAHL2XUij+3HvO8al32mgIO5F7w+rrV9Z7i8S777ylNSUsjLywu5nFVBsdX/RSXYC8+kt9fwxDfbOVsQXbWwaHDg1AXmbzsetvM5u0SirKu80hw7d5Fb3lhVuoZXIGLpvZLgUE6Jn8kopy8U+Rwk1RryC0s4mOt7clCXLl0wm81kZmbKgLQHodb6nUE5ClcjiKgF20+QMW02u094r3AMe2EJU99f7/Xx8tYfPMNP2efCUbwq6V9L97M26zRfbHRf2HHVvtyAKijR3LqNj24lg7gu2V3+voSEBI9LdsezuVuPcc8HG9j4p1EVPteJ85eokRg/H++5jhbBpsNnaZuWEpZz3vRPR9q1n26/OIvDfhWWWJn41moym9bhq3sH+Dw2mufixH3LocRqkxmeUeLt5QcA2JtzOahqDa8t3kvWKe9deZ6MfWl5WMsWq95cto9Hvghzpp5DFF/XDLM/J59Djl6DXcfdu6fLB9Jofg/jPjjsPpHvsykufPsx63TplyGsHN+a0xeKeH7eLia9vSbkU8VSP2+4PfPdTj5cc4iFO4Jb2O3YuYsBHxtt+fmV7clvt5f+PPxvSxn192WV8rrnLhazdHdOxM5veHBQSpmVUhuVUt86brdQSq1RSu1VSv1HKZUY6rkD+dCW2GJ76n9RiY1cA7YcdL63N7+xisHPL67QufaedA/Ox89dKvM6gbbuXNesieYme2X7xYx1QR3f71n/XZ6Xs5V8s9k0by/fz4XCkqDKEM0uFVtZFuCFWWsdkcmc//PBeqa8u5bTQQyIB8Pw4AA8AOxwuf0c8HetdWvgDPCLUE6anJxMbm5ula/VHDh1gSNnL/odSA+Gv/dMa01ubi7Jyclheb3tx+zBwfVV1x88E5Zzx4tQP+YZ02Zz34cbQnpuoLF34c6TPDV7B0/N3uH/4BjxxDfb2R9gV+eby/aT+cR8jp51b42FsgDfywv3MPyFJew9ae9+jdSKroaO2Cml0oFxwNPA/yr7OzUccC44NAN4HPhnsOdOT08nOzubnBzf0f3EGfsfbEdeNY+3/R1/qdjKqfwizieYuJiThNaaE2cvoRSYz3s+RzgdO3cRqw1M55IxmypeVS622jhxvpAGNRNJTjB7PS45OZn09HRgu9djQuX6W1RGaK9K9YdQPgHfbjnGfcPPk1G/hs+/uTdaQ+fH5vF/Y9szuW9zt8cvOlp9/tK7Y8n+HPdkE2/mOZIFjp276PZZy3dpTTkfs9nsCdzevs/OdZkapkR2Vr/R6Rz/AP4AONMr6gNntdbOdywbaOLpiUqpqcBUgGbNmrk9npCQQIsWLfwWYKxjxmPW9HHsPZnHL2csK73t73iAhTtO8Muv1zGsXSrv3dmVS8VWrv7TXBItJnY/Ndbna7+/+iBpKUmM7tjIbzm9ue2p7zmVX8SPj44kNQwflllrDvLo11lM7N2MZ2/sUOHzlXffhxvolVGPKf0zQnr+V5uOMKB1g7AsdxELyyZXljH/WM74zMa8PLGb22MHTl3gu5+OMaFXU+p7eN+11uQVlvDnr7Z6DA6VZdfxPJIsJjIa1OBikZUt2Wfp07J+2F9n4purOXza/zib5zRV7zWR1xfvZUKvpox7eQW7TuSFdXJoKAzrVlJKXQOc1FoHnnjtQmv9pta6p9a6Z2pqaljK9MbS/V4f23jojMdBOtfrS7B9/3/6cmtQeefR7Kfsc3y09hD/WrqP297xPnj87ZZjPPb1tjL3WcuN+yzZdbm1dzLP/p7mXigiJ6+QBz7exC9neu8/9/h1DLJl8OqiPXGZ378u67TH+29+YxXPz9tFj6cWeHw8WhpeV/1jGUNfWALAw59v4ecBXsSDtWp/LkcdY2LhdPTcJT7fcIRdHhJk9pzIc7v+RPp9N7LlMAAYr5S6GkgGagEvAXWUUhZH6yEdcJ9hYoAbXl/ps9tm7YHT9HhqAS9N6FqJpYoe1766IuTn/vY/m7mhW3rpWMeri/eWPrbx0OWxB2ff6vEIfDHBvnSKTWtemL+bF+bv9ltzyy8sIf9SCY1qh2fsJVoVFHkeSI7mGdI7j9svsPkxNgh+zMtn25kB9euhrdwei1T717CWg9b6Ya11utY6A5gALNJaTwIWAz9zHDYF+MqgIrqx+ph2e6HI3q+69oDn2pcvO4+f5+vNR0MuV1Xw0dpDbDjkvgDe2YLK66ee+OZq2jw6J+Djr31lBX2fXRiRsuQXlvDa4r0+P3OuTpyveMA8eu4S2WcKAn7N8pclb8+KlaSQH/aeCjmr6M1l+/jvusN+j/P3Vry/+qDPx19fUnnrO0VDtlJ5/4d9cHov9jGIdwwuj1fHz13ynuMfxPdhzD+Wc/9HG8NTqDD5aO0h9pzII2PabPY4mrlnLhRFrCb28OeeJ2o5a4Cujp27xPajnte/8vTlLrLaAsrmWluuW+WPX/qePHYgyIl5wZg+x77x/JytxwI6fsXeU0Bg4yjnLxWzLyefIx6yZ655ZQWtHvmuzH2uZ/zlzHVeL6D+LnzROMazYPsJPl2fzflLxUx6ew1TfXRZ+vLMdzv5w6db3O4vKrGVVnqyThXw8Y++A8ipILqmIx1zoyI4aK2XaK2vcfy8X2vdW2vdWmt9s9a68pP4y1m+x3PGU99nF/L4N2WzdTx9/u+e8SNzt4ZvwTNXwX5Aikps/Oilb7m8n7+5GrCPEwB0+8v3DHwusCU/bBFe3Ojql91nQH+6PpsSD6+bk1fIxLdWez3X8/N2ebz/g9WHPN7/w95TPPOd57TM0xeKfC7MGKgLhfaW6MHcArYdDe/4x+8/2cyIvy1lwHT3v6W/ltr320/wX8cFLpgLWbS6e+Y6fvfJZoocW50600PD5TWXLtKHPtkc1nOXilDMjYrgEA3Gv7qCT9dnl7nvvR8OsGpfLre9s9bteNc/uidFLjXVBTtOcs8HkR14DrRS9uycHdz8xiqPNe+cvEIe/WJr6W1Pk2tcLx6FJd4nprV85Dt2+FjdNhSefscLhSWlNdnf+fjy/Zjlfd6Ep9aJL5PeXsObyzwnLwx8bhFDnl/i8/k5eYUBt8Cen7eLcS+HPp7jybxtwc2WDrXGv+nwWU6GoburImxas/XIOWat8d1d48tvPtoY0j4OSsV2ADU6lTVqbPGQnfLEN95z+L3VNl1r8rtP5IVtETR/nK+7fE8OKckJdG1ax+NxuxwXwjMeVoz01qLw1gY456eWufHQWTpcUcvjY12fnO/zuYG4UFhCx8fmAZ5TjwPvOw+fgiL/M7l7Pb2AxrWTWfnwiDL3Pzd3J6k1k7hroP8UbG+01nyy7jDXdW1CoiU8db/yoeHcxWJy8i5f9Lwtq379az+Qkmzhqes7haUcoXANrJP6hJZm+02cjgdKyyGCXL9AkVK+UnfbO2u5/rUfwvoaufmFjHxxqdv9nrpwAhWOgWZ/XQDPzoneGbmeUiH/uWRfmXV6QjF363F+/+kWXlq4O2ILSr66eC+9nr6c1nq3j6U58i6Fd4zKZtO8sXRflVqKwx9v42WRbpVIcPAi1OyhSI65uU6TX7TzBKv25XIq3/O6Kt9vPxH0SqbefLvlmMcL8e8/9d2HGqnVQAMVj0twnHfMQs7NL+KaV8LbHeXNNi/JAeX5+mos2nmizDyhC4UlHrOc5m47zvQ5OyMa+PMLS8iYNpsWD0duS9BAvL18P3e8t9Zvt2ek9oSQ4ODFPysxZSwQ2WcKaPPoHP7zo32Q9K5/r/M5yPrLmetKJwQFKtB+8Nz8QjKmzeaHvbl+j918+GzYUhnLZwe5BktPe1GXD5wr950KSzkqwl+qYkU5x1b2nswP2+BqXhC19FAmnRUUlXDXv9dxx3s/AnAy7xIdH5vHG0v3u82xcLaGnAP2Tv66OF19tj6bxT72Ln91kX08UevAF3yMhKdm7ygzIbSyxXVw8JXeGI5mq7cJLaHYl2O/MDozhzypSEth5b5THlPxwD091NtMWU+ue+0HPlmXzcZDZ0prtaG69a2yM69dEwicFxZ/z3edDxBIfev+jzby0VrPWUuuDpy64Hfp8gOnLvCnL7f6PCYQi3ae4P1VWT6PWWdQq2nQXxezeOfJMsHaX93A2T3p/Pw6Jzk+N3cnV/55HusPXh4Lu+BlTOdAuQwxX6/50CebudPL5yX3QlGZvaLb/2mu78L7UFBkZdYa/58dfyqrBVheXA9IL9rpvfYQ6kqHH6+9nMf8u08287Me6SGdJ1jnLhYx8sXg15F/Y+k+xnRsxHof2TwV9YfP7EGnR/O6YT2vv5xxT37MOs19H27kjv4ZLPTx93f6evNRvt58lIm93dfvcjWsXCvt5PlLNKxVduZ0+c/U8j05DGrjeemX8uv0/3PJPn41uCUmk+Kuf9v7+G/rl+G3/Ea489+eL7zeulydmX95hSU89tVWembUK/P4xkNn6dHcfp9rcP1iYzY/Zp3hmRs6u53TFkBr1XWV1Cd9JJ/Eq7huOfj6AIXai+drkHbT4bOlOfA9PdS+LwaQ6eLN64s9d4P5ypE/W1DE9Dk7mfjWas5UwkzkaFiv6NvN9pbXv1dmRfR1nLW9cwXFPPnN9tI8eleeUqTBnlFWPo34ubk7+b7chj2RWDcoErxlMzn9y2VNsxmrDvKbACeE/vY/m/nQS818TwBdav1d5nnE8goFkRrnjOvg4Eu4kiBP5l3uxrj+tR9Kc+A9ZRp0+PPc0q6u5+bu5IuNl7tN/OWL/3TE84XXV478HMfEvIvF1oBqWlWBvwtVuDgXDPzrvJ28+8MBvtx0JKDuKcBr99vvP9lcJt9+0F8rtslSZVm4w95C83QN2xfE0te+xMoSHZEQqRyYuO5Wqgy9n3Zfe8fXINep/CJqJJlLB8SPnr1ETl6h35qur6/GG0v3cc8Q+4Jdzu/Qfpf+77MFxWUm7UVKZbxGqHxN6HPSWgfVwnp9yV5KrPY33GbTvPdDVqjFA+B8mNNCK4uncbJFO0+w7ch5erWo5+EZIhpIcPAinIPJ5fka5Cq/kJu3yXbBmD5nZ2lwcLpYLgvEW/O8qvFWwXTWbn1574esoOYh/HXuLn7es2nAx8cT57jJlH6hTUyL1O5n4jLpVqoi/DUtD58uu9pmHLfCPQpkwuKSCmzmHkxK5Ot+lmapSmasCiy19/yl4jLvYfnZ7/H8cZ79U2CLMwZLWg5xYtBfF3PfsNalt1ft9z9HIZ6U34AoXJxdaeUXaHRatS+Xw2cKymRyLTYwt70yuKamBqrL4/NpWi/y2+7Goj9/tY3bI5C5FufBIfqWEPbHW40/kIwF1010jJxcY6RQV/zwtAihk6/Jg/66P3xNZKxqlFKs2HOKyT52CvTl8Gn3JcY9+c5HTTrcq65WZXHdrRSFy8uLCFuwI7gVSZ12HjvP5sPumxGB7y6paNzDwCha65ADg/9zX/7517M2eH19T2uECc/ivOUQe5ybupQXqfVVhN2tb4d2UZO/ymXfhbinyVOzw7OOkoyzBSeuWw5GrptSEeU3GgdpBRmp/OxoV7E8uSrcPE0EDMe5zvjo8hOhi+vgEKv9j56+ZL76xIWoalyXPrGP6/hvFmSFYYe+eBLXwSFWvbvigNt9OTG845QQFRVIKvLwv8l4QzAkOMQgT7nh0p8q4tk9H3gehBahi+vgIN30QsS+YgO2g40HcR0cZBRXiNh3Q5i3xRV2cR0cJDQIEftOVsJe7fEoroODEEIIzyQ4CCGEcCPBQQghhJu4Dg4yHi2EEJ7FdXBw7tIlhBCirLgODq/G0aYqQggRjLgODkIIITyT4CCEEMKNBAchhBBuJDgIIYRwI8FBCCGEGwkOQggh3BgWHJRSTZVSi5VS25VS25RSDzjur6eU+l4ptcfxf12jyiiEEPHKyJZDCfCQ1vpKoC9wr1LqSmAasFBr3QZY6LgthBCiEhkWHLTWx7TWGxw/5wE7gCbAdcAMx2EzgOuNKaEQQsSvqBhzUEplAN2ANUCa1vqY46HjQJqX50xVSq1TSq3LycmplHIKIUS8MDw4KKVqAp8BD2qtz7s+prXWgMcFkLTWb2qte2qte6amplZCSYUQIn4YGhyUUgnYA8MsrfXnjrtPKKWucDx+BXDSqPIJIUS8MjJbSQHvADu01i+6PPQ1MMXx8xTgq8oumxBCxDuLga89ALgN+Ekptclx3yPAdOC/SqlfAAeBWwwqnxBCxC3DgoPWegXgbbudEZVZFiGEEGUZPiAthBAi+khwEEII4UaCgxBCCDcSHIQQQriR4CCEEMKNBAchhBBuJDgIIYRwI8FBCCGEGwkOQggh3AQcHJRS1SNZECGEENHDb3BQSvVXSm0HdjpuZyqlXo94yYQQQhgmkJbD34GrgFwArfVmYHAkCyWEEMJYAXUraa0Pl7vLGoGyCCGEiBKBrMp6WCnVH9COzXkewL7fsxBCiCoqkJbDPcC9QBPgCNDVcVsIIUQV5bfloLU+BUyqhLIIIYSIEn6Dg1LqPUCXv19rfVdESiSEEMJwgYw5fOvyczJwA3A0MsWpXD3VTu61fMVFkrhIIud1DY7q+hzVDTigG7FHp1Ns6E6qVYEmlXO0NR2mhTpOQ3WGVM5RW13AjA0TNoqwcE7X4Bw1OaIbkKUbccDWiCM0wPtmgSIQFkpork7QWh2hhTpOPZVHPZVHCgWlxxSSwGmdwmldi2zdgH26Mft0Y/KRqU0VVYsLtFHZNFcnaGY6SSrnqKEuUoNLABRhoZBETunanNB1OaIbsEs35aBOw2bwHOVAupU+c72tlPoIWBGxElWiJFVMPZVHNU5RjSLqmPJJURdLHy/UFnbqZmywtWG5rTNrbB24QDUDSxz9FDY6qEP0Ne2gr2k7PUy7qa/ySh+3akUutTmna1CCCRsmEimhtukCtcknSZWUHpurU9hoa80GW1uW2TqzTWegZVK/T9W4RF/TDnqbdtLLtIvOan+Z97RAJ3GaFPJ0NZyBN5lC6pnyqK0Kypxrn+0KfrS1Y51uxzJrF05StzJ/lZjUiFyGmLfQx7SDTLWPVqZjpY/ZtOIMNcnT1blAMhpFEsUkU0Sq6SzJqrj02AKdxHbdnNW2Dqy0dWS9rS2FJFbq76K0dusx8v0EpdoBs7XWrSNTpOD17NlTr1u3LujnZUyb7XZfCgU0Uadoo7LpZDpAZ3WAbqa9VFNFFGsza23tmW3ry1xrL05TKxzFj3lmrPQ27WSMaS1XmdfRSJ0BIMuWxlpbe7br5uzSTdlna8wpavuoEWkacpaWpmO0UkfJVPvobtpDa5O9oXpS12GxtStzbL1YYetMibTqAKjLeUaZ1zPKtJ5Bpp9IVsUUaTM/6Zass7Vlp60Ze3UTDuhGPlsDCZTQVJ2klTpKG5VNd9Meepp2U0ddAGCzrSULrN2ZY+vNXp1eWb9e1GurDjPevJIRpg10MNmz/k/qOmy0tWazrRXbdXMO6jSO6AYUkeDlLJo65JOucuhgOkQHdYiupr10UfuxKBuXdAJLbZl8Z+3NIlt38sr9HbOmjwup7Eqp9Vrrnh4f8xcclFJ52McclOP/48DD5VsURgpncPAkiSJ6mHYzyPQTo03raGU6Rok2sdLWkU+sQ5hn6+Xjj151XUEut1oWMsG8mFR1jos6kaW2TL639mClrSPHqB+W16nHeYaYNjPcvJEhpi3UUgWc0rX4xtqPL60D2KxbEX/dT5ruag+TLQsYZ1pNkiohWzfge2sPvrf1CFtNU2GjrcpmhGkjI83r6ar2YVKaLbYWfGYdzNfWfpyJw0pSGqe53vwD15t/oIPpECXaxFpbe5bYMlli68punU44PpM1uEhv006GmDYzxvwjjdQZCrWFRbZu/Nc6lGW2LlgxGxMcYkGkg0NZmg7qEOPMqxlvWkkzUw6ndU0+sw7mY+sw9ukmIZwzlmj6m7Zxu/l7Rpns7/kiW3c+sw5iqa0LF0mO6KsnUMIQ02auN69glGkDSaqYbbbmvG8dxVfW/hF/faNV4xLjzau4zfw9nUxZ5OlqfGYdxCfWoWzTzYl0kEzlLNeaV3GjeTmdTFkUazPzbD2ZWTKatbp9xF/fWJp+pu3cbp7PKNN6LMrGBltrvrQOYLa1L7nUjuirK2x0U3u5xrya8eaVNFDnOaHr8I+Sm3j26RdDO2cowUEp1d3XSbXWG0IqTQRUbnC4TGGjv2kbE82LuMq0jgRlZaX1St6zjmGhrbvhA0rhVJMCbjIv5zbz97Q2HSVXp/Af6zBmlYzgCKmGlCmFAsabVzLZvIAOpkOc19X4zDqYD6wjq1yQbqGOMdm8gJvNS6mlCthha8oH1lF8YR1IgUEBsZ06xM/My7jZvJQ66gI7bE2ZaR3Nl9YBVSpIl//sn9Y1+a91GB9Zh3FQNzKkTAmUMNy0kZvNS5hn68XzTz8f0nlCDQ6LfZxTa62Hh1SaCDAqOLhqwDluNi9lsuV7mqhcDtoaMsN6FR8gLZcAACAASURBVJ9Yh7j1D8aStuowt5vnc4N5BTVUIZtsrZhZMorZtr6VPkDmnaaH2s1tlu8Za1pLkiphhbUjM62jWWDrEbNB2oyVEaYNTDYvYLD5J4q0mbm23swsGcU63Y5oqaUnU8h480qmmOfT0XSQ87o6n1iHMNM6yrCLZzi0Uke4wzyPG83LHZ/9lrxfMppvo+qzbyfdSl5EQ3BwMmPlKtOP3GmZSy/TbvJ1Mp9YhzDDOposfUXYXy8SLJQw2rSOKZb59DHtpFAn8LW1HzOto/lJtzS6eD7V5xw/Ny9msmUBjdVpsnUDPigZyX+sQ2Omb7yB43e41bKQJiqXo7oeH5aM4D/WYeRQx+ji+WAfB7nDMo+xprWYsbHElslM62iW2rrERKaZwsYQ0xbuNM9liHkLhdrCN7b+zCwZxRbdyujieWVYcFBKdQKuhMttRa31zJBKEwHRFBxcdVL7udMyl2tNq7BgY7GtK+9Zx7DC1oloqfW5SuM0E8yLmWhZRCN1hkO2VD6wjuS/1qGcJcXo4gXFjJWRpvVMMc+nv3l7aYCbYR3N1qgMcJqeahe3WRYw1rSGRGVlubUTH1hHscDWHStmowsYlFTOcKt5EZMsC2mozrLf1oj3raP4NEpb0tW5xE3mZdxhnkcr0zFO6jq8XzKSD60jIj6WEA5GZSs9BgzFHhy+A8YCK7TWPwupNBEQrcHBKZWzTLIsYJJ5AanqPPttjfjYOozPrIMN/+ApbAw0bWWSeSEjHYNsS6yZzLCOZqktM2a7ZFy1Udncbp5f2j2wwdaaGSWjmWPrY3iWWS3yudG8gonmRbQzZXNeV+dTx7jJft3Y0LKFQwIljDWt5XbLfHqadnNBJ/G5dRAzrKOjIh22nTrEBPNibjIvp5YqYJOtJe+VjOE7W9+YmgBrVHD4CcgENmqtM5VSacAHWutRIZUmAqI9ODglUszVpjVMtCyij2knRdrMfFtP/mMdxkpbx0qtHTZVJxhvWsUt5iU0N50kV6fwiXUoH1qHc0inVVo5KlMKBdxkXsbt5vm0NB0nR9fiI+twPrcOqtQuPxM2+ph2cJN5OdeYVpGsitlka8lH1hF8be1XpQZzXXVUB7jDPI/x5lUkqWJWWq/kc9sg5ll7VWproiYFjDOvYaJ5EV1N+yjUFubaejOjZDQbdBuisVXvj1HB4UetdS+l1HpgGJAH7NBatw+pNBEQK8HBVSt1hInmRdxkXk5dlU+uTmGetRezbX1YbbsyIoEiXZ1kpGkD480r6W7aC8AaW3tmlYxkbhzN1XC2lm43z2eEaSMmpdlqy+Bba1++s/WJSHA0Y6Wb2sM48xrGmdfQUJ0lXyfzlXUAH1pHsE1nhP01o1VdzjPBvIQJ5kU0N52kUCew0NaNb6z9WGHrHJFAUYsLjDStZ6x5LYNNW0hSJeyypfOxdRhfWAfGXLdpeZUaHJRSrwEfAROBR4EJwENAPrBJa31nSKWJgFgMDk5JFDHUtIlx5jWMMG2ghirkvK7GGse0+bW2DuwOaY0nzRWcpotpH30ck2icU/l32JrxlbU/31j7GZaGGi0akcs48xquMa+mmyNgZtnS+MHWiR9sHdmiW5Gtg1/jyUIJ7VQ2maZ9DDJtYYBpG7VUAYU6gUW2rnxj7cciWzcukRSB3ypWaLqqfVxn/oFrzKtIVecp0SbW67Yss3Zhg27DVluLkIJFLfLpZMqin2k7A0xbS2caH9H1mWPtzbfWfmyqQpMnKzs4PIA9IDQG/oM9UJwBammtt4RUkgiJ5eDgyh4oNjPYtIX+pq20MJ0AoFib2asbs1c34biux3Fdl/PUoEhbKMZCEsXUVBcd0+9P0UydpJXpKA3VWQAKdQKrbR1YYstkqS2zSvRlR0K6ymGEaQMDTVvpa9peus7WWV2DHbbmZOsGHKU+OboOF3UShY6WVnV1iZpcoqE6S7o6STN1knYqmyTHWjlHdT2WWbuwzGb/JwvauXO2rIaaNzPEtJnOpqzSx/bZriBLN+KwTuWork8+1cnXyZRgJpESklUR9cjjCpVLY5VLO9Nh0tUpAEq0iS26JT/YOrHQ2r1KBQRXRnUrNcceJCYA1bAHiQ+11ntCKk0EVJXgUF4Tcuhm2ksH00GuVAfJUMdppM5QTRV5fc5JXYdDuiEHdRqbbS3ZbGvFDt08brqMwsVCCR1VFp1MWXRUWbQzHaaxyqUhZzArz9+ZQm3hiG5Atk5lp27GT7YWbNEtOajTqIoXpEiqQx5dTPvpovbTyZRFU3WSpiqHWuUWB3R1RtfkuK7HHt2E7bbmbNfN2WhrE5XZUeFm+DwHpVQ34F2gi9Y6anLrqmpw8ExTiwukcJFEVUICJVwikXxdjXyqSRCIMAsl1CWPJGVfTVMBF3QyBSRxjhoxkcsfy6pziZpcpIa6RAIlFJJAoU7gHDWq7EB+ICIRHALZ7MeCPX11AjACWAI8HlJJRBgozlOT89T0sAWTiLQSLORQV957gxSQbF8uRN7/iPMaHJRSo7APRl8NrAU+BqZqrS9UUtmEEEIYxFcb+GFgJdBBaz1ea/1hZQYGpdQYpdQupdRepdS0ynpdIYQQPloORi6sp5QyA68Bo4Bs4Eel1Nda6+1GlUkIIeJJtI6e9Qb2aq33a62LsHdpXWdwmYQQIm5Ea3BoAhx2uZ3tuK+UUmqqUmqdUmpdTk5OpRZOCCGqumgNDn5prd/UWvfUWvdMTY3vWb5CCBFu0RocjgBNXW6nO+4TQghRCaI1OPwItFFKtVBKJWKfY/G1wWUSQoi4EZULlmutS5RS9wHzADPwrtZ6m8HFEkKIuBGVwQFAa/0d9s2FhBBCVLJo7VYSQghhoLgODp2bRP/esEIIYYS4Dg4mWUVZCCE8iuvgkGCO619fCCG8iuurY9emdYwughBCRKW4Dg5KupWEEMKjuA4OQgghPJPgIIQQwk1cBwcl/UpCCOFRXAcHIYQQnklwEEII4Saug4N0KgkhhGdxHRyEEEJ4Ft/BQZoOQgjhUXwHByGEEB7FdXBonVrT6CIIIURUiuvgcE2XxkYXQQgholJcBwchhBCexXVwkAnSQgjhWVwHByGEEJ5JcBBCCOFGgoMQQgg3EhyEEEK4keAghBDCjQQHIYQQbiQ4CCGEcCPBQQghhBsJDkIIIdzEdXCQGdIVYzHJGyhEVRXXwUFUzA3dmhhdBCFEhEhwEEII4UaCgzDcO1N6Gl2EuPerIS0rfI7P/qdfGEoiooUEBxGyGkmWsJwnrVZyWM4jKkBX/BSt4mTzrD+O6xD2c9atngDA/SPahP3coYrr4KBkE+mQvTShK52a1PZ73LwHB5e5fc+QVjx5XUe349b/cST/N6Z92MpXlbVvlBL2c/qKDe3Swv96sSpr+jjuHtSSvU+PDet5qyfaK1qdGtcK+rnf/mZgWMviFNfBQYRuTKdGHu9/ZWK3MrfbNUrhrgEtSm9PG9veY0iuXzOJmsnhaYlUdXWrJ4b9nFp7Dw+/HtYqwHOEqzTx7TfDWwd1fCCVtFAYEhyUUs8rpXYqpbYopb5QStVxeexhpdRepdQupdRVRpQvHP2vVV2Sxex2X9b0cVyb6b71alJC2Y+Zv2tIiwY1KlI0UQFXd3YP+ibJ+Q5aWq0k7uifEfTzlFI8NLod/zM0sIAcSUa1HL4HOmmtuwC7gYcBlFJXAhOAjsAY4HWllPtVKMJSayb5PeYXA1twe7/mAZ/ztVu7V6RIMWXqYM/B9ZeDWni836lbU3sd4eGx0r3ki79rdVot75/fH6YNZ8nvhvLW7WWTAJy1/trVEtyeU79mIrf2aRZ0OeOZxWTi8fHu3acAk/vGxntpSHDQWs/XWpc4bq4G0h0/Xwd8rLUu1FofAPYCvSu7fP1a1S/9ef5vB/PKxG7c1D29zDFd0mtTLSHwuDWuyxVhK1+0e+RqzwN2dfx0h3RqUptdT41hdEfPXVZOzepVD7lssaJpvWohP3fmXX28PtakTjUyGtRg1JVpXo5QPH7tlW73TpLg4NfPeqT7PwhI9tDq9tWtZ5RoGHO4C5jj+LkJcNjlsWzHfW6UUlOVUuuUUutycnJCemFnDaz8TN/6New1r4YpSbRNS+HazMb87ZbMMsf0zKhX+vPUwS35YdrwoF47mmcXv3Bzpv+D8P2BHtoulfEeupjsz/N+Tk/dVeX1a1nf7zGx7oau3icY9mnh+/evVa1iYzd3DPDdwhOeubaYh7ZLDeq5zq9ENF0VIhYclFILlFJbPfy7zuWYR4ESYFaw59dav6m17qm17pmaGtwfwp9gu1jr1UikSZ3ganrROvj67I2d/daAAvng//vO3rxcbnDaqUEA3XbBqIpddr7qkT0z6rLrqTFe+7QDzcL78O7LLYxaju4kj5/9coVp3yiFtY+M8Di+JCAlyeK1S8mbZEcvhLdrj9mlMrnwoSEhly0YEQsOWuuRWutOHv59BaCUugO4BpikL1dBjwBNXU6T7rivUoW7hffenb3c7rvyiuBT1irDhF5N/R6TaA7uY+NsJTkHNntm1PX7HH8X/N+Nblv6c9+W9XwcGXsC6cJJsph55OoOTOxt/3sF8ncrr3/rBux9eiwrpw2nbg17l5+/sJJWK4kvfj2AhrWSecyl++mO/hlxu1ZZ+e9yo9rJJAT4Hdn859G8dmt3tySMmuXmELm+ta7zSbx3D1acUdlKY4A/AOO11gUuD30NTFBKJSmlWgBtgLVGlBF8tyBcH/IXTDwN8kXrF0l5KNjrkypWM//VkFZM6dc8qOwNf2M0yUGM90S7jX8aVfpz35b1ePqGzgE9L9Fiokt6Hf8H+mAxm2gcRKu3Xo0kqiXa33vXFmAwNeWqNGZkNim+uLc/m/48yv/BHtSunsC4Lle4ddHePahFQIkZ5YNIOBk15vAqkAJ8r5TapJR6A0BrvQ34L7AdmAvcq7W2RqoQFpNiZIc03rnjcs1+1t3eB/PclLuOrn1kRJhKdlmwfZeRUNG8+ppJFp64rlPpRaWidLl+jugbyguOs9Ye6wJtcX9zX2QmbRklyWKmTvVEUhxdxRUJ2M66WZLFzK+GeE9n/XhqX4CIprwala3UWmvdVGvd1fHvHpfHntZat9Jat9Naz/F1nopSSvH2lJ4MaWu/AE/q04wBrRuEfL6GtZJ59sbLtb6uTStWqwPjalmutfzGdWJ3eYtpAdS+9oR5tmukRSSZoRKzZWpXd29JxxpP79YVtavxzX0DefqGTm6Pmcv9zRLNJga3Db7i5+zS7duyPlnTx9E2grPXoyFbKSpkTR8XcHMevHcLTewdWMpfJJbuWPfHkWE71+PjO7L/matZ/fAImtf3PCnNYg7tdyh/HfL2XtZMstDZw+zPYe0aBvxagfTfB9o/XBl8XaPbNLT3NfvrUvOWrfTG5B5+X9/T36J7c/9jRJWpdcPoWcOp/NvVOb12QF2eu58ey8y7/GfpT+zdtEz24MqHg8uKrIjoTJkxWL0aidRMsvDoOPd870hpVq86h04X8MLNmSSYFWcuFPH4N9uDCiHhzgIymRSNantvNVzTpTE7j+fxr6X7w/q6TlufsE+Qz5g2u8z9qSlJHDl7sfR2oP2uax4ZwfqDZ/j1rA3hK2SEWEzuActft9xvR7alRpK5dJ2e8jqnB7/Mwu39mkfd+E6TOtXYezLf6GIEZGynRszZetzv9/hyKmvZI5+9sQsATetW4z/rDlO/Ersgo6fKFEUSLSa2PnGV1zz9UF2b2ZgrvFxsnZOe0molcV25HPfHPExKMtKkvvaZ4QlmEw+PDf8KlYFwXrDuHdbK58XLdYA9rVYyV3eOjcmIvxzsfa6Bt8bFAyPbcPegsrPTE82m0s9PPR9jRw0dK+M2rWvsYHHW9HF+j2lS1/MAeijLVYQq0F64f07uEdDvpP1MdOjTsj4v3tLVY8JIpEhwCILrchkKVTq3wdtyBT0dzXFntH9lYjf++rMuPl/DWXMItAf4qevt/ZspEZ43sfMvY1j7yAiypo8rHaOpbD0c76cGft6rKb8d2Zb7hoVnieOGKUmG/V6unH/36okWt4pEqJeFOwe0IGv6OJ8tj9FXpvHenb3cgkuUJtWVfhZapl7u8qzImlztG6XwxuQeLPjfwf4PdhGtWYfhIMEhCE9e14mGKZcDweQ+zXlnSk+v22VOG9ue+b8dXKbPPthxv0BrCuUnJHkqkzPDIRTJCebS2qVRXN+JBLOJB0a2CVsG1NpHRzLDpQ+4ef3ga9CBrgnls4Yb5nHh67oG1vpVSjGsXUO3gVMn59In3ZtVPMkiWPU8dKU4y3mVn6VWApVetxpjOjWidcMUEi1lL4uB1PyrIgkOIVLK3ic/okOa1wu4xWzymk1Q/im+gobPx7zc7xrEnPpGybIT5VNR/bmxW5NK3y3uFwODX0LCW+ph+0YpZS5wvuYEBPve9G5hnwDobbaya/ZcRTSpU425Dw7isWuDm/nrtCaINO+JvZuVSSRY+L/uM4LDVWGfeVdvpo1tz19/dnnQN5BksGD/Tv7PF30kOEQZZ9DwFhD+dVsPGrt0N6Q7urbK78KV5Kj9jCvXx/7hL/vE3LacL/68KyM6hDYT1NP6T66rw/73V5Hf2nLug4O5M5TlmwM4plVqTbKmj/Oagm0JYyZW+0a13GrVrqoneW/Fedvt70MP84qevbFzmczBujUSSXeMMyRZTEwd5HnV32C6eP44rgN/ua4jg9o04J4hrTy2Tl67tXuZDahapbp3W4WSdXi9l54G+/mihwSHIBkZ4e/on8FVHRux/P8up7MNa9+QT+7p53bxcbZmyqf99W/VgBEd0soswVzRZvP1XRvTK4AlMSrK2bXhL8//jcndmfPAIP5naCuPmUyuWWjOmrc3N/r4IgMkJ1TNr1DbNPvnpkMQy7wkWczsf+bqMuNfKUkWtj3hfVuW/gHOK5rUxz7et/5Po8gIYGzhg1/4nsxaI8nCbf0yPLb6nRf8Ye1TSyeZ/fjoSL6pwI5rzm7efc9cHbHNecJNUllDFM4IP7ZTI85dLA74+PL9wr0ygl9bKJzzqP4xwfMCe+H2ws1d+GLjEb+TCxvUTKLDFbV8XtjuHdaKRLOPLCfH/55qwylJFvIK7SvOL3xoKAOmLyp9bNOfR3Eqv5CRLy4re74A3+9wzkWryLLfHRvXZveJfJ8tBYC3bu/J9qPnS2+bTJfr0vN/O5hGtZM97jXer2X9gMdDwD4T2NNsYNf3y/m6k/o0Y2Cb0Cezejp3qodu2mA8e2Nn/njNlV7HdKKRBIcoUCPJwtmCssEhnC2UaB9Qy/Ayya68OtUTuTNMy0n//qrANxSqkWjmQpF9FZf5vx3M2YJibvnXKjpcUcttNd461RP97lvhyRPjO/LY19vK3PfPyT3417J9fPfT8aDPt+2JqyrlQjTqyjSvi7+lpSRTK9nzbOiPAkiOGNI2lQQvEy09BtswpQ4Fcppgg7jFbKJ2Ne+Btl1aTZbtzgn7XKWKqJpt4gj6/VXtgNCXAOjXqj4TezfjuZsup7S6psKW/1yG8nkP5KIwMsQ+/HCoU81+8Xz06g5kTR/nsWYZLF8rs740oSvv3lHxcZZ6NRJpm5ZC/Zr28nubsxIM55jQlR42ls9sWofXJ/XgvmGt+eLX/Uu7QALZGKZGkiXqJq8Fa8ZdvXl7ivuKxoHyVSkKZGXhgK7/YYq/fxjTnk/u6RdVXU7ScgjSLT2bcktP38sjv3lbD48DXGBPwSyfQfLAiLZMeTc8i88+cnV7BrdNZY6f2ubQIJagCLdqieawt2Y+ntqvdCZ1+S91+UmFoVr6+6GAfRD4pQldGdo2+Pfw1+W6Rl5zrHjb3LGG1nUexjh+56iQvDKxG28t31/hlVirov4uuzf6u6g/NKqtzy4tIzp+EsymkLqHI0mCQwT42+ayvESLqTQbw7kJUKjbBk4dbL/4+AsOALPvH8ieE7GxDEE0cB28dA04E3s35aO1hz09xeNzPWlYK5l9z1ztcyyoab3qPHmd+6Ju8eiZGzrz17m7SifDmU2B5w39ZkRgEyd9fQedf05f+3XHOgkOBpo2tj0DHdkaT17XiREdGrrVCkNdoG9K/ww2HT5bZlZ3eR0b16Zj4+hpxsaqZ2/sUroGTqB+N7otLRqUzSSLpcHKcHId4A9Uy9SavHFbD5bu9r1F8AMj2vDSwj1BnTuQiadJFjN//3mm3y1bY5mMORjoniGtSvsYqyWaGdMpfOv+1KuRyIy7elM/iga4YpHWlGa+BLNU9q8Gt6RJnWq8eqvnTK77hrfxu6FRvFj0u6ERO/dvR7VlRHt791/nJrVD2jHPmxu6pQe1UVKskZZDjPrsf/pzqTi8+yDVCNNSFLHsvTt6YdOao+culd730oRuHDt3KagB3oev7sDDVxuzKGGsqWiaqNO1mY35dstR7h3W2uPjD4xow8gAttX85+Tu/Gvpfmp4Wd02XsT3bx8DvLVwe4R5jf3nbuocdQNiwfry3gF8vPYQPZqF/t4Mc9Qy3199sPS+5ARzhRZ1cxWNyyRUFbWrJfDx1IrPeB/UJpVBbYxfhNFoEhwEAD/vFdgmRdGsa9M6Ydl9T1Qtf7m+E7Xn72JQ24pPjIsnEhyiVCXu2ihElda4TjVevKWr0cWIOTIgHeXiM39FiMA4F6GM9S7RaCQtByE8cAblcG/mIi1Cz9b/cWRIu5y1SUthye+G0qyesTvYVUUSHKJUuNeLF8H5WY90th09x+9GtwvL+aryjmHhUJGU60BWaRXBk+AQ5eSiYozkBHPQE9uEqEokOMSA7o601cGSbSEqiXOzqHDO2l4cwcluIvwkOEQp177prk3rsPMvY2J+lc145lwGJVa6Cx++ugN1qie67SRYEeGaKyIqhwSHKOccpJPAENtu79ecA6fy+fVQz7N3o03taglMGxv4nhei6pHgIEQlqJFkKbOJvRDRTuY5RKnY6HwQwp1zpV+Ll13cRGyQlkOUk6+XiDX/ur0Hu47nhWWHP2EcaTkIIcKqVnKCzFiuAiQ4RCnn3gEJAex1K4QQ4Sbtvig1uW9zcvIK+fWwVv4PFkKIMJPgEKWSE8yyWYwQwjDSZyGEEMKNBAchhBBuDA0OSqmHlFJaKdXAcVsppV5WSu1VSm1RSnU3snxCCBGvDAsOSqmmwGjgkMvdY4E2jn9TgX8aUDQhhIh7RrYc/g78gbKTga8DZmq71UAdpVT4Vv4SQggREEOCg1LqOuCI1npzuYeaAIddbmc77vN0jqlKqXVKqXU5OTkRKqkQQsSniKWyKqUWAI08PPQo8Aj2LqWQaa3fBN4E6NmzpyxFJIQQYRSx4KC1HunpfqVUZ6AFsNmxHHU6sEEp1Rs4AjR1OTzdcZ8QQohKpLTBO54rpbKAnlrrU0qpccB9wNVAH+BlrXXvAM6RAxwMsQgNgFMhPjdWye8cH+R3jg8V+Z2ba61TPT0QbTOkv8MeGPYCBcCdgTzJ2y8XCKXUOq11z1CfH4vkd44P8jvHh0j9zoYHB611hsvPGrjXuNIIIYQAmSEthBDCAwkOjoynOCO/c3yQ3zk+ROR3NnxAWgghRPSRloMQQgg3EhyEEEK4ievgoJQao5Ta5VgFdprR5Yk0pVRTpdRipdR2pdQ2pdQDRpepMiilzEqpjUqpb40uS2VRStVRSn2qlNqplNqhlOpndJkiSSn1W8dneqtS6iOlVLLRZYoEpdS7SqmTSqmtLvfVU0p9r5Ta4/i/bjheK26Dg1LKDLyGfSXYK4GJSqkrjS1VxJUAD2mtrwT6AvfGwe8M8ACww+hCVLKXgLla6/ZAJlX491dKNQHuxz6ZthNgBiYYW6qI+Tcwptx904CFWus2wELH7QqL2+AA9Ab2aq33a62LgI+xrwpbZWmtj2mtNzh+zsN+wfC4sGFVoZRKB8YBbxtdlsqilKoNDAbeAdBaF2mtzxpbqoizANWUUhagOnDU4PJEhNZ6GXC63N3XATMcP88Arg/Ha8VzcAh4BdiqSCmVAXQD1hhbkoj7B/al4W1GF6QStQBygPcc3WlvK6VqGF2oSNFaHwFewL43zDHgnNZ6vrGlqlRpWutjjp+PA2nhOGk8B4e4pZSqCXwGPKi1Pm90eSJFKXUNcFJrvd7oslQyC9Ad+KfWuhtwgTB1NUQjRx/7ddiDYmOghlJqsrGlMoZjlYmwzE+I5+AQlyvAKqUSsAeGWVrrz40uT4QNAMY7Fnf8GBiulPrA2CJVimwgW2vtbBV+ij1YVFUjgQNa6xytdTHwOdDf4DJVphPOTdEc/58Mx0njOTj8CLRRSrVQSiViH8D62uAyRZSyr5H+DrBDa/2i0eWJNK31w1rrdMf6XROARVrrKl+j1FofBw4rpdo57hoBbDewSJF2COirlKru+IyPoAoPwHvwNTDF8fMU4KtwnNTwhfeMorUuUUrdB8zDnt3wrtZ6m8HFirQBwG3AT0qpTY77HtFaf2dgmURk/AaY5aj47CfAFY5jkdZ6jVLqU2AD9oy8jVTRZTSUUh8BQ4EGSqls4DFgOvBfpdQvsG9dcEtYXkuWzxBCCFFePHcrCSGE8EKCgxBCCDcSHIQQQriR4CCEEMKNBAchhBBuJDgIEQSlVH2l1CbHv+NKqSOOn/OVUq8bXT4hwkVSWYUIkVLqcSBfa/2C0WURItyk5SBEGCilhjr3i1BKPa6UmqGUWq6UOqiUulEp9Vel1E9KqbmOJUxQSvVQSi1VSq1XSs1zLoEgRDSQ4CBEZLQChgPjgQ+AxVrrzsBFYJwjQLwC/Exr3QN4F3jaqMIKUV7cLp8hRITN0VoXK6V+wr48y1zH/T8BGUA7oBPwvX05IMzYl5sWIipIcBAiMgoBtNY2pVSxvjy4Z8P+vVPANq11ld6+U8Qu6VYSwhi7gFTn3s5KqQSlVEeDyyREKQkOQhjAsTXtz4DnlFKbgU3E1x4EIspJKqsQQgg30nIQQgjhTdZPNgAAAC1JREFURoKDEEIINxIchBBCuJHgIIQQwo0EByGEEG4kOAghhHAjwUEIIYSb/wefOv/flNO5twAAAABJRU5ErkJggg==\n" + }, + "metadata": { + "needs_background": "light" + } + } + ], + "source": [ + "# fiiting using cmaes to the toy data\n", + "\n", + "# Add noise\n", + "values += np.random.normal(0, 10, values.shape)\n", + "\n", + "# Create an object with links to the model and time series\n", + "problem = pints.SingleOutputProblem(model, times, values)\n", + "\n", + "# Select a score function\n", + "score = pints.SumOfSquaresError(problem)\n", + "\n", + "# Select some boundaries\n", + "boundaries = pints.RectangularBoundaries([0, 0], [20, 20])\n", + "\n", + "# Perform an optimization with boundaries and hints\n", + "x0 = 0.01, 0.01\n", + "#sigma0 = [0.01, 100]\n", + "found_parameters, found_value = pints.optimise(\n", + " score,\n", + " x0,\n", + " boundaries = boundaries,\n", + " method=pints.CMAES\n", + " )\n", + "\n", + "# Show score of true solution\n", + "print('Score at true solution: ')\n", + "print(score(real_parameters))\n", + "\n", + "# Compare parameters with original\n", + "print('Found solution: True parameters:' )\n", + "for k, x in enumerate(found_parameters):\n", + " print(pints.strfloat(x) + ' ' + pints.strfloat(real_parameters[k]))\n", + "\n", + "# Show quality of fit\n", + "plt.figure()\n", + "plt.xlabel('Time')\n", + "plt.ylabel('Value')\n", + "plt.plot(times, values, label='Nosiy data')\n", + "plt.plot(times, problem.evaluate(found_parameters), label='Fit')\n", + "plt.legend()\n", + "plt.show()\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.8.2 64-bit ('pints': conda)", + "language": "python", + "name": "python38264bitpintsconda8d0b754a30424fdd8045d82b54ea71e7" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.2-final" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file From 10c8f3522720160227b52762dae8f6809661862d Mon Sep 17 00:00:00 2001 From: alisterde Date: Sun, 8 Nov 2020 16:43:52 +0000 Subject: [PATCH 15/16] style fix --- pints/_optimisers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pints/_optimisers/__init__.py b/pints/_optimisers/__init__.py index ed620cdba..ae25833e7 100644 --- a/pints/_optimisers/__init__.py +++ b/pints/_optimisers/__init__.py @@ -637,7 +637,7 @@ def ask(self): # print('') # Return proposed points (just the one) in the search space to evaluate return [self._proposed] - + def stop(self): """ See :meth:`Optimiser.stop()`. """ From 4369b393ad1f3a77e93c39314b8d3a2d8f011e61 Mon Sep 17 00:00:00 2001 From: alisterde Date: Sun, 8 Nov 2020 17:12:53 +0000 Subject: [PATCH 16/16] Adding lbfgs notebook to documentation --- examples/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/README.md b/examples/README.md index 643671e6b..cb356e689 100644 --- a/examples/README.md +++ b/examples/README.md @@ -34,6 +34,9 @@ relevant code. ### Local optimisers - [Nelder-Mead](./optimisation/nelder-mead.ipynb) +### Quasi-Newton optimisers +- [L-BFGS](./optimisation/lbfgs.ipynb) + ### Further optimisation - [Convenience methods fmin() and curve\_fit()](./optimisation/convenience.ipynb) - [Maximum loglikelihood](./optimisation/maximum-likelihood.ipynb)