last modified April 22, 2012

, ,
1 ,
, MATLAB , Newton-Raphson method
Steepest Descent method

MATLAB
MATLAB

h MATLAB i

:meshgrid, mesh, surf,contour, colorbar, fminsearch, optimset,fprintf

:
min f (x)

(1)

f (x) : Rn R, , x
1 ,

1:
f (x)

5f (x) =

f (x)
x1
f (x)
x2

..
.

f (x)
xn

=0

(2)

 f (x) ,
, closed-form solution

k = 0, 1, 2,

xk+1 = xk + dk

x0 , (dk ) ,
Newton-Raphson method,
:
dk = (52 f (xk ))1 5 f (xk )

(3)

n 1 5f (xk ) (Gradient Vector), 52 f (x)

n n , Hessian matrix, ,

52 f (x) =

2 f (x)
x1 x1
2 f (x)
x2 x1

2 f (x)
x1 x2
2 f (x)
x2 x2

2 f (x)
xn x1

2 f (x)
xn x2

..
.

..
.

...

2 f (x)
x1 xn
2 f (x)
x2 xn

..
.

2 f (x)
xn xn

(4)

Hessian matrix ,
(), ,
, :
,
dk = 5 f (x)

(5)

Steepest Descent direction ,

,

, ,
,

1.
f (x, y) = xex

2 y 2

:
1. X Y (grids), MATLAB

[X,Y]=meshgrid( X , Y )

X Y X Y X Y meshgrid
help meshgrid 2 [X,Y]=meshgrid(-2:2,-1:3)
X Y ,

2: meshgrid
2. X Y Z help meshgrid 3 meshgrid z =
4

f (x, y) = xex

2 y 2

X Y Z

Z = X. exp(X. 2 Y. 2);
mesh(X, Y, Z)
3. meshgrid ,
4. , mesh, surf, contour()

5. Matlab ( Rotate 3D),

, ,

3: meshgrid

 4 4(b) (colorbar),
, , contour
colorbar , , XY ,

, , , 4 ,
XY (x1 vs. x2 ) , ,

axis square

(a)

(b)

4: (a) (mesh)(b) (contour)

X Y (x, y) ,
, , 7 ,
Z Y , X
, mesh

f = @(x, y) x exp(x 2 y 2);

x=-3:0.1:3;
y=-4:0.1:4;
Z=zeros(length(y),length(x));
for i=1:length(x)
for j=1:length(y)
Z(j,i)=f(x(i),y(j));
end
end
mesh(x,y,Z)
xlabel(X),ylabel(Y)

2.
f (x1 , x2 ) = (x1 2)4 + (x1 2)2 x22 + (x2 + 1)2

(6)

1 ,
(,contour), contour
, , ,
contour(X, Y, Z, 100)

,
, , ,
MATLAB ,

3. MATLAB fminsearch
min (x1 2)4 + (x1 2)2 x22 + (x2 + 1)2

x1 ,x2

MATLAB , : (constrained)

(unconstrained) ,
, MATLAB fminsearch,

f = @(x) (x(1) 2) 4 + (x(1) 2) 2 x(2) 2 + (x(2) + 1) 2;

[x, f val] = f minsearch(f, [0, 0]);
MATLAB ,

(function) , fminsearch
,
,
, , ,

, fminsearch
MATLAB
opts=optimset(fminsearch);
opts.Display=iter;
opts.MaxFunEvals=8000;
opts.MaxIter=8000;
opts.TolFun=1e-6;
opts.TolX=1e-6;
f = @(x) (x(1) 2) 4 + (x(1) 2) 2 x(2) 2 + (x(2) + 1) 2;
[x, f val] = f minsearch(f, [0, 0], opts);

,MATLAB optimset
, , MATLAB
opts , opts
2 6
DIsplay iter, ,

, ,
8

 MaxFunEvals , ,
, , , , MATLAB 200* ,

MaxIter , 200*
TolFun , ,

,MATLAB 1e-4 ,2 1e-6

, TolX 1e-4, 1e-6
MATLAB , ,
MATLAB optimization
toolbox , fminuncon,

, ,

, MATLAB file exchange ,

fminsearch fminsearchcon , ,
: http://www.mathworks.com/matlabcentral/fileexchange/8277fminsearchbnd-fminsearchcon optimization toolbox fmincon

4. John DErrico fminsearch , fminsearchcon,

min

x1 ,x2

(x1 2)4 + (x1 2)2 x22 + (x2 + 1)2

1. = {x1 , x2 R| x1 0, x2 0}
2. = {x1 , x2 R| x1 1, x2 2}
3. = {x1 , x2 R| 0 x1 1, 0 x2 2}
2

MATLAB ,1e-4 104

4. = {x1 , x2 R| 0 x1 , x2 2}
5. = {x1 , x2 R| x1 + x2 0.9}
6. = {x1 , x2 R| 1.5 x1 + x2 2}
7. = {x1 , x2 R|

x21 + x22 1}

8. = {x1 , x2 R|

x21 + x22 1, x1 x2 0}

,
, 0 1 ,
fminsearchcon fminsearchcon ,

[x, fval]=fminsearchcon(fun, x0, LB, UB, A, b, nonlcon, options);

fun x0 , LB, U B ,
, LB x U B,A, b , Ax b, A , b
p
nonlcon C(x) 0, x21 + x22 1,
MATLAB @(x)norm(x) 1, g = @(x)norm(x) 1, g

nonlcon ,C(x) ,
, g = @(x)[norm(x) 1; x(1) x(2)]
MATLAB optimization toolbox fmincon,

, , , fminsearchcon
fminsearchcon , ,
,

5. 5 1000 ,

f (x|) = 1 (x|a1 , b1 ) + 2 (x|a2 , b2 )

10

 = {1 , 2 , a1 , b1 , a2 , b2 }, 1 + 2 = 1
(MLE)

L() =

1000
X

ln(1 (x|a1 , b1 ) + 2 (x|a2 , b2 ))

i=1

max

={1 ,2 ,a1 ,b1 ,a2 ,b2 },1 +2 =1,1 >0,2 >0

L()

, 1 + 2 = 1 1 , 2 > 0
0 < 1 < 1 2 1 1 , 5
( x)
opts=optimset(fminsearch);
opts.Display=iter;
opts.MaxFunEvals=8000;
opts.MaxIter=8000;
opts.TolFun=1e-6;
opts.TolX=1e-6;
initial=[0.5 1 10 5 5];
LB=[0 0 0 0 0];
UB=[1 Inf Inf Inf Inf];
L = @(p) sum(log(p(1) betapdf (x, p(2), p(3)) + (1 p(1))
betapdf (x, p(4), p(5))));
[mle, f val] = f minsearchcon(L, initial, LB, U B, [ ], [ ], [ ], opts);

? ,
,

11

 5:
6. (6) 5f (x) Hessian Matrix 52 f (x)

, , ,
,
Hessian Matrix

7. Steepest Descent Method(5) Newton-Raphson Method(3),

(6)

, ( 1),
,
, , , ,
6 steepest descent steepest descent
, , 15 Hessian matrix
, 18 xk1
6 , (), x
,
12

 12 fprintf , C printf, ,
,
, %10.7f, (floating point)
, 7 , 2 , ,
10 , \n

6: Steepest descent
6 , steepest descent , ??
13 xk X, 14 1 ,
, 23,24
, (xk1) (xk),
,
13

 7: Steepest descent :x
, , ,
, Newton-Raphson
Hessian Matrix , , steepest descent ,

1. (3) Steepest Descent (5) Newton-Raphson

, ,
xk+1 = xk + dk

14

2. Newton-Raphson method , [1], 2

3. dk ( (3) (5))

,
[1]

1. ,
2. Newton-Raphson method , (3),

f (xk+1 )
1
f (xk+1 ) = f (xk +dk ) = f (xk )+5f (x)T dk + dTk 52 f (x)dk +O(||dk ||3 )
2

||dk || , O(||dk ||3 ), (3) Newton-Raphson direction

min f (xk+1 )
dk

3. Rosenbrocks Banana
f (x, y) = 100(y x2 )2 + (1 x)2

, (mesh) (contour), ,
4. , (mesh contour)

5. (3) (5) ,

, ,
, ;
, : dk , ,

15

6. MLE

max log L(, )

,

n
Y

L(, ) =

ft (vi ; , )FT (ui ; , )1

i=1

where
ft (v; , ) = v 1 exp(v )
FT (u; , ) = 1 exp(u )

u, v n , UV.txt
http://web.ntpu.edu.tw/ccw/statmath/book.htm ,

7. , ,

, :
X, Y , PDF fX (x) = (x|a1 , b1 ),
fY (x) = (x|a2 , b2 ) Z
Z = XY

Z , , (a, b) Z ,

Z
min
a,b

(f (z) (z|a, b))2 dz

f (z) Z PDF,
Z
f (z) =
z

1
fY (y)fX (z/y) dy
y

Beta fX (x), fY (x) a1 = b1 = a2 = b2 = 2

16

 a, b
Hint:
quad fminsearch
Beta betapdf(x,2,2)
, (function)

8. ,

,
max

={1 ,2 ,1 ,12 ,2 ,22 },1 +2 =1,1 >0,2 >0

L()

L() =

N
X

ln(1 f (x|1 , 12 ) + 2 f (x|2 , 22 ))

i=1

f (x|, 2 ) fminsearchcon = {1 , 1 , 12 , 2 , 22 }

[1] J.E. Dennis, R.B. Schnabel, Numerical Methods for Unconstrained Optimization, Prentice Hall.

17

18