Professional Documents
Culture Documents
Assignmen 4 11! "rogram o #m$%emen &oor'ina e (es)en *e ho' The following minimization problem with L1-regularization
% An example using Coordinate Descent Method % updated by Larasmoyo Nugroho 1837087 % N M A Construct A as a rand matrix. = 512*2; = N/2; = randn(M,N);
% Construct u_exact as a sparse vector. p = floor(0.05*N); u_exact = zeros(N,1); % u_exact has spikes at uniformly random locations with amplitudes % distributed uniformly in [0,1] a = randperm(N); % u_exact(a(1:p)) = a(p+1:2*p)*0.08+5; % 0.08 + 5 u_exact(a(1:p)) = rand(p,1)*N; %randn is fast. % Construct f = A*u_exact. f = A*u_exact; % Precompute B = A'*A. This step is not necessary. B = A'*A; % Initialize lambda. lambda = 10; % Call the main function COORDL1BREG. %[u,Energy] = coordl1breg(A,f,lambda,'B',B,'PlotFun',@myplotfun); [v,Energy] = coordlsl1(A,f,lambda,'B',B,'PlotFun',@myplotfun); % Plot the Energy function. figure(2); semilogy(Energy,'.-'); xlabel('Iteration'); ylabel('Energy'); % COORDL1BREG will call this function every iteration to plot intermediate solutions. % Show the comparison of the output u and u_exact. function myplotfun(v) figure(1); x = 1:length(v); plot(x,v,'.r',x,u_exact,'o'); xlim([1,length(v)]); end end
Energy = zeros(MaxIter,1); end %%% Main Loop %%% for j = 1:MaxIter if NormalizedFlag tilde_v = min(temp + mu, dEnergy = lambda*(v.^2 2*lambda*temp.*(v-tilde_v); else tilde_v = min(temp + mu, dEnergy = lambda*(v.^2 2*lambda*temp.*(v-tilde_v); end
[dvchange,i] = max((dEnergy)); if OutputEnergyFlag Energy(j) = norm(A*v-f,2)^2*lambda + sum(abs(v)); end if ~isempty(PlotFun) PlotFun(v); end dv = (v(i)-tilde_v(i)); if abs(dv) < Tol if Display fprintf('Converged in %d iterations with |du| = %.4e < %.4e.\n',... j,abs(dv),Tol); end break; end temp = temp + dv*B(:,i); if NormalizedFlag temp(i) = temp(i) - dv; else temp(i) = temp(i) - dv*w(i); end v(i) = tilde_v(i); end if OutputEnergyFlag Energy = Energy(1:j); end return;
Property = {'u0','MaxIter','Tol','PlotFun','B','Display'}; for k = 1:2:length(pvlist)-1 if ~ischar(pvlist{k}) error('Invalid property'); end i = find(strcmpi(pvlist{k},Property)); if length(i) ~= 1 error('Invalid property'); elseif ischar(pvlist{k+1}) error('Invalid value'); else assignin('caller',pvlist{k},pvlist{k+1}); end end return;
%to find Dfx = 0 (could be max or %to find fx=0 %first derivative value %update x %value of function(x) %error %iterate
,i-ona))i+moyo+ es !m
fibonacci_moyo(f,-5,5,.05,.02) fplot(f,[-5 2]) title('Fibonacci Method Line Search','Fontsize',10); %Fibonacci Method Line search - Larasmoyo Nugroho 1837087 % clear function X=fibonacci(f,a,b,tol,e) clf %Input - f, the object function % - a, the left endpoint of the interval % - b, the right endpoint of the interval % - tol, length of uncertainty % - e, distinguishability constant %Output - X, x and y coordinates of minimum %Note this function calls the m-file fib.m %If f is defined as an M-file function use the @ notation % call X=fibonacci(@f,a,b,tol,e). %If f is defined as an anonymous function use the % call X=fibonacci(f,a,b,tol,e). %Determine n i=1; F=1; while F<=(b-a)/tol F=fib(i); i=i+1; end %Initialize values n=i-1; A=zeros(1,n-2);B=zeros(1,n-2); A(1)=a; B(1)=b; c=A(1)+(fib(n-2)/fib(n))*(B(1)-A(1)); d=A(1)+(fib(n-1)/fib(n))*(B(1)-A(1)); k=1; %plot()s plot(a,c,'bs') % plotting x hold on plot(b,d,'bx') %Compute Iterates while k<=n-3 if f(c)>f(d) A(k+1)=c; B(k+1)=B(k); c=d; d=A(k+1)+(fib(n-k-1)/fib(n-k))*(B(k+1)-A(k+1)); f_x1=f(d); plot(d,f_x1,'gs'); else A(k+1)=A(k); B(k+1)=d; d=c;
c=A(k+1)+(fib(n-k-2)/fib(n-k))*(B(k+1)-A(k+1)); f_x2=f(c); plot(c,f_x2,'gx') end k=k+1; end %Last iteration using distinguishability constant e if f(c)>f(d) A(n-2)=c; B(n-2)=B(n-3); c=d; d=A(n-2)+(0.5+e)*(B(n-2)-A(n-2)) f_x1=f(d) plot(d,f_x1,'ks') else A(n-2)=A(n-3); B(n-2)=d; d=c; c=A(n-2)+(0.5-e)*(B(n-2)-A(n-2)) f_x2=f(d) plot(c,f_x2,'ko') end
Assignment 6 steepdescentwnewton_moyo.m %Larasmoyo Nugroho 1837087 - steep descent method vs Newton Method for Rosenbrock
clear all clf n=0; %initialize iteration counter eps=1; %initialize error % a=0.001; %set iteration parameter syms alpha; x=[-2 1]'; %set starting value y=[0 0]' %Computation loop using STEEP DESCENT while eps>1e-4&n<10 % g=[100*(x(2)-x(1)^2)^2 + (1-x(1))^2] %g(x) gradf=[2*x(1)-400*x(1)*(-x(1)^2+x(2))-2 ;-200*x(1)^2+200*x(2)]; % gradf=[100*(x(2)-x(1)^2)^2 + (1-x(1))^2]; %gradf(x) x0=x; %save current x as x0 y=x-alpha.*gradf; %x temporary x=y %update x phi=100*(x(2)-x(1)^2)^2 + (1-x(1))^2; %steep descent function alphaoptim=secant_subroutine_moyo(phi) %minimizing steep descent function using secant y=x0-alphaoptim.*gradf; %x+1 x=y; %update x n=n+1; %counter+1 xx(n+1)=x(1); yy(n+1)=x(2); end iteration=n,x,error=eps, alpha_optimal=alphaoptim %display end values clf nn = 30; %# of contours format long g xgrid=[-1:.2:2]; ygrid=[-1.4:.2:4]; [X,Y] = meshgrid(xgrid,ygrid); %Z = (2*X - 1).^2 + 4.*(4 - 1024.*X).^4; Z = 100*(Y-X.^2).^2 + (-X+1).^2; %Then, generate a contour plot of Z. [C,h] = contour(X,Y,Z,nn);hold on plot(xx,yy); clabel(C,h),xlabel('x_1','Fontsize',18),ylabel('x_2','Fontsize',18),title('ro senbrock w steepest descent vs Newton Method ','Fontsize',18); grid on
%Computation loop using NEWTON METHOD while eps>1e-6&n<1000 g=[100*(x(2)-x(1)^2)^2 + (1-x(1))^2] eps=abs(g(1))
%g(x) %error
Jg=[2*x(1)-400*x(1)*(-x(1)^2+x(2))-2 ,-200*x(1)^2+200*x(2)]; %Jacobian y=x-Jg\g; %iterate x=y; %update x n=n+1; %counter+1 xxx(n+1)=x(1); yyy(n+1)=x(2); end n,x,eps
%clear,clf,clc clf nn = 30; %# of contours format long g xgrid=[-1:.2:2]; ygrid=[-1.4:.2:4]; [X,Y] = meshgrid(xgrid,ygrid); %Z = (2*X - 1).^2 + 4.*(4 - 1024.*X).^4; Z = 100*(Y-X.^2).^2 + (-X+1).^2; %Then, generate a contour plot of Z. [C,h] = contour(X,Y,Z,nn);hold on plot(xx,yy,xxx,yyy); clabel(C,h),xlabel('x_1','Fontsize',18),ylabel('x_2','Fontsize',18),title('ro senbrock w steepest descent vs Newton Method ','Fontsize',18); grid on
% take step: xnew = x - alpha*g; % check step if ~isfinite(xnew) display(['Number of iterations: ' num2str(niter)]) error('x is inf or NaN') end % plot current point plot([x(1) xnew(1)],[x(2) xnew(2)],'ko-') refresh drawnow % update termination metrics niter = niter + 1; dx = norm(xnew-x); x = xnew;
end xopt = x; fopt = f2(xopt); niter = niter - 1; % define the gradient of the objective function g = grad(x) g = [2*x(1) + x(2); x(1) + 6*x(2)]; % g=[2*x(1)-400*x(1)*(-x(1)^2+x(2))-2 ;-200*x(1)^2+200*x(2)];
%take y out of the equality eqn %convert sym to inline format %take y out of the equality eqn %convert sym to inline format %convert sym to inline format
%range of subset S x=[-3:0.25:3]; y=[-4:0.25:4]; [X,Y]=meshgrid(x,y); %objective function Z=subs(objective,{x1, x2}, {X, Y}); %substitute both variables using meshgrid data % X.^2+Y.^2; [C,h] = contour(X,Y,Z,30); hold on; %finding minimal point for objective fvalue=min(min(Z)) [row,col]=find(Z==min(min(Z))); xo=x(col) yo=y(row) plot(xo,yo,'r*'); text(xo,yo,['\leftarrowx_m_i_n{objective function} (',num2str(xo),',',num2str(yo),')'],'FontSize',12) %constraint function f=inline(inlineequality); fplot(f,[-3,3]); text(-1,2,['\leftarrowx1.^2+x1*x2+1=0'],'FontSize',12) g=inline(inlineinequality1); fplot(g,[-3,3]); text(2,-1.9,['4*x1-x2.^24>=0\rightarrow'],'FontSize',12,'HorizontalAlignment','right') g=inline(inlineinequality2); fplot(g,[-3,3]);
%Jacobian dtheta1=diff(theta,x1) dtheta2=diff(theta,x2) %xmiu finding process from Jacobian delt1a=solve(dtheta1,x2) %find x2(x1,miu) delt2b=subs(dtheta2,x2,delt1a) %eliminate x2 with x1 x1miu=solve(delt2b,x1) %pop out x1miu x2miu=subs(delt1a,x1,x1miu) %pop out x2miu x2miu=simplify(x2miu) %simplify x2miu ff=objective gg=inequality hh=equality disp(sprintf(' mu x1miu x2miu f(xmiu) alpha=h^2 penalty lagrange_multiplier ')) disp(sprintf(' ------ ------------------------------- ------------ ----------- -------- --------------------')) %put miu into xmiu miu10=[-2:1:7]; for i=1:1:4 miusubstituter=5^miu10(i); x1miu1=subs(x1miu(2),miu,miusubstituter); x2miu1=subs(x2miu(2),miu,miusubstituter); plot(x1miu1(1), x2miu1(1),'r*') %and plot xmiu; text(x1miu1(1), x2miu1(1),['\leftarrowxmu,',num2str(i),' (',num2str(x1miu1(1)),',',num2str(x2miu1(1)),')'],'FontSize',8) % hhx1=subs(hh,x1,x1miu1(1)); hhx=subs(hhx1,x2,x2miu1(1)); alpha=hhx^2; fxa=subs(ff,x1,x1miu1(1)); fx=subs(fxa,x2,x2miu1(1)); penalty=fx+miusubstituter*alpha; lagrangemultiplier=2*miusubstituter*hhx; disp(sprintf('%1.5f (%3.12f,%3.12f) %2.10f %2.10f %2.10f %2.10f',miusubstituter,x1miu1(1),x2miu1(1),fx,alpha,penalty,lagrangemultiplie r)) end % break %Hessian ddtheta11=diff(dtheta1,x1); ddtheta12=diff(dtheta1,x2); ddtheta21=diff(dtheta2,x1);
ddtheta22=diff(dtheta2,x2); hesstheta=[ddtheta11 ddtheta12;ddtheta21 ddtheta22]; %finding the eigenvalue of the Hessian matrix lambda=eig(hesstheta); %put miu, xo, yo to eigenvalues x1miu_hess_miu=subs(lambda,miu,0.1); x1miu_hess_xo=subs(x1miu_hess_miu,x1,xo); x1miu_hess_yo=subs(x1miu_hess_xo,x2,yo);
-arrier+moyo1!m
clear clf syms x1 x2 miu %primal problem % objective=(x1-6).^2+(x2-7).^2 % equality=x1+x2-7 % objective=(x1-2).^4+(x1-2*x2).^2 % equality=x1.^2-x2 % objective=2*x1.^3+15*x2.^2-8*x1*x2+15 % equality=x1.^2+x1*x2+1 % inequality=4*x1-x2.^2-4 %will not be used because the x optimal is inside this constrain objective=1/3*(x1+1).^3+x2 inequality1=-x1+1 %it will be used, because x optimal is inside this constraint inequality2=-x2 %it will be used because the x optimal is inside this constrain inequality=inequality1+inequality2; %composite / augmented / penalized objective function theta=objective-miu/(inequality1+inequality2) %drawing function yequality=solve(inequality,x2) %take y out of the equality eqn inlineequality=char(yequality) %convert sym to inline format x=[-3:0.25:3]; y=[-4:0.25:4]; [X,Y]=meshgrid(x,y); %objective function Z=subs(objective,{x1, x2}, {X, Y}); %substitute both variables using meshgrid data % X.^2+Y.^2; [C,h] = contour(X,Y,Z,30); hold on; %% finding the optimum point inside the constraint fvalue=min(min(Z)) [row,col]=find(Z==min(min(Z))); xo=x(col) yo=y(row) plot(xo,yo,'r*'); text(xo,yo,['\leftarrowx_m_i_n{objective function} (',num2str(xo),',',num2str(yo),')'],'FontSize',16) %constraint function f=inline(inlineequality); fplot(f,[-3,3]); text(-1,2,['\leftarrow-x1+1< =0'],'FontSize',12) title('Barrier Function using KKT Lagrange Multiplier at Optimality',... 'FontWeight','bold') % break
%Jacobian
dtheta1=diff(theta,x1) dtheta2=diff(theta,x2) %xmiu finding process from Jacobian delt1a=solve(dtheta1,x2) %find x2(x1,miu) delt2b=subs(dtheta2,x2,delt1a) %eliminate x2 with x1 x1miu=solve(delt2b,x1) %pop out x1miu x2miu=subs(delt1a,x1,x1miu) %pop out x2miu x2miu=simplify(x2miu) %simplify x2miu ff=objective gg=inequality % hh=equality disp(sprintf(' miu x1miu x2miu f(xmiu) B(xmiu) barrier(theta) miu*B lagrange_multiplier ')) disp(sprintf(' ------ ----------------------- ------------ --------- ------------- ---------- ----------')) %put miu into xmiu miu10=[-2:1:7]; for i=1:1:7 miusubstituter=.1^miu10(i); x1miu1=subs(x1miu(2),miu,miusubstituter); x2miu1=subs(x2miu(2),miu,miusubstituter); plot(x1miu1(1), x2miu1(1),'ko') %and plot xmiu; text(x1miu1(1), x2miu1(1),['\leftarrowxmu,',num2str(i),' (',num2str(x1miu1(1)),',',num2str(x2miu1(1)),')'],'FontSize',6)
% ggx1=subs(gg,x1,x1miu1(1)); ggx=subs(ggx1,x2,x2miu1(1)); alpha=ggx^2; fxa=subs(ff,x1,x1miu1(1)); fx=subs(fxa,x2,x2miu1(1)); penalty=fx+miusubstituter*alpha; lagrangemultiplier=2*miusubstituter*ggx; disp(sprintf('%1.5f (%3.7f,%3.7f) %2.5f %2.10f %2.10f %2.10f %2.10f',miusubstituter,x1miu1(1),x2miu1(1),fx,alpha,penalty,miusubstituter*al pha,lagrangemultiplier)) end % break %Hessian ddtheta11=diff(dtheta1,x1); ddtheta12=diff(dtheta1,x2); ddtheta21=diff(dtheta2,x1); ddtheta22=diff(dtheta2,x2); hesstheta=[ddtheta11 ddtheta12;ddtheta21 ddtheta22]; %finding the eigenvalue of the Hessian matrix lambda=eig(hesstheta);
Assignment 9 and 4 13. 13. CONJUGATE DIRECTION FLETCHER REEVES & QUASI NEWTON BFGS quasi_newton_conjugate_direction_moyo2.m function quasi_newton_conjugate_direction_moyo() clf nn = 30; %# of contours format long g xgrid=[-2:.2:2]; ygrid=[-2:.2:4]; [X,Y] = meshgrid(xgrid,ygrid); %Z = (2*X - 1).^2 + 4.*(4 - 1024.*X).^4; Z = 100*(Y-X.^2).^2 + (-X+1).^2; %Then, generate a contour plot of Z. [C,h] = contour(X,Y,Z,nn);hold on clabel(C,h),xlabel('x_1','Fontsize',10),ylabel('x_2','Fontsize',10),... title('Conjugate Direction Method-FletcherReeves & QuasiNewton BFGS','Fontsize',10); grid on
xstart=[3 2]' k=0; H=eye(2); x=xstart g = gradf(x); d = -H*g; while norm(g)>1e-6 k funct(x) xzero=x; alpha=linesearch_secant('gradf',x,d); x = x + alpha*d; g1 = gradf(x); delx = alpha*d; delg = g1-g; % switch am % case 7 % rank-1 % z = delx - H*delg; % H1 = H + (z*z')/(delg'*z); % case 8 % DFP % H1 = H + (delx*delx')/(delx'*delg) (H*delg)*(H*delg)'/(delg'*H*delg); % case 9 % BFGS H1 = H + (1+(delg'*H*delg)/(delg'*delx))*(delx*delx')/(delx'*delg) ... (H*delg*delx'+(H*delg*delx')')/(delg'*delx); % end k = k+1; if rem(k,6)==0 H1 = eye(2); end H = H1; g = g1; d = -H*g; x % xd = get(l,'xdata'); yd=get(l,'ydata');
% set(l,'xdata',[xd(:); x(1)],'ydata',[yd(:); x(2)]) plot([xzero(1) x(1)],[xzero(2) x(2)],'go-','linewidth',1.5) drawnow end %%Conjugate Direction Method-FletcherReeves n=0; %initialize iteration counter x=xstart; %set starting position value g=gradf(x); direc=-g; eps=norm(g) while eps>1e-6 alpha=linesearch_secant('gradf',x,direc); xzero=x; x = x + alpha*direc; g1 = gradf(x); % fletcher-reeves beta = (g1'*g1)/(g'*g); n = n+1 if rem(n,6)==0 beta = 0; end g = g1; direc = -g + beta*direc; xdirec=xzero; plot([xzero(1) x(1)],[xzero(2) x(2)],'bo-','linewidth',1.5) drawnow eps=norm(g); end xopt=xdirec fopt=funct(xopt) plot(xopt(1),xopt(2),'ro','linewidth',2.5) function f=funct(x) % rosenblatt's "banana" function f = 100*(x(2,:)-x(1,:).^2).^2+(1-x(1,:)).^2; function g=gradf(x) % Gradient of rosenblatt's "banana" function g = [-400*(x(2,:)-x(1,:).^2).*x(1,:)-2*(1-x(1,:)); 200*(x(2,:)-x(1,:).^2) ]; function hf=hessf(x) % Hessian of rosenblatt's "banana" function hf = [-400*(x(2,:)-3*x(1,:).^2)+2 -400*x(1,:); -400*x(1,:) 200]; %-----------------------------------------------------------function alpha=linesearch_secant(grad,x,d) %Line search using secant method %Note: I'm not checking for alpha > 0. epsilon=10^(-5); %line search tolerance max = 200; %maximum number of iterations
alpha_curr=0; alpha=10^(-5); dphi_zero=feval(grad,x)'*d; dphi_curr=dphi_zero; i=0; while abs(dphi_curr)>epsilon*abs(dphi_zero), alpha_old=alpha_curr; alpha_curr=alpha; dphi_old=dphi_curr; dphi_curr=feval(grad,x+alpha_curr*d)'*d; alpha=(dphi_curr*alpha_old-dphi_old*alpha_curr)/(dphi_curr-dphi_old); i=i+1; if (i >= max) & (abs(dphi_curr)>epsilon*abs(dphi_zero)), disp('Line search terminating with number of iterations:'); disp(i); break; end end %while
The codes implementing DFP and Pola !ibiere algorithms are generall" the same# e$cept in following lines%
% DFP H1 = H + (delx*delx')/(delx'*delg) -(H*delg)*(H*delg)' /(delg'*H*delg);
The codes implementing !an -1and Powell algorithms are generall" the same# e$cept in following lines%
% rank-1 z = delx - H*delg; H1 = H + (z*z')/(delg'*z); % powell beta = max(0,g1'*(g1-g)/(g'*g));