"

()046195
'

:5
5.1
5.2
5.3
5.4
5.5

()Least Squares

,HTF : .3

5.1

. ,
:
() = 0 + 1 1 + +
= (0 , 1 , , ) +1 .
:
)(

=1

= )( () = 1 1 () + +

)( , . )( ,
, .
. ,
)( = )(
) . = (1 , , ) , = (1 , ,
( ,)weights ) . = (1 , , )(
, :

)( ,
)( . " "
, "" .

 : 5

(")

:
. ,
. x
( xj j - ):
. .0 () = 1, () = :
. m (x) = log xj , xj , :
. m (x) = xj1 xj2 xj3 :
. ( ) () = cos( ) , sin( ) :
: ( )
. :
. - ( ). () = { [ , ]} :
. )(Splines
Wavelets . -.
. ) :(RBFs: Radial Basis Functions
) () = (
() 0 . 0 :
). () = exp( 2 /

, -.
,
. .
,
(.)feature selection, sparse methods

 (")

5.2

: 5

- )( = )( , .
, . = { , }=1
,
)( :
2

)) ( () = (
=1

, ( )Least Squares:
min over

)) ( () = (
=1

. :
() = 2 +

= 2

= ( ) ,

=1

= ( )( ) ,
=1

=1

.
- ( ,) 0 (?).
:

() = 2 2

. = 0 ,

1

) (

) ) () ( ( = = 1
=1

=1

 : 5

(")

: ,
1

= [ ]

) (1

[=
] ,

) (

:
=

() = 22 , = ,
= 1 = ( )1

:
. .
, .
. - {( )}=1
- ( ) . ""
( ).
. , , = 0
.
.

 : 5

(")

:
)( () . .

)(
1

= )( )(

)(
] [
= )( =

)(

= ) ( = ) ( .
( , ) ,
:

( ) = +

.2 -
, )( )( = )( , )( (),
,:

)(
)(
= )(
() +
)(

)(

[=

)(

)(

, () "

)( 2
[ = )(
]
,=1
= )( ) . () = ( +

 (")

5.3

: 5

ill-posed ,
. (
condition-number).
- , -
( ) , .ill-conditioned
, ""
.
,
. :

)( () = ( ( )) +
=1

)( ( ) , -
.
- :

)( () = ( ( )) +
=1

. " ()Tichonov Regularization

:
2
. () = 2 = 12 + +

"" , .
:
= )(
- ( .) > 0
.
:
: = .1 ,() = 2 .
2
= )( ,
= diag{1 , , } .2 ) ( =1
.

 : 5

(")

.3 ,
) = (0 , 1 , ,

() = 0 + 1 1 + + ,

,0
( () = 12 + 2 .)0 }. = diag{0,1, ,1

:
, :

() = ( ( )) +
=1

() = 2( + ) = 0

) (

) = ( + )1 = ( ( )( ) +
=1

=1

:
= ( + )1
:
. , -
Tradeoff "" ( )

. , ,
. .
) (Validation set
.
. .Tichonov Regularization ,
.Ridge Regression

 : 5

(")

. (1 -regularization) 1 -
,
( ) .
,
.
"" , 1
:

() = 1 =1 | |.
:

() = ( ( )) + 1
=1

( ) .
,
. () .
, 1 - LASSO
Least Absolute Shrinkage and Selection Operator.

: ( )
)( .
)( . :

)( 2
( = )(
)
2
=1

, )( = )( , )( :
= )( ,

)( 2 () 2
( =
()
)
2
2
=1

, .

 (")

5.4

: 5

()Gradient Descent

,
( -).
, , .
,
.
: )( :

)(
)(
( = )(
,,
)
1

, ( ) ,
.

)( min

)( . :
= 0,1,

+1 = ( ),

0 > 0 , .
. ,
.
( ) .
( ,)convex . ,
.

: :

() = ( ( )) +
=1

= 22 +
, ) .( +
. ( ) ,
, .
:
() = ( ) +
9

 : 5

(")

,:
) ( +1 =
) = ( ) + (

) ) ( +1 = ( ) + ( )(
=1

( = 0 ).
:

) ) ( [+1 ] = [ ] + [( )] (
=1

: -
.{ , }=1 ( .)batch
.
, } , { ,

) ) ( +1 = + ( )(

,
, , .

: ?

?

10

 (")

5.5

: 5

( )

( ) 4.3 .
= ( ) + ,
:

2
)) ( = arg min (

=1

, - , .

: ,
: > 0

) . (0,

) (prior .
:
= ( ) +
) (0, 2 - . . -
MAP , : = { , }=1
)|( arg max

, ,
- .:
)} { (,
)} {(

= )(|) = (|{ }=1

)( )|} {( arg max

,
.:

1
1
2
) ({ }|) () = 0 exp ( 2 ( ( )) ) exp ( T 1
2
2
=1

2
) arg max (( ( )) + 2 1
=1

11

 : 5

(")

)( ,
,5.3 , ( . 1 = , = 2
, ). 2 1 = :

, , .
( ) ,
.

12