Lessons from the Netflix Prize Challenge

Robert M. Bell and Yehuda Koren

AT&T Labs Research
180

Park

 Ave, Florham


 Park,
   !" NJ

 $*; C Z/
.P$:" H
!+*"2(O+P +$"2; yX ;q # $
ABSTRACT 
$3X,
$&!;"2Y! 5  '.
+/;$& &!(O
$:%% 1!
# %$&
$'!%$( )!+*,
-.
+/0(1"2"3+
$4
 +c "1 V,
5 , 687,9$:(%(!+*$!;(%<! 5  '"= +<6>!;(? %A@
.<BCDE$GF
R$G
. S$<6cA$" H !
()!`X$:$4l`*;+
$:Df! 5 "2!%) 
H
!;/
 H
$4; !" H "2(OW ' %!
!"2$(/;C! 5   9! ,
 I ( H 
$%:

 $e$(%%,e! 0( +
.&(%$/; !
&"2!XA\
7OuBCB&bV(`d+u
1.(2JC,INTRODUCTION (O 5 ,!
<"2!X<%D 9Y_C|) 5 D!
$4 $!(s!
)
.
$, 
I !,
LKMM;N OBC,E$GF (P
 I  '"2!
L (3Q MMC"2$:$!( >!4,"3((9"3 $(, I (9;X$G$:!( -$G&6j<$:" H !
(Oe!1$(Xu
,.!;"2D
R/;,(,
+ &"2!*$-
$(/- H 
P! 5  SBCDE$GF&T-
$4 XV"2!%L L$(!
H !
3$( 5 !
"3+$!;(3,O!(' c
+u
,!;" H ,$G$!;(U # V/;!;! 5  %W,!;" H ,$G$!;(3$L! H
!%,AQM $(/V %"2*x; /% -XU"2!*$V H 
$%d+
C%,
c
U
H D
(OR
 X,$!;(Y$(< %>
!O!Z"2 (Y[O
 ),

!
V\
]C^`_%aLb! 5 _X,!;(U O6j%,*! H  2*;,
$(%(!+*;+$!;(S +j$" H
!+* Y,Fu
,.c+X 
,d+$*W!e %V]e^`_Xaf $*'O3g>$("3+D U ; % $.$(/2!::!
$*&@D
$:(%/A"2, %!% X(!G
,D %(!;!/A L%

(OG H
!%%S"2!*$c
!"2"2(%$!;(%
5 !
cB,E$GFY.!"2,
Zh H
$:4,! 5Ri Q; M;MMX M;MMY6V$6j
% h(%,6"2, %! 5 !
)!" H X$:(%/l(% 
,.)($/; !
&$(D
u
!Y e@
.c"k!'
 D l V/!jm n+op # <+A(q % H ; ! :$!;(6j,$:/ OC +e,,
C,!;%( 5 !
e$:(O,
,$!;(%
H
$41  *q/;,(,
+ r%( H
 X(O s$:(O,
,.3(t%*(,,u "2!;(%/'(%$/; !

"2,(R$:(< >@:<! 5 !::!
$*c@D
$:(%/% :S! 5 "2, !X h(%$/; !
 %!O!Xu6c
 5 ,!
$4 $!;(s"2D !1 +)$:"Au

 L(GX4 H .>%,
S  *$!
S!e$( 5 D
L
:+$:!( %$ H j"2!;(/
H
!+*U.(
 5 ,!
$4 $!;(<"2!%,:UOC! H $"2$4$(/V,
$Gu
$G"2>(2!<$( 5 !
"v$G"w
!"2"2(%$!;(% 5 !
c%,
L^1(O ,
$:W"2!
L H ,$@S!V Z
/,R! 5  H ,$@ H
 %$,$!;(%
! 5  0%*(,3  *`(x 
U V"2!;.q(%!%Gx$(x %
B,E$GFlT-
$:4, 5 !
%"=m y+oR(1AKM;M;z'{&|e|}6>!
7 ! H mGQ,op I (O/
+$!;(! 5 $( 5 !
"3$!(9!;X6V %$: "2!+*X$AD


 f$(O!1:(O 5 ,!
'"2!% 5 !
< 2
+$:(%/;) "Au
# X
` 
,,
$.$~! 5  ~BC,E$Ft%?!"'%$:(%l! H !
* ; H $(/YD %($:[O 5
!;"m XQ Mop
q:
/;3 :(%/; 5 !
H
%$,$!;(U2R$G
.Y("2!;.e!;*$:!&$
$43BC,E$F H $ 9~,!;" H
, (%$:*;A+l,<$(,:%$(/ BC,69
/;%d+
$4 $!;('"2, !XRD
!;-V*;+
$,&! 5 "2!X
"2!
q (Q MM0"2$$!;(r"2!*$2
$(/' +'6>,
 H ,
5 !
"2  $:(%X$:(%/C! '(,$:/ !
 !O!Y(Y:(O 5 ,!
L"2!X
O!;X~;y;M MM;Mr,
l!;(Q+zO z;zM"2!+*$:, # $ H d~
_X,,$!;(lK<X$L V(2!)X$:$4"'G$ H  ; !;" H :,"2(%u
H
"2$"w!;(3"2, %!%Z +j (qW!" H %',2$(OGOS_,u
!( c"2!.3;;! 5   H !(O$:V,
u$G" H $G
2 +*;~(! 
3"2!XL_XD$:!(1<"2"3
$4,j 5 ,6! 5  $:(%(!+*$!;(%

+$(/%g>!;([%(OGO Z"3 $(3 
(%$(/0"2, !XA%,$:/(  %,*! H  q$(~  H
!O
5 !
c,!;" H ,+<$G$!;(% %!
j(% 
G2!X %"Y.L"2!X$G@ 
!
<(X!;(% U # %$
   H D
(9! 5 !,
*9%3$:&*;,

(%!;(X
(X!;"1$
 ; ; L"2!(OR! 5 !,
*; Y%c*
$-Oe"2!
 2. UTILIZING A COMPLEMENTARY SET
 (& %
,>!
X,
R! 5 "3/;($G%L"2!;(%/V,
R!
Z"2!(/V$G"2 OF MODELS
 !;(~(! D
,C"2!%
 (. U !;%
.W
G "2
# %$ 5 ,Y!;" H :$ +Y %2D (/q!X,,<6j7?$/( 5 H 5 I
5 !
Y,
+"2!+*X$)6V$G f%A$(OY" H 2$4A6V $q *!;$:%$(%/ 5 
!"!;"Y$($(%/ H
 %$,$!;(%3! 5 "2!XA 3,!;" H "2,(
!+*D
W@%$(/ 5 !
,
+"2!+*X$c6V$G 1*D
 5 ,6
$(/  D 0! %,
 $'!%
e6V$((%$(/q(O
O Uq$( +
)!;"Y$(+$!;(
! 5 "3( H
%$,$!;(3, XD %$:,* 2(3$" H
!+*,"2(O-!+*,
Vg>$(Xu
# ! 5 %
 %,
S!;" H :$ +"3+,
 H ,! H ;. 5 !
>"2!+*X$Z$L
*;,
q!;" H :DF H
!OLh H 
$%d+
C%,
c
$(/A! 5 '"2!+*$: "3+D ! 5 yX ;q U <.e$(%/;A,C! 5 H
 %$,$!(C
 D % 
"2$/ eA,, 9O(O! 5 !(O 5 ,!
&
(/;$(%/ 5
!" !;(%G<NX n;zqSaS*;(< Z"2, %!e6cZW O%X
$:) <!;(Y H%H Gu
 %V/;(%,
 x; /% X/;(X
Dx!)[O%$c H ,$G@
$XL%D A $(/1(%$/; !
 %!O!?"2, %!%C!~
,G&! 5 3
.
$, f>!G4,u
 %A,!
 U Y,$(/% P(1 '.X:Y! 5 7/
!(`"Y$; "3(%(9"3D %$(; # A,.)$:" H
!+*"2(OC $*  H X
G0
_$"'G(!G"2!%$(/9 x06V$dX1 H ,
%"! 55 ,!
 ~:,( 5 ,!
<"2!%-6cYnQ M;(`6c<,*(? 5 !
) 
XV!Y e :(%/;; . H X
e(
.W(,$:/ !
W"2, !
$?(!f" ;x
 D %  sQ M H ,
,(l$" H
!+*"2,(  5 !(1 &$Ge6j&,
$G$ G9$:" H !
(OC!q%$$423*;+
$Gu
*; 5 ,
Z!;(%SO
BC,E$Fe6c
X & L@%
.-(( H
!;/
 ,2! 5 "2, !X>W V6>!Y"3$:(2!O!: 5 !
V!d!
+$:*;
@G,
$(%/x($/; !
 !O!9"2!Xe(d+(O 5 ,!
)"2!X
H
$4A!~ 2{&!
>L"=! 5 h #c&# .u]W 
  P6V $D ;%X
W[O$G&X$G,
(OV*W! 5 .
%,%
&$(l %&+X
BC,$:/ !
 !O!"2!%,:~\
7uBB&bt 2"2!;.'!;"2"2!( 5 !
"! 5
!::!
$*A@G,
$(/
Y"2!;.C,,$*')X,,$(/q*,


SIGKDD Explorations Volume 9, Issue 2 Page 75

 !O $:4, 
,d+$!;( %$ H  # 0$G"($/; !
 !O! H%H
!D $(/L! 5 (2$G"!
>Ye%,
 V *D
/;V
$(%/&! 5 (A$G"w!

$:X(O$@ H $G
j! 5 $G"2\
"2!*$bR +>('!<V
3$"2$Gu OA&%,
X(' W+W! 5  c
$(/3\
W,$!(q!(C!;
:+
O $(<!
%D
P! H
 X$,P
+$:(%/; 5 !
Z(Y%(%
+ e$G"} &!;( a-,W$:(3m K+o 5 !
ZX,$bD-L
$:%P$7Z >!+6> ) 5 !


+$(/;>! 5 $:"2$:
e\
($/; !
$:(%/ObL$G"2LO' W"2WD
emGQ;QDo ,F%" H ; !1X$:.$(%/;$ 0,
6V !~$7< '"2!.e!"2"2!;(%
_$"2$::+
O % &%,
 HH
!D ` H
 X$,$!;(V!(1
$(/! 5
"2!+*$',. 5
!" !q6V %! H
 5 D
2"2!
q H $:$4 
 %&"2&$G"Oq$"2$::+
%,
<m z op ! 6>*,
X e
.W! 5  $ 5 +
; S!
Y! H
 %$,Y,,
5 !
A%,
Y6V %!;3
$(/'
$l!+*,

+
$~!($:%,
Y!;(%0 3$G" H%H
!D U0JC%
Y.Y($/ %u $"2; X!+*C(3DO!;(q(O3D (/C,F H d$( 3OA W$( D
u
!
 !O!~"2!Xj H $ q% 1KMY!3nM<($/ X!
$(/A"2!+*$ (OW[O:$Gl! 5  e$G"2c,$:(%/'
U
5 !
3(Of$(/: H
 X$,$!;( S! 5 ($/;(%!
$(/ 3*.A"3 .!
$G
! 5
+$(/;'? 3,
A! 5 $:(O,
,.1m K+oprg>!( [O,(GO Z ,
"2D !%C
A%(<!q  H %
< )!:$G! 5 6> 7`$/( 3. IMPROVING EXISTING
C(%,
G1  7$:(%/% $:"
METHODS

!+* l"2!X$(/A(%j! !:! 6}!;(
,(!" H ~$(1! 5 '%,
W
+$(/; H H 5
(O 5 ,!
3"2!%A,!;" H
$l(tG,
($*` HH
!; r! ! 5  X
&X$
,,$!;( !
'!"'%$:($!;(q ,
,! 5
,!;:!
+$*C@G,
$(%/e6V$ 2 %W"2!
V !;$.$/!!<%(!+*D

Q;j|e H (%$(/17(! 6V(f"2D !%)$(s!
X,
)!`$" H
!+*A ,$

:+(O 5  X
2 +3,F H :$(x %1!;%,
* 
$(/!
l(
[$G
3
+$(/;V"3+
$Fl6V$ l(!Y"2$:$(%/A*; X"3
$GF 5 ,u
!
$4 $!;(`*$:'$(/:
*&X!" H !;$G$!;(f\_C|)b> HH
!+FX$Gu KVg>!;"'%$($(/Y"'G$Gu &*$,6VV! 5  %&+
"3+j6V$G 2 W.>
(7Ouq HH
!+FX$"3$!(3&% %D@(2
XVg>!;"'%$($(/r,F H $$G
$(/t$:( 5 !
"3$!;(}6V$G $:" H $:,$
 % H
!%Dc! 5 6j!)
(%7Ou.q"3
$VRe (q< O6V %,


$(/A,  *$:!
<\ e%$:(
q*$,6b
<$:P)"Y &"3,
R ()$G ,
!
 # %>"3
$GFe
 H X
> %&l"2!. H
!"2$:(%(O 5  X
c! 5  +X X  *X$(%/ h,!
X$:(%/;GO X6je6V$PX"2!;(%.
e!%
W$" H
!+*"2(OL!;(l6j!
!%>:,>$/;(%$@ (O H D
(>$(A %W!,
*; 3C >"2$:/ O 6j,:7(! 6V(~"2!X
c"2D
>(!$;S!
Z H 
V%C$7> %jBC,E$GF<
+$(/; G,
u
($*j,.$:"3+$!;(A"2, %!%-+
j
 [O$G
 <$:(Y!
X,
-!&%6V$
 %)"2$:$(%/2"2(OW(~!3 *!$:1!*;,
@%$(/1\
 ; /% Zm X 3.1 Weights
for
JC%
Y/;!;c$Y!
Neighborhood Models
 %$,'(s%(!;%,
* ?
+$(/OfD
'
H 5 !

XUNX-QMobDC]W.
$, >!G4"3(%(`"3 $( H
!+*$:%<(%! %,

$G"\
"2!+*X$+bj X(! 1W+ch(1$G"Au!
$(O l($/; !
u
"2!XUl!;(~.$"3$(/A:+(O 5 ,!
YmGQMop
!O!A"2!X$:X(O$@L)D>! 5 (%$/; !
$(%/3D'RBY\
bR 
(O 5 ,!
l"2!%,:q+
0/(D
Gx,,,$*l.$"3$(/ ! %,
Y,
<(9!`
+q$"2$:
Gf!1 %$G
Y
+$:(%/! 5 0h
!+*D
Z.
,X
Y +e
:,<$"YG(%!;%`!q"2!;.e!
<
$G"2W$:(BY\
.bW"Y.W  *)(`
+ 1O1P #  H
 X$, 
"2!+*$-!+6>*,
O %W"2!%j+
 H !O!
Vj%,,,$(/).
!;(%/ *&! 5 +Z$V7(`W'6j$/; O ~+*;,
/<! 5  e
+$:(%/;W! 5
!O$:$!;(%j"2!;(/C"3:X,Z! 5 !;G<
: Y"2!*$R%D (,$:/ !
$(/A$G"2
 # % !
1! 5  &]W$(/;V
$!;/O H
$Gl6V ,
)(,$:/ !
u
%!O!~"2!%,:WX!A.  p ;O
\ +  b
+ Pw+ W \.Q+b
$; $(~ ,!
O ,$ %,
"2, !X!;!;/~"2$/; OW&,F(% l! p ;O
,(!" H j % 5 % H D
"v! 5
:$!( $ H   1(~,!
 +
,"2CX!O!;"2 q! 5 $%
)$( H
,$;VR$(%$(/2(%$: &,:%.,
 # q$G"$"2$::+
$$%,(!rO H df,(O
>
!:
! 5 $G"2j%D lc$GD %!OD7A@"2j!
V! FX$(/<"2!*$j"2$/ L
,u
D
; UW ,1
< 1! 5 !
C,$(/3 <($/ X!
e(
[O%$
j %!;%(XZ! 5 :(O 5 ,!
LhW()*(A!X  ! 5 ,!
 5 !
Y6>$/; O$(/1 %2!+*3 *,
/;;1g>!;"2"2!(? !$:,'+
A 
"2$/ <2.6j" H  ?O 2(%!;$l!O$:s6V$G f %A !;Xu TR +
!;(3!

,d+$!;(q!O,2$(Oj(' %V!;GA
:+ 2!;$(
(Xj! 5 ! %,
c"2!+*$:,S_X$"2$:
GO 5 !
V($/; !
 !O!3"2, !X $"2$d+
$GO Ce,R!2X(!C!;"2C,:$(% H
 X$:D!
5 !

!2 H X
e &X/
,) '%,
H
 5 ,
C,$!;(1!3X$d!;/;%   I &$e$" H !
(Oe!q< ',:$(%A*;C!q
"2!+*
"3f
[%$G
1*,
s:
/`("'D
'! 5 (,$:/ !
 >! l,Fu $G"AuC(9,
u H $G@2$:< +Y"3 H
*;(O& 2"2!%,
,:%$!;(! 5 .$"3$(/s %0%,
~$(O,
.~$:(! ,
l
*(O 5
!"
*$(/9 %~"2!
 5 (%"2(O>
:$!;(% $ H TZ
$!

5  X
 "2, %!%>! 5 (q7W-j %"2(2
$(/Y! 5 ,
cl!
c$G"
'! 5 !;(`$C,F
"2Gl*;:%<!3X$:$4Y"2!% $(%u  I (m KOVoc6>l!,
 "2!
3!;" H
 ,($*q H%H
!D s!
,!
H !
+>$( 5 !
"3+$:!(),O!;(C$" H GC %-
$(/ ,"2*  ,&,:$(%&.$"3
# %$e6cC$:" H !
(O 5 !
&D,
e"2!X$:(%/q %Y%,
 e"2!. BC,$:/ !
 !O!u3"2D !%S "2V*,
 H ! H %d+
j
%,
)$(9 <.<, H
!+*$:% 0~"2 /;,
)(%"',
C! 5
+$:(%/;  Dr
1$(O$G$*1(
:$*Gr$" H ~!0$:" H "2(O  I (
I ( H X
$(/A %$: H%H
!;  6j<(GX4 fRpA"2!+*$:,W%,
 H +
$:
U D`%!q(!C
 [O%$
<(%$:(%/3"3(O H 
"2D,
C!


+;
/+
%e! 5 Xf D~
` %<"2!*$C(%; 6> (',F($*j
$:(%$(/&./; # DA:! H
!*$:%Ve,!;(,$:W(

r"2!X$:(%/ 5 !
l6V %$D "2!+*$`?%,
36V$:D %!O!;1!f*!$ $(O$G$*<.$G@$!;( 5 !
2 %l,!;" H X  H
%$,$!;(%t! 6cu
%$ D
! H $(%$:!((*;!f\ H !;$G$*f!
(/;$*+bl
+$:(%/% *D
.(+
Y(,$:/ !
 !O!u'"2, !XP
$V!;"2>!;(Xu
# %$ H
!+*$:% f($:" H !
(O<!O!; 5 !
2 +
($(/0!%',
 ,
(%
/(D
$(/01(%$:[% H ,
 H D$:*;3 <($,?!;" H "2(O 
 %>
$(%/;.u Y$( 5 !
"3+$:!( > : $& j%$:(
)*$,6 Q; # )$"2$::+
$ 5 (D$:!( 6V $ %$G
,G1%,@(V 
! 5  %>%%SJC*D

 %$P6c-c7,e!V!;X
H
!;/
, O(C6V$:
$:(O,
H !:$!;(q6j$/; O %$V
$G
+
O->+
$!;V/;!
$G %"2
H
!;Gl,
*)(1*(l$/;/,
W
!;)$:(q
 $ 5 &
!;"2"2,(%D
'!"2,6V +)%$G,
,(e$:"2$:
$G"2 X
 
%$(%/q!
.X."2 6V ,
A
$ ,
Y(?!" H
 %($*'$:" H $:,$)%,
<,u [(O$ 5 Y V$*W(!$:!(2! 5 $G"8$"2$:
$GOZ_X H%H !;
 *X$!
C\
; /X: O
,( $.!
O %
! 6V$(/) %$:.!
2(A +
D H u  W H 
$%d+
W$,"$: H
%$,  H ,
5 DlOl'Xu
D
(bL$c *$:C!<(X
$: q"2!%>,(O,
 q
!;%(3,F H $$G ,C! 5  )(,$:/ !
 I (~ +; 6>&6j!;%:~6c(OW 

+$(/;
H
 X$:D$:*;&DW!A
,$*< %&6j$/; O %V W$
R$(GO !;"2>! 5 !%
Z"2!%,:-X$:$4'! %,
Z$( 5 !
"3$!;(A!;X $:" H !$ 5 !
c!(X A$"2$d+
$G3!
c$:7;W %CTR +
u
$G"2 ;,
 ;!
S$(X$:*$:%O
$(%/;Z AZ %j("Y,
-! 5
u !;(1!

:$!;(`,!O,2$(O 

SIGKDD Explorations Volume 9, Issue 2 Page 76

 KOVT-
*$:!L($/; !
 !O!Xu 2"2, %!%S%!)(!j!(O JC(c6j Y!) $*V $S$>O'X$:$4$(%/<,!;(X$G$:!(
.
$, 
5 !
e$(O,
D$:!(C"2!;(%/2($/ X!
WaLD `$"2$:
$G~,u >!4,"3((Y"3 $(,j\
]Cj^`b>mQMop # >]Cc^k"2!X $*,
6>(Y(Y$,"}U(YV($/ X!
2lB'\
b$-!;" H %  ?$"2$:
~X
,r!f.(+
 5 ,!
$:4$!;("2!%,p c6V %$:
$(X H (X(OG1! 5  <!;(O(OC! 5 BY\
.bC(1 Y! %,

GX$(/r!r!;( 0$(
*$,6 H !$:(O  %( H%H GX$:(%/s7Ou
$"2$d+
$G$:, d 5 !
3BY\
b O!
A,F%" H ; BCB!;(~
,$dX:W! 5 ]Cc^9 X D
Ol$(O/
$(/3:U$" H !
(

 H%H !;& +V!;X
C$G"2W
&"2!+*$ (3 &(,$:/ !
 *$,6 H !;$(OL! 5  V+A\
!O 
 $/ 2 W(A$(+
bD ;]e^`_Xa
,l!;(O$:(%q X
`"2!+*$:,3 l
 %$:/ G,!

: X
! HH  3!2MX yy;X
6V$G  ! ,
s\
 /X & [O`%D x !
! 5  %
]W$:(%/;&QDObDjhW(1/!
$G " +V$/;(!
V &$"2$d+
$G
! 5  q %
1"2!+*$'6V %(xX,,
"2$($(%/0 %$G
2$(D
H !u
3.2 Latent Factor Models
+(O ,!
3"2!%'"2%
q l/
"2(OA!
5 5 %,
2(
:$!;(`6>$/; O "3l,(1 H (O$:G1
$ H Y,!;(O$(/ "2!+*$:,2,
!q9,
$3! 55  +%
2 3+
`/;!
$G "2$ G
 C$:( 5 !
"3$!;( H
!*$:%qO3 e/
!; H   
(%  5
!;" %'% # %$:e6jO U6>A!O$:3  0,
&
VLt%D@($G$!;(U > %1$(D
H !;:$!;(t6j$/; O2%"!?!(; 6V$G ?3D
u 5 ,!
&*,!
c 0 P(` D 0"2!+*X$)6V$G ?
6V $ '"3& j!+*D
@$(/XZ_X HH !L +Z(Y$G"}  "2!+*$:Du 5 ,!
Z*,!
LS3c #  H
 %$,$!(A$LX!;(%c&7Ou
(!)% 5 %($/; !
L
3OA H 
$%d+
cD
 I (A + $(/3(`$:(%(,
H
!%DV + - . # %)"2!
&$(*!;*  H 

 ; U$6j!:0<.!l$/;(%!
< )($/ X!
 !O!$(%u $L %.$"3+$:!(3! 5   5 ,!
>h.
$/  5 !
6j
q HH
!D
5 !
"3$!;(U .X$(/'6V$G ~ %eX

(OV,:$(%&.$"3; 6j!:~"2$($"2$4C %C
/:
$4 1,!;. 5 %(,$!;(U
BC,*,
 ,:, O W.(%
2(,$:/ !
 !O! 5 !
"'%:)%
f6j$/  *D
/;9! 5
$(/ 5 !
1 %0%($( 5 !
"3+$:*; \
+ b rP\ D  b \
Ob
O d
(,$:/ !


XcBC,$:/ !
 !O!&"2, %!%U"3C(%!U6j!
76j$ 5 *;+
$:$$G # q,Y,!;(O$('e\
P.bu H $G
 5 !
'6V %$D  $'7(! 6V(U
%$G,
V%.(O$:G~"2!;(%/A(,$:/ !
 # )
/:
$4 +$:!( H 
"2,,
e H
,*(OV!*;,
@$(/XA H u
$ -*%<$)~MX MnO )"2$($"2$4 9\
ObWO1(0G,
($(/
 H
! H !>j"2D ! !*;,
!;"2,U %S%$G2%$-R$
.  .Y[O+
)D %"2~\_;~nQ2! 5 m +obD1J %,
2m NXcQMoj% 
6>W
 H :V c6>$/; O 3 *,
/O'e"2!
V/;,(,
6j$/  /
%$(Oc 2 ($:[O, (3%$:(%/'NM 5 ,!
&\lN;M;bD
%"1 Z6V $D t! 6V3%! 6V( H : X$(/0(,$:/ !
 !O!$( 5 !
"3$!(  $l$ 5 D!
"2!%, H
 X$,l %0.~,~6V$G ]e^`_Xa
6V %(f:7$(/1$( 5 !
"3+$:*;2($/; !
lC$:*;(s~,<! 5 ($/ %u MX M;yX
!
ZB'\
bU6>c()!!;" H %DX;
pq>D:+
-+
% I (O,
,.$:(%/;GO  $Z$ 5 ,!
$:4$!;(3"2!%,X(',(,@% 5
!;"
BY\
bl +Y6V$c(3 q. H
 %$,$!;(s
%:q! 5  %
"2!+*$:(%/3!;(%/l 0! 5  %& %
 H !;$'%$G
,$!;(%e!;X$:(% 
5 !
"1
$(? '/;$((%$(/1! 5  A,,$!;(lR$G
. R6>2 (9$:" H
!+*2Du
+ R, V
\ +  b \Kb X
,O% H ,($(/x % 5 !(%$!;(}! 5  %s"2!%
 # 
p ; W
[O+
  H (Gl 1$(f\
bV$" H $$GG`"2V +C 5 ,u
O
!

)
6V( 5
!"$(% H (%(Oc(%!
"3PX$:.
$%%$!;(%V6V$G
  +
()$:(O,
H !:$!;()6j$/ Oe"2!%,:$(%/V %-
:$!;(% $ H 
L
4,
!~"2 (0(0 %'"2'*;+
$d( # $)$)~[O$GY:$"2$G$(/
D6j,(x$G"<(t$G3($/ X!
2 %
!;%/; x9 .q[O+

%" H $!;(USh
$ ,
W"2!%,"2W</;(%,
"'G$*
$:
H
!;%:,"1 (!
"3%$.
$%$!( 5 !
L  5 ,!
LhW!
X$(/;GO  %c
$(/
 (?A"2!X 0O` % 5 !:! 6V$(/XlR$
. 6j2
6w,
<(
"2$:( V \
b1 \p;b "2!+*$: 5 D!
 5
!;" %>.!$:(OW(!
"3%$.
$X$:!(
W
p ;
 + Y?\Z
cb P9\Rcb \n;b
# %C"3 .!
cD (/C$>!)! H 6V$G q %V"2$:$(%/Y*, X(
 %$c (le%!(,2$(OGqA.$"3$(/'$(%(,
H
!%%, # ,(  D A
+$:(%/Y+$S7( 5
!;" V(!
"3X$.
$:%%$!;(U
D6j,(t"2!+*$2
$(/r!
A 5 V%D
$ H $!;(
 5 ,
'!rm +o +P?\G .Db
I " H !
(GO  %$:D %"2 H
!+*$dXR$(O,
H !;:+$:!(Y6>$/; OP +
+
q%,
$* ?%$G
,G 5
!;"= %A
$(/Y
$:%q\
+  bD #  H
$!
2/$:*;(x$(}\nbA *!;$:!*;,
@$(/9Os %
$(7$(/0 
&
(%!& `!(?(~$"2$::+
$"2 X
;<^~!
!+*,
%,
$*$!;( 5  ,!
L!+6j
Xj$:(%*%L6V %(2(!j(%!;/ A
$(/c

! 5  C$:(O,
H !:$!;(~6>$/; OV,F H $$GG1!;%(O 5 !
V
:+$:!(%u  *;$d;3]W"2!+*$:(%/q % H +
"2,D
&
.
$,$!;()$" H $ 0
 %$ H W"2!(/' %e($/ X!
 \
Ob H
!X,q$/($G@ (O1%
,/;$(!
lsN;M 5 ,!

JCX
&,F H ,
$"2(OW ! 6j  +C $e ,"2'$/;(%$@ (OG$"Au ,F%" H ;  &]C^`_%a!;(~ %C.+ 5 U!AMX y;X
H
!+* %
,r
:$*`!? +qx!;("2!
`.(
 Be+%
:G ';%% ,F H
$* H ! 6j,
)! 5 q
$ ,
<e$:(
$(O,
H !d+$!;(36j$/; Oe\
 /X 
$*X$(%/<]C^`_%a 5
!;"v
!;%(2M  H
$!
Y,!;"2e6V$G s(f;%% 0!;" H ,FX$! 5  %A! H $"2$:4$!;(
!Y
!(qM K 5 !
jBC,E$F  H
!;+bS6V$G !%j<"2 ($(/u /!
$G "2 0" H ! O r6j!s (%$:[%vJC($~ 
5 U$(,
 )! 5
(%($(/<$"2Am op !;(sC$:%)" H $:(%/% Z( 2! %,
<$< 9!;(f,F H D$!;(
_! 5 +
P6>3%"2!(.
0 '(,@%e! 5 %$:%$(/~1%, H ,
"3FX$"2$4 $!( ;6V %,
j6jVG,
(+WD6j,(Y@%F%$(%/C  5 ,!

5 ("2,("2!X 5 !
>7uBBY;hCX$G$:!(,%
DA/$(j+
 (q@%F%$(%/'  H 
"2,,
%2mGQ+K+o 5 !
eY
,d+ l (%$:[%;
H !$eOA!"'%$:(%$(/& W!O  :C*X$,6! 5 7uBBx6V$G q % hG,
($*GO O6>e (q.26V$G $(q %C!(*($(O>$:" H  5 !
u
%$/; D
2 1*$,6! 5 :,( 5 ,!
3"2!X
rh!
%$(/GO S6> "'%:$!;(r\
ObC(0$:/($G@(OG$:" H
!+* H
 %$,$!;(9,%
,
(A7;V %c$:(%,.$:"3+W\ c. bSL  H
 %$,$!( O$(,/
$(%/`$G ,
& %A!O $4 s*$,6 H !;$(O)!
) '$(+

"3%AO9~:,( 5 ,!
'"2!%,p 6V +'"2!;%(e!1,$*u *$,6 H !;$(O'$(O!` 2"2!X
 I (O/
+$(/1 3!O j*$:D6 H !;$(O
$(%/s7uBB!;(t 1
$:%~! 5&5 ,!
$4 +$:!( # %$:q(  /;$*,&)l"2!
A!" H ,'"YG$Gu A*$,6w! 5  A% # !
%Y!0 $*;q]C^`_%a! 5 MX y;y 5 !
' %3,.2+X S(%,@$(/  $j,('6j' D

 ,D;,DZ j"2!X %  *u
5
!;" "'%$Gu :*$,6! 5  %0k%%
 D
l$" H
!*;,u $!
L! 5 ,
Z6V$G $(' %j($/ X!
 !O!A! 5  j/$*(A"2!+*$;
"2,(Z$S $*; 'O<$:(O/
$(/)!e 2%$(
Y*$,6 H !;$(O% m op #  (%$ GO 6>c! 6x%,
5 ,!
S!D (/c,!
X$:(%/!

SIGKDD Explorations Volume 9, Issue 2 Page 77

  %'$G" &$&$(%/ H
 %$,  # ,
 5 !
; 6V ( H
%$,u 5 
!""2!+*X$(%/0"2!X3!;(%/? %
,1%$G,
,(2FX # %~@
.
$(%/<  6>W@%
.j!" H %V&*,,!
P \
bc3 'X H (X$(/&!;( \p(r"2!.2!;*$!;%b'FX$2%,(!,2$:" H
!+*X$(%/ %1[O$! 5
! Aq('Y(Y %( H
 %$,S+\G\
bb  # V!"Au  0"2!%,:~x$:(%/r %"!(% H D
5 !($!( # 
H %+$:!(1! 5 \
bj$CX!;(%<O3$G$:(%/A &[O+
 ~,

!
'\
bc! !(FX$' H (%A`"'%$Gu :3"2!X$:(%/! 5  q+1 
!+*D
6j$/; OL !;V"2!+*$Z$:"2$:
S!& O!;X$($(%/) % H
!"1 $(O/
+' q!O $4  H ,
 H ,$*Y,F H
6V$G $($"Au
$G f(,$:/ !
 !O!X<6V$G f %2"2!
A
/$:!(j*$:D6v,F H
 
"2$(
\ + \
b b rR\
b \pN;b O2:(O 5 ,!
W"2!X # W $G
l+FX$:c$(O
!%%j %~%$Gu
W
  (+
*$:D6V~! 5 ,
e  *$!
& +&"2!%,:6V $D 0"2!+*$:,C 
+
D
+
):$7,3!'
+&
/+
%W! 5  C*;:%&! 5  %e
$(/X
,
; PB'\
bC$e ',<! 5 "2!+*$C
?O,
)P # %A!(%u
.(O $:>e$"2$::+
$A"2 %
V,6j(2"2!+*$SR()%SL # $)$(
0*$:D68(?',F H !
 0O06>!
7$(/q6V$ f~!;"Au
H +$l%$(
0
 H
(O$!;(! 5  %q`!
SG,
(+$*GO
$(O/
$(/` 2!O j*$,6 H !;$(O H
%$,$!;(%
,9$:<$/u
O?," H ! %$(%/1"2!%& +A:! 6v,F H
$(/`%$(
f H D
(%$G@ (G2$" H
!*; UR!
L N;M 5 D!
j ;  W,

!
cX
! H 
6V$G $( q!;(O,FA! 5
+$:(%/;2% # l"2!%A$(,:%
!']e^`_Xa>eM yzO
!(%$G$!;(R
.
$,0L!;G4"3(%("3 $(mQMop UB&_C|m op
I (. ~! 5 $(O/
$(/2 &!O R*$,6 H !;$(O 6><(1$(O/
+
(l!"2e
: ~ H ,$:$4 1"2!%,:
 %A$(
0*$:D6 H !;$(O 2C,
Y6j3;X! H < H
$(%$ H  5
!"T-u
hG %!;/ 1 &6V$((%$:(%/3BC,E$Fl(O
1,!;"'%$( ~"3(O~,! 5
D
7 CB&_X|w"2, !fm op # %'B&_C|w"2!%,P
 5
$( 5
!"
DF H $:,$G H +
"2,,
$4$(%/A  1%,
%%c
+ ,
W"2!%,:V D H
%$,$!;(% -6j3(!A Y$G<$: H !;$%:3! $*;3z ny;
$" H
!+*"2,(V6V$G 9q$( +
&!"'$($!;(0! 5  %
 H
%$,$!;(
%,
0 !( ?"2!*$~  %+ %f
  # $`6cO
,W !"'$(%)R &7D1"2(OW%$% `$(1 $e+
u
 "2!+*X$l<$q!O,$d+ x6V$G x6>!?"2!+*$,u 5 D!
3*,!

$;S7OuBB"2!X %
,"2!*! 5 /;!;,, %($/ X!
 !O!
<(. # %l
 H
(O$!(x! 5 0%,
3x$A %
!/ r %
6c+
 5 ,!
$4 +$:!( j(s:,( 5 D!
2"2!X'6V$G  %l%$Gu
6>$/; O 0"1   B'\
b !~+V$ H
 %$,  (+
q*X$,6 H !$( 
+ 

S  BY\
b  I " H !
(OO B&_X|"2!X ]W :$ 5 <
,!;"2"2(%,
V.X."2W
),F H ! ~!A6>!A H V! 5
;  
+  D
5  %7LJC(c$L %c $/; q[O:$G',F H :$$GS
$(/X O6V ,

%,
<  *$!
H X
G9 9!;(? %2$(+
9*$,6 H !$:(O  # $
D
& 
G0,F H
 $G
&! H $($!;(9!;(9 H
!%D )C! 6j*;,

:! 6VV).
$/; O 5 !
6c+
l$(O/
+$!;(q! 5  W$(+
A*$,6 H !;$(O
 $c7$(3! 5 DF H $:,$ 5  X73"33&%$G2%Gc!'!:DCX
$(O!' %e$: 5 ,!
$:4$!;("2!%,pSh
$(/'$:W"2!%,:lO
!3.X.",!;(.
$(C(~
,(<! 5 ,
W!3!O! H ,
+;
h!;(}!;X
f! 5 $( 5 !
"3$!;($ %f((1$:" H $:,$
+ P   B'\
b \zb 5 , %7 Z6V %$: s"3f H X
D +
(Oj %$.!
O S%
! 6V$(/
 
+
 H +,
( Z
 r7;,6j!
X S,; #  H $ O S1
,!;"2"2(%,

.X." H $G:$4,2!;(r!;(Gs!(l! 5  36>!9!%
2! 5 $(Xu
# % H 
"2D,
c
.$"3 2OA/
;X$:,(c%,(>"2$($"2$4,u 5 !
"3$!;(U 3$:,*2 Y %$(
?*$:D6V0D
*AA
$ (%/l! 5  %'!O$: 
/;%:
$4 [O
D

!
 # %Y"2!X
H
! F 5 !
e/;(%,
R$" H :$$G 5 %D7 %/;/.$(/3 C2!;"Au
% l6V$G `NM 5 ,!
 H
 X$,c C.W,c6V$G 1]C^`_%ar!+6>,
$(% x.X."%$$:4,$:(%/s! ,F H :$$G~($:" H $:,$l$( H %q$
 (`M y;zO "2!;.2%$G
%; %$:q 1B,E$GF%?:! 6> xA!9
# !'"2"3+
$4; O C,%
Dl! 5  :(O 5 ,!
"2!X ( !;(%G9 %r6V !1
+ ?6V +`$:( 5 !
"3$!;(U Sl
 >:$ 5 q.X."
~$:" H
!+*  5 !;! 6V$:(%/f %
1%$.$(,qX$
,,$!;(R$G
.  (9,F H :!$G& <"' `%
!;;X,
6V %!l
(O 16V +~$( 5 !
"3+u
!(< (0X H (`  5 !;(+$:!(e! 5  <"2!%
O`"2$(/ $!;(U O!;/;D ,
j6V$G l! %,
c!
j! 5 $" H $$G 5 , %7 # %$
`"2!
3E,FX$q(?
$.$: H
!%$:$.$~"2!X
0hG,
(u $" H $ !;X
e*,
 H !;$G$*<,F H ,
$(,Y! 5 $(O/
$(/3 
$*,O j!;(~ (x
D$(x ~$:" H 1.
,X
1! 5  1!
$:/$( $" H $$G2$(
*$,6 $(!0!;X
3"2!%A(r1*,
/; r
"2!X
j%%A/;$(x,*(r"2!
1X
,rOs$(O
!%,$:(%/?!"Au
 $ 5 e.X."2L c,V6V$:XC
(%/;$(/A$" H :$$G 5  X7
H "2(O$(/ H ,
 H ,,$*'! 5  q+1$(!1 %3"2!%
-$(%/ # )"2!.C :(%/;$(/ H 
W! 5  )B,E$GF~ H
!+* l!2
,$ %,
V e!O R*$,6!
V e%$(
q*$,6&  3"3(9D
Y6V ! H
!*$:%f*,
 5 ,6v
+$(/; I Y$A*,

$7GA +>"3(O'! 5  %!;W,
>%!& +*;e
$ 3
,( $.!
O
3.3 Regularization
]W/:
$4 +$:!( :Xl9(O
V
!;`$(W!
6V $ A!:2! 6C!"'%$:(% <
,!;"2"2(%,
Z.X."!&"37
H 5 !%
3"2, %!% ,%
+; H ,
!;($4 ?
,!;"2"2($!( 5 !
Y %"1 R!
Y,*(
hWU %e"2!%W%,,
$ 1!+*C$:(%X&"3$*e("Y,
j! 5 5 !
C,.!;"2D
j6V ! H
!*$:%l(!Y
$(/+

H 
"2,,
 -"3(0! 5 6V %$D f"Y.)2.$"3+ ? f!;(f
"3(X%"',
L! 5
$(/&\
!
H $G
j! 5
$(%/;bDLg>!;(% [O(OGO
$G2$AD
$$ $" H !
(O'!9 *!;$:s!+*,
'@%$(/` 3
$($(%/ 5. REFERENCES
% &%& %
$(7/e!2$" H
!+*C %)X
,l! 5 $"2$:
u mGQDo<hgc^ _ I C{&|e|Y s{&|e| g> H (  !
7 ! H KM;Mz+%
$G~!
C%  5 !
( +
.($/; !
CD$:!(sm opV_ %
$(7/ 6c6c6& , %$;  % $B,E$GFu{&|e|Cug> H .u KM;M;z O"2p
! 5 .$"3 A$((%,
H
!X,S$>(O$:%!&, 5 .$"3u
$!(l! 5 $(O,
H !;:+$:!(l6>$/; O 5 !
j W( +
.c(%$/; !
c"2!X m K o)]<j^9jL&("!'L{&!
,( 3 I " H
!+* BC,$:/ !
 !O!u
X,
$ !+*3m K+opeJC%
5 ,!
$4 +$!;(?"2!XW H $ G`% wg>!;:!
+$*R$G,
$(/% $#j &%(' )*),+-.+v/

$:X/;V
/
$!( 5 !
>
,/;:+
$:4$!;(UL]W"2!+*X! 5 /;!;D, 0 1.%D3254;768-:9<;=?>@' )A)B=%DD
p;
% ~" H $G
$:jOV %
$(7/2m K+op -S DDYC'&V: OA)&D5
DDl/()<E9O
KM;M;z
 $'!(%$G$!;(Z]c^~ H
!+*$:%Y3!O! 5 !
e,F H !;$G$(/l;u
4. DISCUSSION
JCX
3,F ,
$(,16V$ t 1B,E$GFr!;"
 $G$!;($( 5 !
"3+$!;(26V$G $(3BC,E$F Z.j,CmGQ Mop O6>$*
%
H H D $$!(x ! 6> t +  +Z $G
S
 %.
(/ A$L,A$(O>$(O/
+$!;(2! 5  %c$:" H $:,$
 %l"2!.'% 5 c"2!X>$A(("Y3! 5 "'%$ H : H
,u $(
0*$:D66V$ %$(9 ',F H $$G&
$(/ I (0!;%
&,. R"2!;.
X$,!
C  ! 5  " H ,$:$4~$(
$(/sfX$G,
(O $" H
!+*"2,(Z!+*D
>(%!;(%u!(X$$!(%]Cj^~L6cL
 $:4, 36V$G Xu
 H ,&! 5  Y I ( H +
$:,:
"2!.C$" H
!+*,"2(OV."2 !;XW$:(%X$:(%/Y %&$:" H $:,$V$( 5 !
"3+$!;(1$(1B,E$GF j.W, 

SIGKDD Explorations Volume 9, Issue 2 Page 78

 m +o&])U^9U>Z(F!'{&!
( V._X d3g>!::!
$*AR$:Gu
,
$(/r6V$G HG;!$:(OG|e,
$*B$/; !
 %!O! I (O,
H !:u
$!;( $/; OX I#j &%=KJLJ@JM=D
p;N-S DDC
)<E9OCO=?-:)*93P QIRS KM;M;z

m o&])^9PLp N!'{&!
(9(0gUS!;$(7OO V^~!X$:(%/1]W,u

:$!;(% $ H r+^~%G$ H _X :,! I " H
!*;xhW%
,
! 5 
/]W!"2"2(X,
r_O%.,"2% F#c. &%T2?4;U68-:9
;=?>L' )*)U=D
pV-S DDAF')V: O)&+
5
DD~/W)<E9 KMMz

m n oG%L((%,&(_ (($(%/% V # %YB,E$GF1TZ
$4X X#c. &%

' )A)+Y-./ 0 Y1.%D<T254;Z68-:9[; =?>@' )*)
=D
pV-S DDAC')V:YO)&D5
DD1\
)<E9O KM;MzO

m N+o<_?%(7 B,E$GF^] H %; #
 # $8hjwC!;"2,%

O H ;+$ 5 ,
 !
/ ,$:"2!(.!;X
(KMM;NXQ+KQQ O"2
RKM;MNX

m z oG% WCD
:!O7,
(G%h){&!(.( <h<>!
D %,
0(_G%

]W$:%
Rh(AhW/;!
$G %"2$:j%
"2,6>!
7 5 !
LTR,
5 !
"2$:(%/)g>!;Gu
:!
$*<R$D
$:(%/;X X#c. &%.` `;/a68-:9b;=?>L=cd-S De+
D'f=2
pFcWD


; HH K;M XK;z -Q;;

m y+o&BCDE$GF I (; B,E$GF T-
$4; !
%"2%

6c6c6& (,E$GF H 
$4; !;"q!"2"'%($G

m +o)h<T-+,
7 P I " H
!*$(/W]W/;%d+
$4 A_$(/;%:
ZLL|e,u

!" H !;$G$!;( 5 !
3g>!;:!
$*`R$G,
$(/% V#c. &%V' )A)+
-.+/ 0 1.%DCg254;h68-:9B; =?>@' )A)i=%De+

pj-S DD&$'&V: Ok)&,5
DD2/))
9% KM;MzO

mGQMo)])9_%:7X X%$(!+* th<9^~($ (Y0$:(O!( ]W,u

.
$, fL!;G4"3(%(s^1D %$:(% 5 !
Ag>!d!
+$:*;lR$G,
u
$(/% *#c. &%,`l%m6W%n%;V=D

;(-S DD?
99OAoRD%Op=?-:9go8P QIRYS+ KMMzO

mGQQ,o<O_%
6j
%Y;{)
 H $ IGXO{&!;(%.(2(EG%;]$ X

- I "Au
 g>!;:!
$*9R$D
$:(%/],!;"2"2($!(h/!u

$G "2X (#c. &%$2qQ3=D
p;$-S DD
0 ,r 0 sO 0 t, HH Kyn u.KnO UKMMXQ;

mGQ K+o,!' u (%/(_G%C{&!
(U 0.a-2$(O9jO$:($:D

u

 $ A]C,
l^~!X$(/ 5 !
q],!;"2"2($!(_O%.,"2%
#j &%j4 Qv=D
pw68-:9x-S DDYfcVD,
\,)Y

,3D%-E=2
pEcWD


;XOy; =?>@=cfP QIRS
KMMzO

SIGKDD Explorations Volume 9, Issue 2 Page 79