!"#$% '(( )*$+,+-*./* 01#"-*$ 23
452356
08=$+* 01#"-*$+ 2
H.-$8>=/I8.
4 H.-$8>=/I8. -8 J#>88" #.> -1* J#>88" K/8+9+-*G J#>88" @$/1,-*/-=$* #.> J(L! T U 3 V W S
HG"8$I.: '*<#I8.#< (#-# D,-1 @"#/1* !R88" H.-$8>=/I8. -8 HG"#<# #.> J,A* Q8>*<,.: #.> Q#.#:,.: (#-# D,-1 HG"#<# #.> J,A* (#-# L8$G#-+ (#-# L,<* )#$II8.,.:
6
0#"-=$,.: (#-# D,-1 @"#/1* L<=G*
25 22 24 2T 2U !" 2V 2W
!"#$% M#+,/+ N8$%,.: D,-1 '((+ ,. !"#$% @::$*:#I.: (#-# D,-1 )#,$ '((+ N$,I.: #.> (*"<89,.: !"#$% @""<,/#I8.+ )#$#<<*< )$8/*++,.: ,. !"#$% 78/*9 :&& 0-*('()-3208GG8. )#E*$.+ ,. !"#$% (#-# )$8/*++,.: !"#$% !OP #.> (#-#L$#G*+
2S
08./<=+,8.
08=$+* H.-$8>=/I8. H.-$8>=/I8. -8 J#>88"
HG"8$I.: #.> Q8>*<,.: !-$=/-=$*> (#-#
H.:*+I.: !-$*#G,.: (#-#
&'()*'+,)-. &/)/ 0*12-(('34 5')6 78/*9
08=$+* 08./<=+,8.
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#$
08=$+* 01#"-*$+ 2
H.-$8>=/I8.
4 H.-$8>=/I8. -8 J#>88" #.> -1* J#>88" K/8+9+-*G J#>88" @$/1,-*/-=$* #.> J(L! T U 3 V W S
HG"8$I.: '*<#I8.#< (#-# D,-1 @"#/1* !R88" H.-$8>=/I8. -8 HG"#<# #.> J,A* Q8>*<,.: #.> Q#.#:,.: (#-# D,-1 HG"#<# #.> J,A* (#-# L8$G#-+ (#-# L,<* )#$II8.,.:
6
0#"-=$,.: (#-# D,-1 @"#/1* L<=G*
25 22 24 2T 2U !" 2V 2W
!"#$% M#+,/+ N8$%,.: D,-1 '((+ ,. !"#$% @::$*:#I.: (#-# D,-1 )#,$ '((+ N$,I.: #.> (*"<89,.: !"#$% @""<,/#I8.+ )#$#<<*< )$8/*++,.: ,. !"#$% 78/*9 :&& 0-*('()-3208GG8. )#E*$.+ ,. !"#$% (#-# )$8/*++,.: !"#$% !OP #.> (#-#L$#G*+
2S
08./<=+,8.
08=$+* H.-$8>=/I8. H.-$8>=/I8. -8 J#>88"
HG"8$I.: #.> Q8>*<,.: !-$=/-=$*> (#-#
H.:*+I.: !-$*#G,.: (#-#
&'()*'+,)-. &/)/ 0*12-(('34 5')6 78/*9
08=$+* 08./<=+,8.
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#$
!"#$% '(( )*$+,+-*./* <3 )6'( 26/8)-* =1, 5'>> >-/*3 !
?15 78/*9 ,(-( /3 :&&@( >'3-/4- '3 18-*/A13(
!
?15 )1 8-*('() :&&( )1 'B8*1C- 8-*D1*B/32-
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#;
01#"-*$ X8",/+ 78/*9 :&& 0-*('()-32-
!
:&& F'3-/4-
!
'(( )*$+,+-*./* YA*$A,*D
!
(,+-$,C=-*> )*$+,+-*./*
!
08./<=+,8.
!
J8G*D8$%Z )*$+,+- #. '((
&'()*'+,)-. &/)/ 0*12-(('34 5')6 78/*9
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#E
P,.*#:* K[#G"<* \2] !
G/26 !"#$%&'"(#)'$ 18-*/A13 2*-/)-( / 3-5 *+,-. :&&
L,<*Z "=$"<*/8D?-[-
I've never seen a purple cow. I never hope to see one; But I can tell you, anyhow, I'd rather see than be one.
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#"
P,.*#:* K[#G"<* \4] !
G/26 !"#$%&'"(#)'$ 18-*/A13 2*-/)-( / 3-5 *+,-. :&&
L,<*Z "=$"<*/8D?-[-
I've never seen a purple cow. I never hope to see one; But I can tell you, anyhow, I'd rather see than be one.
Q#""*>'((^2_ \G9>#-#] >
mydata = sc.textFile("purplecow.txt")
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#H
P,.*#:* K[#G"<* \T] !
G/26 !"#$%&'"(#)'$ 18-*/A13 2*-/)-( / 3-5 *+,-. :&&
L,<*Z "=$"<*/8D?-[-
I've never seen a purple cow. I never hope to see one; But I can tell you, anyhow, I'd rather see than be one.
Q#""*>'((^2_ \G9>#-#] > >
mydata = sc.textFile("purplecow.txt") myrdd = mydata.map(lambda s: s.upper())\ .filter(lambda s:s.startswith('I')) Q#""*>'((^4_
L,<-*$*>'((^T_Z \G9$>>]
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#I
P,.*#:* K[#G"<* \U] !
!
78/*9 9--8( )*/29 1D )6- /#"0$! :&& D1* -/26 3-5 :&&
L,<*Z "=$"<*/8D?-[-
I've never seen a purple cow. I never hope to see one; But I can tell you, anyhow, I'd rather see than be one.
K6'>. :&&( .0/0$. '$ )6-'* 8/*-3)( Q#""*>'((^2_ \G9>#-#] > >
mydata = sc.textFile("purplecow.txt") myrdd = mydata.map(lambda s: s.upper())\ .filter(lambda s:s.startswith('I')) Q#""*>'((^4_
L,<-*$*>'((^T_Z \G9$>>]
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#J
P,.*#:* K[#G"<* \3] 2*)'$ 18-*/A13( -M-2,)- )68/*-3) )*/3(D1*B/A13(
!
L,<*Z "=$"<*/8D?-[-
I've never seen a purple cow. I never hope to see one; But I can tell you, anyhow, I'd rather see than be one.
Q#""*>'((^2_ \G9>#-#] > >
>
3
mydata = sc.textFile("purplecow.txt") myrdd = mydata.map(lambda s: s.upper())\ .filter(lambda s:s.startswith('I')) myrdd.count()
I've never seen a purple cow. I never hope to see one; But I can tell you, anyhow, I'd rather see than be one. Q#""*>'((^4_
I'VE NEVER SEEN A PURPLE COW. I NEVER HOPE TO SEE ONE; BUT I CAN TELL YOU, ANYHOW, I'D RATHER SEE THAN BE ONE. L,<-*$*>'((^T_Z \G9$>>]
I'VE NEVER SEEN A PURPLE COW. I NEVER HOPE TO SEE ONE; I'D RATHER SEE THAN BE ONE.
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#L
P,.*#:* K[#G"<* \V] !
G/26 /2A13 *-#-M-2,)-( )6- >'3-/4)*/3(D1*B/A13( ()/*A34 5')6 )6+/(– M9 > >
>
>*F#=<-
I've never seen a purple cow. I never hope to see one; But I can tell you, anyhow, I'd rather see than be one.
Q#""*>'((^2_ \G9>#-#]
mydata = sc.textFile("purplecow.txt") myrdd = mydata.map(lambda s: s.upper())\ .filter(lambda s:s.startswith('I')) myrdd.count() Q#""*>'((^4_
3 >
L,<*Z "=$"<*/8D?-[-
myrdd.count()
L,<-*$*>'((^T_Z \G9$>>]
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#!N
P,.*#:* K[#G"<* \W] !
G/26 /2A13 *-#-M-2,)-( )6- >'3-/4)*/3(D1*B/A13( ()/*A34 5')6 )6+/(– M9
>*F#=<-
L,<*Z "=$"<*/8D?-[-
I've never seen a purple cow. I never hope to see one; But I can tell you, anyhow, I'd rather see than be one.
Q#""*>'((^2_ \G9>#-#]
I've never seen a purple cow. > >
>
mydata = sc.textFile("purplecow.txt") myrdd = mydata.map(lambda s: s.upper())\ .filter(lambda s:s.startswith('I')) myrdd.count()
3
But I can tell you, anyhow, I'd rather see than be one. Q#""*>'((^4_
3 >
I never hope to see one;
myrdd.count()
I'VE NEVER SEEN A PURPLE COW. I NEVER HOPE TO SEE ONE; BUT I CAN TELL YOU, ANYHOW, I'D RATHER SEE THAN BE ONE. L,<-*$*>'((^T_Z \G9$>>]
I'VE NEVER SEEN A PURPLE COW. I NEVER HOPE TO SEE ONE; I'D RATHER SEE THAN BE ONE.
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#!!
01#"-*$ X8",/+ 78/*9 :&& 0-*('()-32-
!
'(( P,.*#:*
!
:&& 0-*('()-32- OC-*C'-5
!
(,+-$,C=-*> )*$+,+-*./*
!
08./<=+,8.
!
J8G*D8$%Z )*$+,+- #. '((
&'()*'+,)-. &/)/ 0*12-(('34 5')6 78/*9
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#!$
'(( )*$+,+-*./* !
0-*('(A34 /3 :&& (/C-( )6- ./)/ P+= .-D/,>) '3 B-B1*=Q
L,<*Z "=$"<*/8D?-[-
I've never seen a purple cow. I never hope to see one; But I can tell you, anyhow, I'd rather see than be one.
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#!;
'(( )*$+,+-*./* !
0-*('(A34 /3 :&& (/C-( )6- ./)/ P+= .-D/,>) '3 B-B1*=Q
L,<*Z "=$"<*/8D?-[-
I've never seen a purple cow. I never hope to see one; But I can tell you, anyhow, I'd rather see than be one.
'((^2_ \G9>#-#]
mydata = sc.textFile("purplecow.txt") > myrdd1 = mydata.map(lambda s: s.upper()) >
'((^4_ \G9$>>2]
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#!E
'(( )*$+,+-*./* !
0-*('(A34 /3 :&& (/C-( )6- ./)/ P+= .-D/,>) '3 B-B1*=Q
L,<*Z "=$"<*/8D?-[-
I've never seen a purple cow. I never hope to see one; But I can tell you, anyhow, I'd rather see than be one.
'((^2_ \G9>#-#]
mydata = sc.textFile("purplecow.txt") > myrdd1 = mydata.map(lambda s: s.upper()) > myrdd1.persist() >
'((^4_ \G9$>>2]
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#!"
'(( )*$+,+-*./* !
0-*('(A34 /3 :&& (/C-( )6- ./)/ P+= .-D/,>) '3 B-B1*=Q
L,<*Z "=$"<*/8D?-[-
I've never seen a purple cow. I never hope to see one; But I can tell you, anyhow, I'd rather see than be one.
'((^2_ \G9>#-#]
mydata = sc.textFile("purplecow.txt") > myrdd1 = mydata.map(lambda s: s.upper()) > myrdd1.persist() > myrdd2 = myrdd1.filter(lambda \ s:s.startswith('I')) >
'((^4_ \G9$>>2]
'((^T_ \G9$>>4]
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#!H
'(( )*$+,+-*./* !
0-*('(A34 /3 :&& (/C-( )6- ./)/ P+= .-D/,>) '3 B-B1*=Q
L,<*Z "=$"<*/8D?-[-
I've never seen a purple cow. I never hope to see one; But I can tell you, anyhow, I'd rather see than be one.
'((^2_ \G9>#-#]
mydata = sc.textFile("purplecow.txt") > myrdd1 = mydata.map(lambda s: s.upper()) > myrdd1.persist() > myrdd2 = myrdd1.filter(lambda \ s:s.startswith('I')) > myrdd2.count() 3 >
I've never seen a purple cow. I never hope to see one; But I can tell you, anyhow, I'd rather see than be one. '((^4_ \G9$>>2]
I'VE NEVER SEEN A PURPLE COW. I NEVER HOPE TO SEE ONE; BUT I CAN TELL YOU, ANYHOW, I'D RATHER SEE THAN BE ONE. '((^T_ \G9$>>4]
I'VE NEVER SEEN A PURPLE COW. I NEVER HOPE TO SEE ONE; I'D RATHER SEE THAN BE ONE.
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#!I
'(( )*$+,+-*./* !
7,+(-R,-3) 18-*/A13( ,(- (/C-. ./)/
L,<*Z "=$"<*/8D?-[-
I've never seen a purple cow. I never hope to see one; But I can tell you, anyhow, I'd rather see than be one.
'((^2_ \G9>#-#]
mydata = sc.textFile("purplecow.txt") > myrdd1 = mydata.map(lambda s: s.upper()) > myrdd1.persist() > myrdd2 = myrdd1.filter(lambda \ s:s.startswith('I')) > myrdd2.count() 3 > myrdd2.count() >
'((^4_ \G9$>>2]
I'VE NEVER SEEN A PURPLE COW. I NEVER HOPE TO SEE ONE; BUT I CAN TELL YOU, ANYHOW, I'D RATHER SEE THAN BE ONE. '((^T_ \G9$>>4]
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#!J
'(( )*$+,+-*./* !
7,+(-R,-3) 18-*/A13( ,(- (/C-. ./)/
L,<*Z "=$"<*/8D?-[-
I've never seen a purple cow. I never hope to see one; But I can tell you, anyhow, I'd rather see than be one.
'((^2_ \G9>#-#]
my data = sc.textFile("purplecow.txt") > myrdd1 = mydata.map(lambda s: s.upper()) > myrdd1.persist() > myrdd2 = myrdd1.filter(lambda \ s:s.startswith('I')) > myrdd2.count() 3 > myrdd2.count() 3 >
'((^4_ \G9$>>2]
I'VE NEVER SEEN A PURPLE COW. I NEVER HOPE TO SEE ONE; BUT I CAN TELL YOU, ANYHOW, I'D RATHER SEE THAN BE ONE. '((^T_ \G9$>>4]
I'VE NEVER SEEN A PURPLE COW. I NEVER HOPE TO SEE ONE; I'D RATHER SEE THAN BE ONE.
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#!L
Q*G8$9 )*$+,+-*./* !
<3#B-B1*= 8-*('()-32- '( / %3440%)'$ )1 78/*9 – HF
.8- *.8=:1 G*G8$9 ,+ #A#,<#C<*` "*$+,+-*> "#$II8.+ D,<< C* /<*#$*> F$8G G*G8$9 – P*#+-
–
$*/*.-<9 =+*> "#$II8.+ /<*#$*> a$+-
X$#.+F8$G#I8.+ D,<< C* $*;*[*/=-*> =+,.: -1* <,.*#:* D1*. .**>*>
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#$N
01#"-*$ X8",/+ 78/*9 :&& 0-*('()-32-
!
'(( P,.*#:*
!
'(( )*$+,+-*./* YA*$A,*D
!
&'()*'+,)-. 0-*('()-32-
!
08./<=+,8.
!
J8G*D8$%Z )*$+,+- #. '((
&'()*'+,)-. &/)/ 0*12-(('34 5')6 78/*9
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#$!
)*$+,+-*./* #.> L#=<-;X8<*$#./* !
:&& S 50%,-,0$! &'()*'+,)-. &/)/(-) – '*+,<,*./9 – '((+
,+ # "$8>=/- 8F -$#/%,.: <,.*#:*
/#. # F$8G -1*,$ C#+* ,F .**>*>
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#$$
(,+-$,C=-*> )*$+,+-*./* !
:&& 8/*AA13( /*- .'()*'+,)-. /2*1(( / 2>,()-*
!
T= .-D/,>)U 8/*AA13( /*- 8-*('()-. '3 B-B1*= '3 GM-2,)1* VWX( '(( ($,A*$
K[*/=-8$
-#+%
$>>b2b5
K[*/=-8$
-#+%
$>>b2b2
K[*/=-8$
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#$;
'(( L#=<-;X8<*$#./* \2] !
Y6/) 6/88-3( 'D / 8/*AA13 8-*('()-. '3 B-B1*= +-21B-( ,3/C/'>/+>-Z
'(( ($,A*$
K[*/=-8$
-#+%
$>>b2b5
c
K[*/=-8$
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#$E
'(( L#=<-;X8<*$#./* \4] !
[6- .*'C-* ()/*)( / 3-5 )/(9 )1 *-21B8,)- )6- 8/*AA13 13 / .'\-*-3) 31.-
!
F'3-/4- '( 8*-(-*C-.U ./)/ '( 3-C-* >1() '(( ($,A*$
K[*/=-8$
-#+%
$>>b2b5
K[*/=-8$
-#+%
$>>b2b2
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#$"
)*$+,+-*./* P*A*<+ !
T= .-D/,>)U )6- persist B-)61. ()1*-( ./)/ '3 B-B1*= 13>= – X1*
cache G*-18> ,+ # +9.8.9G F8$ >*F#=<- \G*G8$9] "*$+,+-
!
[6- persist B-)61. 1\-*( 1)6-* 18A13( 2/>>-. 7)1*/4- F-C->(
!
7)1*/4- F-C->( >-) =1, 213)*1> – !-8$#:*
<8/#I8.
– L8$G#-
,. G*G8$9
– )#$II8.
$*"<,/#I8.
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#$H
)*$+,+-*./* P*A*<+Z !-8$#:* P8/#I8. !
7)1*/4- >12/A13 ] 56-*- '( )6- ./)/ ()1*-.Z
MEMORY_ONLY \>*F#=<-] d +#G* #+ cache – MEMORY_AND_DISK d !-8$* "#$II8.+ 8. >,+% ,F -1*9 >8 .8- a- ,. –
G*G8$9 – 0#<<*> –
!"#$$#%&
DISK_ONLY d !-8$* #<< "#$II8.+ 8. >,+% from pyspark import StorageLevel > myrdd.persist(StorageLevel.DISK_ONLY) >
)9-18.
!/#<#
import org.apache.spark.storage.StorageLevel > myrdd.persist(StorageLevel.DISK_ONLY) >
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#$I
)*$+,+-*./* P*A*<+Z Q*G8$9 L8$G#!
7-*'/>'^/A13 ] =1, 2/3 2611(- )1 (-*'/>'^- )6- ./)/ '3 B-B1*= #.> MEMORY_AND_DISK_SER MEMORY_ONLY_SER
–
– Q=/1 – P*++
G8$* +"#/* *e/,*.-
IG* *e/,*.-
– HF
=+,.: f#A# 8$ !/#<#` /188+* # F#+- +*$,#<,g#I8. <,C$#$9 \*?:? h$98]
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#$J
)*$+,+-*./* P*A*<+Z )#$II8. '*"<,/#I8. !
:-8>'2/A13 ] ()1*- 8/*AA13( 13 )51 31.-(
MEMORY_ONLY_2 – MEMORY_AND_DISK_2 – DISK_ONLY_2 MEMORY_AND_DISK_SER_2 – – DISK_ONLY_2 –
– i8=
/#. #<+8 >*a.* /=+-8G +-8$#:* <*A*<+
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#$L
01#.:,.: )*$+,+-*./* Y"I8.+ !
[1 ()18 8-*('(A34 /3. *-B1C- D*1B B-B1*= /3. .'(9 –
!
rdd.unpersist()
[1 26/34- /3 :&& )1 / .'\-*-3) 8-*('()-32- >-C-> – j."*$+,+-
a$+-
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#;N
(,+% )*$+,+-*./* !
&'(9#8-*('()-. 8/*AA13( /*- ()1*-. '3 >12/> _>-(
'(( ($,A*$
0<,*.-
K[*/=-8$
-#+%
$>>b5
K[*/=-8$
-#+%
$>>b2
$>>b2
K[*/=-8$
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#;!
(,+% )*$+,+-*./* D,-1 '*"<,/#I8. \2] !
0-*('()-32- *-8>'2/A13 B/9-( *-21B8,)/A13 >-(( >'9->= )1 +- 3-2-((/*=
'(( ($,A*$
0<,*.-
K[*/=-8$
-#+%
$>>b5
K[*/=-8$
-#+%
$>>b2
$>>b2
K[*/=-8$ $>>b2
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#;$
(,+% )*$+,+-*./* D,-1 '*"<,/#I8. \4] !
:-8>'2/)-. ./)/ 13 .'(9 5'>> +- ,(-. )1 *-2*-/)- )6- 8/*AA13 'D 81(('+>– N,<<
C* $*/8G"=-*> ,F -1* >#-# ,+ =.#A#,<#C<*
– *?:?`
-1* .8>* ,+ >8D. '((
($,A*$
0<,*.-
K[*/=-8$
-#+%
$>>b5
c
K[*/=-8$
-#+%
$>>b2
$>>b2
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#;;
N1*. #.> N1*$* -8 )*$+,+!
Y6-3 (61,>. =1, 8-*('() / ./)/(-)Z – N1*.
# >#-#+*- ,+ <,%*<9 -8 C* $*;=+*>
– *?:?` !
,-*$#IA* #<:8$,-1G+` G#/1,.* <*#$.,.:
?15 )1 2611(- / 8-*('()-32- >-C-> – Q*G8$9 – !#A* – (,+%
8.<9 d D1*. "8++,C<*` C*+- "*$F8$G#./* +"#/* C9 +#A,.: #+ +*$,#<,g*> 8Ck*/-+ ,. G*G8$9 ,F .*/*++#$9
d /188+* D1*. $*/8G"=-#I8. ,+ G8$* *["*.+,A* -1#. >,+% $*#>
– *?:?`
*["*.+,A* F=./I8.+ 8$ a<-*$,.: <#$:* >#-#+*-+
– '*"<,/#I8.
d /188+* D1*. $*/8G"=-#I8. ,+ G8$* *["*.+,A* -1#.
G*G8$9
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#;E
01#"-*$ X8",/+ 78/*9 :&& 0-*('()-32-
!
'(( P,.*#:*
!
'(( )*$+,+-*./* YA*$A,*D
!
(,+-$,C=-*> )*$+,+-*./*
!
K132>,('13
!
J8G*D8$%Z )*$+,+- #. '((
&'()*'+,)-. &/)/ 0*12-(('34 5')6 78/*9
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#;"
K++*.I#< )8,.-+ !
78/*9 9--8( )*/29 1D -/26 :&&@( >'3-/4– )$8A,>*+
F#=<- -8<*$#./*
!
T= .-D/,>)U -C-*= :&& 18-*/A13 -M-2,)-( )6- -3A*- >'3-/4-
!
> +- ,(-. B,>A8>- AB-(U 8-*('() ') )1 /C1'. *-#21B8,)/A13
!
0-*('()-32- 18A13( – P8/#I8.
d G*G8$9 8.<9` G*G8$9 #.> >,+% ` >,+% 8.<9
– L8$G#-
d ,.;G*G8$9 >#-# /#. C* +*$,#<,g*> -8 +#A* G*G8$9 \C=- #- -1* /8+- 8F "*$F8$G#./*]
– '*"<,/#I8.
d +#A*+ >#-# 8. G=*+ ,. /#+* # .8>* :8*+ >8D.` F8$ k8C $*/8A*$9 D,-18=- $*/8G"=-#I8.
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#;H
01#"-*$ X8",/+ 78/*9 :&& 0-*('()-32-
!
'(( P,.*#:*
!
'(( )*$+,+-*./* YA*$A,*D
!
(,+-$,C=-*> )*$+,+-*./*
!
08./<=+,8.
!
?1B-51*9` 0-*('() /3 :&&
&'()*'+,)-. &/)/ 0*12-(('34 5')6 78/*9
7 08"9$,:1- 4525;4523 0<8=>*$#? @<< $,:1-+ $*+*$A*>? B8- -8 C* $*"$8>=/*> 8$ +1#$*> D,-18=- "$,8$ D$,E*. /8.+*.- F$8G 0<8=>*$#?
!"#;I
