10-SparkBasics

+5:$(. +,#"-.$( /

FC-$5;:*G5C

+5:$(. FC-$5;:*G5C

1 FC-$5;:*G5C -5 O#;55" #C; -,. O#;55" P*5(6(-.E O#;55" =$*,)-.*-:$. #C; OJN! T

FC-$5;:*G5C -5 O#;55"

FE"5$GC7 I.9#G5C#9 J#-# A)-, ="#*,. !R55" U FC-$5;:*G5C -5 FE"#9# #C; O)>. 2 V Q5;.9)C7 #C; Q#C#7)C7 J#-# A)-, FE"#9# #C; O)>. J#-# N5$E#-( W J#-# N)9. K#$GG5C)C7 S

FE"5$GC7 #C; Q5;.9)C7 !-$:*-:$.; J#-#

3

+#"-:$)C7 J#-# A)-, ="#*,. N9:E.

!" // /1 /T /U /2 /V /W

78/*9 :/('2( H5$%)C7 A)-, IJJ( )C !"#$% =77$.7#GC7 J#-# A)-, K#)$ IJJ( H$)GC7 #C; J."956)C7 !"#$% =""9)*#G5C( K#$#99.9 K$5*.(()C7 )C !"#$% !"#$% IJJ K.$()(-.C*. +5EE5C K#B.$C( )C !"#$% !"#$% !LM #C; J#-#N$#E.(

/S

+5C*9:()5C

FC7.(GC7 !-$.#E)C7 J#-#

&'()*'+,)-. &/)/ 0*12-(('34 5')6 78/*9

+5:$(. +5C*9:()5C

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#$

+5:$(. +,#"-.$( /

FC-$5;:*G5C

+5:$(. FC-$5;:*G5C

1 FC-$5;:*G5C -5 O#;55" #C; -,. O#;55" P*5(6(-.E O#;55" =$*,)-.*-:$. #C; OJN! T

FC-$5;:*G5C -5 O#;55"

FE"5$GC7 I.9#G5C#9 J#-# A)-, ="#*,. !R55" U FC-$5;:*G5C -5 FE"#9# #C; O)>. 2 V Q5;.9)C7 #C; Q#C#7)C7 J#-# A)-, FE"#9# #C; O)>. J#-# N5$E#-( W J#-# N)9. K#$GG5C)C7 S

FE"5$GC7 #C; Q5;.9)C7 !-$:*-:$.; J#-#

3

+#"-:$)C7 J#-# A)-, ="#*,. N9:E.

!" // /1 /T /U /2 /V /W

78/*9 :/('2( H5$%)C7 A)-, IJJ( )C !"#$% =77$.7#GC7 J#-# A)-, K#)$ IJJ( H$)GC7 #C; J."956)C7 !"#$% =""9)*#G5C( K#$#99.9 K$5*.(()C7 )C !"#$% !"#$% IJJ K.$()(-.C*. +5EE5C K#B.$C( )C !"#$% !"#$% !LM #C; J#-#N$#E.(

/S

+5C*9:()5C

FC7.(GC7 !-$.#E)C7 J#-#

&'()*'+,)-. &/)/ 0*12-(('34 5')6 78/*9

+5:$(. +5C*9:()5C

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#$

!"#$% '#()*( <3 )6'( 26/8)-* =1, 5'>> >-/*3 !

?15 )1 ()/*) )6- 78/*9 76->>

!

@+1,) )6- 78/*9A13)-B)

!

C-= A132-8)( 1D E-('>'-3) &'()*'+,)-. &/)/(-)( FE&&(G – H,#-

#$. -,.6X – O5A ;5 65: *$.#-. -,.EX – H,#- 5".$#G5C( *#C 65: ".$D5$E A)-, -,.EX !

?15 78/*9 ,(-( )6- 8*'32'8>-( 1D D,32H13/> 8*14*/II'34

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#;

+,#"-.$ Y5")*( 78/*9 :/('2(

!

K6/) '( @8/26- 78/*9L

!

Z()C7 -,. !"#$% !,.99

!

IJJ( [I.()9).C- J)(-$)@:-.; J#-#(.-(\

!

N:C*G5C#9 K$57$#EE)C7 )C !"#

%$!

+5C*9:()5C

!

O5E.A5$% =(()7CE.C-(

&'()*'+,)-. &/)/ 0*12-(('34 5')6 78/*9

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#J

H,#- )( ="#*,. !"#$%X !

!

@8/26- 78/*9 '( / D/() /3. 4-3-*/> -34'3- D1* >/*4-#(2/>./)/ 8*12-(('34 K*'N-3 '3 72/>/ – N:C*G5C#9

!

"$57$#EE)C7 9#C7:#7. -,#- $:C( )C # ]^Q

78/*9 76->> – FC-.$#*G>.

_ D5$ 9.#$C)C7 5$ ;#-# .`"95$#G5C – K6-,5C 5$ !*#9# !

78/*9 @88>'2/H13( – N5$

9#$7. (*#9. ;#-# "$5*.(()C7 – K6-,5Ca !*#9#a 5$ ]#>#

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#M

+,#"-.$ Y5")*( 78/*9 :/('2(

!

H,#- )( ="#*,. !"#$%X

!

P('34 )6- 78/*9 76->>

!

IJJ( [I.()9).C- J)(-$)@:-.; J#-#(.-(\

!

N:C*G5C#9 K$57$#EE)C7 )C !"#

%$!

+5C*9:()5C

!

O5E.A5$% =(()7CE.C-(

&'()*'+,)-. &/)/ 0*12-(('34 5')6 78/*9

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#O

!"#$% !,.99 !

R6- 78/*9 76->> 8*1S'.-( '3)-*/2HS- ./)/ -B8>1*/H13 FET0UG

!

K*'H34 78/*9 /88>'2/H13( 5')61,) )6- (6->> 5'>> +- 21S-*-. >/)-* K6-,5C !,.99b pyspark

$ pyspark

!*#9# !,.99b spark-shell $ spark-shell

Welcome to ____ __ / __/__ ___ _____/ /__ _\ \/ _ \/ _ `/ __/ '_/ /__ / .__/\_,_/_/ /_/\_\ /_/

version 1.3.0

Using Python version 2.7.8 (default, Aug 27 2015 05:23:36) SparkContext available as sc, HiveContext available as sqlCtx. >>>

IPKMb I.#;cP>#9:#-.cK$)C- M55"

Welcome to ____ __ / __/__ ___ _____/ /__ _\ \/ _ \/ _ `/ __/ '_/ /___/ .__/\_,_/_/ /_/\_\ /_/

version 1.3.0

Using Scala version 2.10.4 (Java HotSpot(TM) 64-Bit Server VM, Java 1.7.0_67) Created spark context.. Spark context available as sc. SQL context available as sqlContext. scala>

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#Q

!"#$% +5C-.`!

TS-*= 78/*9 /88>'2/H13 *-W,'*-( / 78/*9 A13)-B) – Y,.

!

E#)C .C-$6 "5)C- -5 -,. !"#$% =KF

78/*9 76->> 8*1S'.-( / 8*-213X4,*-. 78/*9 A13)-B) 2/>>-. sc Using Python version 2.7.8 (default, Aug 27 2015 05:23:36) SparkContext available as sc, HiveContext available as sqlCtx. K6-,5C

>>> sc.appName u'PySparkShell'

… Spark context available as sc. SQL context available as sqlContext. !*#9#

scala> sc.appName res0: String = Spark shell

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#V

+,#"-.$ Y5")*( 78/*9 :/('2(

&'()*'+,)-. &/)/ 0*12-(('34 5')6 78/*9

!

H,#- )( ="#*,. !"#$%X

!

Z()C7 -,. !"#$% !,.99

!

E&&( FE-('>'-3) &'()*'+,)-. &/)/(-)(G

!

N:C*G5C#9 K$57$#EE)C7 H)-, !"#

%$!

+5C*9:()5C

!

O5E.A5$% =(()7CE.C-(

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#Y

IJJ [I.()9).C- J)(-$)@:-.; J#-#(.-\ !

E&& FE-('>'-3) &'()*'+,)-. &/)/(-)G – I.()9).C-

_ )D ;#-# )C E.E5$6 )( 95(-a )- *#C @. $.*$.#-.; – J)(-$)@:-.; _ "$5*.((.; #*$5(( -,. *9:(-.$ – J#-#(.- _ )C)G#9 ;#-# *#C *5E. D$5E # d9. 5$ @. *$.#-.; "$57$#EE#G*#996 !

E&&( /*- )6- D,3./I-3)/> ,3') 1D ./)/ '3 78/*9

!

Z1() 78/*9 8*14*/II'34 213('()( 1D 8-*D1*I'34 18-*/H13( 13 E&&(

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#!"

+$.#GC7 #C IJJ !

R6*-- 5/=( )1 2*-/)- /3 E&& – N$5E

# d9. 5$ (.- 5D d9.( – N$5E ;#-# )C E.E5$6 – N$5E #C5-,.$ IJJ

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#!!

P`#E"9.b = N)9.8@#(.; IJJ N)9.b ":$"9.*5A<-`>

val mydata = sc.textFile("purplecow.txt")

… 15/01/29 06:20:37 INFO storage.MemoryStore: Block broadcast_0 stored as values to memory (estimated size 151.4 KB, free 296.8 MB) >

mydata.count()

… 15/01/29 06:27:37 INFO spark.SparkContext: Job finished: take at :1, took 0.160482078 s 4

I've never seen a purple cow. I never hope to see one; But I can tell you, anyhow, I'd rather see than be one.

IJJb E6;#-#


4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#!$

IJJ e".$#G5C( !

R51 )=8-( 1D E&& 18-*/H13( IJJ

– =*G5C(

!"#

%$_ $.-:$C >#9:.(

'#(. IJJ

?.A IJJ

– Y$#C(D5$E#G5C(

_ ;.dC. # C.A IJJ @#(.; 5C -,. *:$$.C- 5C.[(\

!

018 W,'[\ – H,)*,

-6". 5D 5".$#G5C )( count() X

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#!;

IJJ e".$#G5C(b =*G5C( !

71I- 21II13 /2H13(

IJJ

count() _ $.-:$C -,. C:E@.$ 5D .9.E.C-( – take( n) _ $.-:$C #C #$$#6 5D -,. d$(- ' –

!"#

%$.9.E.C-(

collect() _ $.-:$C #C #$$#6 5D #99 .9.E.C-( – saveAsTextFile( file) _ (#>. -5 -.`–

d9.[(\ >

mydata = sc.textFile("purplecow.txt")

>

>

mydata.count()

>

4

4

for line in mydata.take(2): print line I've never seen a purple cow. I never hope to see one;

>

>

val mydata = sc.textFile("purplecow.txt") mydata.count()

for (line <- mydata.take(2)) println(line) I've never seen a purple cow. I never hope to see one;

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#!J

IJJ e".$#G5C(b Y$#C(D5$E#G5C( !

!

R*/3(D1*I/H13( 2*-/)- / 3-5 E&& D*1I /3 -B'(H34 13-

'#(. IJJ

?.A IJJ

E&&( /*- 'II,)/+>– J#-#

!

)C #C IJJ )( C.>.$ *,#C7.; – Y$#C(D5$E )C (.R:.C*. -5 E5;)D6 -,. ;#-# #( C..;.; 71I- 21II13 )*/3(D1*I/H13(

map(function) _ *$.#-.( # C.A IJJ @6 ".$D5$E)C7 # D:C*G5C 5C

–

.#*, $.*5$; )C -,. @#(. IJJ – filter( function) _ *$.#-.( # C.A IJJ @6 )C*9:;)C7 5$ .`*9:;)C7 .#*, $.*5$; )C -,. @#(. IJJ #**5$;)C7 -5 # @559.#C D:C*G5C

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#!M

P`#E"9.b map #C; filter Y$#C(D5$E#G5C( I've never seen a purple cow. I never hope to see one; But I can tell you, anyhow, I'd rather see than be one. map(lambda line: line.upper())

map(line => line.toUpperCase)

I'VE NEVER SEEN A PURPLE COW. I NEVER HOPE TO SEE ONE; BUT I CAN TELL YOU, ANYHOW, I'D RATHER SEE THAN BE ONE.

filter(lambda line: line.startswith('I'))

filter(line => line.startsWith('I'))

I'VE NEVER SEEN A PURPLE COW. I NEVER HOPE TO SEE ONE; I'D RATHER SEE THAN BE ONE.

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#!O

M#f6 P`.*:G5C [/\ !

&/)/ '3 E&&( '( 31) 8*12-((-. ,3H> /3 !"#$% '( 8-*D1*I-.

N)9.b ":$"9.*5A<-`-


>

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#!Q

M#f6 P`.*:G5C [1\ !

&/)/ '3 E&&( '( 31) 8*12-((-. ,3H> /3 !"#$% '( 8-*D1*I-.

N)9.b ":$"9.*5A<-`-


IJJb E6;#-# >

val mydata = sc.textFile("purplecow.txt")

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#!V

M#f6 P`.*:G5C [T\ !

&/)/ '3 E&&( '( 31) 8*12-((-. ,3H> /3 !"#$% '( 8-*D1*I-.

N)9.b ":$"9.*5A<-`-


IJJb E6;#-# > >

val mydata = sc.textFile("purplecow.txt") val mydata_uc = mydata.map(line => line.toUpperCase()) IJJb E6;#-#g:*

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#!Y

M#f6 P`.*:G5C [U\ !

&/)/ '3 E&&( '( 31) 8*12-((-. ,3H> /3 !"#$% '( 8-*D1*I-.

N)9.b ":$"9.*5A<-`-


IJJb E6;#-# > >

>

val mydata = sc.textFile("purplecow.txt") val mydata_uc = mydata.map(line => line.toUpperCase()) val mydata_filt = mydata_uc.filter(line => line.startsWith("I"))

IJJb E6;#-#g:*

IJJb E6;#-#gd9-

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#$"

M#f6 P`.*:G5C [2\ !

&/)/ '3 E&&( '( 31) 8*12-((-. ,3H> /3 !"#$% '( 8-*D1*I-.

val mydata = sc.textFile("purplecow.txt") > val mydata_uc = mydata.map(line => line.toUpperCase()) > val mydata_filt = mydata_uc.filter(line => line.startsWith("I")) > mydata_filt.count() 3 >

N)9.b ":$"9.*5A<-`-


IJJb E6;#-# I've never seen a purple cow.

I never hope to see one; But I can tell you, anyhow, I'd rather see than be one. IJJb E6;#-#g:* I'VE NEVER SEEN A PURPLE COW.

I NEVER HOPE TO SEE ONE; BUT I CAN TELL YOU, ANYHOW, I'D RATHER SEE THAN BE ONE. IJJb E6;#-#gd9-

I'VE NEVER SEEN A PURPLE COW. I NEVER HOPE TO SEE ONE; I'D RATHER SEE THAN BE ONE.

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#$!

+,#)C)C7 Y$#C(D5$E#G5C( [!*#9#\ !

R*/3(D1*I/H13( I/= +- 26/'3-. )14-)6-* val mydata = sc.textFile("purplecow.txt") > val mydata_uc = mydata.map(line => line.toUpperCase()) > val mydata_filt = mydata_uc.filter(line => line.startsWith("I")) > mydata_filt.count() 3 >

)( .`#*-96 .R:)>#9.C- -5

>

sc.textFile("purplecow.txt").map(line => line.toUpperCase()). filter(line => line.startsWith("I")).count()

3

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#$$

+,#)C)C7 Y$#C(D5$E#G5C( [K6-,5C\ !

7/I- -B/I8>- '3 0=)613 mydata = sc.textFile("purplecow.txt") > mydata_uc = mydata.map(lambda s: s.upper()) > mydata_filt = mydata_uc.filter(lambda s: s.startswith('I')) > mydata_filt.count() 3 >

)( .`#*-96 .R:)>#9.C- -5

>

sc.textFile("purplecow.txt").map(lambda line: line.upper()) \ .filter(lambda line: line.startswith('I')).count()

3

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#$;

IJJ M)C.#7. #C; toDebugString [!*#9#\ 78/*9 I/'3)/'3( -/26 E&&]( '(%)!*) ^ )6- 8*-S'1,( E&&( 13 56'26 ') .-8-3.(

!

!

P(- toDebugString )1 S'-5 )6>'3-/4- 1D /3 E&&

val mydata_filt = sc.textFile("purplecow.txt"). map(line => line.toUpperCase()). filter(line => line.startsWith("I")) > mydata_filt.toDebugString

N)9.b ":$"9.*5A<-`-


IJJh2i

>

(2) | | |

FilteredRDD[7] at filter … MappedRDD[6] at map … purplecow.txt MappedRDD[5] … purplecow.txt HadoopRDD[4] …

IJJhVi

IJJhWi

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#$J

IJJ M)C.#7. #C; toDebugString [K6-,5C\ !

toDebugString 1,)8,) '( 31) .'(8>/=-. /( 3'2->= '3 0=)613

mydata_filt.toDebugString() (1) PythonRDD[8] at RDD at …\n | purplecow.txt MappedRDD[7] at textFile at …[]\n | purplecow.txt HadoopRDD[6] at textFile at …[] >

!

P(- print D1* 8*-_-* 1,)8,)

print mydata_filt.toDebugString() (1) PythonRDD[8] at RDD at … | purplecow.txt MappedRDD[7] at textFile at … | purplecow.txt HadoopRDD[6] at textFile at … >

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#$M

K)".9)C)C7 [/\ !

K6-3 81(('+>-` 78/*9 5'>> 8-*D1*I (-W,-32-( 1D )*/3(D1*I/H13( += *15 (1 31 ./)/ '( ()1*-.

val mydata = sc.textFile("purplecow.txt") > val mydata_uc = mydata.map(line => line.toUpperCase()) > val mydata_filt = mydata_uc.filter(line => line.startsWith("I")) > mydata_filt.take(2) >

N)9.b ":$"9.*5A<-`-


I've never seen a purple cow.

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#$O

K)".9)C)C7 [1\ !

K6-3 81(('+>-` 78/*9 5'>> 8-*D1*I (-W,-32-( 1D )*/3(D1*I/H13( += *15 (1 31 ./)/ '( ()1*-.

val mydata = sc.textFile("purplecow.txt") map(line => > val mydata_uc = mydata. line.toUpperCase()) > val mydata_filt = mydata_uc.filter(line => line.startsWith("I")) > mydata_filt.take(2)

N)9.b ":$"9.*5A<-`-


>

I'VE NEVER SEEN A PURPLE COW.

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#$Q

K)".9)C)C7 [T\ !

K6-3 81(('+>-` 78/*9 5'>> 8-*D1*I (-W,-32-( 1D )*/3(D1*I/H13( += *15 (1 31 ./)/ '( ()1*-.

N)9.b ":$"9.*5A<-`-


val mydata = sc.textFile("purplecow.txt") > val mydata_uc = mydata.map(line => line.toUpperCase()) > val mydata_filt = mydata_uc.filter(line => line.startsWith("I")) > mydata_filt.take(2) >

I'VE NEVER SEEN A PURPLE COW.

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#$V

K)".9)C)C7 [U\ !

K6-3 81(('+>-` 78/*9 5'>> 8-*D1*I (-W,-32-( 1D )*/3(D1*I/H13( += *15 (1 31 ./)/ '( ()1*-.

N)9.b ":$"9.*5A<-`-


val mydata = sc.textFile("purplecow.txt") > val mydata_uc = mydata.map(line => line.toUpperCase()) > val mydata_filt = mydata_uc.filter(line => line.startsWith("I")) > mydata_filt.take(2) I'VE NEVER SEEN A PURPLE COW. >

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#$Y

K)".9)C)C7 [2\ !

K6-3 81(('+>-` 78/*9 5'>> 8-*D1*I (-W,-32-( 1D )*/3(D1*I/H13( += *15 (1 31 ./)/ '( ()1*-.


N)9.b ":$"9.*5A<-`-


I never hope to see one;

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#;"

K)".9)C)C7 [V\ !

K6-3 81(('+>-` 78/*9 5'>> 8-*D1*I (-W,-32-( 1D )*/3(D1*I/H13( += *15 (1 31 ./)/ '( ()1*-.

val mydata = sc.textFile("purplecow.txt") map(line => > val mydata_uc = mydata. line.toUpperCase()) > val mydata_filt = mydata_uc.filter(line => line.startsWith("I")) > mydata_filt.take(2) I'VE NEVER SEEN A PURPLE COW.

N)9.b ":$"9.*5A<-`-


>

I NEVER HOPE TO SEE ONE;

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#;!

K)".9)C)C7 [W\ !

K6-3 81(('+>-` 78/*9 5'>> 8-*D1*I (-W,-32-( 1D )*/3(D1*I/H13( += *15 (1 31 ./)/ '( ()1*-.

N)9.b ":$"9.*5A<-`-



I NEVER HOPE TO SEE ONE;

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#;$

K)".9)C)C7 [S\ !

K6-3 81(('+>-` 78/*9 5'>> 8-*D1*I (-W,-32-( 1D )*/3(D1*I/H13( += *15 (1 31 ./)/ '( ()1*-.

N)9.b ":$"9.*5A<-`-


val mydata = sc.textFile("purplecow.txt") > val mydata_uc = mydata.map(line => line.toUpperCase()) > val mydata_filt = mydata_uc.filter(line => line.startsWith("I")) > mydata_filt.take(2) I'VE NEVER SEEN A PURPLE COW. I NEVER HOPE TO SEE ONE; >

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#;;

+,#"-.$ Y5")*( 78/*9 :/('2(

!

H,#- )( ="#*,. !"#$%X

!

Z()C7 -,. !"#$% !,.99

!

IJJ( [I.()9).C- J)(-$)@:-.; J#-#(.-(\

!

a,32H13/> 0*14*/II'34 '3 78/*9

!

+5C*9:()5C

!

O5E.A5$% =(()7CE.C-(

&'()*'+,)-. &/)/ 0*12-(('34 5')6 78/*9

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#;J

N:C*G5C#9 K$57$#EE)C7 )C !"#$% !

78/*9 .-8-3.( 6-/S'>= 13 )6- 2132-8)( 1D +,%"#$%!' -.$*.!//(%* – N:C*G5C(

#$. -,. D:C;#E.C-#9 :C)- 5D "$57$#EE)C7 – N:C*G5C( ,#>. )C":- #C; 5:-":- 5C96 – ?5 (-#-. 5$ ();. .j.*-( !

C-= 2132-8)( – K#(()C7

D:C*G5C( #( )C":- -5 5-,.$ D:C*G5C( – =C5C6E5:( D:C*G5C(

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#;M

K#(()C7 N:C*G5C( #( K#$#E.-.$( !

Z/3= E&& 18-*/H13( )/9- D,32H13( /( 8/*/I-)-*(

!

0(-,.121.- D1* )6- E&& I/8 18-*/H13 – =""9).(

D:C*G5C fn -5 .#*, $.*5$; )C -,. IJJ

RDD { map( fn(x)) { foreach record in rdd emit fn(record) } }

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#;O

P`#E"9.b K#(()C7 ?#E.; N:C*G5C( !

0=)613 def toUpper(s): return s.upper() > mydata = sc.textFile("purplecow.txt") > mydata.map(toUpper).take(2) >

!

72/>/ def toUpper(s: String): String = { s.toUpperCase } > val mydata = sc.textFile("purplecow.txt") > mydata.map(toUpper).take(2) >

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#;Q

=C5C6E5:( N:C*G5C( !

a,32H13( .-X3-. '3#>'3- 5')61,) /3 '.-3HX-* – '.(-

!

D5$ (,5$-a 5C.85j D:C*G5C(

7,881*)-. '3 I/3= 8*14*/II'34 >/34,/4-( – K6-,5Cb

lambda x: ... – !*#9#b x => ... – ]#># Sb x -> ...

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#;V

P`#E"9.b K#(()C7 =C5C6E5:( N:C*G5C( !

0=)613\ mydata. map(lambda line: line.upper()).take(2)

>

!

72/>/\ mydata. map(line => line.toUpperCase()).take(2)

>

eI mydata. map(_.toUpperCase()).take(2)

>

!*#9# #995A( #C5C6E5:( "#$#E.-.$( :()C7 :C;.$(*5$. [g\

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#;Y

P`#E"9.b ]#>#

]#># W

]#># S

... JavaRDD lines = sc.textFile(" file"); JavaRDD lines_uc = lines.map( new MapFunction() { public String call(String line) { return line.toUpperCase(); } }); ...

... JavaRDD lines = sc.textFile(" file"); JavaRDD lines_uc = lines.map( line -> line.toUpperCase()); ...

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#J"

+,#"-.$ Y5")*( 78/*9 :/('2(

!

H,#- )( ="#*,. !"#$%X

!

Z()C7 -,. !"#$% !,.99

!

IJJ( [I.()9).C- J)(-$)@:-.; J#-#(.-(\

!

N:C*G5C#9 K$57$#EE)C7 H)-, !"#

%$!

A132>,('13

!

O5E.A5$% =(()7CE.C-(

&'()*'+,)-. &/)/ 0*12-(('34 5')6 78/*9

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#J!

P((.CG#9 K5)C-( !

78/*9 2/3 +- ,(-. '3)-*/2HS->= S'/ )6- 78/*9 76->> – K6-,5C

5$ !*#9# – H$)GC7 C5C8)C-.$#*G>. !"#$% #""9)*#G5C( A)99 @. *5>.$.; 9#-.$ !

E&&( FE-('>'-3) &'()*'+,)-. &/)/(-)(G /*- / 9-= 2132-8) '3 78/*9

!

E&& b8-*/H13( – Y$#C(D5$E#G5C(

*$.#-. # C.A IJJ @#(.; 5C #C .`)(GC7 5C. – =*G5C( $.-:$C # >#9:. D$5E #C IJJ !

U/[= TB-2,H13 – Y$#C(D5$E#G5C(

!

#$. C5- .`.*:-.; :CG9 $.R:)$.; @6 #C #*G5C

78/*9 ,(-( D,32H13/> 8*14*/II'34 – K#(()C7

D:C*G5C( #( "#$#E.-.$( – =C5C6E5:( D:C*G5C( )C (:""5$-.; 9#C7:#7.( [K6-,5C #C; !*#9#\

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#J$

+,#"-.$ Y5")*( 78/*9 :/('2(

!

H,#- )( ="#*,. !"#$%X

!

Z()C7 -,. !"#$% !,.99

!

IJJ( [I.()9).C- J)(-$)@:-.; J#-#(.-(\

!

N:C*G5C#9 K$57$#EE)C7 H)-, !"#

%$!

+5C*9:()5C

!

?1I-51*9 @(('43I-3)(

&'()*'+,)-. &/)/ 0*12-(('34 5')6 78/*9

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#J;

!"#$% O5E.A5$%b K)*% k5:$ M#C7:#7. !

c1,* 261'2-\ 0=)613 1* 72/>/ – N5$

-,. !"#$%8@#(.; ,5E.A5$% #(()7CE.C-( )C -,)( *5:$(.a 65: E#6 *,55(. -5 A5$% A)-, .)-,.$ K6-,5C 5$ !*#9#

!

A13S-3H13(\

.pyspark _ K6-,5C (,.99 *5EE#C;( – .scalaspark _ !*#9# (,.99 *5EE#C;( – .py _ K6-,5C !"#$% #""9)*#G5C( – .scala _ !*#9# !"#$% #""9)*#G5C( –

4 +5"6$)7,- 10/0810/2 +95:;.$#< =99 $)7,-( $.(.$>.;< ?5- -5 @. $."$5;:*.; 5$ (,#$.; A)-,5:- "$)5$ A$)B.C *5C(.C- D$5E +95:;.$#< !"#JJ

10-SparkBasics

Recommend Documents