Flume Log Message Generation Automation Example

// create a folder in hdfs :

$ hdfs dfs -mkdir /user/flumeExa

// Create a shell script which generates : Hadoop in real world <n>

hadoop@hadoop:~/Desktop/vow$ cat > loopThrough.sh
rm logfile.log
i=0
while :
do
  echo Hadoop in real world $i >> logfile.log
  i=`expr $i + 1`
  sleep 5
done
^C

// Run the shell script to keep on appending into logfile.log:

hadoop@hadoop:~/Desktop/vow$ sh loopThrough.sh

// create a flume configuration file named : simple-flume.conf
-----------------------------------------------------------------

# Flume Components
agent.sources = tail-source
agent.sinks = hdfs-sink
agent.channels = memory-channel

# Source
agent.sources.tail-source.type = exec
agent.sources.tail-source.command = tail -f logfile.log 
agent.sources.tail-source.channels = memory-channel

# Sink
agent.sinks.hdfs-sink.type = hdfs
agent.sinks.hdfs-sink.hdfs.path = user/flumeExa 
agent.sinks.hdfs-sink.hdfs.fileType = DataStream
agent.sinks.hdfs-sink.channel = memory-channel

# Channel
agent.channels.memory-channel.type = memory



// Run the flume
//Here flume reads the logfile.log content and write it into hdfs location : user/flumeExa

flume-ng agent --conf /home/hadoop/Desktop/vow/ -f /home/hadoop/Desktop/vow/simple-flume.conf -Dflume.root.logger=DEBUG,console -n agent

// Check the hdfs folder

hdfs dfs -ls user/flumeExa/

Found 9 items
-rw-r--r--   1 hadoop supergroup        240 2019-02-22 12:26 user/flumeExa/FlumeData.1550818590651
-rw-r--r--   1 hadoop supergroup        168 2019-02-22 12:27 user/flumeExa/FlumeData.1550818590652
-rw-r--r--   1 hadoop supergroup        168 2019-02-22 12:27 user/flumeExa/FlumeData.1550818623842
-rw-r--r--   1 hadoop supergroup        168 2019-02-22 12:28 user/flumeExa/FlumeData.1550818658881
-rw-r--r--   1 hadoop supergroup        168 2019-02-22 12:28 user/flumeExa/FlumeData.1550818693932
-rw-r--r--   1 hadoop supergroup        168 2019-02-22 12:29 user/flumeExa/FlumeData.1550818728976
-rw-r--r--   1 hadoop supergroup        168 2019-02-22 12:29 user/flumeExa/FlumeData.1550818764024
-rw-r--r--   1 hadoop supergroup        168 2019-02-22 12:30 user/flumeExa/FlumeData.1550818799065
-rw-r--r--   1 hadoop supergroup        168 2019-02-22 12:31 user/flumeExa/FlumeData.1550818834114

// see the content of a file which is created with the help of flume

hdfs dfs -cat user/flumeExa/FlumeData.1550818834114

Hadoop in real world 72
Hadoop in real world 73
Hadoop in real world 74
Hadoop in real world 75
Hadoop in real world 76
Hadoop in real world 77
Hadoop in real world 78

Flume Netcat Logger Example

Step 1:

Prepare below the configuration file

# example.conf: A single-node Flume configuration

# Name the components on this agent

a1.sources = r1
a1.sinks = k1
a1.channels = c1
# Describe/configure the source
a1.sources.r1.type = netcat
a1.sources.r1.bind = localhost
a1.sources.r1.port = 44445
# Describe the sink
a1.sinks.k1.type = logger
# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity =100
# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1


Step 2: Run the flume in on terminal

flume-ng agent  -name a1 --conf /usr/local/flumecode -f  /home/hadoop/flumecode/flume.conf -Dflume.root.logger=DEBUG.console

Step 3:
In another terminal

curl telnet://localhost:44445 I love india ->> the message reaches on the flume logger terminal

Try..Success..Failure Examples

Scala : Try, Success, Failure Examples

// Try..Success..Failure Examples

scala> import scala.util.{Try,Success,Failure}

import scala.util.{Try, Success, Failure}

scala> import scala.io.Source

import scala.io.Source

scala>  def deadline

7(path:String) : Try[List[String]] = {
     |  Try(Source.fromFile(path).getLines.toList)

     |  }

readFile: (path: String)scala.util.Try[List[String]]

scala> val  validFilePath = "E:\\DataSets\\guns.csv"

invalidFilePath: String = E:\DataSets\gunsnotfound.csv

scala> readFile(validFilePath)

res0: scala.util.Try[List[String]] = Success(List("","year","month","intent","police","sex","age","race","hispanic","place","education", "1",2012,"01"
,"Suicide",0,"M",34,"Asian/Pacific Islander",100,"Home",4, "2",2012,"01","Suicide",0,"F",21,"White",100,"Street",3, "3",2012,"01","Suicide",0,"M",60,"
White",100,"Other specified",4, "4",2012,"02","Suicide",0,"M",64,"White",100,"Home",4, "5",2012,"02","Suicide",0,"M",31,"White",100,"Other specified",
2, "6",2012,"02","Suicide",0,"M",17,"Native American/Native Alaskan",100,"Home",1, "7",2012,"02","Undetermined",0,"M",48,"White",100,"Home",2, "8",201
2,"03","Suicide",0,"M",41,"Native American/Native Alaskan",100,"Home",2, "9",2012,"02","Accidental",0,"M",50,"White",100,"Other specified",3, "10",201
2,"02","Suicide",0,"M",NA,"Black",998,"Home",5,...
scala>

scala> val invalidFilePath = "E:\\DataSets\\gunsnotfound.csv"

invalidFilePath: String = E:\DataSets\gunsnotfound.csv

scala> readFile(invalidFilePath)

res1: scala.util.Try[List[String]] = Failure(java.io.FileNotFoundException: E:\DataSets\gunsnotfound.csv (The system cannot find the file specified))


scala>  readFile(filePath) match {
     | case Success(a) => a.foreach(println)
     | case Failure(b) => println(b.getMessage)

     | }

{"athelete":"Michael Phelps","age":19,"country":"United States","year":"2004","closing":"08-29-04","sport":"Swimming","gold":6,"silver":0,"bronze":2,"
total":8}
{"athelete":"Michael Phelps","age":23,"country":"United States","year":"2008","closing":"08-24-08","sport":"Swimming","gold":8,"silver":0,"bronze":0,"
total":8}

 scala> def parseInt(a:String) : Try[Int] = Try(a.toInt)

parseInt: (a: String)scala.util.Try[Int]

scala> parseInt("2")

res5: scala.util.Try[Int] = Success(2)

scala> parseInt("I Love India")

res6: scala.util.Try[Int] = Failure(java.lang.NumberFormatException: For input string: "I Love India")


scala>  parseInt("2") match {
     |  case Success(a) => println(a)
     |  case Failure(b) => println(b.getMessage)

     |  }

2

scala> parseInt("333").toOption

res10: Option[Int] = Some(333)

scala> parseInt("IloveIndia").toOption

res11: Option[Int] = None

Either... Left.. Right

Scala : Either, Left, Right Example

// Either... Left.. Right

scala>  def menu(day:String): Either[Int,String] = {
     |  if (day == "Sunday") Right("Fish Curry")
     |  else if (day == "Monday") Right("Veg Fry")
     |  else Left(0)

     |  }

menu: (day: String)Either[Int,String]

scala> println(menu("Sunday"))

Right(Fish Curry)

scala> println(menu("Monday"))

Right(Veg Fry)

scala> println(menu("Tuesday"))

Left(0)


  menu(input) match {
 case Left(a) => println("Left Answer is : " + a)
 case Right(b) => println("Right Answer is : " + b)

 }

 Right Answer is : Fish Curry

 scala> val c = menu("Monday")

c: Either[Int,String] = Right(Veg Fry)

scala> c.isRight

res3: Boolean = true

scala> c.isLeft

res4: Boolean = false

scala> val c = menu("Monday")

c: Either[Int,String] = Right(Veg Fry)

scala> c.isLeft

res5: Boolean = false

scala>

scala> c.isRight

res6: Boolean = true

scala> val input = "Sunday"

input: String = Sunday

scala> if (menu(input).isLeft) menu(input).left.map( a => println("Answer is : " + a))

res7: Any = ()

scala>  if (menu(input).isRight) menu(input).right.map( b => println("Answer is : " + b))

Answer is : Fish Curry

res1: Any = Right(())

scala> val c = menu("Monday")

c: Either[Int,String] = Right(Veg Fry)

scala> c

res3: Either[Int,String] = Right(Veg Fry)

scala> c.left.toOption

res8: Option[Int] = None

scala> c.right.toOption

res9: Option[String] = Some(Veg Fry)



def divideXByY(x: Int, y: Int): Either[String, Int] = {
      if (y == 0) Left("Dude, can't divide by 0")
      else Right(x / y)
  }

  // a few different ways to use Either, Left, and Right

  println(divideXByY(1, 0))

  println(divideXByY(1, 1))

  divideXByY(1, 0) match {
      case Left(s) => println("Answer: " + s)
      case Right(i) => println("Answer: " + i)
  }

}

Scala : Option Some None Example

Scala : Option Some None Example

Examples for Option..Some..None
-----------------------------------

// Simple Example
scala> def menu(day:String): Option[String] = {
     | day match {
     | case "Sunday" => Some("Fish Curry")
     | case "Monday" => Some("Veg Curry")
     | case none => None
     | }

     | }

menu: (day: String)Option[String]

scala> val foodItem = menu("Monday")

foodItem: Option[String] = Some(Veg Curry)

scala> val foodItem = menu("Sunday").getOrElse("General Food")

foodItem: String = Fish Curry

scala> val foodItem = menu("Monday").getOrElse("General Food")

foodItem: String = Veg Curry

scala> val foodItem = menu("Tuesday").getOrElse("General Food")

foodItem: String = General Food





// Regex Pattern Matching Example

scala> import scala.util.matching.Regex

import scala.util.matching.Regex

scala> val np = new Regex("[0-9]+")

np: scala.util.matching.Regex = [0-9]+

scala> val address = "1024, Vana IT Services"

address: String = 1024, Vana IT Services

scala> val match1 = np.findFirstIn(address)

match1: Option[String] = Some(1024)

scala> val address2 = "Pradham Gardenia"

address2: String = Pradham Gardenia

scala> val match2 = np.findFirstIn(address2)

match2: Option[String] = None

scala> val match1 = np.findFirstIn(address).getOrElse("No House Number found")

match1: String = 1024

scala> val match1 = np.findFirstIn(address2).getOrElse("No House Number found")

match1: String = No House Number found



// Extract numbers from a List and calcualte the sum
scala> def toInt(s:String):Option[Int] ={
     | try{
     | Some(s.toInt)
     | }
     | catch {
     | case e: Exception => None
     | }

     | }

toInt: (s: String)Option[Int]

scala> println(toInt("1").getOrElse(0))
1

scala> println(toInt("a").getOrElse(0))
0

scala> val l1 = List("1","2","Lion","4","Tiger")
l1: List[String] = List(1, 2, Lion, 4, Tiger)

scala> val sum = l1.flatMap(toInt).sum
sum: Int = 7

// class example #1
package com.spark.learning

class Person(firstName:String, lastName:String, email:Option[String]){
  var myemail = email
  def displayOutput = {
    println("First Name : " +  firstName)
    println("Last Name : " + lastName)
    println("Email : " + myemail.getOrElse("No Email"))
  }
}

object OptionSomeNoneExa extends App {
  val p = new Person("Super","Star",None)
  p.displayOutput

  p.myemail = Some("test@test.com")
  p.displayOutput
}


Result:
-------
First Name : Super
Last Name : Star
Email : No Email

First Name : Super
Last Name : Star
Email : test@test.com


// class example #2
package com.spark.learning
case class Address(city:String, state:String, country:String)
class UserInfo(emailID:String, phone:String) {
  var firstName = None: Option[String]
  var lastName = None: Option[String]
  var address = None: Option[Address]
}

object Test extends App {
  var u = new UserInfo("test@test.com","9886177375")
  u.firstName = Some("Sankara ")
  u.lastName = Some("Narayanan")
  u.address = Some(Address("Pallathur","TamilNadu","India"))

  println(s"First Name is : ${u.firstName.getOrElse("First Name Not found")}")
  println(s"Last Name is : ${u.lastName.getOrElse("Last Name Not found")}")

  u.address.foreach { x =>
    println("City is : " +  x.city)
    println("State is : " + x.state)
    println("Country is : " + x.country)
  }


    u = new UserInfo("try@test.com","9886177374")

  u.address = Some(Address("Pallathur","","India"))

  println(s" First Name is : ${u.firstName.getOrElse("First Name Not found")}")
  println(s" Last Name is : ${u.lastName.getOrElse("Last Name Not found")}")

  u.address.foreach { x =>
    println("City is : " +  x.city)
    println("State is : " + x.state)
    println("Country is : " + x.country)
  }
}

Result:
--------

First Name is : Sankara
Last Name is : Narayanan
City is : Pallathur
State is : TamilNadu
Country is : India


First Name is : First Name Not found
Last Name is : Last Name Not found
City is : Pallathur
State is :
Country is : India

How to do groupBy Aggregation in Spark with Scala

How to do groupBy Aggregation in Spark with Scala

Input
file2.csv:
--------------

hdfs dfs -cat /user/file2.csv

s1,d1
s1,d2
s1,d2
s1,d3
s2,d1
s2,d3
s2,d1
s3,d2
s3,d1
s1,d1
s2,d1
s3,d1
s1,d1
s2,d2
s3,d3

// autogenerated column headers

scala> val df = spark.read.format("csv").option("inferSchema","true").load("hdfs://localhost:9000/user/file2.csv")

df: org.apache.spark.sql.DataFrame = [_c0: string, _c1: string]

scala> df.show
+---+---+
|_c0|_c1|
+---+---+
| s1| d1|
| s1| d2|
| s1| d2|
| s1| d3|
| s2| d1|
| s2| d3|
| s2| d1|
| s3| d2|
| s3| d1|
| s1| d1|
| s2| d1|
| s3| d1|
| s1| d1|
| s2| d2|
| s3| d3|
+---+---+

// groupBy aggregation goes here

scala> df.groupBy("_c0","_c1").agg(count("*")).show

+---+---+--------+                                                           
|_c0|_c1|count(1)|
+---+---+--------+
| s3| d2|       1|
| s2| d2|       1|
| s1| d2|       2|
| s1| d1|       3|
| s3| d1|       2|
| s2| d1|       3|
| s3| d3|       1|
| s1| d3|       1|
| s2| d3|       1|
+---+---+--------+

// In order to make our own schema we need to import the following

scala> import org.apache.spark.sql.types._
import org.apache.spark.sql.types._

// Making Schema
scala> val sch = StructType(StructField("School",StringType)::StructField("Department",StringType)::Nil)
sch: org.apache.spark.sql.types.StructType = StructType(StructField(School,StringType,true), StructField(Department,StringType,true))

// applying schema
scala> val df = spark.read.format("csv").option("inferSchema","true").schema(sch).load("hdfs://localhost:9000/user/file2.csv")
df: org.apache.spark.sql.DataFrame = [School: string, Department: string]

scala> df.printSchema
root
 |-- School: string (nullable = true)
 |-- Department: string (nullable = true)

scala> df.show
+------+----------+
|School|Department|
+------+----------+
|    s1|        d1|
|    s1|        d2|
|    s1|        d2|
|    s1|        d3|
|    s2|        d1|
|    s2|        d3|
|    s2|        d1|
|    s3|        d2|
|    s3|        d1|
|    s1|        d1|
|    s2|        d1|
|    s3|        d1|
|    s1|        d1|
|    s2|        d2|
|    s3|        d3|
+------+----------+

// groupBy Aggregation operation
//added alias name for 3rd column
scala> df.groupBy("School","Department").agg(count("*") as "Count").show
+------+----------+-----+
|School|Department|Count|
+------+----------+-----+
|    s3|        d2|    1|
|    s2|        d2|    1|
|    s1|        d2|    2|
|    s1|        d1|    3|
|    s3|        d1|    2|
|    s2|        d1|    3|
|    s3|        d3|    1|
|    s1|        d3|    1|
|    s2|        d3|    1|
+------+----------+-----+

match case example in Scala

match case example in Scala

scala> def f1(x:Int):Unit = {
     | x match{
     | case 1 => println("One")
     | case 2 => println("Two")
     | case 3 => println("Three")
     | case 4 => println("Four")
     | case _ => println("Other Number")
     | }

     | }

f1: (x: Int)Unit

scala> f1(1)
One

scala> f1(2)
Two

scala> f1(_)
res2: Int => Unit = $$Lambda$1067/1853443923@2e1ba142

scala> f1(0)
Other Number