I am trying to create a spark streaming application using the sbt package, I can not understand the cause of this error.
it's kind of a mistake
scala.reflect.internal.MissingRequirementError: java.lang.Object was not found in the compiler mirror. in scala.reflect.internal.MissingRequirementError $ .signal (MissingRequirementError.scala: 16) in scala.reflect.internal.MissingRequirementError $ .notFound (MissingRequirementError.scala: 17) in scala.reflect.internalMrrorgetle .scala: 48) in scala.reflect.internal.Mirrors $ RootsBase.getModuleOrClass (Mirrors.scala: 40) in scala.reflect.internal.Mirrors $ RootsBase.getModuleOrClass (Mirrors.scala: 40)
and here is the code
import org.apache.spark.SparkContext import org.apache.spark._ import org.apache.spark.streaming._ import org.apache.spark.streaming.twitter._ import twitter4j.Status object TrendingHashTags { def main(args: Array[String]): Unit = { val Array(consumerKey, consumerSecret, accessToken, accessTokenSecret, lang, batchInterval, minThreshold, showCount ) = args.take(8) val filters = args.takeRight(args.length - 8) System.setProperty("twitter4j.oauth.consumerKey", consumerKey) System.setProperty("twitter4j.oauth.consumerSecret", consumerSecret) System.setProperty("twitter4j.oauth.accessToken", accessToken) System.setProperty("twitter4j.oauth.accessTokenSecret", accessTokenSecret) val conf = new SparkConf().setAppName("TrendingHashTags") val ssc = new StreamingContext(conf, Seconds(batchInterval.toInt)) val tweets = TwitterUtils.createStream(ssc, None, filters) val tweetsFilteredByLang = tweets.filter{tweet => tweet.getLang() == lang} val statuses = tweetsFilteredByLang.map{ tweet => tweet.getText()} val words = statuses.flatMap{status => status.split("""\s+""")} val hashTags = words.filter{word => word.startsWith("#")} val hashTagPairs = hashTags.map{hashtag => (hashtag, 1)} val tagsWithCounts = hashTagPairs.updateStateByKey( (counts: Seq[Int], prevCount: Option[Int]) => prevCount.map{c => c + counts.sum}.orElse{Some(counts.sum)} ) val topHashTags = tagsWithCounts.filter{ case(t, c) => c > minThreshold.toInt } val sortedTopHashTags = topHashTags.transform{ rdd => rdd.sortBy({case(w, c) => c}, false) } sortedTopHashTags.print(showCount.toInt) ssc.start() ssc.awaitTermination() } }
source share