In [2]:
from pyspark.sql import SparkSession

spark = SparkSession.builder \
    .appName("SentimentAnalysis") \
    .master("local[*]") \
    .config("spark.executor.memory", "12g") \
    .config("spark.driver.memory", "12g") \
    .getOrCreate()
In [6]:
from pyspark.sql.types import StructType, StructField, StringType, IntegerType

schema = StructType([
    StructField("TweetID", IntegerType(), True),
    StructField("Entity", StringType(), True),
    StructField("Sentiment", StringType(), True),
    StructField("Content", StringType(), True)
])

df = spark.read.csv("./Spark/twitter_training.csv", header=False, schema=schema)
df.show()
+-------+-----------+---------+--------------------+
|TweetID|     Entity|Sentiment|             Content|
+-------+-----------+---------+--------------------+
|   2401|Borderlands| Positive|im getting on bor...|
|   2401|Borderlands| Positive|I am coming to th...|
|   2401|Borderlands| Positive|im getting on bor...|
|   2401|Borderlands| Positive|im coming on bord...|
|   2401|Borderlands| Positive|im getting on bor...|
|   2401|Borderlands| Positive|im getting into b...|
|   2402|Borderlands| Positive|So I spent a few ...|
|   2402|Borderlands| Positive|So I spent a coup...|
|   2402|Borderlands| Positive|So I spent a few ...|
|   2402|Borderlands| Positive|So I spent a few ...|
|   2402|Borderlands| Positive|2010 So I spent a...|
|   2402|Borderlands| Positive|                 was|
|   2403|Borderlands|  Neutral|Rock-Hard La Varl...|
|   2403|Borderlands|  Neutral|Rock-Hard La Varl...|
|   2403|Borderlands|  Neutral|Rock-Hard La Varl...|
|   2403|Borderlands|  Neutral|Rock-Hard La Vita...|
|   2403|Borderlands|  Neutral|Live Rock - Hard ...|
|   2403|Borderlands|  Neutral|I-Hard like me, R...|
|   2404|Borderlands| Positive|that was the firs...|
|   2404|Borderlands| Positive|this was the firs...|
+-------+-----------+---------+--------------------+
only showing top 20 rows

In [7]:
df_filtered = df.filter(df["Content"].isNotNull())
df = df.fillna({"Content": "default_value"})
In [8]:
from pyspark.ml.feature import Tokenizer, StopWordsRemover, HashingTF, IDF
import re

def preprocess_text(text):
    text = text.lower()
    text = re.sub(r'<.*?>', '', text)  
    text = re.sub(r'http\S+|www\S+', '', text)  
    text = re.sub(r'\d+', '', text)  
    text = re.sub(r'[^\w\s]', '', text)  
    return text

preprocess_text_udf = spark.udf.register("preprocess_text", preprocess_text)
df = df.withColumn("Content", preprocess_text_udf("Content"))

tokenizer = Tokenizer(inputCol="Content", outputCol="words")
df_words = tokenizer.transform(df)

remover = StopWordsRemover(inputCol="words", outputCol="filtered_words")
df_filtered_words = remover.transform(df_words)

hashingTF = HashingTF(inputCol="filtered_words", outputCol="rawFeatures")
featurizedData = hashingTF.transform(df_filtered_words)
idf = IDF(inputCol="rawFeatures", outputCol="features")
idfModel = idf.fit(featurizedData)
rescaledData = idfModel.transform(featurizedData)

rescaledData.show()
+-------+-----------+---------+--------------------+--------------------+--------------------+--------------------+--------------------+
|TweetID|     Entity|Sentiment|             Content|               words|            filtered|         rawFeatures|            features|
+-------+-----------+---------+--------------------+--------------------+--------------------+--------------------+--------------------+
|   2401|Borderlands| Positive|im getting on bor...|[im, getting, on,...|[im, getting, bor...|(262144,[31015,92...|(262144,[31015,92...|
|   2401|Borderlands| Positive|I am coming to th...|[i, am, coming, t...|[coming, borders,...|(262144,[12409,14...|(262144,[12409,14...|
|   2401|Borderlands| Positive|im getting on bor...|[im, getting, on,...|[im, getting, bor...|(262144,[31015,68...|(262144,[31015,68...|
|   2401|Borderlands| Positive|im coming on bord...|[im, coming, on, ...|[im, coming, bord...|(262144,[12409,31...|(262144,[12409,31...|
|   2401|Borderlands| Positive|im getting on bor...|[im, getting, on,...|[im, getting, bor...|(262144,[12524,31...|(262144,[12524,31...|
|   2401|Borderlands| Positive|im getting into b...|[im, getting, int...|[im, getting, bor...|(262144,[31015,68...|(262144,[31015,68...|
|   2402|Borderlands| Positive|So I spent a few ...|[so, i, spent, a,...|[spent, hours, ma...|(262144,[640,2182...|(262144,[640,2182...|
|   2402|Borderlands| Positive|So I spent a coup...|[so, i, spent, a,...|[spent, couple, h...|(262144,[21823,26...|(262144,[21823,26...|
|   2402|Borderlands| Positive|So I spent a few ...|[so, i, spent, a,...|[spent, hours, so...|(262144,[21823,45...|(262144,[21823,45...|
|   2402|Borderlands| Positive|So I spent a few ...|[so, i, spent, a,...|[spent, hours, ma...|(262144,[640,2182...|(262144,[640,2182...|
|   2402|Borderlands| Positive|2010 So I spent a...|[2010, so, i, spe...|[2010, spent, hou...|(262144,[640,2182...|(262144,[640,2182...|
|   2402|Borderlands| Positive|                 was|               [was]|                  []|      (262144,[],[])|      (262144,[],[])|
|   2403|Borderlands|  Neutral|Rock-Hard La Varl...|[rock-hard, la, v...|[rock-hard, la, v...|(262144,[19937,51...|(262144,[19937,51...|
|   2403|Borderlands|  Neutral|Rock-Hard La Varl...|[rock-hard, la, v...|[rock-hard, la, v...|(262144,[19937,51...|(262144,[19937,51...|
|   2403|Borderlands|  Neutral|Rock-Hard La Varl...|[rock-hard, la, v...|[rock-hard, la, v...|(262144,[19937,51...|(262144,[19937,51...|
|   2403|Borderlands|  Neutral|Rock-Hard La Vita...|[rock-hard, la, v...|[rock-hard, la, v...|(262144,[19937,60...|(262144,[19937,60...|
|   2403|Borderlands|  Neutral|Live Rock - Hard ...|[live, rock, -, h...|[live, rock, -, h...|(262144,[2437,988...|(262144,[2437,988...|
|   2403|Borderlands|  Neutral|I-Hard like me, R...|[i-hard, like, me...|[i-hard, like, me...|(262144,[27971,63...|(262144,[27971,63...|
|   2404|Borderlands| Positive|that was the firs...|[that, was, the, ...|[first, borderlan...|(262144,[16793,11...|(262144,[16793,11...|
|   2404|Borderlands| Positive|this was the firs...|[this, was, the, ...|[first, borderlan...|(262144,[110357,1...|(262144,[110357,1...|
+-------+-----------+---------+--------------------+--------------------+--------------------+--------------------+--------------------+
only showing top 20 rows

In [9]:
from pyspark.ml.feature import StringIndexer
from pyspark.ml.evaluation import MulticlassClassificationEvaluator

indexer = StringIndexer(inputCol="Sentiment", outputCol="label")
df_final = indexer.fit(rescaledData).transform(rescaledData)
In [10]:
train_data, test_data = df_final.randomSplit([0.8, 0.2], seed=1234)
In [15]:
from pyspark.ml.tuning import ParamGridBuilder, CrossValidator
from pyspark.ml.classification import LogisticRegression

lr = LogisticRegression(featuresCol='features', labelCol='label')

paramGrid = (ParamGridBuilder()
             .addGrid(lr.regParam, [0.1, 0.01])  
             .addGrid(lr.elasticNetParam, [0.0, 0.5, 1.0]) 
             .addGrid(lr.maxIter, [10, 50, 100]) 
             .build())

evaluator = MulticlassClassificationEvaluator(labelCol="label", predictionCol="prediction", metricName="accuracy")

crossval = CrossValidator(estimator=lr,
                          estimatorParamMaps=paramGrid,
                          evaluator=evaluator,
                          numFolds = 3)  

cvModel = crossval.fit(train_data)
predictions = cvModel.transform(test_data)
accuracy = evaluator.evaluate(predictions)

print("Logistic Regression Accuracy: %f" % accuracy)

bestModel = cvModel.bestModel
print("Best model parameters:")
print("regParam:", bestModel._java_obj.getRegParam())
print("elasticNetParam:", bestModel._java_obj.getElasticNetParam())
print("maxIter:", bestModel._java_obj.getMaxIter())
Logistic Regression Accuracy: 0.866391
Best model parameters:
regParam: 0.01
elasticNetParam: 0.0
maxIter: 10
In [13]:
from pyspark.ml.tuning import ParamGridBuilder, CrossValidator
from pyspark.ml.classification import NaiveBayes

nb = NaiveBayes(featuresCol='features', labelCol='label')

paramGrid = (ParamGridBuilder()
             .addGrid(nb.smoothing, [0.0, 1.0, 2.0])  
             .build())

crossval = CrossValidator(estimator=nb,
                          estimatorParamMaps=paramGrid,
                          evaluator=evaluator,
                          numFolds=5)  

cvModel = crossval.fit(train_data)

predictions = cvModel.transform(test_data)

accuracy = evaluator.evaluate(predictions)

print("Naive Bayes Classifier Accuracy: %f" % accuracy)

bestModel = cvModel.bestModel
print("Best model parameters:")
print("Smoothing parameter:", bestModel._java_obj.getSmoothing())
Naive Bayes Classifier Accuracy: 0.825080
Best model parameters:
Smoothing parameter: 1.0