# Define pre-processing function
def preprocess_with_stopwords_with_lemmatization(text):
# Lowercase
text = text.lower()
# Replace "5G" or "5 G" with "5 G"
text = re.sub(r'\b(5\s?)g\b', '5g', text)
# Strip white space
text = text.strip()
# Remove spaces between specific punctuation marks
text = re.sub(r'\s*([.,/!])\s*', r'\1', text)
# Remove URLs
text = re.sub(r'https?\s*:\s*\/\s*\/\s*\S+', '', text)
# Initialize inflect engine
p = inflect.engine()
# Replace numbers with words
text = re.sub(r'\d+', lambda x: p.number_to_words(x.group()), text)
# Remove punctuation except exclamation mark
text = re.sub(r'[^\w\s!]', '', text)
# Tokenization
tokens = word_tokenize(text)
# Lemmatization
lemmatizer = WordNetLemmatizer()
tokens = [lemmatizer.lemmatize(word) for word in tokens]
# Join tokens back to string
text = ' '.join(tokens)
return text
# Define pre-processing function
def preprocess_without_stopwords_with_lemmatization(text):
# Lowercase
text = text.lower()
# Replace "5G" or "5 G" with "5 G"
text = re.sub(r'\b(5\s?)g\b', '5g', text)
# Strip white space
text = text.strip()
# Remove spaces between specific punctuation marks
text = re.sub(r'\s*([.,/!])\s*', r'\1', text)
# Remove URLs
text = re.sub(r'https?\s*:\s*\/\s*\/\s*\S+', '', text)
# Initialize inflect engine
p = inflect.engine()
# Replace numbers with words
text = re.sub(r'\d+', lambda x: p.number_to_words(x.group()), text)
# Remove punctuation except exclamation mark
text = re.sub(r'[^\w\s!]', '', text)
# Tokenization
tokens = word_tokenize(text)
# Remove stop words
tokens = [word for word in tokens if word not in stopwords.words('english')]
# Lemmatization
lemmatizer = WordNetLemmatizer()
tokens = [lemmatizer.lemmatize(word) for word in tokens]
# Join tokens back to string
text = ' '.join(tokens)
return text
# Define pre-processing function
def preprocess_with_stopwords_without_lemmatization(text):
# Lowercase
text = text.lower()
# Replace "5G" or "5 G" with "5 G"
text = re.sub(r'\b(5\s?)g\b', '5g', text)
# Strip white space
text = text.strip()
# Remove spaces between specific punctuation marks
text = re.sub(r'\s*([.,/!])\s*', r'\1', text)
# Remove URLs
text = re.sub(r'https?\s*:\s*\/\s*\/\s*\S+', '', text)
# Initialize inflect engine
p = inflect.engine()
# Replace numbers with words
text = re.sub(r'\d+', lambda x: p.number_to_words(x.group()), text)
# Remove punctuation except exclamation mark
text = re.sub(r'[^\w\s!]', '', text)
# Tokenization
tokens = word_tokenize(text)
# Join tokens back to string
text = ' '.join(tokens)
return text
# Define pre-processing function
def preprocess_without_stopwords_without_lemmatization(text):
# Lowercase
text = text.lower()
# Replace "5G" or "5 G" with "5 G"
text = re.sub(r'\b(5\s?)g\b', '5g', text)
# Strip white space
text = text.strip()
# Remove spaces between specific punctuation marks
text = re.sub(r'\s*([.,/!])\s*', r'\1', text)
# Remove URLs
text = re.sub(r'https?\s*:\s*\/\s*\/\s*\S+', '', text)
# Initialize inflect engine
p = inflect.engine()
# Replace numbers with words
text = re.sub(r'\d+', lambda x: p.number_to_words(x.group()), text)
# Remove punctuation except exclamation mark
text = re.sub(r'[^\w\s!]', '', text)
# Tokenization
tokens = word_tokenize(text)
# Remove stop words
tokens = [word for word in tokens if word not in stopwords.words('english')]
# Join tokens back to string
text = ' '.join(tokens)
return text