You can deploy this model and start earning money today!

zzaah/
ppw

Views

No views yet

import requests from bs4 import BeautifulSoup import pandas as pd

def get_article_links(base_url, n_articles=50): articles = [] page = 1

while len(articles) < n_articles:
    # Tambahkan parameter halaman ke URL
    url = f"{base_url}page/{page}/"
    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'html.parser')

    # Ambil semua artikel pada halaman tersebut
    for article in soup.find_all('article'):
        link = article.find('a', href=True)['href']
        articles.append(link)

        # Berhenti jika sudah mendapatkan n_articles
        if len(articles) >= n_articles:
            break

    # Jika tidak ada artikel lagi di halaman, hentikan loop
    if not soup.find_all('article'):
        break

    # Lanjutkan ke halaman berikutnya
    page += 1

return articles[:n_articles]

def get_article_details(article_url): response = requests.get(article_url) soup = BeautifulSoup(response.text, 'html.parser')

# Extract title
title = soup.find('h1', class_='entry-title').get_text(strip=True)

# Extract date
date = soup.find('time', class_='entry-date').get_text(strip=True)

# Extract content
content = soup.find('div', class_='entry-content').get_text(strip=True)

# Extract category
category = soup.find('span', class_='cat-links-content').get_text(strip=True)

return title, date, content, category

URLs for categories Nasional and Jombangku

url_nasional = 'https://wargajombang.com/category/nasional/' url_jombangku = 'https://wargajombang.com/category/jombangku/'

Get 50 articles from each category, navigating through multiple pages

article_links_nasional = get_article_links(url_nasional, n_articles=50) article_links_jombangku = get_article_links(url_jombangku, n_articles=50)

List to store articles data

articles_data = []

Crawl Nasional category articles

for link in article_links_nasional: title, date, content, category = get_article_details(link) articles_data.append({ 'Title': title, 'Date': date, 'Content': content, 'Category': category })

Crawl Jombangku category articles

for link in article_links_jombangku: title, date, content, category = get_article_details(link) articles_data.append({ 'Title': title, 'Date': date, 'Content': content, 'Category': category })

Convert to DataFrame

pd.set_option('display.max_rows', 100) df = pd.DataFrame(articles_data)

Menyimpan DataFrame ke file CSV

df.to_csv('hasil_crawling_wargajombang.csv', index=False) print(df.head(10))

df=pd.read_csv("hasil_crawling_wargajombang.csv") df.head(1000)

import pandas as pd import re

Isi nilai NaN dengan string kosong agar aman

df['Content'] = df['Content'].fillna('')

def remove_url(text): if isinstance(text, str): url = re.compile(r'https?://\S+|www.\S+') return url.sub(r'', text) return text # Jika bukan string, kembalikan nilai asli

def remove_html(text): if isinstance(text, str): html = re.compile(r'<.*?>') return html.sub(r'', text) return text

def remove_emoji(text): if isinstance(text, str): emoji_pattern = re.compile("[" u"\U0001F600-\U0001F64F" # emotikon wajah u"\U0001F300-\U0001F5FF" # simbol & gambar u"\U0001F680-\U0001F6FF" # transportasi & simbol u"\U0001F1E0-\U0001F1FF" # bendera negara "]+", flags=re.UNICODE) return emoji_pattern.sub(r'', text) return text

def remove_numbers(text): if isinstance(text, str): return re.sub(r'\d+', '', text) return text

def remove_symbols(text): if isinstance(text, str): return re.sub(r'[^a-zA-Z0-9\s]', '', text) return text

Terapkan fungsi cleansing untuk kolom 'Content'

df['berita_clean'] = df['Content'].apply(remove_url) df['berita_clean'] = df['berita_clean'].apply(remove_html) df['berita_clean'] = df['berita_clean'].apply(remove_emoji) df['berita_clean'] = df['berita_clean'].apply(remove_symbols) df['berita_clean'] = df['berita_clean'].apply(remove_numbers)

Tampilkan hasil

print(df.head(5))

def case_folding(text): if isinstance(text, str): lowercase_text = text.lower() return lowercase_text else : return text

df ['case_folding'] = df['berita_clean'].apply(case_folding)

df.head(5)

def tokenize(text): tokens = text.split() return tokens

df['tokenize'] = df['case_folding'].apply(tokenize)

df.head(5)

import nltk from nltk.corpus import stopwords

Unduh stopwords jika belum terunduh

nltk.download('stopwords')

Ambil stopwords untuk bahasa Indonesia

stop_words = stopwords.words('indonesian')

Tampilkan beberapa stopwords untuk dicek

print(stop_words[:10])

def remove_stopwords(text): return [word for word in text if word not in stop_words]

df['stopword_removal'] = df['tokenize'].apply(lambda x: ' '.join(remove_stopwords(x)))

df.to_csv("preprocessing-wargajombang.csv", encoding='utf8', index=False) df.head(5)

import pandas as pd from sklearn.feature_extraction.text import TfidfVectorizer

Memuat data

df = pd.read_csv("preprocessing-wargajombang.csv")

Isi NaN dengan string kosong

df['stopword_removal'] = df['stopword_removal'].fillna('')

Pastikan tipe data kolom adalah string

df['stopword_removal'] = df['stopword_removal'].astype(str)

Menginisialisasi TfidfVectorizer

vectorizer = TfidfVectorizer()

Menghitung TF-IDF

tfidf_matrix = vectorizer.fit_transform(df['stopword_removal'])

Menampilkan hasil

print(tfidf_matrix.shape) # Dimensi matriks TF-IDF

import pandas as pd

Membuat DataFrame dari matriks TF-IDF

tfidf_df = pd.DataFrame( tfidf_matrix.toarray(), columns=vectorizer.get_feature_names_out() # Alternatif untuk versi terbaru )

Menampilkan 10 baris pertama

print(tfidf_df.head(10))

import pandas as pd from sklearn.model_selection import train_test_split from sklearn.linear_model import LogisticRegression from sklearn.metrics import classification_report, confusion_matrix, accuracy_score

kolom yang berisi label target

y = df['Category']

Membagi data menjadi data latih dan uji (80% latih, 20% uji)

X_train, X_test, y_train, y_test = train_test_split(tfidf_matrix, y, test_size=0.2, random_state=42)

Membuat model Logistic Regression

logreg = LogisticRegression() logreg.fit(X_train, y_train)

from sklearn.model_selection import train_test_split from sklearn.linear_model import LogisticRegression from sklearn.metrics import classification_report, confusion_matrix import matplotlib.pyplot as plt import seaborn as sns

Menentukan target (y)

y = df['Category'] # Target adalah kategori berita

Pisahkan dataset menjadi set pelatihan dan pengujian

X_train, X_test, y_train, y_test = train_test_split(tfidf_df, y, test_size=0.2, random_state=42)

Inisialisasi model Regresi Logistik

model = LogisticRegression(max_iter=1000)

Latih model

model.fit(X_train, y_train)

Lakukan prediksi pada set pengujian

y_pred = model.predict(X_test)

Evaluasi model

print("Confusion Matrix:") conf_matrix = confusion_matrix(y_test, y_pred) print(conf_matrix)

print("\nHasil Klasifikasi:") class_report = classification_report(y_test, y_pred, zero_division=0) print(class_report)

Menampilkan akurasi

accuracy = (y_pred == y_test).mean() print(f"Akurasi Model: {accuracy:.2f}")

Visualisasi Confusion Matrix

plt.figure(figsize=(8, 6)) sns.heatmap(conf_matrix, annot=True, fmt='d', cmap='Blues', xticklabels=model.classes_, yticklabels=model.classes_) plt.ylabel('Actual') plt.xlabel('Predicted') plt.title('Confusion Matrix') plt.show()

zzaah/ppw