Add application

app.py

@@ -2,10 +2,141 @@ import gradio as gr
 from transformers import AutoModelForSequenceClassification, AutoTokenizer, pipeline
 
 # Buraya İngilizce modelinizi yazın
-model_en = AutoModelForSequenceClassification.from_pretrained("OsBaran/Roberta-Classification-Model")
-tokenizer_en = AutoTokenizer.from_pretrained("roberta-base")
-classifier_en = pipeline("sentiment-analysis", model=model_en, tokenizer=tokenizer_en)
+model = AutoModelForSequenceClassification.from_pretrained("OsBaran/Roberta-Classification-Model")
+tokenizer = AutoTokenizer.from_pretrained("roberta-base")
 
+def predict_with_roberta(model, tokenizer, input_text):
+    # Giriş metnini tokenize et ve tensor'a çevir
+    inputs = tokenizer(input_text, return_tensors="pt", max_length=512, truncation=True).to(device)
+
+    # Model ile tahmin yap
+    with torch.no_grad():
+        outputs = model(**inputs)
+
+    # Logits'leri al ve tahmin yap
+    logits = outputs.logits
+    prediction = torch.argmax(logits, dim=-1).item()  # 0: yanlış, 1: doğru
+    return prediction
+def explain_roberta_prediction(model, tokenizer, input_text):
+    # Tokenize et
+
+    inputs = tokenizer(input_text, return_tensors="pt", max_length=512, truncation=True).to(device)
+    # Model ile tahmin yap
+    with torch.no_grad():
+        outputs = model(**inputs)
+
+    # Logits'leri al
+    logits = outputs.logits
+    probabilities = torch.softmax(logits, dim=-1).cpu().numpy()[0]
+
+    # Tahmin sonucunu ve olasılıkları elde et
+    predicted_class = torch.argmax(logits, dim=-1).item()
+    result = "Doğru" if predicted_class == 1 else "Yanlış"
+    explanation = f"Modelin tahmini: {result} (Olasılık: {probabilities[predicted_class]:.2f})\n"
+
+    # Önemli kelimeleri çıkarma (örnek olarak)
+    tokenized_input = tokenizer.tokenize(tokenizer.decode(inputs['input_ids'][0]))
+    important_tokens = tokenized_input[:10]  # İlk 10 tokeni al
+    explanation += "Modelin kararı aşağıdaki anahtar kelimelere dayanıyor:\n" + ', '.join(important_tokens)
+
+    return explanation
+import shap
+from transformers import (AutoTokenizer,
+                          AutoModelForSequenceClassification,
+                          TextClassificationPipeline)
+pipe = TextClassificationPipeline(model=model, tokenizer=tokenizer, return_all_scores=True, device=device)
+def score_and_visualize(text):
+  prediction = pipe([text])
+  print(prediction[0])
+
+  explainer = shap.Explainer(pipe)
+  shap_values = explainer([text])
+  shap.plots.text(shap_values)
+import requests
+import re
+from collections import Counter
+from sklearn.feature_extraction.text import ENGLISH_STOP_WORDS
+from bs4 import BeautifulSoup
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+api_key = '764e3b45715b449a8aedb8cd8018dfed'
+def fetch_news_from_api(api_key, query, page_size=100):
+    url = f'https://newsapi.org/v2/everything?q={query}&pageSize={page_size}&apiKey={api_key}'
+    response = requests.get(url)
+
+    # API yanıtını kontrol et
+    if response.status_code == 200:
+        articles = response.json().get('articles', [])
+        return articles
+    else:
+        print(f"Error: {response}")
+        return []
+def extract_keywords(text, top_n=5):
+    # 1. Metni temizleme
+    text = re.sub(r'[^\w\s]', '', text.lower())  # Noktalama işaretlerini kaldırma ve küçük harfe çevirme
+
+    # 2. Tokenizasyon
+    words = text.split()
+
+    # 3. Durak kelimeleri kaldırma
+    keywords = [word for word in words if word not in ENGLISH_STOP_WORDS]
+
+    # 4. Anahtar kelimeleri sayma ve en sık geçenleri alma
+    keyword_counts = Counter(keywords)
+    most_common_keywords = keyword_counts.most_common(top_n)
+
+    return [keyword for keyword, _ in most_common_keywords]
+
+from keybert import KeyBERT
+
+kw_model = KeyBERT('all-mpnet-base-v2')  # SBERT kullanarak modeli yükleyin
+
+def extract_keywords_keybert(text, num_keywords=2):
+    keywords = kw_model.extract_keywords(text, keyphrase_ngram_range=(1, 2), stop_words='english', top_n=num_keywords)
+    return [kw[0] for kw in keywords]
+
+
+def filter_trusted_sources(articles, trusted_sources):
+    trusted_articles = []
+    for article in articles:
+        source_name = article['source']['name'].lower()  # Kaynağı küçük harfe çevir
+        if any(trusted_source in source_name for trusted_source in trusted_sources):
+            trusted_articles.append(article)
+    return trusted_articles
+
+def fetch_news_content(link):
+    response = requests.get(link)
+    if response.status_code == 200:
+        soup = BeautifulSoup(response.content, 'html.parser')
+        # Başlık ve içerik çıkarma
+        title = soup.find('h1').get_text() if soup.find('h1') else "Başlık bulunamadı"
+        content = ' '.join([p.get_text() for p in soup.find_all('p')])
+        return title, content
+    else:
+        print(f"Error fetching content: {response.status_code}")
+        return "", ""
+def compare_with_thrusted(input_text, bbc_articles):
+    texts = [input_text] + [article[1] for article in bbc_articles]
+    vectorizer = TfidfVectorizer().fit_transform(texts)
+    vectors = vectorizer.toarray()
+    similarities = cosine_similarity(vectors[0:1], vectors[1:]).flatten()
+    return similarities
+
+from sentence_transformers import SentenceTransformer, util
+def sbert_similarity(input_text, bbc_articles):
+# SBERT modelini yükleyin
+    model = SentenceTransformer('sentence-transformers/all-mpnet-base-v2')
+
+    # Kullanıcı metni ve internetten çekilen metinleri vektörize edin
+    input_embedding = model.encode(input_text, convert_to_tensor=True)
+    news_embeddings = model.encode([news[1] for news in bbc_articles], convert_to_tensor=True)
+
+    # Benzerlikleri hesaplayın
+    cosine_scores = util.pytorch_cos_sim(input_embedding, news_embeddings)
+
+    # En yüksek benzerlik skoru ve karşılık gelen haber
+    max_score, most_similar_news = cosine_scores.max(), bbc_articles[cosine_scores.argmax().item()]
+    print(f"En benzer haber skoru: {max_score:.2f}")
 # Türkçe modelini yükle
 model_tr_name = "dbmdz/bert-base-turkish-cased"  # Buraya Türkçe modelinizi yazın
 model_tr = AutoModelForSequenceClassification.from_pretrained(model_tr_name)
@@ -13,9 +144,61 @@ tokenizer_tr = AutoTokenizer.from_pretrained(model_tr_name)
 classifier_tr = pipeline("sentiment-analysis", model=model_tr, tokenizer=tokenizer_tr)
 
 # Gradio ile API oluştur
-def predict(text, language):
+def predict(input_news, language):
     if language == "en":
-        result = classifier_en(text)
+        keywords = extract_keywords_keybert(input_news)
+        search_query = ' '.join(keywords)
+        news_articles = fetch_news_from_api(api_key, search_query)
+
+        trusted_sources = [
+            "bbc news",
+            "cnn",
+            "reuters.com",
+            "theguardian.com",
+            "time",
+            # Diğer güvenilir kaynaklar...
+        ]
+
+        trusted_articles = filter_trusted_sources(news_articles, trusted_sources)
+        # # Sonuçları yazdır
+        trusted_articles_urls = []
+        for i in trusted_articles:
+            trusted_articles_urls.append(i["url"])
+
+        if trusted_articles:
+            print(f"\nGüvenilir kaynaklardan bulunan haberler:\n")
+            print(trusted_articles_urls)
+            bbc_articles = [fetch_news_content(link) for link in trusted_articles_urls]
+            similarities = compare_with_thrusted(input_news, bbc_articles)
+            sbert_similarity(input_news, bbc_articles)
+            print(similarities)
+            max_similarity = max(similarities)
+            threshold = 0.8
+            if max_similarity > threshold:
+                print(f"Sonuç: Doğru (Benzerlik: {max_similarity:.2f})")
+            else:
+                # Benzerlik bulunmazsa tahmin algoritmasını kullanın ve açıklama sağlayın
+                prediction = predict_with_roberta(model, tokenizer, input_news)
+                explanation = explain_roberta_prediction(model, tokenizer, input_news)
+                # Tahmin sonucunu yazdır
+                # result = "Doğru" if prediction == 1 else "Yanlış"
+                # print(f"Haberin durumu: {result}")
+                return explanation
+                print(explanation)
+
+
+
+
+        else:
+            print("Güvenilir kaynaklardan hiç haber bulunamadı.")
+            prediction = predict_with_roberta(model, tokenizer, input_news)
+            explanation = explain_roberta_prediction(model, tokenizer, input_news)
+            # Tahmin sonucunu yazdır
+            result = "Doğru" if prediction == 1 else "Yanlış"
+            print(f"Haberin durumu: {result}")
+            print("Haberin açıklaması:")
+            print(explanation)
+            return explanation
     elif language == "tr":
         result = classifier_tr(text)
     else: