import os from threading import Thread from dotenv import load_dotenv load_dotenv() import requests from bs4 import BeautifulSoup from newsapi import NewsApiClient import pandas as pd import torch import soundfile as sf from flask import Flask, request, jsonify, send_file from transformers import ( AutoModelForSequenceClassification, AutoTokenizer, pipeline, BartTokenizer, BartForConditionalGeneration, MarianMTModel, MarianTokenizer, BarkModel, AutoProcessor ) # ------------------------- # Global Setup and Environment Variables # ------------------------- NEWS_API_KEY = os.getenv("NEWS_API_KEY") # Set this in your .env file # Set device for Torch models device = "cuda:0" if torch.cuda.is_available() else "cpu" # ------------------------- # Part 1: News Scraping Functions # ------------------------- def fetch_and_scrape_news(company, api_key, count=11, output_file='news_articles.xlsx'): """ Fetch news article URLs related to a given company using News API, scrape each for headline and content, and save the results to an Excel file. """ newsapi = NewsApiClient(api_key=api_key) all_articles = newsapi.get_everything(q=company, language='en', sort_by='relevancy', page_size=count) articles = all_articles.get('articles', []) scraped_data = [] for article in articles: url = article.get('url') if url: scraped_article = scrape_news(url) if scraped_article: scraped_article['url'] = url scraped_data.append(scraped_article) df = pd.DataFrame(scraped_data) df.to_excel(output_file, index=False, header=True) print(f"News scraping complete. Data saved to {output_file}") def scrape_news(url): """ Scrape the news article for headline and content. """ headers = {"User-Agent": "Mozilla/5.0"} response = requests.get(url, headers=headers) if response.status_code != 200: print(f"Failed to fetch the page: {url}") return None soup = BeautifulSoup(response.text, "html.parser") headline = soup.find("h1").get_text(strip=True) if soup.find("h1") else "No headline found" paragraphs = soup.find_all("p") article_text = " ".join(p.get_text(strip=True) for p in paragraphs) return {"headline": headline, "content": article_text} # ------------------------- # Part 2: Sentiment Analysis Setup # ------------------------- sentiment_model_name = "cross-encoder/nli-distilroberta-base" sentiment_model = AutoModelForSequenceClassification.from_pretrained( sentiment_model_name, torch_dtype=torch.float16, device_map="auto" ) sentiment_tokenizer = AutoTokenizer.from_pretrained(sentiment_model_name) classifier = pipeline("zero-shot-classification", model=sentiment_model, tokenizer=sentiment_tokenizer) labels = ["positive", "negative", "neutral"] # ------------------------- # Part 3: Summarization Setup # ------------------------- bart_tokenizer = BartTokenizer.from_pretrained('facebook/bart-large-cnn') bart_model = BartForConditionalGeneration.from_pretrained('facebook/bart-large-cnn') def split_into_chunks(text, tokenizer, max_tokens=1024): words = text.split() chunks = [] current_chunk = [] current_length = 0 for word in words: tokenized_word = tokenizer.encode(word, add_special_tokens=False) if current_length + len(tokenized_word) <= max_tokens: current_chunk.append(word) current_length += len(tokenized_word) else: chunks.append(' '.join(current_chunk)) current_chunk = [word] current_length = len(tokenized_word) if current_chunk: chunks.append(' '.join(current_chunk)) return chunks # ------------------------- # Part 4: Translation Setup (English to Hindi) # ------------------------- translation_model_name = 'Helsinki-NLP/opus-mt-en-hi' trans_tokenizer = MarianTokenizer.from_pretrained(translation_model_name) trans_model = MarianMTModel.from_pretrained(translation_model_name) def translate_text(text): tokens = trans_tokenizer(text, return_tensors="pt", padding=True) translated = trans_model.generate(**tokens) return trans_tokenizer.decode(translated[0], skip_special_tokens=True) # ------------------------- # Part 5: Bark TTS Setup (Hindi) # ------------------------- bark_model = BarkModel.from_pretrained("suno/bark-small").to(device) processor = AutoProcessor.from_pretrained("suno/bark") # ------------------------- # Part 6: Process Company - Main Pipeline Function # ------------------------- def process_company(company): # Step 1: Fetch and scrape news fetch_and_scrape_news(company, NEWS_API_KEY) df = pd.read_excel('news_articles.xlsx') print("Scraped Articles:") print(df) titles, summaries, sentiments, urls = [], [], [], [] for index, row in df.iterrows(): article_text = row.get("content", "") title = row.get("headline", "No title") url = row.get("url", "") chunks = split_into_chunks(article_text, bart_tokenizer) chunk_summaries = [] for chunk in chunks: inputs = bart_tokenizer([chunk], max_length=1024, return_tensors='pt', truncation=True) summary_ids = bart_model.generate(inputs.input_ids, num_beams=4, max_length=130, min_length=30, early_stopping=True) chunk_summary = bart_tokenizer.decode(summary_ids[0], skip_special_tokens=True) chunk_summaries.append(chunk_summary) final_summary = ' '.join(chunk_summaries) sentiment_result = classifier(final_summary, labels) sentiment = sentiment_result["labels"][0] titles.append(title) summaries.append(final_summary) sentiments.append(sentiment) urls.append(url) final_df = pd.DataFrame({ "Title": titles, "Summary": summaries, "Sentiment": sentiments, "URL": urls }) final_df["Translated Summary"] = final_df["Summary"].apply(translate_text) final_df.to_excel('translated_news_articles.xlsx', index=False) print("Final processed data with translations:") print(final_df) # Combine all translated summaries into one text prompt final_translated_text = "\n\n".join(final_df["Translated Summary"].tolist()) # Generate speech from the combined Hindi text using Bark inputs = processor(final_translated_text, return_tensors="pt").to(device) speech_output = bark_model.generate(**inputs) audio_path = "final_summary.wav" sf.write(audio_path, speech_output[0].cpu().numpy(), bark_model.generation_config.sample_rate) return audio_path # ------------------------- # Part 7: Flask Backend Setup # ------------------------- app = Flask(__name__) @app.route("/process", methods=["POST"]) def process_route(): data = request.get_json() company = data.get("company") if not company: return jsonify({"error": "No company provided"}), 400 audio_path = process_company(company) # Return the audio file path as JSON (Gradio will load the file) return jsonify({"audio_path": audio_path}) # ------------------------- # Part 8: Gradio Interface Setup # ------------------------- def gradio_interface(company): # Call the Flask endpoint response = requests.post("http://127.0.0.1:5000/process", json={"company": company}) result = response.json() # Return the audio file path; Gradio's audio output type will read the file. return result.get("audio_path") def launch_gradio(): import gradio as gr iface = gr.Interface( fn=gradio_interface, inputs=gr.Textbox(label="Enter Company Name"), outputs=gr.Audio(type="filepath", label="News Summary Audio (Hindi)"), title="News Summarization & TTS", description="Enter a company name to fetch news, generate a Hindi summary, and listen to the audio." ) iface.launch() # ------------------------- # Main: Run Flask and Gradio # ------------------------- if __name__ == "__main__": # Run the Flask app in a separate thread. flask_thread = Thread(target=lambda: app.run(host="0.0.0.0", port=5000, debug=False, use_reloader=False)) flask_thread.start() # Launch the Gradio interface. launch_gradio()