diff --git a/__pycache__/tasks.cpython-310.pyc b/__pycache__/tasks.cpython-310.pyc new file mode 100644 index 0000000..4cf7354 Binary files /dev/null and b/__pycache__/tasks.cpython-310.pyc differ diff --git a/local-agent.py b/local-agent.py index abe2409..cc174a5 100644 --- a/local-agent.py +++ b/local-agent.py @@ -1,146 +1,83 @@ -from crewai import Agent, Task, Crew -from crewai_tools import SerperDevTool, ScrapeWebsiteTool, SeleniumScrapingTool -from langchain_community.chat_models import ChatOllama -from langchain_openai import ChatOpenAI -from flask import Flask, jsonify -import os - -# export OPENAI_API_BASE=http://the.mk:11434/v1 -# export OPENAI_MODEL_NAME=crewai-llama3 -# export SERPER_API_KEY= - -# class MyAgent(Agent): -# def __init__(self, role="My Math Professor", goal=None, backstory=None, allow_delegation=False, verbose=True, llm=None): -# super().__init__(role=role, goal=goal, backstory=backstory, allow_delegation=allow_delegation, verbose=verbose, llm=llm) -# def to_dict(self): -# return { -# 'role': self.role, -# 'goal': self.goal, -# 'backstory': self.backstory, -# 'allow_delegation': self.allow_delegation, -# 'verbose': self.verbose -# } - - -os.environ["OPENAI_API_KEY"] = "NA" - -llm = ChatOpenAI( - model = "crewai-llama3", - base_url = "http://chat.the.mk:11434/v1", - temperature=0.1) - -# general_agent = MyAgent(role = "Math Professor", -# goal = """Provide the solution to the students that are asking mathematical questions and give them the answer.""", -# backstory = """You are an excellent math professor that likes to solve math questions in a way that everyone can understand your solution""", -# allow_delegation = False, -# verbose = True, -# llm = llm) -# task = Task (description="""what is 3 + 5""", -# agent = general_agent, -# expected_output="A numerical answer.") - -# crew = Crew( -# agents=[general_agent], -# tasks=[task], -# verbose=2 -# ) - - - -# Loading Tools -search_tool = SerperDevTool() -scrape_tool = SeleniumScrapingTool() - - -# Define your agents with roles, goals, tools, and additional attributes -researcher = Agent( - role='Senior Research Analyst', - goal='Uncover cutting-edge developments in AI and data science', - backstory=( - "You are a Senior Research Analyst at a leading tech think tank." - "Your expertise lies in identifying emerging trends and technologies in AI and data science." - "You have a knack for dissecting complex data and presenting actionable insights." - "Always search the web first and make the determination for the best 4 Links" - "if you are going to use Read website content tool replace the search positional argument: to 'website_url'" - # "Don't include single or double quotes in any or the searches or parameters for the tools including website_url or css_element" - "For any web searches, be sure to scrape the website content from the Link in the search" - ), - verbose=True, - allow_delegation=False, - tools=[search_tool, scrape_tool], - max_rpm=100 -) - -writer = Agent( - role='Tech Content Strategist', - goal='Craft compelling content on tech advancements', - backstory=( - "You are a renowned Tech Content Strategist, known for your insightful and engaging articles on technology and innovation." - "With a deep understanding of the tech industry, you transform complex concepts into compelling narratives." - "Always search the web first and make the determination for the best 4 Links" - "if you are going to use Read website content tool replace the search positional argument: to 'website_url'" - # "Don't include single or double quotes in any or the searches or parameters for the tools including website_url or css_element" - "For any web searches, be sure to scrape the website content from the Link in the search" - ), - verbose=True, - allow_delegation=True, - tools=[search_tool, scrape_tool], - cache=False, # Disable cache for this agent -) - -# Create tasks for your agents -task1 = Task( - description=( - "Conduct a comprehensive analysis of the latest advancements in AI in 2024." - "Identify key trends, breakthrough technologies, and potential industry impacts." - "Compile your findings in a detailed report." - "Make sure to check with a human if the draft is good before finalizing your answer." - ), - expected_output='A comprehensive full report on the latest AI advancements in 2024, leave nothing out', - agent=researcher, - human_input=True, -) - -task2 = Task( - description=( - "Using the insights from the researcher's report, develop an engaging blog post that highlights the most significant AI advancements." - "Your post should be informative yet accessible, catering to a tech-savvy audience." - "Aim for a narrative that captures the essence of these breakthroughs and their implications for the future." - ), - expected_output='A compelling three paragraphs blog post formatted as markdown with headings, subheadings, and a main thesis about the latest AI advancements in 2024', - agent=writer -) - -# Instantiate your crew with a sequential process -crew = Crew( - agents=[researcher, writer], - tasks=[task1, task2], - verbose=4 -) +from flask import Flask, jsonify, render_template, request +from flask_socketio import SocketIO, emit +from pymongo import MongoClient +from bson.objectid import ObjectId +import threading +from tasks import generate_content +import html app = Flask(__name__) -def get_agents(cls): - return [agent.__dict__ for agent in cls.agents] +# MongoDB Atlas connection +client = MongoClient("mongodb+srv://maheshkommareddi:Yu2L6pQKyJgcTb9a@cluster0.qadl40g.mongodb.net/?retryWrites=true&w=majority&appName=Cluster0") +db = client.content_generation +app.config['SECRET_KEY'] = 'secret_key' +socketio = SocketIO(app) -def get_tasks(cls): - return [{**task.__dict__, 'expected_output': task.expected_output} for task in cls.tasks] +# Periodically fetch and emit task updates +def fetch_and_emit_updates(): + while True: + updates = db.task_updates.find({}) + for update in updates: + socketio.emit('agent_update', { + 'task_id': update['task_id'], + 'status': update['status'], + 'message': update['message'] + }) + db.task_updates.delete_one({"_id": update["_id"]}) # Remove the update after emitting + socketio.sleep(5) +# Start the periodic update thread +threading.Thread(target=fetch_and_emit_updates).start() -@app.route('/agents', methods=['GET']) -def get_agents_route(): - return jsonify({'agents': get_agents(crew)}) +# Route for the chat interface +@app.route('/chat') +def chat(): + return render_template('chat.html') -@app.route('/tasks', methods=['GET']) -def get_tasks_route(): - return jsonify({'tasks': get_tasks(crew)}) +# SocketIO event handler for new content generation requests +@socketio.on('generate_content') +def handle_generate_content(data): + agenda = data['agenda'] + task = generate_content.apply_async(args=[agenda]) + task_id = task.id + emit('task_queued', {'task_id': task_id, 'agenda': agenda}) -@app.route('/kickoff', methods=['GET']) -def get_kickoff_route(): - result = crew.kickoff() - print(result) - return result +@app.route('/', methods=['GET', 'POST']) +def index(): + # Fetch all existing blog entries from MongoDB Atlas + blog_entries = list(db.content.find({}, {"_id": 1, "agenda": 1, "blog_post": 1})) + if request.method == 'POST': + agenda = request.form.get('agenda', 'quantum gravitational sensing for underground detection') + + # Queue the task for background processing + task = generate_content.apply_async(args=[agenda]) + + # Save the task ID to MongoDB Atlas + content = { + "agenda": agenda, + "task_id": str(task.id), + "status": "PENDING" + } + db.content.insert_one(content) + + return render_template('processing.html', agenda=agenda, task_id=task.id) + + return render_template('index.html', blog_entries=blog_entries) + +@app.route('/blog/', methods=['GET']) +def blog_post(id): + # Fetch the blog entry from MongoDB Atlas + blog_entry = db.content.find_one({"_id": ObjectId(id)}) + + if blog_entry: + if blog_entry.get('status') == 'COMPLETED': + return render_template('result.html', report=html.unescape(blog_entry['report']), blog_post=blog_entry['blog_post']) + else: + return render_template('result.html', report=html.unescape(blog_entry['report']), blog_post=blog_entry['blog_post']) + else: + return "Blog entry not found", 404 if __name__ == '__main__': - app.run(debug=True, port=5001) + socketio.run(app, debug=True, port=5001) diff --git a/tasks.py b/tasks.py new file mode 100644 index 0000000..4cb5cdd --- /dev/null +++ b/tasks.py @@ -0,0 +1,143 @@ +from celery import Celery +from crewai import Agent, Task, Crew, tasks +from crewai_tools import SerperDevTool, ScrapeWebsiteTool, SeleniumScrapingTool +from langchain_openai import ChatOpenAI +from pymongo import MongoClient +import langchain_core +import html +import os +import logging + +# Set up logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +client = MongoClient("mongodb+srv://maheshkommareddi:Yu2L6pQKyJgcTb9a@cluster0.qadl40g.mongodb.net/?retryWrites=true&w=majority&appName=Cluster0") +db = client.content_generation + +# Initialize Celery +app = Celery('tasks', broker='amqp://guest:guest@localhost:5672//') + +# Load environment variables +os.environ["OPENAI_API_KEY"] = "sk-kkk" +os.environ["OPENAI_MODEL_NAME"] = "anthropic.claude-3-sonnet-20240229-v1:0" +os.environ["OPENAI_API_BASE"] = "http://chat.the.mk:1337" + +# Initialize LLM +llm = ChatOpenAI( + model="anthropic.claude-3-sonnet-20240229-v1:0", + base_url="http://chat.the.mk:1337", + temperature=0.1 +) + +# Load tools +search_tool = SerperDevTool() +scrape_tool = SeleniumScrapingTool() +scrape_tool_bare = ScrapeWebsiteTool() + +# Function to update task status in MongoDB +def update_task_status(task_id, status, message): + try: + db.task_updates.insert_one({"task_id": task_id, "status": status, "message": message}) + logger.info(f"Updated task status: {task_id}, {status}, {message}") + except Exception as e: + logger.error(f"Error updating task status: {e}") + +# Define tasks +@app.task +def generate_content(agenda): + def researcher_callback(output): + if isinstance(output, langchain_core.agents.AgentFinish): + update_task_status(app.current_task.request.id, f"researcher", "Agent finished") + elif output and output[0]: + print(output[0]) + if output[0][0]: + update_task_status(app.current_task.request.id, f"researcher", output[0][0].log) + + def writer_callback(output): + if isinstance(output, langchain_core.agents.AgentFinish): + update_task_status(app.current_task.request.id, f"writer", "Agent finished") + elif output and output[0]: + print(output[0]) + if output[0][0]: + update_task_status(app.current_task.request.id, f"writer", output[0][0].log) + + researcher = Agent( + role='Senior Research Analyst', + goal='Find way to explain ' + agenda, + backstory=( + "You are a Senior Research Analyst at a leading tech think tank." + f"Your expertise lies in identifying {agenda}." + "You have a knack for dissecting complex data and presenting actionable insights." + "Always search the web first and make the determination for the best 4 Links, but exclude PDFs" + "For any web searches, be sure to scrape the website content from the Link in the search" + ), + verbose=True, + allow_delegation=False, + tools=[search_tool, scrape_tool, scrape_tool_bare], + max_rpm=100, + step_callback=researcher_callback + ) + + writer = Agent( + role='Tech Content Strategist', + goal='Craft compelling content on ' + agenda, + backstory=( + "You are a renowned Tech Content Strategist, known for your insightful and engaging articles on science and innovation." + "With a deep understanding of the tech industry, you transform complex concepts into compelling narratives." + "For any web searches, be sure to scrape the website content from the Link in the search, but exclude PDFs" + ), + verbose=True, + allow_delegation=True, + tools=[search_tool, scrape_tool, scrape_tool_bare], + cache=False, # Disable cache for this agent + step_callback=writer_callback + ) + + task1 = Task( + description=( + f"Conduct a comprehensive analysis of the latest in {agenda}" + "Identify key trends, breakthrough technologies, and potential industry impacts." + "Compile your findings in a detailed report and include references and links to the source material." + ), + expected_output=f"A comprehensive full report on {agenda} in 2024, leave nothing out", + agent=researcher, + ) + + task2 = Task( + description=( + f"Using the insights from the researcher's report, develop an engaging blog post that highlights the most significant {agenda} ideas." + "Your post should be informative yet accessible, catering to a tech-savvy audience." + "Aim for a narrative that captures the essence of these breakthroughs and their implications for the future." + "Keep asking for research and revise until the minimum 5000 words are met" + "Include the research in the entirety along with the resulting report" + "Include at least five links to external pages or PDFs with an appropriate anchor tag in the final report" + ), + expected_output=f"A compelling ten paragraphs blog post formatted as html to place inside the body tag with headings, subheadings, and a main thesis about the latest {agenda}", + agent=writer + ) + + # Define the main callback for the crew + def main_callback(output: tasks.task_output.TaskOutput): + update_task_status(app.current_task.request.id, 1, output.description) + + # Instantiate your crew with a sequential process + crew = Crew( + agents=[researcher, writer], + tasks=[task1, task2], + verbose=4, + task_callback=main_callback + ) + + + result = crew.kickoff() + + # Save the result to MongoDB Atlas + content = { + "agenda": agenda, + "report": html.unescape(result), + "blog_post": html.unescape(result) + } + db.content.insert_one(content) + + return result diff --git a/templates/chat.html b/templates/chat.html new file mode 100644 index 0000000..d75cb9d --- /dev/null +++ b/templates/chat.html @@ -0,0 +1,58 @@ + + + + Content Generation Chat + + + + + +

Content Generation Chat

+
+ + + + \ No newline at end of file diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..f4f8b91 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,30 @@ + + + + Content Generation + + +

Content Generation

+
+ + + +
+ + {% if blog_entries %} +

Existing Blog Entries

+ + {% endif %} + + \ No newline at end of file diff --git a/templates/processing.html b/templates/processing.html new file mode 100644 index 0000000..f6821c0 --- /dev/null +++ b/templates/processing.html @@ -0,0 +1,13 @@ + + + + Content Generation Processing + + + +

Content Generation Processing

+

Your request for generating content on "{{ agenda }}" is being processed.

+

Task ID: {{ task_id }}

+

This page will refresh every 5 seconds to check the status.

+ + \ No newline at end of file diff --git a/templates/result.html b/templates/result.html new file mode 100644 index 0000000..22e2d0a --- /dev/null +++ b/templates/result.html @@ -0,0 +1,13 @@ + + + + Content Generation Result + + +

Content Generation Result

+

Report

+ {{ report|safe }} + + + \ No newline at end of file