From fc2c279836c1db3b4606bd1febc0eda38513a73a Mon Sep 17 00:00:00 2001 From: Mahesh Kommareddi Date: Sat, 15 Jun 2024 13:41:25 -0400 Subject: [PATCH] Various updates --- __pycache__/tasks.cpython-310.pyc | Bin 0 -> 4940 bytes local-agent.py | 207 +++++++++++------------------- tasks.py | 143 +++++++++++++++++++++ templates/chat.html | 58 +++++++++ templates/index.html | 30 +++++ templates/processing.html | 13 ++ templates/result.html | 13 ++ 7 files changed, 329 insertions(+), 135 deletions(-) create mode 100644 __pycache__/tasks.cpython-310.pyc create mode 100644 tasks.py create mode 100644 templates/chat.html create mode 100644 templates/index.html create mode 100644 templates/processing.html create mode 100644 templates/result.html diff --git a/__pycache__/tasks.cpython-310.pyc b/__pycache__/tasks.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4cf7354e1b8123f0d22dc0c94cf150a4fb662f78 GIT binary patch literal 4940 zcmcIoOLH5?5#ARTAV`7_QKDqY()tlAh)Iy5WZ71vSTs#lF_tMMluM<(Ad8&=u)sdx znFT4Jz*MTFt8Td@wyLC~uQ{ZWzp~eybW0_vRK*pS^YtwFuxuRz5{;SJneLwMuX}nn znVv3d_?U&L`R7xb_BZMr{}j-oG~rG}Ww)%xtZ*`#a;H?g5Kc!Gx1!p1IOEQMmmSW!v+8%wom0Q_?mVCGoWOU1 z6`w#NpMQ3epL(vli@d^0k97AmKixUwzQ%Pn!OCp%iQ%4QQ+%#-j!j?JSVgsF(3(}P zx$7FAVih*a=GloSru#ZCJ<{CsybOAQPk=gXft`G;xo_~Zoj2Jjw)j|sl((3@qg7A8 z!2Y$Wp4l5bDsD0H(vPJoliDz*SFU{+Z-;o)09;3 zVXs=+=C!DM?b7AT@4GX{gnYcdv)TINAL#SfkKmU=lrSE;rG_VYy)Qy{GUX8*<)wYW zeUp4V{#f{>(AdCN#vsG9sn*e-86$0^j|`@NWj+TeQ}da{F!rU!%#p6B@ul|Erf2sN zBs-d=rG>#&UE4QxZ8(GZHnT=1<{0^$2bi^Q{???qd23JK)2fASs9PjF)dQBNaKkqRw<;hwnFGmm{^Ci5{*O0{2d*pGvl$guQQWb_%%|E6_`EJ7ByB>jwwAgDMO@sYNTe%WRoME zO+6{?o0vJxDr)8oW=tH;D8s?Cm|>+Bn^QdI!Q+H#Ej%?xCOi4W+%MqiDK)l;r>Cj) z)EHUnna$3mg^t}RHVyU~JNww$FQ%oDIkH;Ds4%iy20J$@ve%zj`=wFo*~D{ge`2Ji z&Y*-of{uDM+e;mEJKr<|9MQ zr&g{=r$mN&1QZg|Mw+-X0%p%>;%W-f)Oth$R7;J2aD3&_c&B{8BS$Qu7D`#$Sy^z& zJ>j>x_@<(gqtguHK(;v(q~zC=K$-;BQZM#-Rtdehh3F09Izc3}f$KPx?xb3JORZGy z&rBqtY|8Hofe8vUSyAvWDEDLrqNC6g0Wz@(=zfi)ll0SGKUJ8_$!idvojB62jzkqU zJimL1Oh!&h@|*lc?ML{$(jVd+n3HYnJxZ@a zoJU_7LtLO%l?VmTF-2$csfS8_NRO95hSgUq_0pQ*;tmcMWPx9gUE-|AUO15HZ~=y8 z&Ym}L(!}8pdm-%a3>y!UzJtTV@tjvIbi5P~L+&w^Q@G!D(smGckzV3%4a)^_Lly}n zcW_=w2Ydqt%LUB=E%6G#08Mn--YyC1#_)jCgpw?f5~+e_`AO6Z`9lY}%X2(v+9Pv= z330%bB=#C12X7g)+NrFqhuDgAa@;{9?(FdfU?(Kh#q!AOay862%pV0Ya$*uIsWiC( z2su}r+t`R)avJ>wUk*R33+C<5--dq2#_*O#2_o~a?wa6&t`$sWUr1Ri?@{G zs4!3_NVgby59^{AWz#s;dxuaNF5V-@Y>ai-NeRKN0B%3GA-ra)%*(?tw{)&3tl@3J zHAe_QlqGvH&X!j&TY~E_K^bvCjvsrY*$lvyNV@n6CI)jMyF$nFbA&? zhQ^zWJpmI0yIj`(PtN=yYN{o%N+Cdq>2icI+{Fl%f_q)6rur>}Cae}G$cG5TC1vwo zLRl>E!b4*^LDWM?$=U`42M8+VOJr;^zY1USOX}efa|G`tA`ht`&Lc$dCWmYv{w1dZ zR9#6yp@0rpz}XR$7LZvRURZx=e*jB(z!mTqo~5l((YHiN1_p=|$}cSk*&LP7B;yFS zK$Rly3}qT0)lWT7^;tn39mDi7*no40+6K(# z^iaow5t3dTi6&IZMlp`)0g_J~Gy@;ytK7~%%w>{0ZO=k-ICNj}Q?mUFYoJN|Xz@Q~-F$$0#-0c%FJK0vO2x!m%=!GyHvy z%W=9uh=8T=TDYyXrE`RY+=Wzcd(vsA5p0k+I6HxHLK@Bs;@n6W;t`V6zS{X&iTIQK z#!-i?=^-+5Z6qSxYXHWI!8pjp&yX5^P*c#2glar_8YCT8cpZ!BBdt?7sL86v_6w@D zsN-VEbw-$2mrds2tJ8Uhjyh=b&g`*Hy#de(5~`%( zI!70#JS({6d+6NBi$N-o-C{TJyGgT|S>!^u)Q^!=a4#40B(XR?y2T0fs*~adaebFa zjmYanmO)%~%fFQWtcjn~kVW?!+KmK^#Rg4%sj8cO*@7#WB!2@^GD`S2bXzwJ(C-Y} z&`Y|3Kl>lX+<4qN9Q&6sQ#Q)z+y61jWusI?>)-Pg!?KA3t@f4#^t2Y;dT=k_BD', methods=['GET']) +def blog_post(id): + # Fetch the blog entry from MongoDB Atlas + blog_entry = db.content.find_one({"_id": ObjectId(id)}) + + if blog_entry: + if blog_entry.get('status') == 'COMPLETED': + return render_template('result.html', report=html.unescape(blog_entry['report']), blog_post=blog_entry['blog_post']) + else: + return render_template('result.html', report=html.unescape(blog_entry['report']), blog_post=blog_entry['blog_post']) + else: + return "Blog entry not found", 404 if __name__ == '__main__': - app.run(debug=True, port=5001) + socketio.run(app, debug=True, port=5001) diff --git a/tasks.py b/tasks.py new file mode 100644 index 0000000..4cb5cdd --- /dev/null +++ b/tasks.py @@ -0,0 +1,143 @@ +from celery import Celery +from crewai import Agent, Task, Crew, tasks +from crewai_tools import SerperDevTool, ScrapeWebsiteTool, SeleniumScrapingTool +from langchain_openai import ChatOpenAI +from pymongo import MongoClient +import langchain_core +import html +import os +import logging + +# Set up logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +client = MongoClient("mongodb+srv://maheshkommareddi:Yu2L6pQKyJgcTb9a@cluster0.qadl40g.mongodb.net/?retryWrites=true&w=majority&appName=Cluster0") +db = client.content_generation + +# Initialize Celery +app = Celery('tasks', broker='amqp://guest:guest@localhost:5672//') + +# Load environment variables +os.environ["OPENAI_API_KEY"] = "sk-kkk" +os.environ["OPENAI_MODEL_NAME"] = "anthropic.claude-3-sonnet-20240229-v1:0" +os.environ["OPENAI_API_BASE"] = "http://chat.the.mk:1337" + +# Initialize LLM +llm = ChatOpenAI( + model="anthropic.claude-3-sonnet-20240229-v1:0", + base_url="http://chat.the.mk:1337", + temperature=0.1 +) + +# Load tools +search_tool = SerperDevTool() +scrape_tool = SeleniumScrapingTool() +scrape_tool_bare = ScrapeWebsiteTool() + +# Function to update task status in MongoDB +def update_task_status(task_id, status, message): + try: + db.task_updates.insert_one({"task_id": task_id, "status": status, "message": message}) + logger.info(f"Updated task status: {task_id}, {status}, {message}") + except Exception as e: + logger.error(f"Error updating task status: {e}") + +# Define tasks +@app.task +def generate_content(agenda): + def researcher_callback(output): + if isinstance(output, langchain_core.agents.AgentFinish): + update_task_status(app.current_task.request.id, f"researcher", "Agent finished") + elif output and output[0]: + print(output[0]) + if output[0][0]: + update_task_status(app.current_task.request.id, f"researcher", output[0][0].log) + + def writer_callback(output): + if isinstance(output, langchain_core.agents.AgentFinish): + update_task_status(app.current_task.request.id, f"writer", "Agent finished") + elif output and output[0]: + print(output[0]) + if output[0][0]: + update_task_status(app.current_task.request.id, f"writer", output[0][0].log) + + researcher = Agent( + role='Senior Research Analyst', + goal='Find way to explain ' + agenda, + backstory=( + "You are a Senior Research Analyst at a leading tech think tank." + f"Your expertise lies in identifying {agenda}." + "You have a knack for dissecting complex data and presenting actionable insights." + "Always search the web first and make the determination for the best 4 Links, but exclude PDFs" + "For any web searches, be sure to scrape the website content from the Link in the search" + ), + verbose=True, + allow_delegation=False, + tools=[search_tool, scrape_tool, scrape_tool_bare], + max_rpm=100, + step_callback=researcher_callback + ) + + writer = Agent( + role='Tech Content Strategist', + goal='Craft compelling content on ' + agenda, + backstory=( + "You are a renowned Tech Content Strategist, known for your insightful and engaging articles on science and innovation." + "With a deep understanding of the tech industry, you transform complex concepts into compelling narratives." + "For any web searches, be sure to scrape the website content from the Link in the search, but exclude PDFs" + ), + verbose=True, + allow_delegation=True, + tools=[search_tool, scrape_tool, scrape_tool_bare], + cache=False, # Disable cache for this agent + step_callback=writer_callback + ) + + task1 = Task( + description=( + f"Conduct a comprehensive analysis of the latest in {agenda}" + "Identify key trends, breakthrough technologies, and potential industry impacts." + "Compile your findings in a detailed report and include references and links to the source material." + ), + expected_output=f"A comprehensive full report on {agenda} in 2024, leave nothing out", + agent=researcher, + ) + + task2 = Task( + description=( + f"Using the insights from the researcher's report, develop an engaging blog post that highlights the most significant {agenda} ideas." + "Your post should be informative yet accessible, catering to a tech-savvy audience." + "Aim for a narrative that captures the essence of these breakthroughs and their implications for the future." + "Keep asking for research and revise until the minimum 5000 words are met" + "Include the research in the entirety along with the resulting report" + "Include at least five links to external pages or PDFs with an appropriate anchor tag in the final report" + ), + expected_output=f"A compelling ten paragraphs blog post formatted as html to place inside the body tag with headings, subheadings, and a main thesis about the latest {agenda}", + agent=writer + ) + + # Define the main callback for the crew + def main_callback(output: tasks.task_output.TaskOutput): + update_task_status(app.current_task.request.id, 1, output.description) + + # Instantiate your crew with a sequential process + crew = Crew( + agents=[researcher, writer], + tasks=[task1, task2], + verbose=4, + task_callback=main_callback + ) + + + result = crew.kickoff() + + # Save the result to MongoDB Atlas + content = { + "agenda": agenda, + "report": html.unescape(result), + "blog_post": html.unescape(result) + } + db.content.insert_one(content) + + return result diff --git a/templates/chat.html b/templates/chat.html new file mode 100644 index 0000000..d75cb9d --- /dev/null +++ b/templates/chat.html @@ -0,0 +1,58 @@ + + + + Content Generation Chat + + + + + +

Content Generation Chat

+
+ + + + \ No newline at end of file diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..f4f8b91 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,30 @@ + + + + Content Generation + + +

Content Generation

+
+ + + +
+ + {% if blog_entries %} +

Existing Blog Entries

+
    + {% for entry in blog_entries %} + {% if entry.blog_post %} +
  • + +

    {{ entry.agenda }}

     + 
    {{ entry.blog_post[:100] }}...
    + +
    • + {% endif %} + {% endfor %} +
+ {% endif %} + + \ No newline at end of file diff --git a/templates/processing.html b/templates/processing.html new file mode 100644 index 0000000..f6821c0 --- /dev/null +++ b/templates/processing.html @@ -0,0 +1,13 @@ + + + + Content Generation Processing + + + +

Content Generation Processing

+

Your request for generating content on "{{ agenda }}" is being processed.

+

Task ID: {{ task_id }}

+

This page will refresh every 5 seconds to check the status.

+ + \ No newline at end of file diff --git a/templates/result.html b/templates/result.html new file mode 100644 index 0000000..22e2d0a --- /dev/null +++ b/templates/result.html @@ -0,0 +1,13 @@ + + + + Content Generation Result + + +

Content Generation Result

+

Report

+ {{ report|safe }} + + + \ No newline at end of file