From 5dd630ed7a0fd73ee1a8db164988144e0e2411f2 Mon Sep 17 00:00:00 2001 From: Mashrfee Aryan Date: Sun, 2 Aug 2026 14:58:01 -0500 Subject: [PATCH 1/4] Implement initial Gemini system prompt --- automation/prompt.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 automation/prompt.py diff --git a/automation/prompt.py b/automation/prompt.py new file mode 100644 index 0000000..adc4353 --- /dev/null +++ b/automation/prompt.py @@ -0,0 +1,28 @@ +SYSTEM_PROMPT = """ +You are Friday. + +You are controlling a Windows computer. + +The user will give you a goal. + +You will continuously receive screenshots of the computer. + +Your job is to accomplish the goal by writing Python code. + +The code you write will be executed immediately. + +After execution, you will receive another screenshot. + +Based on the new screenshot, continue writing Python until the task is complete. + +Guidelines: + +- Use Python. +- Use any installed Python libraries. +- If another library is required, install it. +- Think step by step. +- Only output executable Python code. +- Do not explain your reasoning. +- Do not wrap your code in Markdown. +- Do not output anything except Python. +""" \ No newline at end of file From 05ad985395215075f9f2a9868abcb90797d9d31f Mon Sep 17 00:00:00 2001 From: Mashrfee Aryan Date: Sun, 2 Aug 2026 14:58:38 -0500 Subject: [PATCH 2/4] feat(model): implement Gemini model interface --- automation/model.py | 66 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 automation/model.py diff --git a/automation/model.py b/automation/model.py new file mode 100644 index 0000000..7096902 --- /dev/null +++ b/automation/model.py @@ -0,0 +1,66 @@ +""" +model.py + +Maintains a continuous conversation with Gemini. + +Its only job is to: +1. Send the goal and latest screenshot to Gemini. +2. Keep the conversation alive. +3. Return the Python code Gemini generates. +""" + +import os + +from google import genai + + +class Model: + def __init__(self): + api_key = os.getenv("GEMINI_API_KEY") + + if not api_key: + raise ValueError("GEMINI_API_KEY not found.") + + self.client = genai.Client(api_key=api_key) + self.model = "gemini-2.5-flash" + + # Stores the conversation history so Gemini remembers + # the goal and previous screenshots. + self.history = [] + + def send( + self, + system_prompt: str, + user_prompt: str, + screenshot, + ) -> str: + """ + Sends the latest computer state to Gemini and returns + the Python code it generates. + """ + + self.history.append( + { + "role": "user", + "parts": [ + {"text": f"{system_prompt}\n\n{user_prompt}"}, + screenshot, + ], + } + ) + + response = self.client.models.generate_content( + model=self.model, + contents=self.history, + ) + + self.history.append( + { + "role": "model", + "parts": [ + {"text": response.text}, + ], + } + ) + + return response.text \ No newline at end of file From 3dbfa65ebd9d583612184dc0d8c30c35fbbad9b2 Mon Sep 17 00:00:00 2001 From: Mashrfee Aryan Date: Sun, 2 Aug 2026 14:59:29 -0500 Subject: [PATCH 3/4] feat(computer): implement computer interaction module --- automation/computer.py | 59 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 automation/computer.py diff --git a/automation/computer.py b/automation/computer.py new file mode 100644 index 0000000..adcd009 --- /dev/null +++ b/automation/computer.py @@ -0,0 +1,59 @@ +""" +computer.py + +Handles all interaction with the local computer. +""" + +import subprocess +from pathlib import Path + +import mss +from PIL import Image + + +class Computer: + def __init__(self): + self.generated_file = Path("generated.py") + self.screenshot_file = Path("screenshot.png") + + def take_screenshot(self) -> Path: + """ + Captures the current screen. + Returns the path to the screenshot. + """ + + with mss.mss() as sct: + monitor = sct.monitors[1] + screenshot = sct.grab(monitor) + + image = Image.frombytes( + "RGB", + screenshot.size, + screenshot.rgb, + ) + + image.save(self.screenshot_file) + + return self.screenshot_file + + def execute(self, code: str) -> dict: + """ + Saves and executes the Python code generated by the AI. + """ + + self.generated_file.write_text( + code, + encoding="utf-8", + ) + + result = subprocess.run( + ["python", str(self.generated_file)], + capture_output=True, + text=True, + ) + + return { + "stdout": result.stdout, + "stderr": result.stderr, + "returncode": result.returncode, + } \ No newline at end of file From f969e4be6d7043ea980d12068538916183eddf16 Mon Sep 17 00:00:00 2001 From: Mashrfee Aryan Date: Sun, 2 Aug 2026 14:59:58 -0500 Subject: [PATCH 4/4] feat(main): implement Friday execution loop --- automation/main.py | 59 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 automation/main.py diff --git a/automation/main.py b/automation/main.py new file mode 100644 index 0000000..f5dbc8d --- /dev/null +++ b/automation/main.py @@ -0,0 +1,59 @@ +""" +main.py + +Friday's main loop. +""" + +from computer import Computer +from model import Model +from prompt import SYSTEM_PROMPT + + +def main(): + computer = Computer() + model = Model() + + goal = input("What would you like Friday to do?\n> ") + + while True: + # Take the latest screenshot + screenshot = computer.take_screenshot() + + # Build the user prompt + user_prompt = f""" +Current Goal: +{goal} + +This is the latest screenshot of the computer. + +Figure out the next step. + +If the task is complete, simply output: + +TASK_COMPLETE + +Otherwise, output ONLY executable Python code. +""" + + # Ask Gemini what to do next + response = model.send( + SYSTEM_PROMPT, + user_prompt, + screenshot, + ) + + # Stop if the task is finished + if response.strip() == "TASK_COMPLETE": + print("Task completed.") + break + + # Execute the generated Python + result = computer.execute(response) + + # Optional: print any errors for debugging + if result["stderr"]: + print(result["stderr"]) + + +if __name__ == "__main__": + main() \ No newline at end of file