carbathrv/andy repository overview

docker model pull carbathrv/andy

from openai import OpenAI

client = OpenAI(base_url="http://localhost:12434/engines/v1", api_key="unused")

SYSTEM = """You control an Android phone for the user. Choose the single next action that best advances the user's goal on the current screen.
Reply with exactly one JSON object copied from the allowed actions, and nothing else.
Actions: tap an element, type into a text field (the text itself is filled in later), scroll the screen up/down/left/right, go back, go home, open an app, wait for loading, done when the goal is complete, or handoff to ask the user when you cannot proceed.
Example reply format: {"op":"tap","id":0}"""

screen = """Goal: Watch the video on how to make molten chocolate cake in the Kitchen Stories app.
Current app: com.ajnsnewmedia.kitchenstories
Previous actions:
1. opened app Kitchen Stories
Screen: change since the last action unknown · 6 elements
Screen elements (@x,y = centre, % of screen width,height):
[1] FrameLayout #content @50,50
  [2] View "Our Latest Recipes" @43,75
  [3] Button "See all" @89,75 (click)
  [4] ImageView "Today, Tab 1 of 3" @16,94 (click,selected)
  [5] ImageView "Search, Tab 2 of 3" @50,94 (click)
  [6] ImageView "Profile , Tab 3 of 3" @83,94 (click)
Allowed actions:
{"op":"tap","id":3}
{"op":"tap","id":4}
{"op":"tap","id":5}
{"op":"tap","id":6}
{"op":"scroll","dir":"up"}
{"op":"scroll","dir":"down"}
{"op":"back"}
{"op":"home"}
{"op":"wait"}
{"op":"done"}
{"op":"handoff"}"""

r = client.chat.completions.create(
    model="carbathrv/andy",
    messages=[{"role": "system", "content": SYSTEM}, {"role": "user", "content": screen}],
    temperature=0,
    max_tokens=48,
    extra_body={"chat_template_kwargs": {"enable_thinking": False}},
)
print(r.choices[0].message.content)   # {"op":"tap","id":5}