superbizon007/linux-gui-agent repository overview

cd apps/linux_gui_agent
docker build \
  --build-context a2a_pkg=../../packages/a2a_agent \
  -f Dockerfile.ubuntu \
  -t linux_gui_agent:ubuntu .

docker run -d \
  --name gui_agent \
  -p 443:443 \
  -p 6901:6901 \
  -p 5901:5901 \
  -e VLM_URL=https://api.openai.com/v1 \
  -e VLM_API_KEY=sk-... \
  -e VLM_MODEL=gpt-4o \
  linux_gui_agent:ubuntu --wait

docker build \
  --build-context a2a_pkg=../../packages/a2a_agent \
  -f Dockerfile.dev \
  -t linux_gui_agent:dev .

docker run -d \
  --name gui_agent_dev \
  -p 5901:5901 \
  -p 6901:6901 \
  -p 6001:6001 \
  linux_gui_agent:dev --wait

curl -k https://localhost/ca.crt -o linux-gui-agent-ca.crt
sudo security add-trusted-cert -d -r trustRoot \
  -k /Library/Keychains/System.keychain linux-gui-agent-ca.crt

curl -k https://localhost/ca.crt -o /usr/local/share/ca-certificates/linux-gui-agent-ca.crt
sudo update-ca-certificates

curl -k https://localhost/ca.crt -o linux-gui-agent-ca.crt
certutil -addstore Root linux-gui-agent-ca.crt

curl -sk -X POST https://localhost/ \
  -H "Content-Type: application/json" \
  -d '{
    "jsonrpc": "2.0",
    "id": "1",
    "method": "message/send",
    "params": {
      "message": {
        "role": "user",
        "messageId": "m1",
        "parts": [{"kind": "text", "text": "Open Chrome, go to example.com, and tell me the page title"}]
      }
    }
  }'

curl -sk -X POST https://localhost/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "desktop-agent",
    "messages": [{"role": "user", "content": "Open a terminal and run ls /"}]
  }'

curl -sk https://localhost/.well-known/agent-card.json

curl -sk -N -X POST https://localhost/ \
  -H "Content-Type: application/json" \
  -d '{
    "jsonrpc": "2.0",
    "id": "1",
    "method": "message/stream",
    "params": {
      "message": {
        "role": "user",
        "messageId": "m1",
        "parts": [{"kind": "text", "text": "Take a screenshot and describe the desktop"}]
      }
    }
  }'

$TRAJECTORY_DIR/
  20260313_181523/              # run timestamp (YYYYMMDD_HHMMSS)
    summary.md                  # task, result, and final plan summary
    turn_0000/
      screenshot.png            # desktop state at start of step
      llm_0_request.json        # messages sent to the VLM
      llm_0_response.json       # raw VLM reply + parsed action/args/reasoning
      locator.json              # locator model request + response (when LOCATOR_URL is set)
    turn_0001/
      ...
  20260313_182041/              # next run
    ...

docker run -d \
  -e VLM_URL=... -e VLM_API_KEY=... \
  -e TRAJECTORY_DIR=/data/trajectory \
  -v /tmp/agent-logs:/data/trajectory \
  linux_gui_agent:ubuntu --wait

┌──────────────────────────────────────────────┐
│               Docker Container               │
│                                              │
│  Xfce desktop (DISPLAY :1)                  │
│    Chromium browser                          │
│    xfce4-terminal, mousepad, evince          │
│    flatpak + Flathub                         │
│         │                                    │
│  X11Client (python-xlib + wmctrl + xdotool)  │
│                                              │
│  LangGraph (graph.py)                        │
│    screenshot ──► X11Client.screenshot()     │
│    call_vlm   ──► VLM (OpenAI-compatible)    │
│    execute_action                            │
│       │   Locator model (optional)           │
│    generate_summary                          │
│         │                                    │
│  TigerVNC (:5901) + noVNC (:6901)           │
│                                              │
│  FastAPI Server (:443 HTTPS)                 │
│    ├── A2A router  (POST /)                  │
│    └── OpenAI router (/v1/...)               │
└──────────────────────────────────────────────┘