maheshvaikri/snapllm repository overview

Load Model A
Use Model A
Unload Model A
Load Model B
Use Model B

Load Model A
Load Model B
Select A or B without an avoidable reload while both remain resident

# Clone the repository
git clone https://github.com/snapllm/snapllm.git
cd snapllm

# Build with CUDA support
build_gpu.bat

build_cpu.bat

chmod +x build.sh
./build.sh gpu

# Check the build
build_gpu/bin/snapllm --help

# Start the server
build_gpu/bin/snapllm --server --port 6930

Start_Server.bat

build_gpu\bin\snapllm.exe --server --host 127.0.0.1 --port 6930

build_gpu\bin\snapllm.exe --server --host 127.0.0.1 --port 6930 --load-model mymodel D:\Models\mymodel.gguf

chmod +x start_server.sh
./start_server.sh

mkdir -p workspace models
export SNAPLLM_API_KEY='replace-with-a-random-value-at-least-32-characters'
docker compose up --build

# Start with a model pre-loaded
./snapllm --server --port 6930 --load-model mymodel /path/to/model.gguf

# Or start empty and load models via API
./snapllm --server --port 6930

# Load first model
curl -X POST http://localhost:6930/api/v1/models/load \
  -H "Content-Type: application/json" \
  -d '{
    "model_id": "llama3",
    "file_path": "/models/llama-3-8b-instruct.Q5_K_M.gguf"
  }'

# Load second model
curl -X POST http://localhost:6930/api/v1/models/load \
  -H "Content-Type: application/json" \
  -d '{
    "model_id": "gemma",
    "file_path": "/models/gemma-2-9b-it.Q5_K_M.gguf"
  }'

# Select gemma without reloading it if it is still resident
curl -X POST http://localhost:6930/api/v1/models/switch \
  -H "Content-Type: application/json" \
  -d '{"model_id": "gemma"}'

# OpenAI-compatible chat endpoint
curl -X POST http://localhost:6930/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "gemma",
    "messages": [
      {"role": "user", "content": "Explain quantum computing in simple terms"}
    ],
    "max_tokens": 256
  }'

cd desktop-app
npm install
npm run dev
# Open http://localhost:9780

http://localhost:6930

POST /api/v1/models/load

{
  "model_id": "medical-llm",
  "file_path": "D:/Models/medicine-llm.Q8_0.gguf",
  "model_type": "auto"
}

{
  "status": "success",
  "message": "Model loaded: medical-llm",
  "model": "medical-llm",
  "model_type": "Text LLM",
  "load_time_ms": 2500.5,
  "active": true
}

POST /v1/chat/completions

{
  "model": "medical-llm",
  "messages": [
    {"role": "system", "content": "You are a medical assistant."},
    {"role": "user", "content": "What are the symptoms of diabetes?"}
  ],
  "max_tokens": 512,
  "temperature": 0.7,
  "stream": false
}

{
  "id": "chatcmpl-abc123",
  "object": "chat.completion",
  "created": 1234567890,
  "model": "medical-llm",
  "choices": [
    {
      "index": 0,
      "message": {
        "role": "assistant",
        "content": "Common symptoms of diabetes include..."
      },
      "finish_reason": "stop"
    }
  ],
  "usage": {
    "prompt_tokens": 25,
    "completion_tokens": 150,
    "total_tokens": 175,
    "tokens_per_second": 64.5
  }
}

POST /api/v1/models/load

{
  "model_id": "gemma-vision",
  "file_path": "/models/gemma-3-4b-it.gguf",
  "model_type": "vision",
  "mmproj_path": "/models/mmproj-gemma-3-4b.gguf"
}

POST /api/v1/vision/generate

{
  "prompt": "Describe this image in detail",
  "images": ["<base64-encoded-image>"],
  "max_tokens": 256
}

{
  "status": "success",
  "response": "The image shows a beautiful sunset over mountains...",
  "model": "gemma-vision",
  "generation_time_s": 2.5,
  "tokens_per_second": 57.5
}

POST /api/v1/contexts/ingest

{
  "content": "Your large document text here...",
  "model_id": "llama3",
  "name": "company-handbook",
  "ttl_seconds": 86400
}

{
  "status": "success",
  "context_id": "ctx_abc123",
  "token_count": 5000,
  "storage_size_mb": 125.5,
  "tier": "hot",
  "ingest_time_ms": 1500.5
}

POST /api/v1/contexts/ctx_abc123/query

{
  "query": "What is the vacation policy?",
  "max_tokens": 100
}

{
  "status": "success",
  "context_id": "ctx_abc123",
  "response": "According to the handbook, employees receive...",
  "cache_hit": true,
  "latency_ms": 15.2
}

+--------------------------------------------------------------------+
|                          SnapLLM Server                             |
+--------------------------------------------------------------------+
|  Model A (vPID 1)  |  Model B (vPID 2)  |  Model C (vPID 3)  | ... |
+---------+----------+---------+----------+---------+----------+-----+
          |                    |                    |
          +--------------------+--------------------+
                               |
                               v
+--------------------------------------------------------------------+
|                    Unified Memory Manager                           |
|   HOT (GPU VRAM)    |   WARM (CPU RAM)    |    COLD (SSD)           |
+--------------------------------------------------------------------+
|               HTTP API Server (OpenAI-compatible)                   |
+--------------------------------------------------------------------+

Client -> HTTP API -> Router -> Model Manager -> Active Model -> Response
                           |                      ^
                           v                      |
                             vPID Cache (L1/L2) 

SnapLLM_Workspace/
|-- index.json                          # Model registry
|-- medicine-llm/
|   `-- Q8_0/
|       `-- workspace.bin               # Engine-managed workspace
|-- legal-llama/
|   `-- Q4_K_M/
|       `-- workspace.bin
|-- gemma-3-4b/
|   `-- Q5_K_M/
|       `-- workspace.bin
|-- diffusion/
|   `-- sd15/
|       `-- workspace.bin
`-- contexts/                           # vPID L2 KV caches
    |-- hot/
    |-- warm/
    `-- cold/

cd desktop-app
npm install
npm run dev
# Open http://localhost:9780

./snapllm --server [OPTIONS]

Options:
  --port PORT              Server port (default: 6930)
  --host HOST              Bind address (default: 127.0.0.1)
  --cors-origin ORIGIN     Allow an exact browser origin (repeatable)
  --workspace-root PATH    Workspace directory
  --ui-dir PATH            Web UI directory (auto-detected)
  --load-model NAME PATH   Pre-load a model

./snapllm --load-model NAME PATH [OPTIONS]

Options:
  --prompt TEXT            Generate text from prompt
  --max-tokens N           Maximum tokens to generate
  --temperature N          Sampling temperature
  --multi-model-test       Run multi-model switching benchmark
  --list-models            List all loaded models
  --stats                  Show cache statistics

./snapllm --load-diffusion NAME PATH [OPTIONS]

Options:
  --generate-image PROMPT  Generate image from text
  --output PATH            Output image path
  --width N                Image width (default: 512)
  --height N               Image height (default: 512)
  --steps N                Sampling steps (default: 20)
  --cfg-scale N            CFG scale (default: 7.0)
  --seed N                 Random seed (-1 for random)
  --negative PROMPT        Negative prompt

./snapllm --multimodal MODEL MMPROJ [OPTIONS]

Options:
  --image PATH             Input image file
  --vision-prompt TEXT     Prompt with <__media__> marker
  --max-tokens N           Maximum tokens to generate

# Clone the repository
git clone https://github.com/snapllm/snapllm.git
cd snapllm

# Build debug version
mkdir build && cd build
cmake .. -DCMAKE_BUILD_TYPE=Debug
cmake --build .

# Run tests
ctest --output-on-failure

MIT License

Copyright (c) 2024-2026 SnapLLM Contributors

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.