Development Guide¶

git clone https://github.com/smg-project/smg.git
cd smg

# Install rustup
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh

# The toolchain CI uses, pinned for this checkout
rustup toolchain install 1.98.0
rustup override set 1.98.0

# Nightly, used only for rustfmt
rustup toolchain install nightly --profile minimal --component rustfmt

pip install pre-commit
pre-commit install                          # hooks that run on every commit
pre-commit install --hook-type commit-msg   # DCO sign-off and no-AI-attribution checks
pre-commit install --hook-type pre-push     # optional: branch-name check before pushing

# Debug build of the whole workspace
cargo build --locked

# Only the gateway binary: target/debug/smg
cargo build --locked --bin smg

# Optimized build (`make build` runs `cargo build --release`)
cargo build --locked --release

error: cannot update the lock file .../Cargo.lock because --locked was passed to prevent this

# Self-hosted engines only: no third-party provider routers
cargo build --locked -p smg --no-default-features --features grpc-client,jemalloc-stats --bin smg

# Self-hosted engines plus Anthropic
cargo build --locked -p smg --no-default-features \
  --features grpc-client,jemalloc-stats,provider-anthropic --bin smg

Anthropic routing is not compiled into this build; rebuild with the `provider-anthropic` Cargo feature

smg/
├── model_gateway/              # The gateway: package `smg`, binaries `smg` and `amg`
│   ├── src/
│   │   ├── main.rs             # CLI parsing and startup
│   │   ├── lib.rs              # Library root
│   │   ├── server.rs           # HTTP server and route table
│   │   ├── app_context.rs      # Shared application state
│   │   ├── config/             # RouterConfig types, builder, and validation
│   │   ├── routers/            # HTTP, gRPC and ZMQ, PD, and external-provider routing
│   │   ├── policies/           # Load-balancing policies
│   │   ├── worker/             # Worker registry, health, circuit breakers, load monitoring
│   │   ├── service_discovery/  # Kubernetes worker discovery
│   │   ├── mesh/               # HA mesh adapters (crates/mesh does the gossip)
│   │   ├── mesh_discovery/     # Router-peer discovery for the mesh
│   │   ├── middleware/         # Auth, request IDs, concurrency limits, admission scheduler
│   │   ├── rate_limit/         # Per-tenant token and request rate limiting
│   │   ├── endpoints/          # Non-inference endpoints: models, tokenize, parse, conversations, responses
│   │   ├── workflow/           # Registration workflows for workers, tokenizers, MCP servers, WASM modules
│   │   ├── observability/      # Logging, metrics, and tracing
│   │   ├── wasm/               # WASM plugin support
│   │   ├── health.rs           # Liveness and readiness endpoints
│   │   ├── tenant.rs           # Tenant identity
│   │   ├── rl_adapter.rs       # Glue to the RL control plane (crates/rl)
│   │   └── version.rs          # Version information
│   ├── tests/                  # Integration tests
│   └── benches/                # Criterion benchmarks
├── crates/                     # Library crates (directory: package name)
│   ├── auth/                   # smg-auth: control-plane authentication and authorization
│   ├── data_connector/         # data-connector: conversation and response storage backends
│   ├── engine_zmq_client/      # engine-zmq-client: ZMQ transport for same-host engine connections
│   ├── external_router/        # smg-external-router: third-party provider routers
│   ├── grpc_client/            # smg-grpc-client: engine gRPC clients, .proto files, Python and Go packages
│   ├── kv_index/               # kv-index: radix trees for cache-aware routing
│   ├── mcp/                    # smg-mcp: Model Context Protocol client
│   ├── mesh/                   # smg-mesh: HA mesh gossip and state synchronization
│   ├── mm_rdma/                # smg-mm-rdma: multimodal pixel RDMA (NIXL) transport
│   ├── mock_worker/            # mock-worker: mock HTTP, gRPC, and ZMQ workers for local and scale testing
│   ├── multimodal/             # llm-multimodal: image, video, and audio preprocessing
│   ├── protocols/              # openai-protocol: OpenAI-compatible API types
│   ├── radix_tree/             # smg-radix-tree: prefix-membership radix tree
│   ├── reasoning_parser/       # reasoning-parser: reasoning output extraction
│   ├── rl/                     # smg-rl: RL control plane
│   ├── tokenizer/              # llm-tokenizer: tokenizers, caching, and chat templates
│   ├── tool_parser/            # tool-parser: tool-call parsing
│   ├── wasm/                   # smg-wasm: WebAssembly runtime for plugins
│   └── workflow/               # wfaas: workflow engine for multi-step operations
├── bindings/
│   ├── python/                 # PyPI package `smg`: `smg launch` / `smg serve` and the Rust extension
│   └── golang/                 # Go SDK over a Rust FFI library
├── clients/
│   ├── rust/                   # smg-client: Rust HTTP client
│   ├── python/                 # smg-client: Python client (types generated from OpenAPI)
│   ├── java/                   # Generated Java types
│   └── openapi-gen/            # Generates the OpenAPI spec from the protocol types
├── grpc_servicer/              # smg-grpc-servicer: Python gRPC servicers for vLLM, SGLang, TokenSpeed, MLX
├── e2e_test/                   # End-to-end tests against real engines (pytest)
├── deploy/helm/smg/            # Helm chart
├── docker/                     # Dockerfiles for the gateway and engine images
├── examples/                   # RL and WASM plugin examples
├── scripts/                    # Dev and CI scripts (e2e-test.sh, install_opencv.sh, ci_*.sh, ...)
├── Cargo.toml                  # Workspace manifest and shared lints
├── Cargo.lock                  # Committed lockfile
├── clippy.toml, rustfmt.toml   # Rust lint and format configuration
├── ruff.toml, mypy.ini         # Python lint configuration
└── Makefile                    # Development shortcuts (make help)

# Terminal 1: two mock HTTP workers on 127.0.0.1:9000 and 127.0.0.1:9001
cargo run --locked -p mock-worker -- --http-count 2

# Terminal 2: the gateway, round-robin across them
cargo run --locked --bin smg -- launch \
  --worker-urls http://127.0.0.1:9000 http://127.0.0.1:9001 \
  --policy round_robin --log-level debug

# Terminal 3: a request
curl http://127.0.0.1:30000/v1/chat/completions \
  -H 'Content-Type: application/json' \
  -d '{"model": "mock-model", "messages": [{"role": "user", "content": "Hello"}]}'

# Format (nightly, because rustfmt.toml uses unstable options)
cargo +nightly fmt --all

# Check formatting without changing files, as CI does
cargo +nightly fmt --all -- --check

# Lint the way CI and the pre-commit hook do
cargo clippy --locked --all-targets --all-features -- -D warnings

# Apply clippy's machine-applicable fixes, then re-run the lint above
cargo clippy --locked --all-targets --all-features --fix --allow-dirty

# Self-hosted-only build (no provider routers)
cargo clippy --locked -p smg -p smg-external-router --no-default-features \
  --features smg/grpc-client,smg/jemalloc-stats --all-targets -- -D warnings

# Anthropic-only build
cargo clippy --locked -p smg -p smg-external-router --no-default-features \
  --features smg/grpc-client,smg/jemalloc-stats,smg/provider-anthropic --all-targets -- -D warnings

ruff check e2e_test/ bindings/python/ scripts/
ruff format --check e2e_test/ bindings/python/ scripts/
mypy e2e_test/ --config-file mypy.ini
mypy bindings/python/ --config-file mypy.ini

git commit -s -m "feat(policies): add a weighted policy"

git rebase HEAD~N --signoff   # N = the number of commits to sign
git push --force-with-lease

# Everything (what `make test` and CI run)
cargo test --locked

# The gateway's unit tests only
cargo test --locked -p smg --lib

# One integration test binary: routing_tests.rs and tests/routing/
cargo test --locked -p smg --test routing_tests

# Tests whose names contain a string, with their output shown
cargo test --locked -p smg round_robin -- --nocapture

#[cfg(test)]
mod tests {
    use super::*;
    use crate::worker::{BasicWorkerBuilder, WorkerType};

    #[test]
    fn test_round_robin_selection() {
        let policy = RoundRobinPolicy::new();
        let workers: Vec<Arc<dyn Worker>> = vec![/* three BasicWorkerBuilder workers */];

        // Should select workers in order: 0, 1, 2, 0, 1, 2, ...
        let info = SelectWorkerInfo::default();
        assert_eq!(policy.select_worker(&workers, &info), Some(0));
        assert_eq!(policy.select_worker(&workers, &info), Some(1));
        assert_eq!(policy.select_worker(&workers, &info), Some(2));
        assert_eq!(policy.select_worker(&workers, &info), Some(0));
    }
}

use crate::common::{AppTestContext, TestRouterConfig, TestWorkerConfig};

#[tokio::test]
async fn test_round_robin_distribution() {
    let config = TestRouterConfig::round_robin(3100);
    let ctx =
        AppTestContext::new_with_config(config, TestWorkerConfig::healthy_workers(19001, 3))
            .await;

    let app = ctx.create_app();
    // ... send 30 POST /generate requests through `app` and count the 200s ...

    assert_eq!(
        success_count, num_requests,
        "All requests should succeed with round robin"
    );

    ctx.shutdown().await;
}

docker run -d -e POSTGRES_PASSWORD=test -e POSTGRES_DB=smg_test -p 5432:5432 postgres:16
DATA_CONNECTOR_TEST_POSTGRES_URL=postgres://postgres:test@localhost:5432/smg_test \
  cargo test --locked -p data-connector --test postgres_integration -- --ignored --test-threads=1

bash crates/wasm/tests/fixtures/build_fixtures.sh   # adds the wasm32-wasip2 target if missing
cargo test --locked -p smg-wasm

python3 -m venv .venv && source .venv/bin/activate
pip install maturin pytest
make python-dev          # maturin develop: a debug build of bindings/python

cd bindings/python
pytest -q tests

# One-time setup, in the virtualenv that has the engine installed
make python-dev                       # the harness launches `python3 -m smg.launch_router`
make generate-python-types            # generate the smg_client types (uses uvx)
bash scripts/ci_install_e2e_deps.sh   # pip install e2e_test/ and clients/python/

# One suite, on vLLM, 1-GPU tests only
E2E_RUNTIME=vllm E2E_ENGINE=vllm E2E_GPU_TIER=1 ROUTER_LOCAL_MODEL_PATH=$HOME/models \
  pytest e2e_test/chat_completions -s -vv

pip install ./e2e_test
PYTHONPATH=e2e_test pytest -q --noconftest e2e_test/infra e2e_test/fixtures

cd bindings/golang
make test   # builds the Rust FFI library (release), then runs go test ./...

cargo bench --locked --bench manual_policy_benchmark
cargo bench --locked --bench tool_parser_benchmark

// model_gateway/src/policies/my_policy.rs (illustrative)
use std::sync::Arc;

use super::{get_healthy_worker_indices, LoadBalancingPolicy, SelectWorkerInfo};
use crate::worker::Worker;

#[derive(Debug, Default)]
pub struct MyPolicy;

impl LoadBalancingPolicy for MyPolicy {
    fn select_worker(
        &self,
        workers: &[Arc<dyn Worker>],
        _info: &SelectWorkerInfo,
    ) -> Option<usize> {
        // Return an index into `workers`, or None when no worker is available.
        get_healthy_worker_indices(workers).first().copied()
    }

    fn name(&self) -> &'static str {
        "my_policy"
    }

    fn as_any(&self) -> &dyn std::any::Any {
        self
    }
}

# --log-level takes debug, info (default), warn, or error
cargo run --locked --bin smg -- launch --worker-urls http://127.0.0.1:9000 --log-level debug

# RUST_LOG replaces the whole filter, which allows per-module and trace levels
RUST_LOG=info,smg::policies=trace \
  cargo run --locked --bin smg -- launch --worker-urls http://127.0.0.1:9000

cargo build --locked --bin smg
lldb target/debug/smg -- launch --worker-urls http://127.0.0.1:9000

cargo install flamegraph
cargo flamegraph --bin smg -- launch --worker-urls http://127.0.0.1:9000

git clone https://github.com/smg-project/smg-docs.git
cd smg-docs
pnpm install
pnpm dev     # local dev server
pnpm lint    # prettier + eslint
pnpm check   # type checks
pnpm build   # production build

brew install openssl@3
export OPENSSL_DIR="$(brew --prefix openssl@3)"

# Install OpenCV (Homebrew on macOS, apt on Debian/Ubuntu)
make opencv-deps

# ...or skip the feature for a quick lint
cargo clippy --locked --workspace --all-targets -- -D warnings