initial commit

fly.toml

@@ -0,0 +1,25 @@
+app = "vllm-demo"
+primary_region = "ord"
+
+[experimental]
+entrypoint = "/start.sh"
+
+[build]
+image = "vllm/vllm-openai:v0.4.2"
+
+[[mounts]]
+source = "models"
+destination = "/root/.cache/huggingface"
+
+[http_service]
+internal_port = 8000
+force_https = true
+
+[[vm]]
+size = 'l40s'
+gpus = 1
+
+[[files]]
+guest_path = "/start.sh"
+local_path = "start.sh"
+

start.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+set -ex
+ldconfig
+exec python3 -m vllm.entrypoints.openai.api_server