commit 0066470e8569fb5e3a072aea5bd7a9d7f18ab9b7
Author: Daniel Graña <dangra@gmail.com>
Date:   Mon May 20 21:28:10 2024 -0300

    initial commit

diff --git a/fly.toml b/fly.toml
new file mode 100644
index 0000000..a7e8a74
--- /dev/null
+++ b/fly.toml
@@ -0,0 +1,25 @@
+app = "vllm-demo"
+primary_region = "ord"
+
+[experimental]
+entrypoint = "/start.sh"
+
+[build]
+image = "vllm/vllm-openai:v0.4.2"
+
+[[mounts]]
+source = "models"
+destination = "/root/.cache/huggingface"
+
+[http_service]
+internal_port = 8000
+force_https = true
+
+[[vm]]
+size = 'l40s'
+gpus = 1
+
+[[files]]
+guest_path = "/start.sh"
+local_path = "start.sh"
+
diff --git a/start.sh b/start.sh
new file mode 100644
index 0000000..a6e329b
--- /dev/null
+++ b/start.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+set -ex
+ldconfig
+exec python3 -m vllm.entrypoints.openai.api_server