wip(tpu): training working, tgi not working

.github/workflows/debug-dind.yml

@@ -17,55 +17,59 @@ jobs:
           apt-get update -y
           apt-get install -y docker.io
 
-      - name: Create test server Dockerfile
+      - name: Run test
         run: |
-          cat << EOF > Dockerfile
-          FROM us-central1-docker.pkg.dev/tpu-pytorch-releases/docker/xla:r2.4.0_3.10_tpuvm
+          python debug-dind-locally/test.py
+
+    #   - name: Create test server Dockerfile
+    #     run: |
+    #       cat << EOF > Dockerfile
+    #       FROM us-central1-docker.pkg.dev/tpu-pytorch-releases/docker/xla:r2.4.0_3.10_tpuvm
 
-          WORKDIR /app
-          RUN pip install fastapi uvicorn
+    #       WORKDIR /app
+    #       RUN pip install fastapi uvicorn
 
-          COPY server.py .
+    #       COPY server.py .
 
-          EXPOSE 80
-          CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "80"]
-          EOF
+    #       EXPOSE 80
+    #       CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "80"]
+    #       EOF
 
-      - name: Create minimal test server
-        run: |
-          cat << EOF > server.py
-          from fastapi import FastAPI
-          from pydantic import BaseModel
+    #   - name: Create minimal test server
+    #     run: |
+    #       cat << EOF > server.py
+    #       from fastapi import FastAPI
+    #       from pydantic import BaseModel
 
-          app = FastAPI()
+    #       app = FastAPI()
 
-          class GenerateRequest(BaseModel):
-              inputs: str
+    #       class GenerateRequest(BaseModel):
+    #           inputs: str
 
-          @app.post("/generate")
-          async def generate(request: GenerateRequest):
-              return {
-                  "generated_text": "Hello World!",
-                  "request_received": request.dict()
-              }
-          EOF
+    #       @app.post("/generate")
+    #       async def generate(request: GenerateRequest):
+    #           return {
+    #               "generated_text": "Hello World!",
+    #               "request_received": request.dict()
+    #           }
+    #       EOF
 
-      - name: Build and run test container
-        run: |
-          docker build -t test-tgi-server .
-          docker run -d -p 80:80 --name test-server test-tgi-server
+    #   - name: Build and run test container
+    #     run: |
+    #       docker build -t test-tgi-server .
+    #       docker run -d -p 80:80 --name test-server test-tgi-server
 
-          sleep 5
+    #       sleep 5
 
-          docker logs -f test-server &
+    #       docker logs -f test-server &
 
-          sleep 10
+    #       sleep 10
 
-          # Test the endpoint
-          curl --max-time 30 localhost:80/generate \
-            -X POST \
-            -d '{"inputs":"test message"}' \
-            -H 'Content-Type: application/json'
+    #       # Test the endpoint
+    #       curl --max-time 30 localhost:80/generate \
+    #         -X POST \
+    #         -d '{"inputs":"test message"}' \
+    #         -H 'Content-Type: application/json'
 
-          # Clean up
-          docker stop test-server
+    #       # Clean up
+    #       docker stop test-server

Dockerfile

@@ -0,0 +1,9 @@
+FROM us-central1-docker.pkg.dev/tpu-pytorch-releases/docker/xla:r2.4.0_3.10_tpuvm
+
+WORKDIR /app
+RUN pip install fastapi uvicorn
+
+COPY server.py .
+
+EXPOSE 80
+CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "80"]

debug-dind-locally/test.py

@@ -0,0 +1,86 @@
+# test_setup.py
+import os
+import subprocess
+import time
+
+def create_dockerfile():
+    dockerfile_content = """FROM us-central1-docker.pkg.dev/tpu-pytorch-releases/docker/xla:r2.4.0_3.10_tpuvm
+
+WORKDIR /app
+RUN pip install fastapi uvicorn
+
+COPY server.py .
+
+EXPOSE 80
+CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "80"]
+"""
+    with open("Dockerfile", "w") as f:
+        f.write(dockerfile_content)
+
+def create_server():
+    server_content = """from fastapi import FastAPI
+from pydantic import BaseModel
+
+app = FastAPI()
+
+class GenerateRequest(BaseModel):
+    inputs: str
+
+@app.post("/generate")
+async def generate(request: GenerateRequest):
+    return {
+        "generated_text": "Hello World!",
+        "request_received": request.dict()
+    }
+"""
+    with open("server.py", "w") as f:
+        f.write(server_content)
+
+def run_test():
+    # Create necessary files
+    create_dockerfile()
+    create_server()
+
+    try:
+        # Build container
+        print("Building container...")
+        subprocess.run(["docker", "build", "-t", "test-tgi-server", "."], check=True)
+
+        # Run container with explicit network binding
+        print("Starting container...")
+        subprocess.run([
+            "docker", "run", "-d",
+            "-p", "0.0.0.0:80:80",  # Explicitly bind to 0.0.0.0
+            "--name", "test-server",
+            "--network", "host",  # Use host networking
+            "test-tgi-server"
+        ], check=True)
+
+        # Wait for server to start
+        print("Waiting for server to start...")
+        time.sleep(5)
+
+        # Show logs
+        print("Container logs:")
+        subprocess.run(["docker", "logs", "test-server"], check=True)
+
+        # Test endpoint using 0.0.0.0
+        print("\nTesting endpoint...")
+        curl_command = [
+            "curl", "--max-time", "30", "http://0.0.0.0:80/generate",
+            "-X", "POST",
+            "-d", '{"inputs":"test message"}',
+            "-H", "Content-Type: application/json"
+        ]
+        subprocess.run(curl_command, check=True)
+
+    except subprocess.CalledProcessError as e:
+        print(f"Error occurred: {e}")
+    finally:
+        # Cleanup
+        print("\nCleaning up...")
+        subprocess.run(["docker", "stop", "test-server"], check=False)
+        subprocess.run(["docker", "rm", "test-server"], check=False)
+
+if __name__ == "__main__":
+    run_test()

server.py

@@ -0,0 +1,14 @@
+from fastapi import FastAPI
+from pydantic import BaseModel
+
+app = FastAPI()
+
+class GenerateRequest(BaseModel):
+    inputs: str
+
+@app.post("/generate")
+async def generate(request: GenerateRequest):
+    return {
+        "generated_text": "Hello World!",
+        "request_received": request.dict()
+    }