removed the torch_dtype

Signed-off-by: Dipankar Das <[email protected]>

app.py

@@ -4,7 +4,7 @@ from fastapi.responses import RedirectResponse
 
 from transformers import AutoTokenizer, AutoModelForCausalLM
 import transformers
-import torch
+# import torch
 
 model = "tiiuae/falcon-7b"
 
@@ -13,11 +13,11 @@ pipeline = transformers.pipeline(
     "text-generation",
     model=model,
     tokenizer=tokenizer,
-    torch_dtype=torch.bfloat16,
     trust_remote_code=True,
     device_map="auto",
 )
 
+    # torch_dtype=torch.bfloat16,
 
 app = FastAPI()
 

logger.log

@@ -0,0 +1,43 @@
+Downloading shards: 100%|██████████| 2/2 [03:33<00:00, 99.67s/it] 
+Downloading shards: 100%|██████████| 2/2 [03:33<00:00, 106.81s/it]
+Traceback (most recent call last):
+  File "/usr/local/bin/uvicorn", line 8, in <module>
+    sys.exit(main())
+  File "/usr/local/lib/python3.9/site-packages/click/core.py", line 1130, in __call__
+    return self.main(*args, **kwargs)
+  File "/usr/local/lib/python3.9/site-packages/click/core.py", line 1055, in main
+    rv = self.invoke(ctx)
+  File "/usr/local/lib/python3.9/site-packages/click/core.py", line 1404, in invoke
+    return ctx.invoke(self.callback, **ctx.params)
+  File "/usr/local/lib/python3.9/site-packages/click/core.py", line 760, in invoke
+    return __callback(*args, **kwargs)
+  File "/usr/local/lib/python3.9/site-packages/uvicorn/main.py", line 437, in main
+    run(app, **kwargs)
+  File "/usr/local/lib/python3.9/site-packages/uvicorn/main.py", line 463, in run
+    server.run()
+  File "/usr/local/lib/python3.9/site-packages/uvicorn/server.py", line 60, in run
+    return asyncio.run(self.serve(sockets=sockets))
+  File "/usr/local/lib/python3.9/asyncio/runners.py", line 44, in run
+    return loop.run_until_complete(main)
+  File "uvloop/loop.pyx", line 1517, in uvloop.loop.Loop.run_until_complete
+  File "/usr/local/lib/python3.9/site-packages/uvicorn/server.py", line 67, in serve
+    config.load()
+  File "/usr/local/lib/python3.9/site-packages/uvicorn/config.py", line 458, in load
+    self.loaded_app = import_from_string(self.app)
+  File "/usr/local/lib/python3.9/site-packages/uvicorn/importer.py", line 21, in import_from_string
+    module = importlib.import_module(module_str)
+  File "/usr/local/lib/python3.9/importlib/__init__.py", line 127, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1030, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1007, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 986, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 680, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 850, in exec_module
+  File "<frozen importlib._bootstrap>", line 228, in _call_with_frames_removed
+  File "/home/user/app/./app.py", line 12, in <module>
+    pipeline = transformers.pipeline(
+  File "/usr/local/lib/python3.9/site-packages/transformers/pipelines/__init__.py", line 788, in pipeline
+    framework, model = infer_framework_load_model(
+  File "/usr/local/lib/python3.9/site-packages/transformers/pipelines/base.py", line 278, in infer_framework_load_model
+    raise ValueError(f"Could not load model {model} with any of the following classes: {class_tuple}.")
+ValueError: Could not load model tiiuae/falcon-7b with any of the following classes: (<class 'transformers.models.auto.modeling_auto.AutoModelForCausalLM'>,).