Create inference.py

inference.py

@@ -0,0 +1,30 @@
+import sagemaker
+import boto3
+from sagemaker.huggingface import HuggingFaceModel
+
+try:
+	role = sagemaker.get_execution_role()
+except ValueError:
+	iam = boto3.client('iam')
+	role = iam.get_role(RoleName='sagemaker_execution_role')['Role']['Arn']
+
+# Hub Model configuration. https://huggingface.co/models
+hub = {
+	'HF_MODEL_ID':'Pandago/llama_3.1_infer_pdf',
+	'HF_TASK':'undefined'
+}
+
+# create Hugging Face Model Class
+huggingface_model = HuggingFaceModel(
+	transformers_version='4.37.0',
+	pytorch_version='2.1.0',
+	py_version='py310',
+	env=hub,
+	role=role, 
+)
+
+# deploy model to SageMaker Inference
+predictor = huggingface_model.deploy(
+	initial_instance_count=1, # number of instances
+	instance_type='ml.m5.xlarge' # ec2 instance type
+)