import time import json import requests import gradio as gr STYLE = """ .no-border { border: none !important; } .group-border { padding: 10px; border-width: 1px; border-radius: 10px; border-color: gray; border-style: solid; box-shadow: 1px 1px 3px; } .control-label-font { font-size: 13pt !important; } .control-button { background: none !important; border-color: #69ade2 !important; border-width: 2px !important; color: #69ade2 !important; } .center { text-align: center; } .right { text-align: right; } .no-label { padding: 0px !important; } .no-label > label > span { display: none; } .small-big { font-size: 12pt !important; } """ def avaliable_providers(): providers = [] headers = { "Content-Type": "application/json", } endpoint_url = "https://api.endpoints.huggingface.cloud/v2/provider" response = requests.get(endpoint_url, headers=headers) providers = {} for provider in response.json()['vendors']: if provider['status'] == 'available': regions = {} availability = False for region in provider['regions']: if region["status"] == "available": regions[region['name']] = { "label": region['label'], "computes": region['computes'] } availability = True if availability: providers[provider['name']] = regions return providers providers = avaliable_providers() def update_regions(provider): avalialbe_regions = [] regions = providers[provider] for region, attributes in regions.items(): avalialbe_regions.append(f"{region}[{attributes['label']}]") return gr.Dropdown.update( choices=avalialbe_regions, value=avalialbe_regions[0] if len(avalialbe_regions) > 0 else None ) def update_compute_options(provider, region): avalialbe_compute_options = [] computes = providers[provider][region.split("[")[0].strip()]["computes"] for compute in computes: if compute['status'] == 'available': accelerator = compute['accelerator'] numAccelerators = compute['numAccelerators'] memoryGb = compute['memoryGb'] architecture = compute['architecture'] instanceType = compute['instanceType'] pricePerHour = compute['pricePerHour'] type = f"{numAccelerators}vCPU {memoryGb} · {architecture}" if accelerator == "cpu" else f"{numAccelerators}x {architecture}" avalialbe_compute_options.append( f"{compute['accelerator'].upper()} [{compute['instanceSize']}] · {type} · {instanceType} · ${pricePerHour}/hour" ) return gr.Dropdown.update( choices=avalialbe_compute_options, value=avalialbe_compute_options[0] if len(avalialbe_compute_options) > 0 else None ) def submit( hf_account_input, hf_token_input, endpoint_name_input, provider_selector, region_selector, repository_selector, task_selector, framework_selector, compute_selector, min_node_selector, max_node_selector, security_selector, custom_kernel, max_input_length, max_tokens, max_batch_prefill_token, max_batch_total_token ): compute_resources = compute_selector.split("·") accelerator = compute_resources[0][:3].strip() size_l_index = compute_resources[0].index("[") - 1 size_r_index = compute_resources[0].index("]") size = compute_resources[0][size_l_index : size_r_index].strip() type = compute_resources[-2].strip() payload = { "accountId": hf_account_input.strip(), "compute": { "accelerator": accelerator.lower(), "instanceSize": size[1:], "instanceType": type, "scaling": { "maxReplica": int(max_node_selector), "minReplica": int(min_node_selector) } }, "model": { "framework": framework_selector.lower(), "image": { "custom": { "health_route": "/health", "env": { "DISABLE_CUSTOM_KERNELS": "true" if custom_kernel == "Enabled" else "false", "MAX_BATCH_PREFILL_TOKENS": str(max_batch_prefill_token), "MAX_BATCH_TOTAL_TOKENS": str(max_batch_total_token), "MAX_INPUT_LENGTH": str(max_input_length), "MAX_TOTAL_TOKENS": str(max_tokens), "MODEL_ID": repository_selector.lower(), # QUANTIZE: 'bitsandbytes' | 'gptq'; }, "url": "ghcr.io/huggingface/text-generation-inference:1.0.1", } }, "repository": repository_selector.lower(), # "revision": "main", "task": task_selector.lower() }, "name": endpoint_name_input.strip().lower(), "provider": { "region": region_selector.split("[")[0].lower(), "vendor": provider_selector.lower() }, "type": security_selector.lower() } print(payload) payload = json.dumps(payload) print(payload) headers = { "Authorization": f"Bearer {hf_token_input.strip()}", "Content-Type": "application/json", } endpoint_url = f"https://api.endpoints.huggingface.cloud/v2/endpoint/"#{hf_account_input.strip()}" print(endpoint_url) response = requests.post(endpoint_url, headers=headers, data=payload) if response.status_code == 400: return f"{response.text}. Malformed data in {payload}" elif response.status_code == 401: return "Invalid token" elif response.status_code == 409: return f"Endpoint {endpoint_name_input} already exists" elif response.status_code == 202: return f"Endpoint {endpoint_name_input} created successfully on {provider_selector.lower()} using {repository_selector.lower()}@main.\nPlease check out the progress at https://ui.endpoints.huggingface.co/endpoints." else: return f"something went wrong {response.status_code} = {response.text}" with gr.Blocks(css=STYLE) as hf_endpoint: with gr.Tab("Hugging Face", elem_classes=["no-border"]): gr.Markdown("# Deploy LLM on 🤗 Hugging Face Inference Endpoint", elem_classes=["center"]) with gr.Column(elem_classes=["group-border"]): with gr.Row(): with gr.Column(): gr.Markdown("""### Hugging Face account ID (name)""") hf_account_input = gr.Textbox(show_label=False, elem_classes=["no-label", "small-big"]) with gr.Column(): gr.Markdown("### Hugging Face access token") hf_token_input = gr.Textbox(show_label=False, type="password", elem_classes=["no-label", "small-big"]) with gr.Row(): with gr.Column(): gr.Markdown("""### Target model Model from the Hugging Face hub""") repository_selector = gr.Textbox( value="NousResearch/Nous-Hermes-Llama2-13b", interactive=False, show_label=False, elem_classes=["no-label", "small-big"] ) with gr.Column(): gr.Markdown("""### Target model version(branch) Branch name of the Model""") revision_selector = gr.Textbox( value=f"main", interactive=False, show_label=False, elem_classes=["no-label", "small-big"] ) with gr.Column(elem_classes=["group-border"]): with gr.Column(): gr.Markdown("""### Endpoint name Name for your new endpoint""") endpoint_name_input = gr.Textbox(show_label=False, elem_classes=["no-label", "small-big"]) with gr.Row(): with gr.Column(): gr.Markdown("""### Cloud Provider""") provider_selector = gr.Dropdown( choices=providers.keys(), interactive=True, show_label=False, elem_classes=["no-label", "small-big"] ) with gr.Column(): gr.Markdown("""### Cloud Region""") region_selector = gr.Dropdown( [], value="", interactive=True, show_label=False, elem_classes=["no-label", "small-big"] ) with gr.Row(visible=False): with gr.Column(): gr.Markdown("### Task") task_selector = gr.Textbox( value="text-generation", interactive=False, show_label=False, elem_classes=["no-label", "small-big"] ) with gr.Column(): gr.Markdown("### Framework") framework_selector = gr.Textbox( value="PyTorch", interactive=False, show_label=False, elem_classes=["no-label", "small-big"] ) with gr.Column(): gr.Markdown("""### Compute Instance Type""") compute_selector = gr.Dropdown( [], value="", interactive=True, show_label=False, elem_classes=["no-label", "small-big"] ) with gr.Row(): with gr.Column(): gr.Markdown("""### Min Number of Nodes""") min_node_selector = gr.Number( value=1, interactive=True, show_label=False, elem_classes=["no-label", "small-big"] ) with gr.Column(): gr.Markdown("""### Max Number of Nodes""") max_node_selector = gr.Number( value=1, interactive=True, show_label=False, elem_classes=["no-label", "small-big"] ) with gr.Column(): gr.Markdown("""### Security Level""") security_selector = gr.Radio( choices=["Protected", "Public", "Private"], value="Public", interactive=True, show_label=False, elem_classes=["no-label", "small-big"] ) with gr.Column(elem_classes=["group-border"]): with gr.Accordion("Serving Container", open=False, elem_classes=["no-border"]): with gr.Column(): gr.Markdown("""### Container Type Text Generation Inference is an optimized container for text generation task""") _ = gr.Textbox("Text Generation Inference", show_label=False, elem_classes=["no-label", "small-big"]) with gr.Row(): with gr.Column(): gr.Markdown("""### Custom Cuda Kernels TGI uses custom kernels to speed up inference for some models. You can try disabling them if you encounter issues.""") custom_kernel = gr.Dropdown( value="Enabled", choices=["Enabled", "Disabled"], interactive=True, show_label=False, elem_classes=["no-label", "small-big"] ) with gr.Column(): gr.Markdown("""### Quantization Quantization can reduce the model size and improve latency, with little degradation in model accuracy.""") _ = gr.Dropdown( value="None", choices=["None", "Bitsandbytes", "GPTQ"], interactive=True, show_label=False, elem_classes=["no-label", "small-big"] ) with gr.Row(): with gr.Column(): gr.Markdown("""### Max Input Length (per Query) Increasing this value can impact the amount of RAM required. Some models can only handle a finite range of sequences.""") max_input_length = gr.Number( value=1024, interactive=True, show_label=False, elem_classes=["no-label", "small-big"] ) with gr.Column(): gr.Markdown("""### Max Number of Tokens (per Query) The larger this value, the more memory each request will consume and the less effective batching can be.""") max_tokens = gr.Number( value=1512, interactive=True, show_label=False, elem_classes=["no-label", "small-big"] ) with gr.Row(): with gr.Column(): gr.Markdown("""### Max Batch Prefill Tokens Number of prefill tokens used during continuous batching. It can be useful to adjust this number since the prefill operation is memory-intensive and compute-bound.""") max_batch_prefill_token = gr.Number( value=2048, interactive=True, show_label=False, elem_classes=["no-label", "small-big"] ) with gr.Column(): gr.Markdown("""### Max Batch Total Tokens Number of tokens that can be passed before forcing waiting queries to be put on the batch. A value of 1000 can fit 10 queries of 100 tokens or a single query of 1000 tokens.""") max_batch_total_token = gr.Number( value=None, interactive=True, show_label=False, elem_classes=["no-label", "small-big"] ) submit_button = gr.Button( value="Submit", elem_classes=["control-label-font", "control-button"] ) status_txt = gr.Textbox( value="any status update will be displayed here", interactive=False, elem_classes=["no-label"] ) provider_selector.change(update_regions, inputs=provider_selector, outputs=region_selector) region_selector.change(update_compute_options, inputs=[provider_selector, region_selector], outputs=compute_selector) submit_button.click( submit, inputs=[ hf_account_input, hf_token_input, endpoint_name_input, provider_selector, region_selector, repository_selector, task_selector, framework_selector, compute_selector, min_node_selector, max_node_selector, security_selector, custom_kernel, max_input_length, max_tokens, max_batch_prefill_token, max_batch_total_token], outputs=status_txt) with gr.Tab("AWS", elem_classes=["no-border"]): gr.Markdown("# Deploy LLM on 🤗 Hugging Face Inference Endpoint", elem_classes=["center"]) with gr.Tab("GCP", elem_classes=["no-border"]): gr.Markdown("# Deploy LLM on 🤗 Hugging Face Inference Endpoint", elem_classes=["center"]) with gr.Tab("Azure", elem_classes=["no-border"]): gr.Markdown("# Deploy LLM on 🤗 Hugging Face Inference Endpoint", elem_classes=["center"]) with gr.Tab("Lambdalabs", elem_classes=["no-border"]): gr.Markdown("# Deploy LLM on 🤗 Hugging Face Inference Endpoint", elem_classes=["center"]) hf_endpoint.launch(enable_queue=True, debug=True)