|
--- |
|
library_name: transformers.js |
|
tags: |
|
- pose-estimation |
|
license: agpl-3.0 |
|
--- |
|
|
|
YOLOv8s-pose with ONNX weights to be compatible with Transformers.js. |
|
|
|
## Usage (Transformers.js) |
|
|
|
If you haven't already, you can install the [Transformers.js](https://huggingface.co./docs/transformers.js) JavaScript library from [NPM](https://www.npmjs.com/package/@xenova/transformers) using: |
|
```bash |
|
npm i @xenova/transformers |
|
``` |
|
|
|
**Example:** Perform pose-estimation w/ `Xenova/yolov8s-pose`. |
|
|
|
```js |
|
import { AutoModel, AutoProcessor, RawImage } from '@xenova/transformers'; |
|
|
|
// Load model and processor |
|
const model_id = 'Xenova/yolov8s-pose'; |
|
const model = await AutoModel.from_pretrained(model_id); |
|
const processor = await AutoProcessor.from_pretrained(model_id); |
|
|
|
// Read image and run processor |
|
const url = 'https://huggingface.co./datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg'; |
|
const image = await RawImage.read(url); |
|
const { pixel_values } = await processor(image); |
|
|
|
// Set thresholds |
|
const threshold = 0.3; // Remove detections with low confidence |
|
const iouThreshold = 0.5; // Used to remove duplicates |
|
const pointThreshold = 0.3; // Hide uncertain points |
|
|
|
// Predict bounding boxes and keypoints |
|
const { output0 } = await model({ images: pixel_values }); |
|
|
|
// Post-process: |
|
const permuted = output0[0].transpose(1, 0); |
|
// `permuted` is a Tensor of shape [ 8400, 56 ]: |
|
// - 8400 potential detections |
|
// - 56 parameters for each box: |
|
// - 4 for the bounding box dimensions (x-center, y-center, width, height) |
|
// - 1 for the confidence score |
|
// - 17 * 3 = 51 for the pose keypoints: 17 labels, each with (x, y, visibilitiy) |
|
|
|
// Example code to format it nicely: |
|
const results = []; |
|
const [scaledHeight, scaledWidth] = pixel_values.dims.slice(-2); |
|
for (const [xc, yc, w, h, score, ...keypoints] of permuted.tolist()) { |
|
if (score < threshold) continue; |
|
|
|
// Get pixel values, taking into account the original image size |
|
const x1 = (xc - w / 2) / scaledWidth * image.width; |
|
const y1 = (yc - h / 2) / scaledHeight * image.height; |
|
const x2 = (xc + w / 2) / scaledWidth * image.width; |
|
const y2 = (yc + h / 2) / scaledHeight * image.height; |
|
results.push({ x1, x2, y1, y2, score, keypoints }) |
|
} |
|
|
|
|
|
// Define helper functions |
|
function removeDuplicates(detections, iouThreshold) { |
|
const filteredDetections = []; |
|
|
|
for (const detection of detections) { |
|
let isDuplicate = false; |
|
let duplicateIndex = -1; |
|
let maxIoU = 0; |
|
|
|
for (let i = 0; i < filteredDetections.length; ++i) { |
|
const filteredDetection = filteredDetections[i]; |
|
const iou = calculateIoU(detection, filteredDetection); |
|
if (iou > iouThreshold) { |
|
isDuplicate = true; |
|
if (iou > maxIoU) { |
|
maxIoU = iou; |
|
duplicateIndex = i; |
|
} |
|
} |
|
} |
|
|
|
if (!isDuplicate) { |
|
filteredDetections.push(detection); |
|
} else if (duplicateIndex !== -1 && detection.score > filteredDetections[duplicateIndex].score) { |
|
filteredDetections[duplicateIndex] = detection; |
|
} |
|
} |
|
|
|
return filteredDetections; |
|
} |
|
|
|
function calculateIoU(detection1, detection2) { |
|
const xOverlap = Math.max(0, Math.min(detection1.x2, detection2.x2) - Math.max(detection1.x1, detection2.x1)); |
|
const yOverlap = Math.max(0, Math.min(detection1.y2, detection2.y2) - Math.max(detection1.y1, detection2.y1)); |
|
const overlapArea = xOverlap * yOverlap; |
|
|
|
const area1 = (detection1.x2 - detection1.x1) * (detection1.y2 - detection1.y1); |
|
const area2 = (detection2.x2 - detection2.x1) * (detection2.y2 - detection2.y1); |
|
const unionArea = area1 + area2 - overlapArea; |
|
|
|
return overlapArea / unionArea; |
|
} |
|
|
|
const filteredResults = removeDuplicates(results, iouThreshold); |
|
|
|
// Display results |
|
for (const { x1, x2, y1, y2, score, keypoints } of filteredResults) { |
|
console.log(`Found person at [${x1}, ${y1}, ${x2}, ${y2}] with score ${score.toFixed(3)}`) |
|
for (let i = 0; i < keypoints.length; i += 3) { |
|
const label = model.config.id2label[Math.floor(i / 3)]; |
|
const [x, y, point_score] = keypoints.slice(i, i + 3); |
|
if (point_score < pointThreshold) continue; |
|
console.log(` - ${label}: (${x.toFixed(2)}, ${y.toFixed(2)}) with score ${point_score.toFixed(3)}`); |
|
} |
|
} |
|
``` |
|
|
|
<details> |
|
|
|
<summary>See example output</summary> |
|
|
|
``` |
|
Found person at [533.1403350830078, 39.96531672477722, 645.8853149414062, 296.1657429695129] with score 0.739 |
|
- nose: (443.99, 91.98) with score 0.970 |
|
- left_eye: (449.84, 85.01) with score 0.968 |
|
- right_eye: (436.28, 86.54) with score 0.839 |
|
- left_ear: (458.69, 87.08) with score 0.822 |
|
- right_ear: (427.88, 89.20) with score 0.317 |
|
- left_shoulder: (471.29, 128.05) with score 0.991 |
|
- right_shoulder: (421.84, 127.22) with score 0.788 |
|
- left_elbow: (494.03, 174.09) with score 0.976 |
|
- right_elbow: (405.83, 162.81) with score 0.367 |
|
- left_wrist: (505.29, 232.06) with score 0.955 |
|
- right_wrist: (411.89, 213.05) with score 0.470 |
|
- left_hip: (469.48, 217.49) with score 0.978 |
|
- right_hip: (438.79, 216.48) with score 0.901 |
|
- left_knee: (474.03, 283.00) with score 0.957 |
|
- right_knee: (448.00, 287.90) with score 0.808 |
|
- left_ankle: (472.06, 339.67) with score 0.815 |
|
- right_ankle: (447.15, 340.44) with score 0.576 |
|
Found person at [0.03232002258300781, 57.89646775722503, 156.35095596313477, 370.9132190942764] with score 0.908 |
|
- nose: (60.48, 105.82) with score 0.975 |
|
- left_eye: (64.86, 100.59) with score 0.952 |
|
- right_eye: (55.12, 100.60) with score 0.855 |
|
- left_ear: (73.04, 101.96) with score 0.820 |
|
- right_ear: (51.07, 103.28) with score 0.482 |
|
- left_shoulder: (85.74, 137.77) with score 0.996 |
|
- right_shoulder: (42.04, 137.63) with score 0.988 |
|
- left_elbow: (101.10, 190.45) with score 0.988 |
|
- right_elbow: (25.75, 186.44) with score 0.937 |
|
- left_wrist: (115.93, 250.05) with score 0.975 |
|
- right_wrist: (7.39, 233.44) with score 0.918 |
|
- left_hip: (80.15, 242.20) with score 0.999 |
|
- right_hip: (52.69, 239.82) with score 0.999 |
|
- left_knee: (93.29, 326.00) with score 0.999 |
|
- right_knee: (57.42, 329.04) with score 0.998 |
|
- left_ankle: (100.24, 413.83) with score 0.992 |
|
- right_ankle: (50.47, 417.93) with score 0.988 |
|
Found person at [106.16920471191406, 8.419264698028565, 515.0135803222656, 530.6886708259583] with score 0.819 |
|
- nose: (134.03, 111.15) with score 0.921 |
|
- left_eye: (137.51, 100.95) with score 0.824 |
|
- right_eye: (131.82, 97.53) with score 0.489 |
|
- left_ear: (147.19, 92.96) with score 0.792 |
|
- left_shoulder: (188.28, 127.51) with score 0.993 |
|
- right_shoulder: (181.81, 149.32) with score 0.995 |
|
- left_elbow: (258.49, 199.10) with score 0.984 |
|
- right_elbow: (181.43, 251.27) with score 0.988 |
|
- left_wrist: (311.74, 257.93) with score 0.979 |
|
- right_wrist: (129.68, 284.38) with score 0.984 |
|
- left_hip: (267.43, 299.85) with score 1.000 |
|
- right_hip: (277.05, 307.50) with score 1.000 |
|
- left_knee: (232.15, 427.54) with score 0.999 |
|
- right_knee: (278.99, 453.09) with score 0.999 |
|
- left_ankle: (352.68, 457.89) with score 0.990 |
|
- right_ankle: (362.15, 554.69) with score 0.993 |
|
Found person at [425.3855133056641, 73.76281919479369, 640.6651306152344, 502.32841634750366] with score 0.876 |
|
- nose: (416.15, 149.68) with score 0.996 |
|
- left_eye: (430.34, 139.56) with score 0.984 |
|
- right_eye: (412.88, 142.56) with score 0.976 |
|
- left_ear: (446.59, 142.21) with score 0.843 |
|
- right_ear: (398.82, 144.52) with score 0.740 |
|
- left_shoulder: (436.54, 197.92) with score 0.999 |
|
- right_shoulder: (362.94, 210.20) with score 0.996 |
|
- left_elbow: (460.06, 293.80) with score 0.992 |
|
- right_elbow: (352.33, 262.09) with score 0.966 |
|
- left_wrist: (491.33, 364.20) with score 0.986 |
|
- right_wrist: (402.62, 272.23) with score 0.956 |
|
- left_hip: (429.79, 354.94) with score 0.999 |
|
- right_hip: (383.27, 372.77) with score 0.999 |
|
- left_knee: (461.07, 437.73) with score 0.998 |
|
- right_knee: (410.89, 522.05) with score 0.995 |
|
- left_ankle: (460.74, 552.53) with score 0.966 |
|
- right_ankle: (429.00, 560.54) with score 0.940 |
|
``` |
|
</details> |
