59:I[603186,["/_next/static/chunks/41aagivimsyys.js","/_next/static/chunks/1w6b9x4ox-2ab.js","/_next/static/chunks/2du4f4hdcx8vy.js","/_next/static/chunks/3lydxrl6slrf2.js","/_next/static/chunks/1la1pzs9j8idc.js","/_next/static/chunks/2hrea-dwwjjr_.js","/_next/static/chunks/28ougzph2g_t9.js","/_next/static/chunks/3r5yd4k4rms8-.js","/_next/static/chunks/125flzb3dd6q2.js","/_next/static/chunks/0wbn6hf2c6k2t.js","/_next/static/chunks/3pjc6r7w9qxan.js","/_next/static/chunks/39vtz3iby1cf_.js","/_next/static/chunks/2v5t_o330ua3s.js","/_next/static/chunks/3trxs7mom31q3.js","/_next/static/chunks/3nk0wrm9knx3m.js","/_next/static/chunks/1kxuebyfge68i.js","/_next/static/chunks/22e07luyn9bfp.js","/_next/static/chunks/1_lyn-lih218f.js","/_next/static/chunks/1dwajbnfjnvzx.js","/_next/static/chunks/0bvim9_g0hu68.js","/_next/static/chunks/3c7e5ojau5akn.js","/_next/static/chunks/0gf5i4fmb2vxz.js","/_next/static/chunks/3a57oltrmz5kh.js","/_next/static/chunks/3j9dwf14_qrhp.js","/_next/static/chunks/21k90yd6lrzsc.js","/_next/static/chunks/3-5mrnq0kjby0.js","/_next/static/chunks/1r40kzfox4d83.js","/_next/static/chunks/24_gg9g_4g22i.js","/_next/static/chunks/0l0c166aeafbb.js","/_next/static/chunks/0czpc-f8wjri8.js","/_next/static/chunks/3xaq4o65p_dw0.js","/_next/static/chunks/2dyt8rg18m736.js","/_next/static/chunks/43b9k5vjei3d1.js","/_next/static/chunks/3eatuedmz1636.js","/_next/static/chunks/2pqhypz2vo8mu.js","/_next/static/chunks/1nz6e-lpv7lb-.js","/_next/static/chunks/2l525havh4_yi.js","/_next/static/chunks/0fpx6rmsgi4um.js"],"PageMetadata"] 5a:I[831915,["/_next/static/chunks/41aagivimsyys.js","/_next/static/chunks/1w6b9x4ox-2ab.js","/_next/static/chunks/2du4f4hdcx8vy.js","/_next/static/chunks/3lydxrl6slrf2.js","/_next/static/chunks/1la1pzs9j8idc.js","/_next/static/chunks/2hrea-dwwjjr_.js","/_next/static/chunks/28ougzph2g_t9.js","/_next/static/chunks/3r5yd4k4rms8-.js","/_next/static/chunks/125flzb3dd6q2.js","/_next/static/chunks/0wbn6hf2c6k2t.js","/_next/static/chunks/3pjc6r7w9qxan.js","/_next/static/chunks/39vtz3iby1cf_.js","/_next/static/chunks/2v5t_o330ua3s.js","/_next/static/chunks/3trxs7mom31q3.js","/_next/static/chunks/3nk0wrm9knx3m.js","/_next/static/chunks/1kxuebyfge68i.js","/_next/static/chunks/22e07luyn9bfp.js","/_next/static/chunks/1_lyn-lih218f.js","/_next/static/chunks/1dwajbnfjnvzx.js","/_next/static/chunks/0bvim9_g0hu68.js","/_next/static/chunks/3c7e5ojau5akn.js","/_next/static/chunks/0gf5i4fmb2vxz.js","/_next/static/chunks/3a57oltrmz5kh.js","/_next/static/chunks/3j9dwf14_qrhp.js","/_next/static/chunks/21k90yd6lrzsc.js","/_next/static/chunks/3-5mrnq0kjby0.js","/_next/static/chunks/1r40kzfox4d83.js","/_next/static/chunks/24_gg9g_4g22i.js","/_next/static/chunks/0l0c166aeafbb.js","/_next/static/chunks/0czpc-f8wjri8.js","/_next/static/chunks/3xaq4o65p_dw0.js","/_next/static/chunks/2dyt8rg18m736.js","/_next/static/chunks/43b9k5vjei3d1.js","/_next/static/chunks/3eatuedmz1636.js","/_next/static/chunks/2pqhypz2vo8mu.js","/_next/static/chunks/1nz6e-lpv7lb-.js","/_next/static/chunks/2l525havh4_yi.js","/_next/static/chunks/0fpx6rmsgi4um.js"],"SearchResultsImage"] 5b:I[66106,["/_next/static/chunks/41aagivimsyys.js","/_next/static/chunks/1w6b9x4ox-2ab.js","/_next/static/chunks/2du4f4hdcx8vy.js","/_next/static/chunks/3lydxrl6slrf2.js","/_next/static/chunks/1la1pzs9j8idc.js","/_next/static/chunks/2hrea-dwwjjr_.js","/_next/static/chunks/28ougzph2g_t9.js","/_next/static/chunks/3r5yd4k4rms8-.js","/_next/static/chunks/125flzb3dd6q2.js","/_next/static/chunks/0wbn6hf2c6k2t.js","/_next/static/chunks/3pjc6r7w9qxan.js","/_next/static/chunks/39vtz3iby1cf_.js","/_next/static/chunks/2v5t_o330ua3s.js","/_next/static/chunks/3trxs7mom31q3.js","/_next/static/chunks/3nk0wrm9knx3m.js","/_next/static/chunks/1kxuebyfge68i.js","/_next/static/chunks/22e07luyn9bfp.js","/_next/static/chunks/1_lyn-lih218f.js","/_next/static/chunks/1dwajbnfjnvzx.js","/_next/static/chunks/0bvim9_g0hu68.js","/_next/static/chunks/3c7e5ojau5akn.js","/_next/static/chunks/0gf5i4fmb2vxz.js","/_next/static/chunks/3a57oltrmz5kh.js","/_next/static/chunks/3j9dwf14_qrhp.js","/_next/static/chunks/21k90yd6lrzsc.js","/_next/static/chunks/3-5mrnq0kjby0.js","/_next/static/chunks/1r40kzfox4d83.js","/_next/static/chunks/24_gg9g_4g22i.js","/_next/static/chunks/0l0c166aeafbb.js","/_next/static/chunks/0czpc-f8wjri8.js","/_next/static/chunks/3xaq4o65p_dw0.js","/_next/static/chunks/2dyt8rg18m736.js","/_next/static/chunks/43b9k5vjei3d1.js","/_next/static/chunks/3eatuedmz1636.js","/_next/static/chunks/2pqhypz2vo8mu.js","/_next/static/chunks/1nz6e-lpv7lb-.js","/_next/static/chunks/2l525havh4_yi.js","/_next/static/chunks/0fpx6rmsgi4um.js"],""] 5c:I[461459,["/_next/static/chunks/41aagivimsyys.js","/_next/static/chunks/1w6b9x4ox-2ab.js","/_next/static/chunks/2du4f4hdcx8vy.js","/_next/static/chunks/3lydxrl6slrf2.js","/_next/static/chunks/1la1pzs9j8idc.js","/_next/static/chunks/2hrea-dwwjjr_.js","/_next/static/chunks/28ougzph2g_t9.js","/_next/static/chunks/3r5yd4k4rms8-.js","/_next/static/chunks/125flzb3dd6q2.js","/_next/static/chunks/0wbn6hf2c6k2t.js","/_next/static/chunks/3pjc6r7w9qxan.js","/_next/static/chunks/39vtz3iby1cf_.js","/_next/static/chunks/2v5t_o330ua3s.js","/_next/static/chunks/3trxs7mom31q3.js","/_next/static/chunks/3nk0wrm9knx3m.js","/_next/static/chunks/1kxuebyfge68i.js","/_next/static/chunks/22e07luyn9bfp.js","/_next/static/chunks/1_lyn-lih218f.js","/_next/static/chunks/1dwajbnfjnvzx.js","/_next/static/chunks/0bvim9_g0hu68.js","/_next/static/chunks/3c7e5ojau5akn.js","/_next/static/chunks/0gf5i4fmb2vxz.js","/_next/static/chunks/3a57oltrmz5kh.js","/_next/static/chunks/3j9dwf14_qrhp.js","/_next/static/chunks/21k90yd6lrzsc.js","/_next/static/chunks/3-5mrnq0kjby0.js","/_next/static/chunks/1r40kzfox4d83.js","/_next/static/chunks/24_gg9g_4g22i.js","/_next/static/chunks/0l0c166aeafbb.js","/_next/static/chunks/0czpc-f8wjri8.js","/_next/static/chunks/3xaq4o65p_dw0.js","/_next/static/chunks/2dyt8rg18m736.js","/_next/static/chunks/43b9k5vjei3d1.js","/_next/static/chunks/3eatuedmz1636.js","/_next/static/chunks/2pqhypz2vo8mu.js","/_next/static/chunks/1nz6e-lpv7lb-.js","/_next/static/chunks/2l525havh4_yi.js","/_next/static/chunks/0fpx6rmsgi4um.js"],"ArtifactBadges"] 64:I[946424,["/_next/static/chunks/41aagivimsyys.js","/_next/static/chunks/1w6b9x4ox-2ab.js","/_next/static/chunks/2du4f4hdcx8vy.js","/_next/static/chunks/3lydxrl6slrf2.js","/_next/static/chunks/1la1pzs9j8idc.js","/_next/static/chunks/2hrea-dwwjjr_.js","/_next/static/chunks/28ougzph2g_t9.js","/_next/static/chunks/3r5yd4k4rms8-.js","/_next/static/chunks/125flzb3dd6q2.js","/_next/static/chunks/0wbn6hf2c6k2t.js","/_next/static/chunks/3pjc6r7w9qxan.js","/_next/static/chunks/39vtz3iby1cf_.js","/_next/static/chunks/2v5t_o330ua3s.js","/_next/static/chunks/3trxs7mom31q3.js","/_next/static/chunks/3nk0wrm9knx3m.js","/_next/static/chunks/1kxuebyfge68i.js","/_next/static/chunks/22e07luyn9bfp.js","/_next/static/chunks/1_lyn-lih218f.js","/_next/static/chunks/1dwajbnfjnvzx.js","/_next/static/chunks/0bvim9_g0hu68.js","/_next/static/chunks/3c7e5ojau5akn.js","/_next/static/chunks/0gf5i4fmb2vxz.js","/_next/static/chunks/3a57oltrmz5kh.js","/_next/static/chunks/3j9dwf14_qrhp.js","/_next/static/chunks/21k90yd6lrzsc.js","/_next/static/chunks/3-5mrnq0kjby0.js","/_next/static/chunks/1r40kzfox4d83.js","/_next/static/chunks/24_gg9g_4g22i.js","/_next/static/chunks/0l0c166aeafbb.js","/_next/static/chunks/0czpc-f8wjri8.js","/_next/static/chunks/3xaq4o65p_dw0.js","/_next/static/chunks/2dyt8rg18m736.js","/_next/static/chunks/43b9k5vjei3d1.js","/_next/static/chunks/3eatuedmz1636.js","/_next/static/chunks/2pqhypz2vo8mu.js","/_next/static/chunks/1nz6e-lpv7lb-.js","/_next/static/chunks/2l525havh4_yi.js","/_next/static/chunks/0fpx6rmsgi4um.js"],"default"] 56:["$","$L59",null,{"hasPartnerEndpoint":false}] 5d:T60c1,# Retail Object Detection ## Description: RetailObjectDetection detects retail items within an image; it classifies objects as retail or not. This model is ready for commercial use. ## References: ### Citations - Tobin, Josh, et al. "Domain randomization for transferring deep neural networks from simulation to the real world." 2017 *IEEE/RSJ international conference on intelligent robots and systems (IROS)*. IEEE, 2017. - Morrical, Nathan, et al. "NViSII: A scriptable tool for photorealistic image generation." *arXiv preprint arXiv:2105.13962 (2021)*. ### Using TAO Pre-trained Models - Get [TAO Container](https://ngc.nvidia.com/catalog/containers/nvidia:tao:tao-toolkit) - Get other purpose-built models from the NGC model registry: - [TrafficCamNet](https://ngc.nvidia.com/catalog/models/nvidia:tao:trafficcamnet) - [PeopleNet](https://ngc.nvidia.com/catalog/models/nvidia:tao:peoplenet) - [PeopleNet-Transformer](https://ngc.nvidia.com/catalog/models/nvidia:tao:peoplenet_transformer) - [DashCamNet](https://ngc.nvidia.com/catalog/models/nvidia:tao:dashcamnet) - [FaceDetectIR](https://ngc.nvidia.com/catalog/models/nvidia:tao:facedetectir) - [VehicleMakeNet](https://ngc.nvidia.com/catalog/models/nvidia:tao:vehiclemakenet) - [VehicleTypeNet](https://ngc.nvidia.com/catalog/models/nvidia:tao:vehicletypenet) - [PeopleSegNet](https://ngc.nvidia.com/catalog/models/nvidia:tao:peoplesegnet) - [PeopleSemSegNet](https://ngc.nvidia.com/catalog/models/nvidia:tao:peoplesemsegnet) - [License Plate Detection](https://ngc.nvidia.com/catalog/models/nvidia:tao:lpdnet) - [License Plate Recognition](https://ngc.nvidia.com/catalog/models/nvidia:tao:lprnet) - [PoseClassificationNet](https://ngc.nvidia.com/catalog/models/nvidia:tao:poseclassificationnet) - [Facial Landmark](https://ngc.nvidia.com/catalog/models/nvidia:tao:fpenet) - [FaceDetect](https://ngc.nvidia.com/catalog/models/nvidia:tao:facenet) - [2D Body Pose Estimation](https://ngc.nvidia.com/catalog/models/nvidia:tao:bodyposenet) - [ActionRecognitionNet](https://ngc.nvidia.com/catalog/models/nvidia:tao:actionrecognitionnet) - [People ReIdentification](https://ngc.nvidia.com/catalog/models/nvidia:tao:reidentificationnet) - [PointPillarNet](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/pointpillarnet) - [CitySegFormer](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/citysemsegformer) - [Retail Object Detection](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/retail_object_detection) - [Retail Object Embedding](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/retail_object_recognition) - [Optical Inspection](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/optical_inspection) - [Optical Character Detection](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/ocdnet) - [Optical Character Recognition](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/ocrnet) - [PCB Classification](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/pcb_classification) - [PeopleSemSegFormer](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplesemsegformer) ## Model Architecture: **Architecture Type:** Convolution Neural Network (CNN)
**Network Architecture:** EfficientDet, DINO-FAN_base
## Input: **Input Type(s):** Image
**Input Format(s):** Red, Green, Blue (RGB)
**Input Parameters:** 2D
**Other Properties Related to Input:** RGB Fixed Resolution: 416x416 and 960x544 (W x H); No minimum bit depth, alpha, or gamma.
The logos on retail items were smudged.

## Output: **Output Type(s):** Label(s)
**Output Format:** Label: Text String
**Other Properties Related to Output:** Category Label(s): returns a single category.
The logos on retail items were smudged.

## Software Integration: **Runtime Engine(s):** * TAO - 5.2
* DeepStream 6.1 or later
**Supported Hardware Architecture(s):**
* Ampere
* Jetson
* Hopper
* Lovelace
* Pascal
* Turing
* Volta
**Supported Operating System(s):**
* Linux
* Linux 4 Tegra
## Model Version(s): - trainable_binary_v1.0 - deployable_binary_v1.0 ## Training & Evaluation: ## Training Dataset: **Data Collection Method by dataset:**
* Automatic/Sensors
**Labeling Method by dataset:**
* Human
**Properties:**
320,000 proprietary synthetic images of objects found in retail settings randomizing several simulation domains including: - light types, light intensities - object sizes, orientations, and locations - camera locations - background textures - flying distractors The background textures are real images sampled from: - Proprietary real images - images taken from a retail checkout counter - HDRI texture maps create by NVIDIA Omniverse Each synthetic image contains 1 target retail item. This dataset is set up to simulate the diverse environments in the real world and to have the detector learn to extract retail items from noisy backgrounds. The logos on retail items were smudged.

| dataset | total #images | train #images | val #images | | ------------------------------------------------- | ------------- | ------------- | ----------- | | Synthetic data | 1,500,000 | 1,425,000 | 75,000 | | Real data - checkout counter 45 overhead | 107 | 85 | 23 | | Real data - shelf | 107 | 85 | 22 | | Real data - conveyor belt | 106 | 84 | 22 | | Real data - basket | 106 | 84 | 22 | | Real data - checkout counter barcode scanner view | 125 | 100 | 25 | | Real data - checkout counter overhead | 98 | 80 | 18 | #### Fine-tuning Data This model is fine-tuned on about 600 real proprietary images from 6 different real environments. In each environment, only 1 image per item is collected. The fine tuning data are captured under random camera heights and field of views. All fine tuning data were collected indoor, having retail items placed at the checkout counter, shelf, baskets, and conveyor belt. The camera is typically set up at approximately 10 feet height, 45-degree angle off the vertical axis and has close field-of-view. This content was chosen to decrease the simulation-to-reality gap of the model trained on synthetic data, and to improve the accuracy and the robustness of the model. The logos on retail items were smudged.

#### Fine-tuning Data Ground-truth Labeling Guidelines The fine tuning data are created by labeling ground-truth bounding-boxes and categories by human-labelers. The following guidelines were used while labeling the training data for NVIDIA Retail Detection models. If you are looking to transfer-learn or to fine-tune the models to adapt to your target environment and classes, please follow the guidelines below for better model accuracy. 1. All objects that fall under the definition of retail items and are larger than the smallest bounding-box limit for the corresponding class (height >= 10px OR width >= 10px) are labeled with the appropriate class label. 2. Occlusion: For partially occluded objects that are visible approximately 60% or are marked as visible objects with a bounding box around the visible part of the object. These objects are marked as partially occluded. Objects under 60% visibility are not annotated. 3. Truncation: An object, at the edge of the frame, which is 60% or more visible is marked with the truncation flag. 4. Each frame is not required to have an object. ## Evaluation Dataset: **Data Collection Method by dataset:**
* Automatic/Sensors
**Labeling Method by dataset:**
* Human
**Properties:**
15,000 proprietary real-world images of objects found in retail settings. #### Methodology and KPI AP50 is calculated using intersection-over-union (IOU) criterion greater than 0.5. The KPI for the evaluation data are reported in the table below. Model is evaluated based on AP50 and AR0.5:0.95. Both AR and AP numbers are based on 100 maximum detections each image. Please note that “unseen items” measurements are irrelevant to the 100-class detection model. #### Binary-class Retail Item Detection Model | scene | seen items result (AP50) | seen items result (AR MaxDets=100) | unseen items result (AP50) | unseen items result (AR MaxDets=100) | | ------------------------------------- | ------------------------ | ---------------------------------- | --------------------------- | ------------------------------------- | | checkout counter 45 degree overhead | 0.960 | 0.791 | 0.959 | 0.753 | | shelf | 0.983 | 0.888 | 0.978 | 0.841 | | conveyor belt | 1.000 | 0.921 | 0.995 | 0.887 | | basket | 0.956 | 0.851 | 0.959 | 0.861 | | checkout counter barcode scanner view | 0.858 | 0.789 | 0.744 | 0.655 | | checkout counter overhead | 0.990 | 0.915 | 0.993 | 0.910 | | **overall (mean of all scenes)** | **0.959** | **0.859** | **0.938** | **0.818** | ## Inference: **Engine:** Tensor(RT)
**Test Hardware:**
- Jetson AGX Xavier - Xavier NX - Orin - Orin NX - NVIDIA T4 - Ampere GPU - A2 - A30 - L4 - T4 - DGX H100 - DGX A100 - DGX H100 - L40 - JAO 64GB - Orin NX16GB - Orin Nano 8GB The inference is run on the provided unpruned model at FP16 precision. The model input resolution is 416x416. The inference performance is run using [trtexec](https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#trtexec-ovr) on Jetson AGX Orin 64GB and A10. The performance shown here is the inference only performance. The end-to-end performance with streaming video data might slightly vary depending on other bottlenecks in the hardware and software. | model | device | batch size | latency (ms) | images per second | | --------------------------------- | -------------------- | ---------- | ------------ | ----------------- | | Retail Item Detection (binary) | Jetson AGX Orin 64GB | 1 | 10.43 | 96 | | Retail Item Detection (binary) | Jetson AGX Orin 64GB | 16 | 131.79 | 121 | | Retail Item Detection (binary) | Jetson AGX Orin 64GB | 32 | 258.44 | 124 | | Retail Item Detection (binary) | Tesla A10 | 1 | 4.27 | 234 | | Retail Item Detection (binary) | Tesla A10 | 16 | 44.94 | 356 | | Retail Item Detection (binary) | Tesla A10 | 64 | 174.46 | 367 | ## How to use this model ### Instructions to use unpruned model with TAO In order to use these models as pretrained weights for transfer learning, please use the snippet below as template for the model component of the experiment spec file to train a Efficientdet-TF2 model. For more information on the experiment spec file, please refer to the [RetailDetector notebook and the EfficientDdet-TFtf2 TAO doc]. ``` % spec file model: name: 'efficientdet-d5' data: loader: prefetch_size: 4 shuffle_file: True num_classes: 101 # switch to 2 for RetailDetector_binary model image_size: '416x416' max_instances_per_image: 10 train_tfrecords: - [train tfrecords] val_tfrecords: - [validation tfrecords] val_json_file: [validation annotation json file path] train: num_examples_per_epoch: 10000 # change to train set size ... evaluate: num_samples: 500 # change to test set size label_map: # label map file here ... ``` ### Instructions to deploy these models with DeepStream Here is an example of using the Retail Item Embedder together with the Retail Item Detector [TODO: add Retail Item Embedder url here] for an end-to-end video analytic application. To do so, deploy these models with [DeepStream SDK](https://developer.nvidia.com/deepstream-sdk). DeepStream SDK is a streaming analytic toolkit to accelerate building AI-based video analytic applications. DeepStream supports direct integration of these models into the deepstream sample app. To deploy these models with [DeepStream 6.2](https://deepstreamsdk.gitlab-master-pages.nvidia.com/docs-template/text/DS_ref_app_deepstream.html?highlight=attach), please follow the instructions below: [Download](https://developer.nvidia.com/deepstream-sdk) and install DeepStream SDK. The installation instructions for DeepStream are provided in [DeepStream development guide](https://docs.nvidia.com/metropolis/deepstream/dev-guide/index.html). The config files for the purpose-built models are located in: /opt/nvidia/deepstream is the default DeepStream installation directory. This path will be different if you are installing in a different directory. The sample config files are provided in NVIDIA-AI-IOT(TODO: Update the URL when deepstream_tao_apps are merged with???). Assume the repo is cloned under `$DS_TAO_APPS_HOME`, in `$DS_TAO_APPS_HOME/configs/retailDetector_tao`, ```bash # Binary-class detector (the primary GIE) inference setting pgie_retailDetector_binary_config.yml pgie_retailDetector_binary_config.txt ``` Key Parameters in `pgie_retailDetector_100_tao_config.yml` ```prototxt property: gpu-id: 0 net-scale-factor: 1 offsets: 0;0;0 model-color-format: 0 tlt-model-key: nvidia_tlt tlt-encoded-model: ../../models/retailDetector/retailDetector_100.etlt model-engine-file: ../../models/retailDetector/retailDetector_100.etlt_b1_gpu0_fp16.engine labelfile-path: ../../models/retailDetector/retailDetector_100_labels.txt network-input-order: 1 infer-dims: 3;416;416 maintain-aspect-ratio: 1 batch-size: 1 ## 0=FP32, 1=INT8, 2=FP16 mode network-mode: 2 num-detected-classes: 100 interval: 0 cluster-mode: 3 output-blob-names: num_detections;detection_boxes;detection_scores;detection_classes parse-bbox-func-name: NvDsInferParseCustomEfficientDetTAO custom-lib-path: ../../post_processor/libnvds_infercustomparser_tao.so #Use the config params below for NMS clustering mode class-attrs-all: pre-cluster-threshold: 0.5 ``` In order to decode the bounding box information from the EfficientDet output tensor, the custom parser function and library have to be specified. To inference the model, please run: ```bash cd $DS_TAO_APPS_HOME/configs/retailDetector_tao $DS_TAO_APPS_HOME/apps/tao_detection/ds-tao-detection -c retailDetector_100_config.txt -i file://$DS_TAO_APPS_HOME/samples/streams/retailDetector_h264.mp4 ``` The "Deploying to DeepStream" chapter of [TAO User Guide](https://docs.nvidia.com/tao/tao-toolkit/index.html) provides more details. ### Technical blogs - [Access the latest in Vision AI development workflows with NVIDIA TAO Toolkit 5.0](https://developer.nvidia.com/blog/access-the-latest-in-vision-ai-model-development-workflows-with-nvidia-tao-toolkit-5-0/) - [Improve accuracy and robustness of vision ai models with vision transformers and NVIDIA TAO](https://developer.nvidia.com/blog/improve-accuracy-and-robustness-of-vision-ai-apps-with-vision-transformers-and-nvidia-tao/) - [Train like a ‘pro’ without being an AI expert using TAO AutoML](https://developer.nvidia.com/blog/training-like-an-ai-pro-using-tao-automl/) - [Create Custom AI models using NVIDIA TAO Toolkit with Azure Machine Learning ](https://developer.nvidia.com/blog/creating-custom-ai-models-using-nvidia-tao-toolkit-with-azure-machine-learning/) - [Developing and Deploying AI-powered Robots with NVIDIA Isaac Sim and NVIDIA TAO](https://developer.nvidia.com/blog/developing-and-deploying-ai-powered-robots-with-nvidia-isaac-sim-and-nvidia-tao/) - Learn endless ways to adapt and supercharge your AI workflows with TAO - [Whitepaper](https://developer.nvidia.com/tao-toolkit-usecases-whitepaper/1-introduction) - [Customize Action Recognition with TAO and deploy with DeepStream](https://developer.nvidia.com/blog/developing-and-deploying-your-custom-action-recognition-application-without-any-ai-expertise-using-tao-and-deepstream/) - Read the 2 part blog on training and optimizing 2D body pose estimation model with TAO - [Part 1](https://developer.nvidia.com/blog/training-optimizing-2d-pose-estimation-model-with-tao-toolkit-part-1) | [Part 2](https://developer.nvidia.com/blog/training-optimizing-2d-pose-estimation-model-with-tao-toolkit-part-2) - Learn how to train [real-time License plate detection and recognition app](https://developer.nvidia.com/blog/creating-a-real-time-license-plate-detection-and-recognition-app) with TAO and DeepStream. - Model accuracy is extremely important, learn how you can achieve [state of the art accuracy for classification and object detection models](https://developer.nvidia.com/blog/preparing-state-of-the-art-models-for-classification-and-object-detection-with-tao-toolkit/) using TAO ### Suggested reading - More information on about TAO Toolkit and pre-trained models can be found at the [NVIDIA Developer Zone](https://developer.nvidia.com/tao-toolkit) - [TAO documentation](https://docs.nvidia.com/tao/tao-toolkit/index.html) - Read the [TAO getting Started](https://docs.nvidia.com/tao/tao-toolkit/text/tao_toolkit_quick_start_guide.html) guide and [release notes](https://docs.nvidia.com/tao/tao-toolkit/text/release_notes.html). - If you have any questions or feedback, please refer to the discussions on [TAO Toolkit Developer Forums](https://forums.developer.nvidia.com/c/accelerated-computing/intelligent-video-analytics/tao-toolkit/17) - Deploy your models for video analytics application using DeepStream. Learn more about [DeepStream SDK](https://developer.nvidia.com/deepstream-sdk) - Deploy your models in [Riva](https://developer.nvidia.com/riva) for ConvAI use case. ## Ethical Considerations: NVIDIA Retail Object Detection model detects retail items. However, no additional information such as people and other distractors in the background are inferred. Training and evaluation dataset mostly consists of North American content. An ideal training and evaluation dataset would additionally include content from other geographies. NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal model team to ensure this model meets requirements for the relevant industry and use case and addresses unforeseen product misuse. For more detailed information on ethical considerations for this model, please see the Model Card++ Promise and the Explainability, Bias, Safety & Security, and Privacy Subcards. Please report security vulnerabilities or NVIDIA AI Concerns [here](https://www.nvidia.com/en-us/support/submit-security-vulnerability)5e:T4ec,| Field | Response | | -- | -- | | Intended Application(s) & Domain(s): | Detecting retail objects | | Model Type: | Object Detection | | Intended Users: | Developers working in smart spaces, retail, and industrial applications with people. | | Output: | Bounding Box(es), Category Label(s), Confidence Score(s) | | Describe how the model works: | Detects retail items within an image and classifies object as retail item or not. | | Technical Limitations: | Model should not be used in low-light, low contrast lighting conditions, with warped or motion-induced images, or very small and very large objects. Model is not as accurate with monochrome or infrared images. Model accuracy may vary with occlusion or truncation. | | Verified to have met prescribed NVIDIA standards: | Yes | | Performance Metrics: | Intersection over Union (IoU), Mean Average Precision (mAP) | | Potential Known Risks: | This model may fail to detect a retail object. | | Licensing: | [https://developer.download.nvidia.com/licenses/tao_toolkit_21-08_models_eula.pdf?t=eyJscyI6ImdzZW8iLCJsc2QiOiJodHRwczovL3d3dy5nb29nbGUuY29tLyJ9](https://developer.download.nvidia.com/licenses/tao_toolkit_21-08_models_eula.pdf?t=eyJscyI6ImdzZW8iLCJsc2QiOiJodHRwczovL3d3dy5nb29nbGUuY29tLyJ9) |5f:T497,| Field | Response | | -- | -- | | Generatable or reverse engineerable personally-identifiable information (PII)? | None | | Protected classes used to create this model? | None | | Was consent obtained for any PII used? | Yes | | How often is dataset reviewed? | Before Release | | Is a mechanism in place to honor data subject right of access or deletion of personal data? | Yes | | If PII collected for the development of the model, was it collected directly by NVIDIA? | Yes, PII was collected by NVIDIA. | | If PII collected for the development of the model by NVIDIA, do you maintain or have access to disclosures made to data subjects? | Yes | | If PII collected for the development of this AI model, was it minimized to only what was required? | Yes | | Is there provenance for all datasets used in training? | Yes | | Does data labeling (annotation, metadata) comply with privacy laws? | Yes | | Is data compliant with data subject requests for data correction or removal, if such a request was made? | Yes | | Applicable NVIDIA Privacy Policy | [https://www.nvidia.com/en-us/about-nvidia/privacy-policy/](https://www.nvidia.com/en-us/about-nvidia/privacy-policy/) |57:[["$","div",".3",{"className":"relative","children":[["$","div",null,{"className":"relative flex min-h-[194px] w-full flex-col overflow-hidden p-md sm:p-ml bg-manitoulinLightBlack","children":[["$","$L5a",null,{"alt":"","className":"object-cover object-center !left-1/2 z-10 !w-1/2 brightness-20 lg:brightness-75","fill":true,"priority":true,"src":"https://assets.ngc.nvidia.com/products/api-catalog/images/retail-object-detection.jpg","style":{"maskImage":"linear-gradient(to left, rgba(0,0,0,1), rgba(0,0,0,0))"}}],["$","div",null,{"className":"z-30 mx-auto flex w-full max-w-screen-xl min-w-0 flex-1 flex-col items-start justify-between gap-md lg:flex-row lg:gap-ml","children":["$","section",null,{"aria-describedby":"resource-description","aria-labelledby":"resource-name","className":"flex w-full min-w-0 flex-col lg:flex-1","children":["$","div",null,{"className":"flex flex-col gap-1.5 whitespace-pre-wrap","children":[["$","$L5b",null,{"className":"w-fit text-ml text-n300 lowercase hover:text-n050","href":"/nvidia","children":"NVIDIA"}],["$","div",null,{"className":"flex flex-col items-start gap-3 lg:flex-row lg:items-center","children":[["$","h1",null,{"className":"max-w-full text-lg leading-heading font-medium break-words text-primary","data-testid":"resource-name","id":"resource-name","children":"retail-object-detection"}],["$","$L5c",null,{"artifact":{"artifactType":"ENDPOINT","name":"retail-object-detection","displayName":"retail-object-detection","publisher":"nvidia","shortDescription":"EfficientDet-based object detection network to detect 100 specific retail objects from an input video.","logo":"https://assets.ngc.nvidia.com/products/api-catalog/images/retail-object-detection.jpg","labels":["NVIDIA NIM","Object Detection","TAO Toolkit","computer vision","cv","image","nimType:endpoint:nim_type_preview","usecase:endpoint:usecase_object_detect","video","vlm"],"attributes":[{"key":"DEPRECATION","value":"04/15/2026"},{"key":"PREVIEW","value":"true"}],"updatedDate":"2024-08-26T16:47:19.149Z","bias":"| Field | Response |\n| -- | -- |\n|Participation considerations from adversely impacted groups [(protected classes)](https://www.senate.ca.gov/content/protected-classes) in model design and testing: | Unknown |\n| Measures taken to mitigate against unwanted bias: | No measures taken to mitigate against unwanted bias.|","canGuestDownload":true,"createdDate":"2024-05-28T00:13:28.818Z","description":"$5d","explainability":"$5e","isPublic":true,"isReadOnly":true,"orgName":"qc69jvmznzxy","privacy":"$5f","safetyAndSecurity":"| Field | Response |\n| -- | -- |\n| Model Application(s): | Object Detection at checkout counters |\n| Describe the life-critical application (if present). | Not Applicable |\n| Use Case Restrictions: | Abide by https://www.nvidia.com/en-us/data-center/products/nvidia-ai-enterprise/eula/ |\n| Describe access restrictions (if any): | The Principle of least privilege (PoLP) is applied limiting access for dataset generation and model development. Restrictions enforce dataset access during training, and dataset license constraints adhered to. |"}}],"$undefined"]}],"$L60","$L61"]}]}]}],"$L62"]}],"$L63"]}]] 65:T84d,import os import sys import uuid import zipfile import requests nvai_url="https://ai.api.nvidia.com/v1/cv/nvidia/retail-object-detection" header_auth = f"Bearer $NVIDIA_API_KEY" def _upload_asset(input, description): assets_url = "https://api.nvcf.nvidia.com/v2/nvcf/assets" headers = { "Authorization": header_auth, "Content-Type": "application/json", "accept": "application/json", } s3_headers = { "x-amz-meta-nvcf-asset-description": description, "content-type": "video/mp4", } payload = {"contentType": "video/mp4", "description": description} response = requests.post(assets_url, headers=headers, json=payload, timeout=30) response.raise_for_status() asset_url = response.json()["uploadUrl"] asset_id = response.json()["assetId"] response = requests.put( asset_url, data=input, headers=s3_headers, timeout=300, ) response.raise_for_status() return uuid.UUID(asset_id) if __name__ == "__main__": """Uploads a video or image of your choosing to the NVCF API and sends a request to the Retail Object Detection model. The response is saved to a local directory. Note: You must set up an environment variable, NVIDIA_API_KEY. """ if len(sys.argv) != 3: print("Usage: python test.py ") sys.exit(1) asset_id = _upload_asset(open(sys.argv[1], "rb"), "Input Video") inputs = {"input_video": f"{asset_id}", "threshold": <%- request.threshold %>} asset_list = f"{asset_id}" headers = { "Content-Type": "application/json", "NVCF-INPUT-ASSET-REFERENCES": asset_list, "NVCF-FUNCTION-ASSET-IDS": asset_list, "Authorization": header_auth, } response = requests.post(nvai_url, headers=headers, json=inputs) with open(f"{sys.argv[2]}.zip", "wb") as out: out.write(response.content) with zipfile.ZipFile(f"{sys.argv[2]}.zip", "r") as z: z.extractall(sys.argv[2]) print(f"Output saved to {sys.argv[2]}") print(os.listdir(sys.argv[2]))66:T960,const fs = require('fs'); const decompress = require('decompress'); const nvai_url="https://ai.api.nvidia.com/v1/cv/nvidia/retail-object-detection" const header_auth = `Bearer $NVIDIA_API_KEY`; async function _upload_asset(input, description) { const assets_url = "https://api.nvcf.nvidia.com/v2/nvcf/assets"; const headers = { "Authorization": header_auth, "Content-Type": "application/json", "accept": "application/json", }; const s3_headers = { "x-amz-meta-nvcf-asset-description": description, "content-type": "video/mp4", }; const payload = { "contentType": "video/mp4", "description": description }; const response = await fetch( assets_url, { method: 'POST', body: JSON.stringify(payload), headers: headers } ); const data = await response.json(); const asset_url = data["uploadUrl"]; const asset_id = data["assetId"]; const fileData = fs.readFileSync(input); await fetch( asset_url, { method: 'PUT', body: fileData, headers: s3_headers } ); return asset_id.toString(); } (async () => { if (process.argv.length != 4) { console.log("Usage: node test.js "); process.exit(1); } // Upload specified user asset const asset_id = await _upload_asset(`${process.argv[2]}`, "Input Video"); // Metadata for the request const inputs = { "input_video": asset_id, "threshold": <%- request.threshold %> }; const asset_list = asset_id; const headers = { "Content-Type": "application/json", "NVCF-INPUT-ASSET-REFERENCES": asset_list, "NVCF-FUNCTION-ASSET-IDS": asset_list, "Authorization": header_auth }; // Make the request to nvcf const response = await fetch(nvai_url, { method: 'POST', body: JSON.stringify(inputs), headers: headers }); // Gather the binary response data const arrayBuffer = await response.arrayBuffer(); const buffer = Buffer.from(arrayBuffer); const zipname = `${process.argv[3]}.zip`; fs.writeFileSync(zipname, buffer); // Unzip the response synchronously await decompress(zipname, process.argv[3]); // Log the output directory and its contents console.log(`Response saved to ${process.argv[3]}`); console.log(fs.readdirSync(process.argv[3])); })();67:T7d8,#!/bin/bash set -e # Check arguments if [ "$#" -ne 2 ]; then echo "Usage: ./test.sh " exit 1 fi header_auth="Bearer $NVIDIA_API_KEY" assets_url="https://api.nvcf.nvidia.com/v2/nvcf/assets" nvai_url="https://ai.api.nvidia.com/v1/cv/nvidia/retail-object-detection" function upload_asset { local input=$1 local description=$2 response=$(curl -s -X POST $assets_url \ -H "Authorization: ${header_auth}" \ -H "Content-Type: application/json" \ -H "accept: application/json" \ -d '{"contentType": "video/mp4", "description": "'"${description}"'"}') asset_url=$(echo ${response} | jq -r .uploadUrl) asset_id=$(echo ${response} | jq -r .assetId) curl -s -X PUT -H "x-amz-meta-nvcf-asset-description: ${description}" -H "content-type: video/mp4" \ --data-binary "@${input}" ${asset_url} echo ${asset_id} } asset_id=$(upload_asset "$1" "Input Video") inputs='{"input_video": "'"${asset_id}"'", "threshold": <%- request.threshold %>}' asset_list="${asset_id}" # The response will include a location header that curl doesn't trivially # redirect to due to its complexity. We need to extract the location header # and then download the file from that location. location_command="curl -D - -s -X POST $nvai_url \ -H \"Content-Type: application/json\" \ -H \"NVCF-INPUT-ASSET-REFERENCES: ${asset_list}\" \ -H \"NVCF-FUNCTION-ASSET-IDS: ${asset_list}\" \ -H \"Authorization: ${header_auth}\" \ -d '${inputs}' | grep location | awk '{print \$2}'" # Remove any newlines, carriage returns, spaces, quotes, and commas from the location header location=$(eval ${location_command} | tr -d '\n' | tr -d '\r' | tr -d ' ' | tr -d '"' | tr -d ',') # The download command will download the file from the location header download_command="curl -s '${location}' > $2.zip" # Download the .zip file response=$(eval ${download_command}) # Unzip the file unzip -q $2.zip -d $2 echo "Response saved to $2.zip" echo $(ls $2) 58:["$","$L64",null,{"spec":{"openAPISpec":{"openapi":"3.1.0","info":{"title":"Retail Object Detection","description":"The NVIDIA NIM REST API for retail object detection.","version":"1.0.0","termsOfService":"https://nvidia.com/legal/terms-of-use","contact":{"name":"NVIDIA Support","url":"https://help.nvidia.com/"},"license":{"name":"NVIDIA AI Foundation Models Community License","url":"https://docs.nvidia.com/ai-foundation-models-community-license.pdf"}},"servers":[{"url":"https://ai.api.nvidia.com/v1/"}],"tags":[{"name":"Retail Object Detection API","description":"Run inference on the model"}],"paths":{"/cv/nvidia/retail-object-detection":{"post":{"tags":["Retail Object Detection API"],"summary":"Request response from the model","description":"Invokes inference using the model parameters. This POST should be used in conjunction with the NVCF API which allows for the upload of large assets. \nYou can find details on how to use NVCF Asset APIs here: https://docs.api.nvidia.com/cloud-functions/reference/createasset","operationId":"nvidia-retail-object-detection-infer","parameters":[{"in":"header","name":"NVCF-INPUT-ASSET-REFERENCES","schema":{"type":"string","maxLength":370,"format":"uuid"},"required":false,"description":"String of asset IDs separated by commas. Data is uploaded to AWS S3 using NVCF Asset APIs and associated with these asset IDs.The presigned URL allows for secure and temporary access to the S3 bucket for uploading the video. Once the asset is requested, an asset ID is generated for it. Please include this asset ID in this header and to use the uploaded video in a prompt."}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/VideoRequest"}}},"required":true},"responses":{"200":{"description":"Invocation is fulfilled","content":{"application/json":{"schema":{"properties":{"inference_time":{"description":"End to end processing time of the video in seconds","title":"Inference Time","type":"number"},"metadata":{"description":"Output metadata from the model : FrameID, Bounding Box, Score, Label","items":{"properties":{"frameId":{"description":"Frame id of the input video","minimum":0,"maximum":10000,"title":"Frameid","type":"integer"},"bbox":{"description":"Bounding boxes detected [xmin, ymin, xmax, ymax]","items":{"properties":{"leftX":{"description":"Top left X coordinate","maximum":10000,"minimum":0,"title":"Leftx","type":"integer"},"topY":{"description":"Top left Y coordinate","maximum":10000,"minimum":0,"title":"Topy","type":"integer"},"rightX":{"description":"Bottom right X coordinate","maximum":10000,"minimum":0,"title":"Rightx","type":"integer"},"bottomY":{"description":"Bottom right Y coordinate","maximum":10000,"minimum":0,"title":"Bottomy","type":"integer"}},"required":["leftX","topY","rightX","bottomY"],"title":"Bbox","type":"object"},"maxItems":10000,"title":"Bbox","type":"array"},"confidence":{"description":"Confidence score of the detection","items":{"type":"number"},"maxItems":10000,"title":"Confidence","type":"array"},"type":{"description":"Label of the detection","items":{"type":"string","maxLength":256,"format":"^[a-zA-Z-]{1,64}$"},"maxItems":10000,"title":"Type","type":"array"}},"required":["frameId","bbox","confidence","type"],"title":"Metadata","type":"object"},"maxItems":10000,"title":"Metadata","type":"array"}},"required":["inference_time","metadata"],"title":"Response","type":"object"}}}},"202":{"description":"Result is pending. Client should poll using the requestId.\n","content":{"application/json":{"example":{},"schema":{}}},"headers":{"NVCF-REQID":{"description":"requestId required for pooling","schema":{"type":"string","format":"uuid","maxLength":36}},"NVCF-STATUS":{"description":"Invocation status","schema":{"type":"string","format":"^[a-zA-Z-]{1,64}$","maxLength":64}}}},"402":{"description":"Payment Required","content":{"application/json":{"schema":{"properties":{"detail":{"type":"string","maxLength":256,"format":"^[a-zA-Z-]{1,64}$","description":"Contains specific information related to the error and why it occurred.","example":"You have reached your limit of credits."}},"type":"object","title":"PaymentRequiredError"}}}},"422":{"description":"Validation failed, provided entity could not be processed.","content":{"application/json":{"schema":{"properties":{"type":{"type":"string","format":"^.{1, 128}$","maxLength":128,"description":"Error type"},"title":{"type":"string","format":"^.{1, 128}$","maxLength":128,"description":"Error title"},"status":{"type":"integer","minimum":100,"maximum":999,"description":"Error status code"},"detail":{"type":"string","format":"^.{1, 1024}$","maxLength":1024,"description":"Detailed information about the error"},"instance":{"type":"string","format":"^.{1, 256}$","maxLength":256,"description":"Function instance used to invoke the request"},"requestId":{"type":"string","format":"uuid","maxLength":36,"description":"UUID of the request"}},"type":"object","required":["type","title","status","detail","instance","requestId"],"title":"InvokeError"},"example":{"type":"urn:nvcf-worker-service:problem-details:unprocessable-entity","title":"Unprocessable Entity","status":422,"detail":"string","instance":"/v2/nvcf/pexec/functions/c0b05e1d-3847-467a-80d5-85e3e5b05f17","requestId":"3fa85f64-5717-4562-b3fc-2c963f66afa6"}}}},"500":{"description":"The invocation ended with an error.","content":{"application/json":{"schema":{"properties":{"type":{"type":"string","format":"^.{1, 128}$","maxLength":128,"description":"Error type"},"title":{"type":"string","format":"^.{1, 128}$","maxLength":128,"description":"Error title"},"status":{"type":"integer","minimum":100,"maximum":999,"description":"Error status code"},"detail":{"type":"string","format":"^.{1, 1024}$","maxLength":1024,"description":"Detailed information about the error"},"instance":{"type":"string","format":"^.{1, 256}$","maxLength":256,"description":"Function instance used to invoke the request"},"requestId":{"type":"string","format":"uuid","maxLength":36,"description":"UUID of the request"}},"type":"object","required":["type","title","status","detail","instance","requestId"],"title":"InvokeError"},"example":{"type":"urn:nvcf-worker-service:problem-details:internal-server-error","title":"Internal Server Error","status":500,"detail":"string","instance":"/v2/nvcf/pexec/functions/c0b05e1d-3847-467a-80d5-85e3e5b05f17","requestId":"3fa85f64-5717-4562-b3fc-2c963f66afa6"}}}}},"x-nvai-meta":{"name":"Run retail object detection on input video or image to generate a .json file containing hits","returns":"Returns a json file with object identification results for each frame.","path":"run","examples":[],"templates":[{"title":"Default","requestEjs":{"python":"$65","node":"$66","shell":"$67"}}]}}},"/status/{requestId}":{"get":{"tags":["Retail Object Detection API"],"summary":"Gets the result of an earlier function invocation request that returned a status of 202.","operationId":"nvidia-retail-object-detection-statuspolling","parameters":[{"name":"requestId","in":"path","description":"requestId to poll results","required":true,"schema":{"type":"string","format":"uuid","maxLength":36}}],"responses":{"200":{"description":"Invocation is fulfilled","content":{"application/json":{"schema":{"properties":{"inference_time":{"description":"End to end processing time of the video in seconds","title":"Inference Time","type":"number"},"metadata":{"description":"Output metadata from the model : FrameID, Bounding Box, Score, Label","items":{"properties":{"frameId":{"description":"Frame id of the input video","minimum":0,"maximum":10000,"title":"Frameid","type":"integer"},"bbox":{"description":"Bounding boxes detected [xmin, ymin, xmax, ymax]","items":{"properties":{"leftX":{"description":"Top left X coordinate","maximum":10000,"minimum":0,"title":"Leftx","type":"integer"},"topY":{"description":"Top left Y coordinate","maximum":10000,"minimum":0,"title":"Topy","type":"integer"},"rightX":{"description":"Bottom right X coordinate","maximum":10000,"minimum":0,"title":"Rightx","type":"integer"},"bottomY":{"description":"Bottom right Y coordinate","maximum":10000,"minimum":0,"title":"Bottomy","type":"integer"}},"required":["leftX","topY","rightX","bottomY"],"title":"Bbox","type":"object"},"maxItems":10000,"title":"Bbox","type":"array"},"confidence":{"description":"Confidence score of the detection","items":{"type":"number"},"maxItems":10000,"title":"Confidence","type":"array"},"label":{"description":"Label of the detection","items":{"type":"string","maxLength":256,"format":"^[a-zA-Z-]{1,64}$"},"maxItems":10000,"title":"Type","type":"array"}},"required":["frameId","bbox","confidence","type"],"title":"Metadata","type":"object"},"maxItems":10000,"title":"Metadata","type":"array"}},"required":["inference_time","metadata"],"title":"Response","type":"object"}}}},"202":{"description":"Result is pending. Client should poll using the requestId.\n","content":{"application/json":{"example":{},"schema":{}}},"headers":{"NVCF-REQID":{"description":"requestId required for pooling","schema":{"type":"string","format":"uuid","maxLength":36}},"NVCF-STATUS":{"description":"Invocation status","schema":{"type":"string","format":"^[a-zA-Z-]{1,64}$","maxLength":64}}}},"422":{"description":"The invocation ended with an error.","content":{"application/json":{"schema":{"properties":{"type":{"type":"string","format":"^.{1, 128}$","maxLength":128,"description":"Error type"},"title":{"type":"string","format":"^.{1, 128}$","maxLength":128,"description":"Error title"},"status":{"type":"integer","minimum":100,"maximum":999,"description":"Error status code"},"detail":{"type":"string","format":"^.{1, 1024}$","maxLength":1024,"description":"Detailed information about the error"},"instance":{"type":"string","format":"^.{1, 256}$","maxLength":256,"description":"Function instance used to invoke the request"},"requestId":{"type":"string","format":"uuid","maxLength":36,"description":"UUID of the request"}},"type":"object","required":["type","title","status","detail","instance","requestId"],"title":"InvokeError"},"example":{"type":"urn:nvcf-worker-service:problem-details:unprocessable-entity","title":"Unprocessable Entity","status":422,"detail":"string","instance":"/v2/nvcf/pexec/functions/c0b05e1d-3847-467a-80d5-85e3e5b05f17","requestId":"3fa85f64-5717-4562-b3fc-2c963f66afa6"}}}},"500":{"description":"The invocation ended with an error.","content":{"application/json":{"schema":{"properties":{"type":{"type":"string","format":"^.{1, 128}$","maxLength":128,"description":"Error type"},"title":{"type":"string","format":"^.{1, 128}$","maxLength":128,"description":"Error title"},"status":{"type":"integer","minimum":100,"maximum":999,"description":"Error status code"},"detail":{"type":"string","format":"^.{1, 1024}$","maxLength":1024,"description":"Detailed information about the error"},"instance":{"type":"string","format":"^.{1, 256}$","maxLength":256,"description":"Function instance used to invoke the request"},"requestId":{"type":"string","format":"uuid","maxLength":36,"description":"UUID of the request"}},"type":"object","required":["type","title","status","detail","instance","requestId"],"title":"InvokeError"},"example":{"type":"urn:nvcf-worker-service:problem-details:internal-server-error","title":"Internal Server Error","status":500,"detail":"string","instance":"/v2/nvcf/pexec/functions/c0b05e1d-3847-467a-80d5-85e3e5b05f17","requestId":"3fa85f64-5717-4562-b3fc-2c963f66afa6"}}}}}}}},"security":[{"Token":[]}],"components":{"securitySchemes":{"Token":{"type":"http","scheme":"bearer"}},"schemas":{"Errors":{"properties":{"type":{"type":"string","format":"^.{1, 128}$","maxLength":128,"description":"Error type"},"title":{"type":"string","format":"^.{1, 128}$","maxLength":128,"description":"Error title"},"status":{"type":"integer","minimum":100,"maximum":999,"description":"Error status code"},"detail":{"type":"string","format":"^.{1, 1024}$","maxLength":1024,"description":"Detailed information about the error"},"instance":{"type":"string","format":"^.{1, 256}$","maxLength":256,"description":"Function instance used to invoke the request"},"requestId":{"type":"string","format":"uuid","maxLength":36,"description":"UUID of the request"}},"type":"object","required":["type","title","status","detail","instance","requestId"],"title":"InvokeError"},"PaymentRequiredError":{"properties":{"detail":{"type":"string","maxLength":256,"format":"^[a-zA-Z-]{1,64}$","description":"Contains specific information related to the error and why it occurred.","example":"You have reached your limit of credits."}},"type":"object","title":"PaymentRequiredError"},"Bbox":{"properties":{"leftX":{"description":"Top left X coordinate","maximum":10000,"minimum":0,"title":"Leftx","type":"integer"},"topY":{"description":"Top left Y coordinate","maximum":10000,"minimum":0,"title":"Topy","type":"integer"},"rightX":{"description":"Bottom right X coordinate","maximum":10000,"minimum":0,"title":"Rightx","type":"integer"},"bottomY":{"description":"Bottom right Y coordinate","maximum":10000,"minimum":0,"title":"Bottomy","type":"integer"}},"required":["leftX","topY","rightX","bottomY"],"title":"Bbox","type":"object"},"Metadata":{"properties":{"frameId":{"description":"Frame id of the input video","minimum":0,"maximum":10000,"title":"Frameid","type":"integer"},"bbox":{"description":"Bounding boxes detected [xmin, ymin, xmax, ymax]","items":{"properties":{"leftX":{"description":"Top left X coordinate","maximum":10000,"minimum":0,"title":"Leftx","type":"integer"},"topY":{"description":"Top left Y coordinate","maximum":10000,"minimum":0,"title":"Topy","type":"integer"},"rightX":{"description":"Bottom right X coordinate","maximum":10000,"minimum":0,"title":"Rightx","type":"integer"},"bottomY":{"description":"Bottom right Y coordinate","maximum":10000,"minimum":0,"title":"Bottomy","type":"integer"}},"required":["leftX","topY","rightX","bottomY"],"title":"Bbox","type":"object"},"maxItems":10000,"title":"Bbox","type":"array"},"confidence":{"description":"Confidence score of the detection","items":{"type":"number"},"maxItems":10000,"title":"Confidence","type":"array"},"label":{"description":"Label of the detection","items":{"type":"string","maxLength":256,"format":"^[a-zA-Z-]{1,64}$"},"maxItems":10000,"title":"Type","type":"array"}},"required":["frameId","bbox","confidence","label"],"title":"Metadata","type":"object"},"Response":{"properties":{"inference_time":{"description":"End to end processing time of the video in seconds","title":"Inference Time","type":"number"},"metadata":{"description":"Output metadata from the model : FrameID, Bounding Box, Score, Label","items":{"properties":{"frameId":{"description":"Frame id of the input video","minimum":0,"maximum":10000,"title":"Frameid","type":"integer"},"bbox":{"description":"Bounding boxes detected [xmin, ymin, xmax, ymax]","items":{"properties":{"leftX":{"description":"Top left X coordinate","maximum":10000,"minimum":0,"title":"Leftx","type":"integer"},"topY":{"description":"Top left Y coordinate","maximum":10000,"minimum":0,"title":"Topy","type":"integer"},"rightX":{"description":"Bottom right X coordinate","maximum":10000,"minimum":0,"title":"Rightx","type":"integer"},"bottomY":{"description":"Bottom right Y coordinate","maximum":10000,"minimum":0,"title":"Bottomy","type":"integer"}},"required":["leftX","topY","rightX","bottomY"],"title":"Bbox","type":"object"},"maxItems":10000,"title":"Bbox","type":"array"},"confidence":{"description":"Confidence score of the detection","items":{"type":"number"},"maxItems":10000,"title":"Confidence","type":"array"},"label":{"description":"Label of the detection","items":{"type":"string","maxLength":256,"format":"^[a-zA-Z-]{1,64}$"},"maxItems":10000,"title":"Type","type":"array"}},"required":["frameId","bbox","confidence","label"],"title":"Metadata","type":"object"},"maxItems":10000,"title":"Metadata","type":"array"}},"required":["inference_time","metadata"],"title":"Response","type":"object"},"VideoRequest":{"additionalProperties":false,"properties":{"input_video":{"description":"Asset ID of the input video","format":"uuid","maxLength":36,"title":"Input Video","type":"string"},"threshold":{"default":0.9,"description":"Confidence threshold for detections in range 0.1-1.0","maximum":1,"minimum":0.1,"title":"Threshold","type":"number"}},"required":["input_video"],"title":"VideoRequest","type":"object"}}}},"namespace":"qc69jvmznzxy","updatedDate":"2026-04-24T21:04:20.695Z","nvcfFunctionId":"c0b05e1d-3847-467a-80d5-85e3e5b05f17","createdDate":"2024-05-28T00:13:29.266Z","attributes":{"apiDocsUrl":"https://docs.api.nvidia.com/nim/reference/nvidia-retail-object-detection","termsOfUse":"GOVERNING TERMS: Your use of this API is governed by the NVIDIA API Trial Service Terms of Use; and the use of this model is governed by the NVIDIA AI Foundation Models Community License.\n","deprecationNotice":"This API will be deprecated on 04/15/2026. It will no longer be supported after 04/15/2026. Please use qwen/qwen3.5-122b-a10b instead. For more information, visit our API Reference.","showUnavailableBanner":false,"cta":{"type":"Apply to Self-Host","url":"https://www.nvidia.com/en-us/ai/nim-notifyme/"}},"artifactName":"retail-object-detection"}}]