45:I[603186,["/_next/static/chunks/27gl9k2703ugd.js","/_next/static/chunks/3pd35vh8xvq70.js","/_next/static/chunks/0ru0lffoswy0r.js","/_next/static/chunks/3ku5jwxt0m0j7.js","/_next/static/chunks/24mqrfe34h1_g.js","/_next/static/chunks/1la1pzs9j8idc.js","/_next/static/chunks/3-axx3uk1c0-h.js","/_next/static/chunks/14bn3mp7hfp6x.js","/_next/static/chunks/1_xj2c5utin8v.js","/_next/static/chunks/2tdv_pqf9hlyu.js","/_next/static/chunks/44n8gpf9x1nn2.js","/_next/static/chunks/1e7h6z92_w2-c.js","/_next/static/chunks/1p9dpv--te6vb.js","/_next/static/chunks/1geiruffics9x.js","/_next/static/chunks/1lfygqrocy8iw.js","/_next/static/chunks/0q47y720ck5-9.js","/_next/static/chunks/287gwk2yxvb9p.js","/_next/static/chunks/3mi_myijg06x1.js","/_next/static/chunks/2_c926k09c1th.js","/_next/static/chunks/0bvim9_g0hu68.js","/_next/static/chunks/3xav1ddquf5gq.js","/_next/static/chunks/0ayh-bmdnfery.js","/_next/static/chunks/34kl4tv6yifcg.js","/_next/static/chunks/0nm6za7z70maq.js","/_next/static/chunks/44e22u2t839t0.js","/_next/static/chunks/0czpc-f8wjri8.js","/_next/static/chunks/2aq42x9vabhfb.js","/_next/static/chunks/24_gg9g_4g22i.js","/_next/static/chunks/00yjb-caoi49j.js","/_next/static/chunks/1l0z2zsen9vs2.js","/_next/static/chunks/0lt_qhq2z_c0z.js","/_next/static/chunks/01auidv8wbc03.js","/_next/static/chunks/434amnen8yqdi.js","/_next/static/chunks/1yd7p-515l2jc.js","/_next/static/chunks/2pqhypz2vo8mu.js","/_next/static/chunks/2tbc_4m20jg4-.js","/_next/static/chunks/2-18150gleij4.js","/_next/static/chunks/295ffwyuqa4b6.js"],"PageMetadata"] 46:I[831915,["/_next/static/chunks/27gl9k2703ugd.js","/_next/static/chunks/3pd35vh8xvq70.js","/_next/static/chunks/0ru0lffoswy0r.js","/_next/static/chunks/3ku5jwxt0m0j7.js","/_next/static/chunks/24mqrfe34h1_g.js","/_next/static/chunks/1la1pzs9j8idc.js","/_next/static/chunks/3-axx3uk1c0-h.js","/_next/static/chunks/14bn3mp7hfp6x.js","/_next/static/chunks/1_xj2c5utin8v.js","/_next/static/chunks/2tdv_pqf9hlyu.js","/_next/static/chunks/44n8gpf9x1nn2.js","/_next/static/chunks/1e7h6z92_w2-c.js","/_next/static/chunks/1p9dpv--te6vb.js","/_next/static/chunks/1geiruffics9x.js","/_next/static/chunks/1lfygqrocy8iw.js","/_next/static/chunks/0q47y720ck5-9.js","/_next/static/chunks/287gwk2yxvb9p.js","/_next/static/chunks/3mi_myijg06x1.js","/_next/static/chunks/2_c926k09c1th.js","/_next/static/chunks/0bvim9_g0hu68.js","/_next/static/chunks/3xav1ddquf5gq.js","/_next/static/chunks/0ayh-bmdnfery.js","/_next/static/chunks/34kl4tv6yifcg.js","/_next/static/chunks/0nm6za7z70maq.js","/_next/static/chunks/44e22u2t839t0.js","/_next/static/chunks/0czpc-f8wjri8.js","/_next/static/chunks/2aq42x9vabhfb.js","/_next/static/chunks/24_gg9g_4g22i.js","/_next/static/chunks/00yjb-caoi49j.js","/_next/static/chunks/1l0z2zsen9vs2.js","/_next/static/chunks/0lt_qhq2z_c0z.js","/_next/static/chunks/01auidv8wbc03.js","/_next/static/chunks/434amnen8yqdi.js","/_next/static/chunks/1yd7p-515l2jc.js","/_next/static/chunks/2pqhypz2vo8mu.js","/_next/static/chunks/2tbc_4m20jg4-.js","/_next/static/chunks/2-18150gleij4.js","/_next/static/chunks/295ffwyuqa4b6.js"],"SearchResultsImage"] 47:I[430295,["/_next/static/chunks/27gl9k2703ugd.js","/_next/static/chunks/3pd35vh8xvq70.js","/_next/static/chunks/0ru0lffoswy0r.js","/_next/static/chunks/3ku5jwxt0m0j7.js","/_next/static/chunks/24mqrfe34h1_g.js","/_next/static/chunks/1la1pzs9j8idc.js","/_next/static/chunks/3-axx3uk1c0-h.js","/_next/static/chunks/14bn3mp7hfp6x.js","/_next/static/chunks/1_xj2c5utin8v.js","/_next/static/chunks/2tdv_pqf9hlyu.js","/_next/static/chunks/44n8gpf9x1nn2.js","/_next/static/chunks/1e7h6z92_w2-c.js","/_next/static/chunks/1p9dpv--te6vb.js","/_next/static/chunks/1geiruffics9x.js","/_next/static/chunks/1lfygqrocy8iw.js","/_next/static/chunks/0q47y720ck5-9.js","/_next/static/chunks/287gwk2yxvb9p.js","/_next/static/chunks/3mi_myijg06x1.js","/_next/static/chunks/2_c926k09c1th.js","/_next/static/chunks/0bvim9_g0hu68.js","/_next/static/chunks/3xav1ddquf5gq.js","/_next/static/chunks/0ayh-bmdnfery.js","/_next/static/chunks/34kl4tv6yifcg.js","/_next/static/chunks/0nm6za7z70maq.js","/_next/static/chunks/44e22u2t839t0.js","/_next/static/chunks/0czpc-f8wjri8.js","/_next/static/chunks/2aq42x9vabhfb.js","/_next/static/chunks/24_gg9g_4g22i.js","/_next/static/chunks/00yjb-caoi49j.js","/_next/static/chunks/1l0z2zsen9vs2.js","/_next/static/chunks/0lt_qhq2z_c0z.js","/_next/static/chunks/01auidv8wbc03.js","/_next/static/chunks/434amnen8yqdi.js","/_next/static/chunks/1yd7p-515l2jc.js","/_next/static/chunks/2pqhypz2vo8mu.js","/_next/static/chunks/2tbc_4m20jg4-.js","/_next/static/chunks/2-18150gleij4.js","/_next/static/chunks/295ffwyuqa4b6.js"],""] 48:I[299632,["/_next/static/chunks/27gl9k2703ugd.js","/_next/static/chunks/3pd35vh8xvq70.js","/_next/static/chunks/0ru0lffoswy0r.js","/_next/static/chunks/3ku5jwxt0m0j7.js","/_next/static/chunks/24mqrfe34h1_g.js","/_next/static/chunks/1la1pzs9j8idc.js","/_next/static/chunks/3-axx3uk1c0-h.js","/_next/static/chunks/14bn3mp7hfp6x.js","/_next/static/chunks/1_xj2c5utin8v.js","/_next/static/chunks/2tdv_pqf9hlyu.js","/_next/static/chunks/44n8gpf9x1nn2.js","/_next/static/chunks/1e7h6z92_w2-c.js","/_next/static/chunks/1p9dpv--te6vb.js","/_next/static/chunks/1geiruffics9x.js","/_next/static/chunks/1lfygqrocy8iw.js","/_next/static/chunks/0q47y720ck5-9.js","/_next/static/chunks/287gwk2yxvb9p.js","/_next/static/chunks/3mi_myijg06x1.js","/_next/static/chunks/2_c926k09c1th.js","/_next/static/chunks/0bvim9_g0hu68.js","/_next/static/chunks/3xav1ddquf5gq.js","/_next/static/chunks/0ayh-bmdnfery.js","/_next/static/chunks/34kl4tv6yifcg.js","/_next/static/chunks/0nm6za7z70maq.js","/_next/static/chunks/44e22u2t839t0.js","/_next/static/chunks/0czpc-f8wjri8.js","/_next/static/chunks/2aq42x9vabhfb.js","/_next/static/chunks/24_gg9g_4g22i.js","/_next/static/chunks/00yjb-caoi49j.js","/_next/static/chunks/1l0z2zsen9vs2.js","/_next/static/chunks/0lt_qhq2z_c0z.js","/_next/static/chunks/01auidv8wbc03.js","/_next/static/chunks/434amnen8yqdi.js","/_next/static/chunks/1yd7p-515l2jc.js","/_next/static/chunks/2pqhypz2vo8mu.js","/_next/static/chunks/2tbc_4m20jg4-.js","/_next/static/chunks/2-18150gleij4.js","/_next/static/chunks/295ffwyuqa4b6.js"],"default"] 49:I[400402,["/_next/static/chunks/27gl9k2703ugd.js","/_next/static/chunks/3pd35vh8xvq70.js","/_next/static/chunks/0ru0lffoswy0r.js","/_next/static/chunks/3ku5jwxt0m0j7.js","/_next/static/chunks/24mqrfe34h1_g.js","/_next/static/chunks/1la1pzs9j8idc.js","/_next/static/chunks/3-axx3uk1c0-h.js","/_next/static/chunks/14bn3mp7hfp6x.js","/_next/static/chunks/1_xj2c5utin8v.js","/_next/static/chunks/2tdv_pqf9hlyu.js","/_next/static/chunks/44n8gpf9x1nn2.js","/_next/static/chunks/1e7h6z92_w2-c.js","/_next/static/chunks/1p9dpv--te6vb.js","/_next/static/chunks/1geiruffics9x.js","/_next/static/chunks/1lfygqrocy8iw.js","/_next/static/chunks/0q47y720ck5-9.js","/_next/static/chunks/287gwk2yxvb9p.js","/_next/static/chunks/3mi_myijg06x1.js","/_next/static/chunks/2_c926k09c1th.js","/_next/static/chunks/0bvim9_g0hu68.js","/_next/static/chunks/3xav1ddquf5gq.js","/_next/static/chunks/0ayh-bmdnfery.js","/_next/static/chunks/34kl4tv6yifcg.js","/_next/static/chunks/0nm6za7z70maq.js","/_next/static/chunks/44e22u2t839t0.js","/_next/static/chunks/0czpc-f8wjri8.js","/_next/static/chunks/2aq42x9vabhfb.js","/_next/static/chunks/24_gg9g_4g22i.js","/_next/static/chunks/00yjb-caoi49j.js","/_next/static/chunks/1l0z2zsen9vs2.js","/_next/static/chunks/0lt_qhq2z_c0z.js","/_next/static/chunks/01auidv8wbc03.js","/_next/static/chunks/434amnen8yqdi.js","/_next/static/chunks/1yd7p-515l2jc.js","/_next/static/chunks/2pqhypz2vo8mu.js","/_next/static/chunks/2tbc_4m20jg4-.js","/_next/static/chunks/2-18150gleij4.js","/_next/static/chunks/295ffwyuqa4b6.js"],"EndpointCTA"] 4a:I[331699,["/_next/static/chunks/27gl9k2703ugd.js","/_next/static/chunks/3pd35vh8xvq70.js","/_next/static/chunks/0ru0lffoswy0r.js","/_next/static/chunks/3ku5jwxt0m0j7.js","/_next/static/chunks/24mqrfe34h1_g.js","/_next/static/chunks/1la1pzs9j8idc.js","/_next/static/chunks/3-axx3uk1c0-h.js","/_next/static/chunks/14bn3mp7hfp6x.js","/_next/static/chunks/1_xj2c5utin8v.js","/_next/static/chunks/2tdv_pqf9hlyu.js","/_next/static/chunks/44n8gpf9x1nn2.js","/_next/static/chunks/1e7h6z92_w2-c.js","/_next/static/chunks/1p9dpv--te6vb.js","/_next/static/chunks/1geiruffics9x.js","/_next/static/chunks/1lfygqrocy8iw.js","/_next/static/chunks/0q47y720ck5-9.js","/_next/static/chunks/287gwk2yxvb9p.js","/_next/static/chunks/3mi_myijg06x1.js","/_next/static/chunks/2_c926k09c1th.js","/_next/static/chunks/0bvim9_g0hu68.js","/_next/static/chunks/3xav1ddquf5gq.js","/_next/static/chunks/0ayh-bmdnfery.js","/_next/static/chunks/34kl4tv6yifcg.js","/_next/static/chunks/0nm6za7z70maq.js","/_next/static/chunks/44e22u2t839t0.js","/_next/static/chunks/0czpc-f8wjri8.js","/_next/static/chunks/2aq42x9vabhfb.js","/_next/static/chunks/24_gg9g_4g22i.js","/_next/static/chunks/00yjb-caoi49j.js","/_next/static/chunks/1l0z2zsen9vs2.js","/_next/static/chunks/0lt_qhq2z_c0z.js","/_next/static/chunks/01auidv8wbc03.js","/_next/static/chunks/434amnen8yqdi.js","/_next/static/chunks/1yd7p-515l2jc.js","/_next/static/chunks/2pqhypz2vo8mu.js","/_next/static/chunks/2tbc_4m20jg4-.js","/_next/static/chunks/2-18150gleij4.js","/_next/static/chunks/295ffwyuqa4b6.js"],"ModelContent"] 51:I[946424,["/_next/static/chunks/27gl9k2703ugd.js","/_next/static/chunks/3pd35vh8xvq70.js","/_next/static/chunks/0ru0lffoswy0r.js","/_next/static/chunks/3ku5jwxt0m0j7.js","/_next/static/chunks/24mqrfe34h1_g.js","/_next/static/chunks/1la1pzs9j8idc.js","/_next/static/chunks/3-axx3uk1c0-h.js","/_next/static/chunks/14bn3mp7hfp6x.js","/_next/static/chunks/1_xj2c5utin8v.js","/_next/static/chunks/2tdv_pqf9hlyu.js","/_next/static/chunks/44n8gpf9x1nn2.js","/_next/static/chunks/1e7h6z92_w2-c.js","/_next/static/chunks/1p9dpv--te6vb.js","/_next/static/chunks/1geiruffics9x.js","/_next/static/chunks/1lfygqrocy8iw.js","/_next/static/chunks/0q47y720ck5-9.js","/_next/static/chunks/287gwk2yxvb9p.js","/_next/static/chunks/3mi_myijg06x1.js","/_next/static/chunks/2_c926k09c1th.js","/_next/static/chunks/0bvim9_g0hu68.js","/_next/static/chunks/3xav1ddquf5gq.js","/_next/static/chunks/0ayh-bmdnfery.js","/_next/static/chunks/34kl4tv6yifcg.js","/_next/static/chunks/0nm6za7z70maq.js","/_next/static/chunks/44e22u2t839t0.js","/_next/static/chunks/0czpc-f8wjri8.js","/_next/static/chunks/2aq42x9vabhfb.js","/_next/static/chunks/24_gg9g_4g22i.js","/_next/static/chunks/00yjb-caoi49j.js","/_next/static/chunks/1l0z2zsen9vs2.js","/_next/static/chunks/0lt_qhq2z_c0z.js","/_next/static/chunks/01auidv8wbc03.js","/_next/static/chunks/434amnen8yqdi.js","/_next/static/chunks/1yd7p-515l2jc.js","/_next/static/chunks/2pqhypz2vo8mu.js","/_next/static/chunks/2tbc_4m20jg4-.js","/_next/static/chunks/2-18150gleij4.js","/_next/static/chunks/295ffwyuqa4b6.js"],"default"] 42:["$","$L45",null,{"hasPartnerEndpoint":false}] 4b:T30b0,# Pretrained Grounding DINO with Commercial License ## Description Open vocabulary object detection is a computer vision technique that can detect one or multiple objects in a frame based on the text input. Object detection recognizes the individual objects in an image and places bounding boxes around the object. This model card contains pre-trained weights for the [Grounding DINO](https://arxiv.org/abs/2303.05499) object detection networks pretrained on the commercial dataset. Note that the model in this model card can be used for commercial purpose. ## References - Zhang, H., Li, F., Liu, S., Zhang, L., Su, H., Zhu, J., Ni, L., Shum, H.: DINO: DETR with Improved DeNoising Anchor Boxes for End-to-End Object Detection. - Ziu, S., Zeng, Z., Ren, T., Li, F., Zhang, H., Yang, J., Li, C., Yang, J., Su, H., Zhu, J., Zhang, L.: Grounding DINO: Marrying DINO with Grounded Pre-Training for Open-Set Object Detection. - Devlin, J., Chang, M., Lee, K., Toutanova, K.: BERT: PRe-training of Deep Bidirectional Transformers for Language Understanding. - Liu, Z., Lin, Y., Cao, Y., Hu, H., Wei, Y., Zhang, Z., Lin, S., Guo, B.: Swin Transformer: Hierarchical Vision Transformer using Shifted Windows. ## License The licenses to use this model is covered by the Model EULA. By downloading the unpruned or pruned version of the model, you accept the terms and conditions of these [licenses](https://www.nvidia.com/en-us/data-center/products/nvidia-ai-enterprise/eula/) ## Model Architecture **Architecture Type:** Transformer-based Network Architecture **Network Architecture** - Backbone: Window attention based Transformers called Swin-Tiny and BERT-Base text encoder. - Encoder: 6 layers of Multi-headed attention with deformable attention layers. - Decoders: 6 layers of Multi-headed attention with deformable attention layers. **More Details** - The models in this instance are object detectors that take RGB images and list of phrases as input and produce bounding boxes and classes as output. More specifically, this model was trained with [Swin-Tiny backbone](https://arxiv.org/pdf/2103.14030) that was trained using a supervised manner on NVIDIA proprietary data called NVImageNetv2, which allows commercial usage. In addition, BERT-Base was used as the starting weight for text tower. Finally, Grounding DINO was trained end-to-end on about 1.8M images that were collected from publicly available datasets. Note that we ensured that all the raw images used during training have commercial licenses to ensure safe commercial usage. ### Input **Input Type(s):** Image and list of captions of tokenized through HuggingFace
**Input Format(s):** Red, Green, Blue (RGB) and tokenized inputs. Can support any input resolution and images do not need any additional pre-processing (e.g. alpha channels or bits)
**Input Parameters:** Multiple dimensions. See below for detailed input shapes
**Other Properties Related to Input:**
- `inputs`: B X 3 X 544 X 960 (Batch Size x Channel x Height x Width) - `input_ids`: B x 256 (Batch Size x Max Token Length ) - `attention_mask`: B x 256 (Batch Size x Max Token Length ) - `position_ids`: B x 256 (Batch Size x Max Token Length ) - `token_type_ids`: B x 256 (Batch Size x Max Token Length ) - `text_token_mask`: B x 256 x 256 (Batch Size x Max Token Length x Max Token Length) - Because ONNX / TensorRT can't take string as input, we've offloaded tokenizer outside of the model graph. See [TAO-Deploy](https://github.com/NVIDIA/tao_deploy) repo on running tokenization through HuggingFace. building change

### Output **Output Type(s):** Bounding Boxes and Confidence Scores for each detected object in the input image.
**Output Formats:** One Dimensional (1D), Two Dimensional (2D) vectors
**Other Properties Related to Output:**
- `pred_logits`: B x 900 (Batch Size x Number of Queries) - `pred_boxes`: B x 900 x 4 (Batch Size x Number of Queries x Coordinates in `cxcywh` format) ## Software Integration **Runtime Engine(s):** * TAO - 5.5.0
**Supported Hardware Architecture(s):**
* NVIDIA Ampere
* NVIDIA Jetson
* NVIDIA Hopper
* NVIDIA Lovelace
* NVIDIA Pascal
* NVIDIA Turing
* NVIDIA Volta
**Supported Operating System(s):**
* Linux
* Linux 4 Tegra
## Model Versions - **grounding_dino_swin_tiny_commercial_trainable_v1.0** - Pre-trained Swin-Tiny Grounding DINO model for finetune. - **grounding_dino_swin_tiny_commercial_deployable_v1.0** - Swin-Tiny Grounding DINO model deployable. ## Training & Evaluation: This model was trained using the `grounding_dino` entrypoint in TAO. The training algorithm optimizes the network to minimize the localization and contrastive embedding loss between text and visual features. ## Using this Model These models need to be used with NVIDIA hardware and software. For hardware, the models can run on any NVIDIA GPU including NVIDIA Jetson devices. These models can only be used with [Train Adapt Optimize (TAO) Toolkit](https://developer.nvidia.com/tao-toolkit), or [TensorRT](https://developer.nvidia.com/tensorrt). The intended use for these models is detecting objects in a color (RGB) image. The model can be used to detect objects from photos and videos by using appropriate video or image decoding and pre-processing. These models are intended for training and fine-tune with the TAO Toolkit and your datasets for object detection. High-fidelity models can be trained with new use cases. A Jupyter Notebook is available as a part of the pytorch [TAO container](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/containers/tao-toolkit/tags) and can be used to re-train. The models are also intended for easy edge deployment using TensorRT. ## Training Dataset ### Training Data Grounding DINO was pretrained on wide range of commercial datasets where the annotations were either human generated or pseudo-labeled. The model was trained on 1,815,237 images and 14,794,974 instances of both object detection (OD) and grounding annotations. Please refer to below section for details of every dataset used to train Grounding DINO. | Dataset | Data Collection Method by dataset | Labeling Method by dataset | # of Images|# of Annotations| |--------------------------------|-----------------------------------|----------------------------|------------|----------------| | Subset of OpenImagesv5 | Unknown | Automated. Pseudo-labeled raw images with Objects365 trained CO-DETR. |803,826 | 7,345,546| | Localized Narrative OpenImages | Unknown | Automated. Pseudo-labeled raw images and global captions with Grounding DINO. | 670,553 | 6,098,908| | Subset of LVIS | Unknown | Human-labeled (only contains commercial subset). | 30,740 | 391,840 | | Subset of Mixed Grounding | Unknown | Human-labeled (only contains commercial subset). | 150,668 | 777,178 | | Subset of RefCOCO | Unknown | Human-labeled (only contains commercial subset). | 36,459 | 36,459 | | Subset of RefCOCO+ | Unknown | Human-labeled (only contains commercial subset). | 36,302 | 36,302 | | Subset of RefCOCOg | Unknown | Human-labeled (only contains commercial subset). | 23,718 | 23,718 | | Subset of gRefCOCO | Unknown | Human-labeled (only contains commercial subset). | 62,971 | 85,023 | ## Evaluation Data **Data Collection Method by dataset:**
- Unknown
**Labeling Method by dataset:**
- Human.
**Properties:**
- COCO validation dataset with 5,000 images. ### Methodology and KPI The key performance indicator is the mean average precision (mAP), following the standard evaluation protocol for object detection. The KPI for the evaluation data are: |model|precision|mAP|mAP50|mAP75|mAPs|mAPm|mAPl| |---|---|---|---|---|---|---|---| |grounding_dino_swin_tiny|BF16|46.1|59.9|51.0|30.5|49.3|60.8| ## Model Limitation Grounding DINO was trained on images collected from the web and text data of everyday noun phrases. The model might not perform well on different data distributions. Conducting further fine-tuning on the target domain is recommended to get a higher mAP. ## Technical Blogs - [Train like a ‘pro’ without being an AI expert using TAO AutoML](https://developer.nvidia.com/blog/training-like-an-ai-pro-using-tao-automl/) - [Create Custom AI models using NVIDIA TAO Toolkit with Azure Machine Learning ](https://developer.nvidia.com/blog/creating-custom-ai-models-using-nvidia-tao-toolkit-with-azure-machine-learning/) - [Developing and Deploying AI-powered Robots with NVIDIA Isaac Sim and NVIDIA TAO](https://developer.nvidia.com/blog/developing-and-deploying-ai-powered-robots-with-nvidia-isaac-sim-and-nvidia-tao/) - Learn endless ways to adapt and supercharge your AI workflows with TAO - [Whitepaper](https://developer.nvidia.com/tao-toolkit-usecases-whitepaper/1-introduction) - [Customize Action Recognition with TAO and deploy with DeepStream](https://developer.nvidia.com/blog/developing-and-deploying-your-custom-action-recognition-application-without-any-ai-expertise-using-tao-and-deepstream/) - Read the two-part blog on training and optimizing 2D body pose estimation model with TAO - [Part 1](https://developer.nvidia.com/blog/training-optimizing-2d-pose-estimation-model-with-tao-toolkit-part-1) | [Part 2](https://developer.nvidia.com/blog/training-optimizing-2d-pose-estimation-model-with-tao-toolkit-part-2) - Learn how to train a [real-time License plate detection and recognition app](https://developer.nvidia.com/blog/creating-a-real-time-license-plate-detection-and-recognition-app) with TAO and DeepStream. - Model accuracy is extremely important; learn how you can achieve [state of the art accuracy for classification and object detection models](https://developer.nvidia.com/blog/preparing-state-of-the-art-models-for-classification-and-object-detection-with-tao-toolkit/) using TAO. ## Suggested Reading - More information on TAO Toolkit and pre-trained models can be found at the [NVIDIA Developer Zone](https://developer.nvidia.com/tao-toolkit) - Refer to the [TAO documentation](https://docs.nvidia.com/tao/tao-toolkit/index.html) - Read the [TAO Toolkit Quick Start Guide](https://docs.nvidia.com/tao/tao-toolkit/index.html) and [release notes](https://docs.nvidia.com/tao/tao-toolkit/text/release_notes.html). - If you have any questions or feedback, please refer to the discussions on the [TAO Toolkit Developer Forums](https://forums.developer.nvidia.com/c/accelerated-computing/intelligent-video-analytics/tao-toolkit/17) - Deploy your models for video analytics application using the [DeepStream SDK](https://developer.nvidia.com/deepstream-sdk). - Deploy your models in [Riva](https://developer.nvidia.com/riva) for ConvAI use case. ## Ethical Considerations NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their supporting model team to ensure this model meets requirements for the relevant industry and use case and addresses unforeseen product misuse. For more detailed information on ethical considerations for this model, please see the Model Card++ Promise and the Explainability, Bias, Safety & Security, and Privacy Subcards. Please report security vulnerabilities or NVIDIA AI Concerns [here](https://www.nvidia.com/en-us/support/submit-security-vulnerability/).4c:T504,| Field | Response | | -- | -- | | Generatable or reverse engineerable personally-identifiable information (PII)? | None | | Protected classes used to create this model? | Not Applicable (No PII) | | Was consent obtained for any PII used? | Not Applicable (No PII) | | How often is dataset reviewed? | Before Release | | Is a mechanism in place to honor data subject right of access or deletion of personal data? | No | | If PII collected for the development of the model, was it collected directly by NVIDIA? |Not Applicable | | If PII collected for the development of the model by NVIDIA, do you maintain or have access to disclosures made to data subjects? | No | | If PII collected for the development of this AI model, was it minimized to only what was required? | Not Applicable | | Is there provenance for all datasets used in training? | No | | Does data labeling (annotation, metadata) comply with privacy laws? | Yes | | Is data compliant with data subject requests for data correction or removal, if such a request was made? | No, not possible with externally-sourced data. | | Applicable NVIDIA Privacy Policy | [https://www.nvidia.com/en-us/data-center/products/nvidia-ai-enterprise/eula/](https://www.nvidia.com/en-us/data-center/products/nvidia-ai-enterprise/eula/) |4d:T103d,import os import sys import uuid import zipfile import time import requests nvai_url="https://ai.api.nvidia.com/v1/cv/nvidia/nv-grounding-dino" nvai_polling_url = "https://api.nvcf.nvidia.com/v2/nvcf/pexec/status/" header_auth = f"Bearer {os.getenv('NVIDIA_PERSONAL_API_KEY')}" UPLOAD_ASSET_TIMEOUT = 300 # Timeout (in secs) to upload asset MAX_RETRIES = 5 # Max num of retries while polling DELAY_BTW_RETRIES = 1 # adding 1s delay between each polls def _upload_asset(input, description): assets_url = "https://api.nvcf.nvidia.com/v2/nvcf/assets" headers = { "Authorization": header_auth, "Content-Type": "application/json", "accept": "application/json", } s3_headers = { "x-amz-meta-nvcf-asset-description": description, "content-type": "video/mp4", } payload = {"contentType": "video/mp4", "description": description} response = requests.post(assets_url, headers=headers, json=payload, timeout=60) response.raise_for_status() asset_url = response.json()["uploadUrl"] asset_id = response.json()["assetId"] response = requests.put( asset_url, data=input, headers=s3_headers, timeout=UPLOAD_ASSET_TIMEOUT, ) response.raise_for_status() return uuid.UUID(asset_id) if __name__ == "__main__": """Uploads a video or image of your choosing to the NVCF API and sends a request to the Grounding Dino Object Detection model. The response is saved to a local directory. Note: You must set up an environment variable, NGC_PERSONAL_API_KEY. """ if len(sys.argv) != 4: print("Usage: python test.py ") sys.exit(1) asset_id = _upload_asset(open(sys.argv[2], "rb"), "Input Video") inputs = { "model": "Grounding-Dino", "messages": [ { "role": "user", "content": [ { "type": "text", "text": f"{sys.argv[1]}" }, { "type": "media_url", "media_url": { "url": f"data:image/jpeg;asset_id,{asset_id}" } } ] } ], "threshold": 0.3 } asset_list = f"{asset_id}" headers = { "Content-Type": "application/json", "NVCF-INPUT-ASSET-REFERENCES": asset_list, "NVCF-FUNCTION-ASSET-IDS": asset_list, "Authorization": header_auth, } response = requests.post(nvai_url, headers=headers, json=inputs) if response.status_code == 200: # evaluation complete, output video ready with open(f"{sys.argv[3]}.zip", "wb") as out: out.write(response.content) with zipfile.ZipFile(f"{sys.argv[3]}.zip", "r") as z: z.extractall(sys.argv[3]) elif response.status_code == 202: # pending evaluation print("Pending evaluation ...") nvcf_reqid = response.headers['NVCF-REQID'] nvai_polling_url = nvai_polling_url + nvcf_reqid # Polling to check if the response is ready while( MAX_RETRIES ): print(f'Polling ...') headers_polling = { "accept": "application/json", "Authorization": header_auth } response_polling = requests.get(nvai_polling_url, headers=headers_polling) if response_polling.status_code == 202: # evaluation pending print('Result is not yet ready.') MAX_RETRIES -= 1 time.sleep(DELAY_BTW_RETRIES) continue elif response_polling.status_code == 200: # evaluation complete, output video ready print('Result ready!') with open(f"{sys.argv[3]}.zip", "wb") as out: out.write(response_polling.content) break else: print(f"Unexpected response status: {response_polling.status_code}") with zipfile.ZipFile(f"{sys.argv[3]}.zip", "r") as z: z.extractall(sys.argv[3]) print(f"Output saved to {sys.argv[3]}") print(os.listdir(sys.argv[3]))4e:T13df,const fs = require('fs'); const decompress = require('decompress'); const { setTimeout } = require('timers/promises'); const nvai_url="https://ai.api.nvidia.com/v1/cv/nvidia/nv-grounding-dino" const nvai_polling_url = "https://api.nvcf.nvidia.com/v2/nvcf/pexec/status/" const header_auth = `Bearer ${process.env.NVIDIA_PERSONAL_API_KEY}`; // Constants for polling const MAX_RETRIES = 5; const DELAY_BTW_RETRIES = 1000; // in milliseconds (1 second) async function _upload_asset(input, description) { const assets_url = "https://api.nvcf.nvidia.com/v2/nvcf/assets"; const headers = { "Authorization": header_auth, "Content-Type": "application/json", "accept": "application/json", }; const s3_headers = { "x-amz-meta-nvcf-asset-description": description, "content-type": "video/mp4", }; const payload = { "contentType": "video/mp4", "description": description }; const response = await fetch( assets_url, { method: 'POST', body: JSON.stringify(payload), headers: headers } ); const data = await response.json(); const asset_url = data["uploadUrl"]; const asset_id = data["assetId"]; const fileData = fs.readFileSync(input); await fetch( asset_url, { method: 'PUT', body: fileData, headers: s3_headers } ); return asset_id.toString(); } (async () => { if (process.argv.length != 5) { console.log("Usage: node test.js "); process.exit(1); } // Upload specified user asset const asset_id = await _upload_asset(`${process.argv[3]}`, "Input Video"); // Metadata for the request const inputs = { "model": "Grounding-Dino", "messages": [ { "role": "user", "content": [ { "type": "text", "text": `${process.argv[2]}` }, { "type": "media_url", "media_url": { "url": `data:image/jpeg;asset_id,${asset_id}` } } ] } ], "threshold": 0.3 } const asset_list = asset_id; const headers = { "Content-Type": "application/json", "NVCF-INPUT-ASSET-REFERENCES": asset_list, "NVCF-FUNCTION-ASSET-IDS": asset_list, "Authorization": header_auth }; // Make the request to nvcf const response = await fetch(nvai_url, { method: 'POST', body: JSON.stringify(inputs), headers: headers }); if (response.status === 200) { // Evaluation complete, output ready // Gather the binary response data const arrayBuffer = await response.arrayBuffer(); const buffer = Buffer.from(arrayBuffer); const zipname = `${process.argv[4]}.zip`; fs.writeFileSync(zipname, buffer); // Unzip the response synchronously await decompress(zipname, process.argv[4]); // Log the output directory and its contents console.log(`Response saved to ${process.argv[4]}`); console.log(fs.readdirSync(process.argv[4])); } else if (response.status === 202) { // Pending evalutation console.log("Pending evaluation ..."); const nvcf_reqid = response.headers.get('NVCF-REQID'); let retries = MAX_RETRIES; while (retries > 0) { console.log('Polling ...'); const pollingResponse = await fetch(`${nvai_polling_url}${nvcf_reqid}`, { method: 'GET', headers: { "accept": "application/json", "Authorization": header_auth } }); if (pollingResponse.status === 202) { // Evaluation still pending console.log('Result is not yet ready.'); retries -= 1; await setTimeout(DELAY_BTW_RETRIES); } else if (pollingResponse.status === 200) { // Evaluation complete, output ready console.log('Result ready!'); const arrayBuffer = await pollingResponse.arrayBuffer(); const buffer = Buffer.from(arrayBuffer); const zipname = `${process.argv[4]}.zip`; fs.writeFileSync(zipname, buffer); // Unzip the response synchronously await decompress(zipname, process.argv[4]); // Log the output directory and its contents console.log(`Response saved to ${process.argv[4]}`); console.log(fs.readdirSync(process.argv[4])); break; } else { console.error('Unexpected response status:', pollingResponse.status); break } if (retries === 0) { console.error('Max retries reached. Evaluation not completed.'); } } } else { console.error('Unexpected response status:', response.status); } })();4f:Tf2e,#!/bin/bash set -e # Check arguments if [ "$#" -ne 3 ]; then echo "Usage: ./test.sh " exit 1 fi # Constants MAX_RETRIES=5 DELAY_BTW_RETRIES=1 # in seconds header_auth="Bearer $NVIDIA_PERSONAL_API_KEY" assets_url="https://api.nvcf.nvidia.com/v2/nvcf/assets" nvai_url="https://ai.api.nvidia.com/v1/cv/nvidia/nv-grounding-dino" nvai_polling_url="https://api.nvcf.nvidia.com/v2/nvcf/pexec/status/" function upload_asset { local input=$1 local description=$2 response=$(curl -s -X POST $assets_url \ -H "Authorization: ${header_auth}" \ -H "Content-Type: application/json" \ -H "accept: application/json" \ -d '{"contentType": "video/mp4", "description": "'"${description}"'"}') asset_url=$(echo ${response} | jq -r .uploadUrl) asset_id=$(echo ${response} | jq -r .assetId) curl -s -X PUT -H "x-amz-meta-nvcf-asset-description: ${description}" -H "content-type: video/mp4" \ --data-binary "@${input}" ${asset_url} echo ${asset_id} } asset_id=$(upload_asset "$2" "Input Video") inputs='{"model": "Grounding-Dino", "messages": [{"role": "user", "content": [{"type": "text", "text": "'"${1}"'"}, {"type": "media_url", "media_url": {"url": "data:image/jpeg;asset_id,'"${asset_id}"'" }}]}], "threshold": 0.3}' asset_list="${asset_id}" # The response will include a location header that curl doesn't trivially # redirect to due to its complexity. We need to extract the location header # and then download the file from that location. response=$(curl -D - -s -w "%{http_code}" -X POST $nvai_url \ -H "Content-Type: application/json" \ -H "NVCF-INPUT-ASSET-REFERENCES: ${asset_list}" \ -H "NVCF-FUNCTION-ASSET-IDS: ${asset_list}" \ -H "Authorization: ${header_auth}" \ -d "${inputs}") # Extracting status code status_code=$(echo "$response" | grep HTTP | awk '{print $2}') # Remove any newlines, carriage returns, spaces, quotes, and commas from the location header location=$(echo "$response" | grep -i location | awk '{print $2}' | tr -d '\n' | tr -d '\r' | tr -d ' ' | tr -d '"' | tr -d ',') if [ -n "$location" ]; then # location non empty, Evaluation complete, download the file curl -s "$location" -o "$3.zip" # Unzip the file unzip -q "$3.zip" -d "$3" echo "Response saved to $3.zip" echo $(ls "$3") exit 0 elif [ -z "$location" ]; then # location empty, Pending evaluation, need polling echo "Pending evaluation ..." retries=$MAX_RETRIES while [ $retries -gt 0 ]; do echo "Polling ..." nvcf_reqid=$(echo "$response" | grep -i nvcf-reqid | awk '{print $2}' | tr -d '\n' | tr -d '\r' | tr -d ' ' | tr -d '"' | tr -d ',') polling_url="${nvai_polling_url}${nvcf_reqid}" polling_response=$(curl -D - -s -w "%{http_code}" -X GET $polling_url \ -H "accept: application/json" \ -H "Authorization: ${header_auth}") # Extracting location polling_location=$(echo "$polling_response" | grep -i location | awk '{print $2}' | tr -d '\n' | tr -d '\r' | tr -d ' ' | tr -d '"' | tr -d ',') if [ -z "$polling_location" ]; then echo "Result is not yet ready." retries=$((retries - 1)) sleep $DELAY_BTW_RETRIES elif [ -n "$polling_location" ]; then echo "Result ready!" curl -s "$polling_location" -o "$3.zip" unzip -q "$3.zip" -d "$3" echo "Response saved to $3.zip" echo $(ls "$3") exit 0 else echo "Unexpected response status: $polling_status" exit 1 fi done if [ $retries -eq 0 ]; then echo "Max retries reached. Evaluation not completed." exit 1 fi else echo "Unexpected response status: $status_code" exit 1 fi50:Tbeb,data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD//gAQTGF2YzU3LjEwNy4xMDD/2wBDAAcFBQYFBAcGBQYIBwcIChELCgkJChUPEAwRGBUaGRgVGBcbHichGx0lHRcYIi4iJSgpKywrGiAvMy8qMicqKyr/2wBDAQcICAoJChQLCxQqHBgcKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKir/wAARCABAAHIDASIAAhEBAxEB/8QAHAAAAgIDAQEAAAAAAAAAAAAABAUDBgABAgcI/8QANBAAAgEDAwIEBQQBAwUAAAAAAQIDAAQRBRIhMUEGE1FhFCJxgaEykbHRwQcj4RUmQmLx/8QAGgEAAgMBAQAAAAAAAAAAAAAAAwQBAgUABv/EACgRAAIBAwMCBgMBAAAAAAAAAAABAgMRIQQSMSJBEyNRcaHwFDJhwf/aAAwDAQACEQMRAD8AIW29q7Fv7U3WzyMgcV2LL2r02480xN8P7Vnke1Ofg8DkVr4PjpUbjsifyPatiCm/wftWxZn0qNyOFIgqRIPaj5o4raIy3DrGg7mqlrXjGC23RWhw3qOWP+B96FUrwprIelRnUfSO7m5t7JC08gB7L3NV7UPE43iKNlhD8KCfmf6CqXquuX9xMyrJ5QIByvLnIB/UenXsPvSmxLLqsEjE5Mq7mJJJycck81mVa1eqm49K+TTpaelTfVl/BbLm7upFLyyJbL3aQ5b+v/tJbnWNMttzvNLeuOCQMjv9u5rNXWC3vxvgjkeTGGm3MqADn5c8mlt/dXElpNbSOrJjlEhCqpHIxj6VkwhKfU+DRbUcDP4zU5Ys2tjFZwkZDzuB+BQgR7iXNxqsFyyfMYYgMfzQVuFu0DzqJWCoF384G0DAH1Boh08ry5AoAjdWOBjjOD+DRlp2o7rg3WW61iWTw7ZSyNIVA3EnA96yiBLtG0tyODxWUrun6h9q9D6NS1jEnlZUPtyEyM49celSiw9q8lPjHULPxWNQ1BI4rubbsiTLpsAKE+nJB4Bqx6x4w8RWd6kdujL8oZQbXIkBzyfp9a9BLURir3MJaVt2ZdZrILA54GFPJqG6aysovMvLqC3Tj5pJQo56da8u1rxj4gvZYbDULmKCCUgTI0Plhkz3AOT0qPUtM0K+1FZP+pNaq4GZpImSNz3AbJ554zjp0oT1UrXisB1o4Ws5ZPVrZLW8LC0uIpyuN3lOGx+1J/EOvWWgstuf929kZUSEdmbpn0zXnOl2Ys/OME95GclUjVmVihJ2khRz07Gs1Pw9HBfNPdzXccs0SsPOG9gcAg7iMjjpVZaqajdotDRwvlm9X1G61OSSa5vQYwrEJC3AAHr9+2Kr91bKHSNEI+UtjGOMj+6iGnXB3C2vWCsDuQPjjABBwO+BQpsL0+XGlwGKqQu+aTOOOM4pb8qF+cjLoPhGXUYNw6lOVRB1/wDUVxFCodH81UO4cBeRz61zJZalGzOTaMyoN26d+g4yf7oVrXULpmUQWcxxkql0w/Ga5amDjyT4MrjvWER762MjlFOQWBx2ahr3T9PkinmhnZ5EhVly5wWJwwx9DQ/iWSUaZFJLhZVClgvQHIz/ADSFbubMe2TcpUB1PQc0DTy8povVj5iYdo7boNuB+nk454Y/3Tezto7x5YXZTujYDJ744/NVi2aXZshkKN5jrkexWjtPubiHfPGxUlDuJA6g84/amozxYBOF2Oo4FmiWUnlwGP3rKXWt+RZwgnpGv8VlZTckzRSQ/wDGt0vk6PdW86OqJtVYzltrHcT68N/NehaNqd74n08PbwSLMo2MScJ5gXafXv29q8W1nRbjwf4hOm3+yVGVZI5kHDq3f7HI+1ei6D47Tw/4Oktbdlubyad5F5/RuAzu+46DrntT9WPjU3BcMR2pNSfYq3jwXOneJ7u4uizMZSjwgfpCqNpB9CO/tXekeJf+3r6zubczwX0amNWfi3cZyV46fpPrkGlGvzT615013LvuJCGaRu+Og+mO1C2O2Gxij3ZwtMQjtiodkVb7osmi+J7WxVodeS9ukgx8K0VyymL1GNwGPzUPi/xva6nCj6YL2G5UoC8sm4lV7ZyT04xVbuGDykdjSqZc7kbtxUSgmWi2W4ak80QaVEu4yMiRFG8f5P2NZE/xBLafqcyMv/gx37fqG5FU+0u59PkzExKHqh6GnsFxZasoLArOvcHa6/Q96QqU3H2+9hiMkxnJNrMSvmO1vAy7Sy/7bkfWhY9cWzuzNf2F1bsV2k8MvXr2rQl1Ky5Rlv4h2b5ZB9+hoi112zuW8p2MMnQxTDb/AMUvZ2/W6/n3/Aq9znxHOtzowmjOUdQ68dsrSOwtibMzHABBPXrjp+ad63ctDZjyURiVKhWUFcEgdKT27CJ4/i7LyY5CGPlOU3DOM7TkH8Uxp7qnhA6uZcm7BQZpM9ppP5FF3TmGyaNcL5rmNPUZOc/TrQdg6C3llM0aSGdwokzgjjJ4BxU7PdlS8Vv8R7wSBwf25H7UdSXDBuLZFbki2iHog/iso+GwcW8YkIVto3D0OKylHONxhJjn/U3X7PxHrdn8G/mpFZnY20p5RBYkY+uKrWivnS19dzZ596yaN2mV85UAgr9ay3jFshROATnHpWrGO3gQk7oJl+cEAke9QxoI0C7i2O7d63uJ6VonIq5UhmPOaEukwQ/rwaLlGVPNRvFvtTIR8pO38VDLoXPyu0gce3NQYaNw8bFWHQg9KJYnlW/UvGaiYUJq5YZ2GvkYjveO3mAfyKZyx2t/GPiY1fPRx6exFVNkqewvJ7WdVibKswBQ9DSs6PeOGFjPsyyXcsMT28bsPKCbRuXdn0/jrS29Ko0GzA3sGIzyAK41S8IuI2jxlcYyPY/3S0yOWaR23HaTn8f5q9JbadiJ5nckhuXt44ipwcFv3P8AxRHx0UjeYYikuMK8Zwc9jQTkAhDj5VAx9q6tY1a7j6gBskew5q/CK9y0G9AJD8sOp96yo0s96BmC5IyeaykegaP/2Q==43:[["$","div",".3",{"className":"relative","children":[["$","div",null,{"className":"relative flex min-h-[194px] w-full flex-col overflow-hidden p-md sm:p-ml bg-manitoulinLightBlack","children":[["$","$L46",null,{"alt":"","className":"object-cover object-center !left-1/2 z-10 !w-1/2 brightness-20 lg:brightness-75","fill":true,"priority":true,"src":"https://assets.ngc.nvidia.com/products/api-catalog/images/nv-grounding-dino.jpg","style":{"maskImage":"linear-gradient(to left, rgba(0,0,0,1), rgba(0,0,0,0))"}}],["$","div",null,{"className":"z-30 mx-auto flex w-full max-w-screen-xl min-w-0 flex-1 flex-col items-start justify-between gap-md lg:flex-row lg:gap-ml","children":["$","section",null,{"aria-describedby":"resource-description","aria-labelledby":"resource-name","className":"flex w-full min-w-0 flex-col lg:flex-1","children":["$","div",null,{"className":"flex flex-col gap-1.5 whitespace-pre-wrap","children":[["$","$L47",null,{"className":"w-fit text-ml text-n300 lowercase hover:text-n050","href":"/nvidia","children":"NVIDIA"}],["$","div",null,{"className":"flex flex-col items-start gap-3 lg:flex-row lg:items-center","children":[["$","h1",null,{"className":"max-w-full text-lg leading-heading font-medium break-words text-primary","data-testid":"resource-name","id":"resource-name","children":"nv-grounding-dino"}],[["$","span","DEPRECATION",{"className":"nv-badge nv-badge--kind-solid nv-badge--color-red h-[18px] py-0","data-testid":"nv-badge","ref":"$undefined","children":["$undefined","Deprecated"]}],["$","span","PREVIEW",{"className":"nv-badge nv-badge--kind-solid nv-badge--color-purple h-[18px] py-0","data-testid":"nv-badge","ref":"$undefined","children":["$undefined","Free Endpoint"]}]],"$undefined"]}],["$","p",null,{"className":"max-w-[85ch] text-ms text-primary","id":"resource-description","children":"Grounding dino is an open vocabulary zero-shot object detection model."}],["$","div",null,{"className":"flex w-full flex-col items-start gap-md lg:flex-row lg:items-end lg:justify-between","children":[["$","div",null,{"className":"flex w-full min-w-0 flex-wrap gap-sm lg:flex-1","children":["$","$L48",null,{"artifactType":"ENDPOINT","density":"standard","labels":["NVIDIA NIM","Object Detection","computer vision","deepstream","nimType:endpoint:nim_type_preview","usecase:endpoint:usecase_object_detect"],"searchRoutePath":"/models"}]}],["$","div",null,{"className":"z-30 flex w-full flex-row justify-start gap-ms sm:justify-end lg:w-auto lg:shrink-0","children":["$undefined",["$","$L49",null,{"cta":{"type":"Apply to Self-Host","url":"https://www.nvidia.com/en-us/ai/nim-notifyme/"},"isNIMAccessRestricted":"$undefined","isNIMAvailable":false}]]}]]}]]}]}]}],["$","div",null,{"className":"pointer-events-none absolute right-0 bottom-0 left-0 z-20 h-30 bg-gradient-to-t to-transparent from-manitoulinLightBlack via-manitoulinLightBlack/70"}]]}],["$","div",null,{"className":"px-4 mx-auto flex max-w-screen-xl flex-col pt-0 lg:px-0","data-body-background":"light-black","children":["$","section",null,{"className":"flex w-full flex-col justify-between rounded-4 border-manitoulinBorderColor bg-manitoulinDarkBlack sm:rounded-6 lg:rounded-8 p-ms lg:p-lg","children":["$","$L4a",null,{"artifact":{"artifactType":"ENDPOINT","name":"nv-grounding-dino","displayName":"nv-grounding-dino","publisher":"nvidia","shortDescription":"Grounding dino is an open vocabulary zero-shot object detection model.","logo":"https://assets.ngc.nvidia.com/products/api-catalog/images/nv-grounding-dino.jpg","labels":"$43:0:props:children:0:props:children:1:props:children:props:children:props:children:3:props:children:0:props:children:props:labels","attributes":[{"key":"DEPRECATION","value":"04/15/2026"},{"key":"PREVIEW","value":"true"}],"updatedDate":"2025-03-26T16:21:21.125Z","bias":"| Field | Response |\n| -- | -- |\n|Participation considerations from adversely impacted groups [(protected classes)](https://www.senate.ca.gov/content/protected-classes) in model design and testing: | None of the Above |\n| Measures taken to mitigate against unwanted bias: | No measures taken to mitigate against unwanted bias.|","canGuestDownload":true,"createdDate":"2024-08-09T16:42:36.460Z","description":"$4b","explainability":"| Field | Response |\n| -- | -- |\n| Intended Application(s) & Domain(s): | Detecting Objects |\n| Model Type: | Object Detection |\n| Intended Users: | The model is intended for developers that build multimodal object detection application. |\n| Output: | Bounding Box, Confidence Scores, and Noun Phrases |\n| Describe how the model works: | Extracts features from provided images using provided list of nouns. |\n| Technical Limitations: | The model may have difficulties in non-Flickr style data like medical, satellite, and industrial data. |\n| Verified to have met prescribed NVIDIA standards: | Yes |\n| Performance Metrics: | Mean Average Precision (mAP) |\n| Licensing: | [EULA](https://commercialtype.com/eula) |","isPublic":true,"isReadOnly":true,"orgName":"qc69jvmznzxy","privacy":"$4c","safetyAndSecurity":"| Field | Response |\n| -- | -- |\n| Model Application(s): | Object Detection (General Purpose) |\n| Describe the life-critical application (if present). | None: Not within Operational Design Domain |\n| Use Case Restrictions: | Abide by [https://www.nvidia.com/en-us/data-center/products/nvidia-ai-enterprise/eula/\"](https://www.nvidia.com/en-us/data-center/products/nvidia-ai-enterprise/eula/) |\n| Describe access restrictions (if any): | The Principle of least privilege (PoLP) is applied limiting access for dataset generation and model development. |"},"endpointName":"nv-grounding-dino","endpointSpec":{"openAPISpec":{"openapi":"3.1.0","info":{"title":"NV Grounding Dino Object Detection","description":"The NVIDIA NIM REST API for NV Grounding Dino object detection.","version":"1.0.0","termsOfService":"https://nvidia.com/legal/terms-of-use","contact":{"name":"NVIDIA Support","url":"https://help.nvidia.com/"},"license":{"name":"NVIDIA AI Foundation Models Community License","url":"https://docs.nvidia.com/ai-foundation-models-community-license.pdf"}},"servers":[{"url":"https://ai.api.nvidia.com/v1"}],"tags":[{"name":"NV Grounding Dino Object Detection API","description":"Run inference using the model"}],"paths":{"/cv/nvidia/nv-grounding-dino":{"post":{"tags":["NV Grounding Dino API"],"summary":"Run inference on the input image/video for a given text prompt.","description":"Runs inference on the specified input file for given text prompt. The input image can be Base64 encoded string in the image URL or it can be specified using the file identifier of an uploaded file.","operationId":"nvidia-nv-grounding-dino-infer","parameters":[{"in":"header","name":"NVCF-INPUT-ASSET-REFERENCES","schema":{"type":"string","maxLength":370,"format":"uuid"},"required":false,"description":"String of asset IDs separated by commas. Data is uploaded to AWS S3 using NVCF Asset APIs and associated with these asset IDs.The presigned URL allows for secure and temporary access to the S3 bucket for uploading the video. Once the asset is requested, an asset ID is generated for it. Please include this asset ID in this header and to use the uploaded video in a prompt."}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RequestModel"}}},"required":true},"responses":{"200":{"description":"Successful Response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InferResponse"}}}},"202":{"description":"Result is pending. Client should poll using the requestId.\n","content":{"application/json":{"example":{},"schema":{}}},"headers":{"NVCF-REQID":{"description":"requestId required for pooling","schema":{"type":"string","format":"uuid","maxLength":36}},"NVCF-STATUS":{"description":"Invocation status","schema":{"type":"string","format":"^[a-zA-Z-]{1,64}$","maxLength":64}}}},"402":{"description":"Payment Required","content":{"application/json":{"schema":{"$ref":"#/components/schemas/PaymentRequiredError"}}}},"422":{"description":"Validation failed, provided entity could not be processed.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Errors"},"example":{"type":"urn:nvcf-worker-service:problem-details:unprocessable-entity","title":"Unprocessable Entity","status":422,"detail":"string","instance":"/v2/nvcf/pexec/functions/c0b05e1d-3847-467a-80d5-85e3e5b05f17","requestId":"3fa85f64-5717-4562-b3fc-2c963f66afa6"}}}},"500":{"description":"The invocation ended with an error.","content":{"application/json":{"schema":{"properties":{"type":{"type":"string","format":"^.{1, 128}$","maxLength":128,"description":"Error type"},"title":{"type":"string","format":"^.{1, 128}$","maxLength":128,"description":"Error title"},"status":{"type":"integer","minimum":100,"maximum":999,"description":"Error status code"},"detail":{"type":"string","format":"^.{1, 1024}$","maxLength":1024,"description":"Detailed information about the error"},"instance":{"type":"string","format":"^.{1, 256}$","maxLength":256,"description":"Function instance used to invoke the request"},"requestId":{"type":"string","format":"uuid","maxLength":36,"description":"UUID of the request"}},"type":"object","required":["type","title","status","detail","instance","requestId"],"title":"InvokeError"},"example":{"type":"urn:nvcf-worker-service:problem-details:internal-server-error","title":"Internal Server Error","status":500,"detail":"string","instance":"/v2/nvcf/pexec/functions/c0b05e1d-3847-467a-80d5-85e3e5b05f17","requestId":"3fa85f64-5717-4562-b3fc-2c963f66afa6"}}}}},"x-nvai-meta":{"examples":[{"input":{"image":"https://assets.ngc.nvidia.com/products/api-catalog/nv-grounding-dino/input/forklift.png","text":"factory worker, forklift, pallet"},"output":{"image":"https://assets.ngc.nvidia.com/products/api-catalog/nv-grounding-dino/output/forklift.jpg"}},{"input":{"image":"https://assets.ngc.nvidia.com/products/api-catalog/nv-grounding-dino/input/cats.png","text":"tallest cat, right most cat"},"output":{"image":"https://assets.ngc.nvidia.com/products/api-catalog/nv-grounding-dino/output/cats.jpg"}},{"input":{"image":"https://assets.ngc.nvidia.com/products/api-catalog/nv-grounding-dino/input/people.png","text":"people wearing glasses"},"output":{"image":"https://assets.ngc.nvidia.com/products/api-catalog/nv-grounding-dino/output/glasses.jpg"}},{"input":{"image":"https://assets.ngc.nvidia.com/products/api-catalog/nv-grounding-dino/input/warehouse.png","text":"person in hardhat, robotic arm, magenta boxes"},"output":{"image":"https://assets.ngc.nvidia.com/products/api-catalog/nv-grounding-dino/output/warehouse.jpg"}}],"templates":[{"requestEjs":{"python":"$4d","node":"$4e","shell":"$4f"}}]}}},"/status/{requestId}":{"get":{"tags":["NV-Grounding-Dino Object Detection Inference API"],"summary":"Gets the result of an earlier function invocation request that returned a status of 202.","operationId":"getFunctionInvocationResult","parameters":[{"name":"requestId","in":"path","description":"The requestId to poll results.","required":true,"schema":{"type":"string","format":"uuid","maxLength":36}}],"responses":{"200":{"description":"Invocation is fulfilled.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InferResponse"}}}},"202":{"description":"Result is pending. Client should poll using the requestId.","content":{"application/json":{"example":{},"schema":{}}},"headers":{"NVCF-REQID":{"description":"requestId required for pooling.","schema":{"type":"string","format":"uuid","maxLength":36}},"NVCF-STATUS":{"description":"Invocation status.","schema":{"type":"string","format":"^[a-zA-Z-]{1,64}$","maxLength":64}}}},"422":{"description":"The invocation ended with an error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Errors"},"example":{"type":"urn:nvcf-worker-service:problem-details:unprocessable-entity","title":"Unprocessable Entity","status":422,"detail":"string","instance":"/v2/nvcf/pexec/functions/TBD","requestId":"3fa85f64-5717-4562-b3fc-2c963f66afa6"}}}},"500":{"description":"The invocation ended with an error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Errors"},"example":{"type":"urn:nvcf-worker-service:problem-details:internal-server-error","title":"Internal Server Error","status":500,"detail":"string","instance":"/v2/nvcf/pexec/functions/TBD","requestId":"3fa85f64-5717-4562-b3fc-2c963f66afa6"}}}}}}}},"components":{"schemas":{"BoundingBox":{"properties":{"phrase":{"type":"string","maxLength":1024,"pattern":"^.*$","title":"Phrase","description":"Type of Object detected.","examples":["Car"]},"bboxes":{"items":{"items":{"type":"integer","maximum":10000000000,"exclusiveMinimum":-1,"format":"int32"},"type":"array","maxItems":4,"minItems":0,"description":"List of coordinates for Each object."},"type":"array","maxItems":1000000,"title":"Bboxes","description":"Bounding box coordinates of each detection.","examples":[[[1212,304,55,154],[307,322,17,45],[290,316,16,42],[61,307,76,200],[0,257,130,422],[195,294,121,275]]]},"confidence":{"items":{"type":"number","maximum":1,"exclusiveMinimum":0},"type":"array","maxItems":1000000,"title":"Confidence","description":"Confidence of each detection.","examples":[[0.933861494064331,0.8327290296554565,0.7375573039054871,0.865410578250885,0.795340359210968,0.8262916803359985]]}},"additionalProperties":false,"type":"object","required":["phrase","bboxes","confidence"],"title":"BoundingBox"},"Choice":{"properties":{"index":{"type":"integer","maximum":1000000000,"minimum":0,"format":"int32","title":"Index","description":"Index of the message.","examples":[1]},"message":{"$ref":"#/components/schemas/Message-Output"},"finish_reason":{"type":"string","maxLength":128,"pattern":"^.*$","title":"Finish Reason","description":"Reason for Finish.","examples":["stop"]},"timestamp":{"type":"integer","maximum":9223372036854776000,"minimum":0,"format":"int64","title":"Timestamp","description":"Time stamp of the Frame in nanosecond.","examples":[0,333333333]}},"additionalProperties":false,"type":"object","required":["index","message","finish_reason","timestamp"],"title":"Choice"},"Content":{"properties":{"frameNo":{"type":"integer","maximum":9223372036854776000,"minimum":0,"format":"int64","title":"Frameno","description":"Frame no. of the media","examples":[0]},"frameWidth":{"type":"integer","maximum":1000000,"minimum":16,"format":"int32","title":"Framewidth","description":"Resolution width of the inferenced Frame.","examples":[1280]},"frameHeight":{"type":"integer","maximum":1000000,"minimum":16,"format":"int32","title":"Frameheight","description":"Resolution height of the inferenced Frame.","examples":[720]},"message":{"type":"string","maxLength":1024,"pattern":"^.*$","title":"Message","description":"Prompt for detection.","examples":["car, bus"]},"boundingBoxes":{"items":{"$ref":"#/components/schemas/BoundingBox"},"type":"array","maxItems":1000000,"title":"Boundingboxes","description":"List of bounding boxes detected."}},"additionalProperties":false,"type":"object","required":["frameNo","frameWidth","frameHeight","message","boundingBoxes"],"title":"Content"},"Errors":{"properties":{"type":{"type":"string","format":"^.{1, 128}$","maxLength":128,"description":"Error type"},"title":{"type":"string","format":"^.{1, 128}$","maxLength":128,"description":"Error title"},"status":{"type":"integer","minimum":100,"maximum":999,"format":"int32","description":"Error status code"},"detail":{"type":"string","format":"^.{1, 1024}$","maxLength":1024,"description":"Detailed information about the error"},"instance":{"type":"string","format":"^.{1, 256}$","maxLength":256,"description":"Function instance used to invoke the request"},"requestId":{"type":"string","format":"uuid","maxLength":36,"description":"UUID of the request"}},"type":"object","required":["type","title","status","detail","instance","requestId"],"title":"InvokeError"},"MediaUrl":{"properties":{"url":{"type":"string","maxLength":20480000,"pattern":"^.*$","title":"Url","description":"Base64 encoded image data in the form of `data:image/{format};base64,{base64encodedimage}`. If the size of an image is more than 200KB, it needs to be uploaded using Files API. Once uploaded you can refer to it using the following format: `data:image/{format};asset_id,{asset_id}`.","example":"$50"}},"additionalProperties":false,"type":"object","required":["url"],"title":"MediaUrl"},"InferResponse":{"properties":{"created":{"type":"integer","maximum":9223372036854776000,"minimum":0,"format":"int64","title":"Created","description":"Time for message creation.","examples":[1717778251]},"model":{"type":"string","maxLength":256,"pattern":"^.*$","title":"Model","description":"Name of the model used for inference.","examples":["Grounding Dino"]},"id":{"anyOf":[{"type":"string","maxLength":1024,"pattern":"^.*$","description":"Asset id for which inference is performed.","examples":["Grounding Dino"]}],"title":"Id"},"object":{"type":"string","maxLength":1024,"pattern":"^.*$","title":"Object","description":"Type of Inference performed.","examples":["0cfa0669-369f-4092-a855-281c3823c6e8"]},"choices":{"items":{"$ref":"#/components/schemas/Choice"},"type":"array","maxItems":1000000,"title":"Choices","description":"List of result for current frame."},"usage":{"$ref":"#/components/schemas/Usage"}},"additionalProperties":false,"type":"object","required":["created","model","object","choices","usage"],"title":"InferResponse"},"Message-Input":{"properties":{"role":{"type":"string","maxLength":512,"format":"str","title":"Role","description":"Role user or assistant.","examples":["user","assistant"]},"content":{"items":{"anyOf":[{"$ref":"#/components/schemas/TextContent"},{"$ref":"#/components/schemas/UrlContent"}]},"type":"array","maxItems":32,"title":"Content","description":"List of text content and url content."}},"additionalProperties":false,"type":"object","required":["role","content"],"title":"Message"},"Message-Output":{"properties":{"role":{"type":"string","maxLength":128,"format":"str","title":"Role","description":"Role of user/assistant.","examples":["user","assistant"]},"content":{"allOf":[{"$ref":"#/components/schemas/Content"}],"description":"Content of the message."}},"additionalProperties":false,"type":"object","required":["role","content"],"title":"Message"},"PaymentRequiredError":{"properties":{"detail":{"type":"string","maxLength":256,"format":"^[a-zA-Z-]{1,64}$","description":"Contains specific information related to the error and why it occurred.","example":"You have reached your limit of credits."}},"type":"object","title":"PaymentRequiredError"},"RequestModel":{"properties":{"model":{"type":"string","maxLength":256,"pattern":"^.*$","title":"Model","description":"Name of the model used.","examples":["Grounding Dino"]},"messages":{"items":{"$ref":"#/components/schemas/Message-Input"},"type":"array","maxItems":1000000,"title":"Messages","description":"Requests for inference."},"threshold":{"type":"number","default":0.3,"maximum":1,"minimum":0.1,"title":"Threshold","description":"Threshold used for detection.","examples":[0.7]}},"additionalProperties":false,"type":"object","required":["model","messages","threshold"],"title":"RequestModel"},"TextContent":{"properties":{"type":{"type":"string","const":"text","title":"Type","description":"The type of content part."},"text":{"type":"string","maxLength":1024,"format":"str","title":"Text","description":"Prompt for Grounding Dino.","example":"person, cars, bus, backpack"}},"additionalProperties":false,"type":"object","required":["type","text"],"title":"TextContent"},"UrlContent":{"properties":{"type":{"type":"string","const":"media_url","title":"Type","description":"The type of content part."},"media_url":{"$ref":"#/components/schemas/MediaUrl"}},"additionalProperties":false,"type":"object","required":["type","media_url"],"title":"UrlContent"},"Usage":{"properties":{"queryResponseTime":{"type":"integer","maximum":1000000000000000,"minimum":0,"format":"int32","title":"Queryresponsetime","description":"The total time taken for inference processing in milliseconds.","examples":[240]},"additionalInfo":{"anyOf":[{"type":"object"}],"title":"Additionalinfo"}},"additionalProperties":false,"type":"object","required":["queryResponseTime"],"title":"Usage"}},"securitySchemes":{"Token":{"type":"http","scheme":"bearer"}}},"security":[{"Token":[]}]},"namespace":"qc69jvmznzxy","updatedDate":"2026-04-24T21:04:20.266Z","nvcfFunctionId":"50128865-af25-4a3f-b175-d75ce3c72bd9","createdDate":"2024-08-09T16:42:36.871Z","attributes":{"apiDocsUrl":"https://docs.api.nvidia.com/nim/reference/nvidia-nv-grounding-dino","termsOfUse":"GOVERNING TERMS: Your use of this API is governed by the NVIDIA API Trial Service Terms of Use; and the use of this model is governed by the NVIDIA AI Foundation Models Community License.\n","deprecationNotice":"This API will be deprecated on 04/15/2026. It will no longer be supported after 04/15/2026. Please use openai/gpt-oss-120b instead. For more information, visit our API Reference.","showUnavailableBanner":false,"cta":"$43:0:props:children:0:props:children:1:props:children:props:children:props:children:3:props:children:1:props:children:1:props:cta"},"artifactName":"nv-grounding-dino"},"isLoginRequired":true,"playgroundElement":"$undefined","publisherName":"nvidia","searchParams":{},"tabs":[{"label":"Experience","bare":true,"icon":"play","kind":"default","id":"experience","href":"/nvidia/nv-grounding-dino","aliases":["/nvidia/nv-grounding-dino/experience"]}]}]}]}]]}]] 44:["$","$L51",null,{"spec":"$43:0:props:children:1:props:children:props:children:props:endpointSpec"}]