3b:I[603186,["/_next/static/chunks/0410xx54a0zpq.js","/_next/static/chunks/3pd35vh8xvq70.js","/_next/static/chunks/32q37d2vm9h_s.js","/_next/static/chunks/14sl9ye0a60vb.js","/_next/static/chunks/1la1pzs9j8idc.js","/_next/static/chunks/24mqrfe34h1_g.js","/_next/static/chunks/3-axx3uk1c0-h.js","/_next/static/chunks/14bn3mp7hfp6x.js","/_next/static/chunks/1_xj2c5utin8v.js","/_next/static/chunks/2tdv_pqf9hlyu.js","/_next/static/chunks/44n8gpf9x1nn2.js","/_next/static/chunks/1e7h6z92_w2-c.js","/_next/static/chunks/2g-r6rokjl78a.js","/_next/static/chunks/1geiruffics9x.js","/_next/static/chunks/1lfygqrocy8iw.js","/_next/static/chunks/0q47y720ck5-9.js","/_next/static/chunks/1ttnpl_ih9zxl.js","/_next/static/chunks/3mi_myijg06x1.js","/_next/static/chunks/2_c926k09c1th.js","/_next/static/chunks/0bvim9_g0hu68.js","/_next/static/chunks/3xav1ddquf5gq.js","/_next/static/chunks/0ayh-bmdnfery.js","/_next/static/chunks/34kl4tv6yifcg.js","/_next/static/chunks/0nm6za7z70maq.js","/_next/static/chunks/44e22u2t839t0.js","/_next/static/chunks/0czpc-f8wjri8.js","/_next/static/chunks/2aq42x9vabhfb.js","/_next/static/chunks/24_gg9g_4g22i.js","/_next/static/chunks/00yjb-caoi49j.js","/_next/static/chunks/1l0z2zsen9vs2.js","/_next/static/chunks/2drnqf6qy04sf.js","/_next/static/chunks/01auidv8wbc03.js","/_next/static/chunks/2azed6ymy4h96.js","/_next/static/chunks/1yd7p-515l2jc.js","/_next/static/chunks/2pqhypz2vo8mu.js","/_next/static/chunks/1i60vi-4s7c4l.js","/_next/static/chunks/23fi4qujvabmc.js","/_next/static/chunks/295ffwyuqa4b6.js"],"PageMetadata"] 3c:I[831915,["/_next/static/chunks/0410xx54a0zpq.js","/_next/static/chunks/3pd35vh8xvq70.js","/_next/static/chunks/32q37d2vm9h_s.js","/_next/static/chunks/14sl9ye0a60vb.js","/_next/static/chunks/1la1pzs9j8idc.js","/_next/static/chunks/24mqrfe34h1_g.js","/_next/static/chunks/3-axx3uk1c0-h.js","/_next/static/chunks/14bn3mp7hfp6x.js","/_next/static/chunks/1_xj2c5utin8v.js","/_next/static/chunks/2tdv_pqf9hlyu.js","/_next/static/chunks/44n8gpf9x1nn2.js","/_next/static/chunks/1e7h6z92_w2-c.js","/_next/static/chunks/2g-r6rokjl78a.js","/_next/static/chunks/1geiruffics9x.js","/_next/static/chunks/1lfygqrocy8iw.js","/_next/static/chunks/0q47y720ck5-9.js","/_next/static/chunks/1ttnpl_ih9zxl.js","/_next/static/chunks/3mi_myijg06x1.js","/_next/static/chunks/2_c926k09c1th.js","/_next/static/chunks/0bvim9_g0hu68.js","/_next/static/chunks/3xav1ddquf5gq.js","/_next/static/chunks/0ayh-bmdnfery.js","/_next/static/chunks/34kl4tv6yifcg.js","/_next/static/chunks/0nm6za7z70maq.js","/_next/static/chunks/44e22u2t839t0.js","/_next/static/chunks/0czpc-f8wjri8.js","/_next/static/chunks/2aq42x9vabhfb.js","/_next/static/chunks/24_gg9g_4g22i.js","/_next/static/chunks/00yjb-caoi49j.js","/_next/static/chunks/1l0z2zsen9vs2.js","/_next/static/chunks/2drnqf6qy04sf.js","/_next/static/chunks/01auidv8wbc03.js","/_next/static/chunks/2azed6ymy4h96.js","/_next/static/chunks/1yd7p-515l2jc.js","/_next/static/chunks/2pqhypz2vo8mu.js","/_next/static/chunks/1i60vi-4s7c4l.js","/_next/static/chunks/23fi4qujvabmc.js","/_next/static/chunks/295ffwyuqa4b6.js"],"SearchResultsImage"] 3d:I[430295,["/_next/static/chunks/0410xx54a0zpq.js","/_next/static/chunks/3pd35vh8xvq70.js","/_next/static/chunks/32q37d2vm9h_s.js","/_next/static/chunks/14sl9ye0a60vb.js","/_next/static/chunks/1la1pzs9j8idc.js","/_next/static/chunks/24mqrfe34h1_g.js","/_next/static/chunks/3-axx3uk1c0-h.js","/_next/static/chunks/14bn3mp7hfp6x.js","/_next/static/chunks/1_xj2c5utin8v.js","/_next/static/chunks/2tdv_pqf9hlyu.js","/_next/static/chunks/44n8gpf9x1nn2.js","/_next/static/chunks/1e7h6z92_w2-c.js","/_next/static/chunks/2g-r6rokjl78a.js","/_next/static/chunks/1geiruffics9x.js","/_next/static/chunks/1lfygqrocy8iw.js","/_next/static/chunks/0q47y720ck5-9.js","/_next/static/chunks/1ttnpl_ih9zxl.js","/_next/static/chunks/3mi_myijg06x1.js","/_next/static/chunks/2_c926k09c1th.js","/_next/static/chunks/0bvim9_g0hu68.js","/_next/static/chunks/3xav1ddquf5gq.js","/_next/static/chunks/0ayh-bmdnfery.js","/_next/static/chunks/34kl4tv6yifcg.js","/_next/static/chunks/0nm6za7z70maq.js","/_next/static/chunks/44e22u2t839t0.js","/_next/static/chunks/0czpc-f8wjri8.js","/_next/static/chunks/2aq42x9vabhfb.js","/_next/static/chunks/24_gg9g_4g22i.js","/_next/static/chunks/00yjb-caoi49j.js","/_next/static/chunks/1l0z2zsen9vs2.js","/_next/static/chunks/2drnqf6qy04sf.js","/_next/static/chunks/01auidv8wbc03.js","/_next/static/chunks/2azed6ymy4h96.js","/_next/static/chunks/1yd7p-515l2jc.js","/_next/static/chunks/2pqhypz2vo8mu.js","/_next/static/chunks/1i60vi-4s7c4l.js","/_next/static/chunks/23fi4qujvabmc.js","/_next/static/chunks/295ffwyuqa4b6.js"],""] 3e:I[299632,["/_next/static/chunks/0410xx54a0zpq.js","/_next/static/chunks/3pd35vh8xvq70.js","/_next/static/chunks/32q37d2vm9h_s.js","/_next/static/chunks/14sl9ye0a60vb.js","/_next/static/chunks/1la1pzs9j8idc.js","/_next/static/chunks/24mqrfe34h1_g.js","/_next/static/chunks/3-axx3uk1c0-h.js","/_next/static/chunks/14bn3mp7hfp6x.js","/_next/static/chunks/1_xj2c5utin8v.js","/_next/static/chunks/2tdv_pqf9hlyu.js","/_next/static/chunks/44n8gpf9x1nn2.js","/_next/static/chunks/1e7h6z92_w2-c.js","/_next/static/chunks/2g-r6rokjl78a.js","/_next/static/chunks/1geiruffics9x.js","/_next/static/chunks/1lfygqrocy8iw.js","/_next/static/chunks/0q47y720ck5-9.js","/_next/static/chunks/1ttnpl_ih9zxl.js","/_next/static/chunks/3mi_myijg06x1.js","/_next/static/chunks/2_c926k09c1th.js","/_next/static/chunks/0bvim9_g0hu68.js","/_next/static/chunks/3xav1ddquf5gq.js","/_next/static/chunks/0ayh-bmdnfery.js","/_next/static/chunks/34kl4tv6yifcg.js","/_next/static/chunks/0nm6za7z70maq.js","/_next/static/chunks/44e22u2t839t0.js","/_next/static/chunks/0czpc-f8wjri8.js","/_next/static/chunks/2aq42x9vabhfb.js","/_next/static/chunks/24_gg9g_4g22i.js","/_next/static/chunks/00yjb-caoi49j.js","/_next/static/chunks/1l0z2zsen9vs2.js","/_next/static/chunks/2drnqf6qy04sf.js","/_next/static/chunks/01auidv8wbc03.js","/_next/static/chunks/2azed6ymy4h96.js","/_next/static/chunks/1yd7p-515l2jc.js","/_next/static/chunks/2pqhypz2vo8mu.js","/_next/static/chunks/1i60vi-4s7c4l.js","/_next/static/chunks/23fi4qujvabmc.js","/_next/static/chunks/295ffwyuqa4b6.js"],"default"] 3f:I[400402,["/_next/static/chunks/0410xx54a0zpq.js","/_next/static/chunks/3pd35vh8xvq70.js","/_next/static/chunks/32q37d2vm9h_s.js","/_next/static/chunks/14sl9ye0a60vb.js","/_next/static/chunks/1la1pzs9j8idc.js","/_next/static/chunks/24mqrfe34h1_g.js","/_next/static/chunks/3-axx3uk1c0-h.js","/_next/static/chunks/14bn3mp7hfp6x.js","/_next/static/chunks/1_xj2c5utin8v.js","/_next/static/chunks/2tdv_pqf9hlyu.js","/_next/static/chunks/44n8gpf9x1nn2.js","/_next/static/chunks/1e7h6z92_w2-c.js","/_next/static/chunks/2g-r6rokjl78a.js","/_next/static/chunks/1geiruffics9x.js","/_next/static/chunks/1lfygqrocy8iw.js","/_next/static/chunks/0q47y720ck5-9.js","/_next/static/chunks/1ttnpl_ih9zxl.js","/_next/static/chunks/3mi_myijg06x1.js","/_next/static/chunks/2_c926k09c1th.js","/_next/static/chunks/0bvim9_g0hu68.js","/_next/static/chunks/3xav1ddquf5gq.js","/_next/static/chunks/0ayh-bmdnfery.js","/_next/static/chunks/34kl4tv6yifcg.js","/_next/static/chunks/0nm6za7z70maq.js","/_next/static/chunks/44e22u2t839t0.js","/_next/static/chunks/0czpc-f8wjri8.js","/_next/static/chunks/2aq42x9vabhfb.js","/_next/static/chunks/24_gg9g_4g22i.js","/_next/static/chunks/00yjb-caoi49j.js","/_next/static/chunks/1l0z2zsen9vs2.js","/_next/static/chunks/2drnqf6qy04sf.js","/_next/static/chunks/01auidv8wbc03.js","/_next/static/chunks/2azed6ymy4h96.js","/_next/static/chunks/1yd7p-515l2jc.js","/_next/static/chunks/2pqhypz2vo8mu.js","/_next/static/chunks/1i60vi-4s7c4l.js","/_next/static/chunks/23fi4qujvabmc.js","/_next/static/chunks/295ffwyuqa4b6.js"],"EndpointCTA"] 40:I[331699,["/_next/static/chunks/0410xx54a0zpq.js","/_next/static/chunks/3pd35vh8xvq70.js","/_next/static/chunks/32q37d2vm9h_s.js","/_next/static/chunks/14sl9ye0a60vb.js","/_next/static/chunks/1la1pzs9j8idc.js","/_next/static/chunks/24mqrfe34h1_g.js","/_next/static/chunks/3-axx3uk1c0-h.js","/_next/static/chunks/14bn3mp7hfp6x.js","/_next/static/chunks/1_xj2c5utin8v.js","/_next/static/chunks/2tdv_pqf9hlyu.js","/_next/static/chunks/44n8gpf9x1nn2.js","/_next/static/chunks/1e7h6z92_w2-c.js","/_next/static/chunks/2g-r6rokjl78a.js","/_next/static/chunks/1geiruffics9x.js","/_next/static/chunks/1lfygqrocy8iw.js","/_next/static/chunks/0q47y720ck5-9.js","/_next/static/chunks/1ttnpl_ih9zxl.js","/_next/static/chunks/3mi_myijg06x1.js","/_next/static/chunks/2_c926k09c1th.js","/_next/static/chunks/0bvim9_g0hu68.js","/_next/static/chunks/3xav1ddquf5gq.js","/_next/static/chunks/0ayh-bmdnfery.js","/_next/static/chunks/34kl4tv6yifcg.js","/_next/static/chunks/0nm6za7z70maq.js","/_next/static/chunks/44e22u2t839t0.js","/_next/static/chunks/0czpc-f8wjri8.js","/_next/static/chunks/2aq42x9vabhfb.js","/_next/static/chunks/24_gg9g_4g22i.js","/_next/static/chunks/00yjb-caoi49j.js","/_next/static/chunks/1l0z2zsen9vs2.js","/_next/static/chunks/2drnqf6qy04sf.js","/_next/static/chunks/01auidv8wbc03.js","/_next/static/chunks/2azed6ymy4h96.js","/_next/static/chunks/1yd7p-515l2jc.js","/_next/static/chunks/2pqhypz2vo8mu.js","/_next/static/chunks/1i60vi-4s7c4l.js","/_next/static/chunks/23fi4qujvabmc.js","/_next/static/chunks/295ffwyuqa4b6.js"],"ModelContent"] 47:I[539001,["/_next/static/chunks/0410xx54a0zpq.js","/_next/static/chunks/3pd35vh8xvq70.js","/_next/static/chunks/32q37d2vm9h_s.js","/_next/static/chunks/14sl9ye0a60vb.js","/_next/static/chunks/1la1pzs9j8idc.js","/_next/static/chunks/24mqrfe34h1_g.js","/_next/static/chunks/3-axx3uk1c0-h.js","/_next/static/chunks/14bn3mp7hfp6x.js","/_next/static/chunks/1_xj2c5utin8v.js","/_next/static/chunks/2tdv_pqf9hlyu.js","/_next/static/chunks/44n8gpf9x1nn2.js","/_next/static/chunks/1e7h6z92_w2-c.js","/_next/static/chunks/2g-r6rokjl78a.js","/_next/static/chunks/1geiruffics9x.js","/_next/static/chunks/1lfygqrocy8iw.js","/_next/static/chunks/0q47y720ck5-9.js","/_next/static/chunks/1ttnpl_ih9zxl.js","/_next/static/chunks/3mi_myijg06x1.js","/_next/static/chunks/2_c926k09c1th.js","/_next/static/chunks/0bvim9_g0hu68.js","/_next/static/chunks/3xav1ddquf5gq.js","/_next/static/chunks/0ayh-bmdnfery.js","/_next/static/chunks/34kl4tv6yifcg.js","/_next/static/chunks/0nm6za7z70maq.js","/_next/static/chunks/44e22u2t839t0.js","/_next/static/chunks/0czpc-f8wjri8.js","/_next/static/chunks/2aq42x9vabhfb.js","/_next/static/chunks/24_gg9g_4g22i.js","/_next/static/chunks/00yjb-caoi49j.js","/_next/static/chunks/1l0z2zsen9vs2.js","/_next/static/chunks/2drnqf6qy04sf.js","/_next/static/chunks/01auidv8wbc03.js","/_next/static/chunks/2azed6ymy4h96.js","/_next/static/chunks/1yd7p-515l2jc.js","/_next/static/chunks/2pqhypz2vo8mu.js","/_next/static/chunks/1i60vi-4s7c4l.js","/_next/static/chunks/23fi4qujvabmc.js","/_next/static/chunks/295ffwyuqa4b6.js"],"default"] 48:I[946424,["/_next/static/chunks/0410xx54a0zpq.js","/_next/static/chunks/3pd35vh8xvq70.js","/_next/static/chunks/32q37d2vm9h_s.js","/_next/static/chunks/14sl9ye0a60vb.js","/_next/static/chunks/1la1pzs9j8idc.js","/_next/static/chunks/24mqrfe34h1_g.js","/_next/static/chunks/3-axx3uk1c0-h.js","/_next/static/chunks/14bn3mp7hfp6x.js","/_next/static/chunks/1_xj2c5utin8v.js","/_next/static/chunks/2tdv_pqf9hlyu.js","/_next/static/chunks/44n8gpf9x1nn2.js","/_next/static/chunks/1e7h6z92_w2-c.js","/_next/static/chunks/2g-r6rokjl78a.js","/_next/static/chunks/1geiruffics9x.js","/_next/static/chunks/1lfygqrocy8iw.js","/_next/static/chunks/0q47y720ck5-9.js","/_next/static/chunks/1ttnpl_ih9zxl.js","/_next/static/chunks/3mi_myijg06x1.js","/_next/static/chunks/2_c926k09c1th.js","/_next/static/chunks/0bvim9_g0hu68.js","/_next/static/chunks/3xav1ddquf5gq.js","/_next/static/chunks/0ayh-bmdnfery.js","/_next/static/chunks/34kl4tv6yifcg.js","/_next/static/chunks/0nm6za7z70maq.js","/_next/static/chunks/44e22u2t839t0.js","/_next/static/chunks/0czpc-f8wjri8.js","/_next/static/chunks/2aq42x9vabhfb.js","/_next/static/chunks/24_gg9g_4g22i.js","/_next/static/chunks/00yjb-caoi49j.js","/_next/static/chunks/1l0z2zsen9vs2.js","/_next/static/chunks/2drnqf6qy04sf.js","/_next/static/chunks/01auidv8wbc03.js","/_next/static/chunks/2azed6ymy4h96.js","/_next/static/chunks/1yd7p-515l2jc.js","/_next/static/chunks/2pqhypz2vo8mu.js","/_next/static/chunks/1i60vi-4s7c4l.js","/_next/static/chunks/23fi4qujvabmc.js","/_next/static/chunks/295ffwyuqa4b6.js"],"default"] 38:["$","$L3b",null,{"hasPartnerEndpoint":false}] 41:T2035,## Nemotron-4-340B-Reward ### Model Overview The Nemotron-4-340B-Reward is a multi-dimensional Reward Model that can be used as part of a synthetic data generation pipeline to create training data that helps researchers and developers build their own LLMs; Nemotron-4-340B-Reward consists of the Nemotron-4-340B-Base model and a linear layer that converts the final layer representation of the end-of-response token into five scalar values, each corresponding to a [HelpSteer2](https://arxiv.org/abs/2406.08673) attribute. It supports a context length of up to 4,096 tokens. Given a conversation with multiple turns between user and assistant, it rates the following attributes (typically between 0 and 4) for every assistant turn. 1. **Helpfulness**: Overall helpfulness of the response to the prompt. 2. **Correctness**: Inclusion of all pertinent facts without errors. 3. **Coherence**: Consistency and clarity of expression. 4. **Complexity**: Intellectual depth required to write response (i.e. whether the response can be written by anyone with basic language competency or requires deep domain expertise). 5. **Verbosity**: Amount of detail included in the response, relative to what is asked for in the prompt. Nonetheless, if you are only interested in using it as a conventional reward model that outputs a singular scalar, we recommend using the weights ```[0, 0, 0, 0, 0.3, 0.74, 0.46, 0.47, -0.33]``` to do elementwise multiplication with the predicted attributes (which outputs 9 float values in line with [Llama2-13B-SteerLM-RM](https://huggingface.co/nvidia/Llama2-13B-SteerLM-RM) but the first four are not trained or used) Under the NVIDIA Open Model License, NVIDIA confirms: * Models are commercially usable. * You are free to create and distribute Derivative Models. * NVIDIA does not claim ownership to any outputs generated using the Models or Derivative Models. ### License: [NVIDIA Open Model License](https://developer.download.nvidia.com/licenses/nvidia-open-model-license-agreement-june-2024.pdf) ### Intended use Nemotron-4 340B Reward Model is a pretrained Reward Model intended for use in English Synthetic Data Generation and English Reinforcement Learning from AI Feedback (RLAIF). Nemotron-4 340B-Reward can be used in the alignment stage to align pretrained models to human preferences. It can also be used in cases like Reward-Model-as-a-Judge. **Model Developer:** NVIDIA **Model Input:** Text only **Input Format:** String **Model Output:** Scalar Values (List of 9 Floats) **Output Format:** Float **Model Dates:** Nemotron-4-340B-Reward was trained between December 2023 and May 2024 **Data Freshness:** The pretraining data has a cutoff of June 2023 ### Required Hardware BF16 Inference: - 16x H100 (2x H100 Nodes) - 16x A100 (2x A100 80GB Nodes) ### Usage: You can use the model with [NeMo Aligner](https://github.com/NVIDIA/NeMo-Aligner) following [SteerLM training user guide](https://docs.nvidia.com/nemo-framework/user-guide/latest/modelalignment/steerlm.html). 1. Spin up an inference server within the [NeMo Aligner container](https://github.com/NVIDIA/NeMo-Aligner/blob/main/Dockerfile) ``` python /opt/NeMo-Aligner/examples/nlp/gpt/serve_reward_model.py \ rm_model_file=Nemotron-4-340B-Reward \ trainer.num_nodes=2 \ trainer.devices=8 \ ++model.tensor_model_parallel_size=8 \ ++model.pipeline_model_parallel_size=2 \ inference.micro_batch_size=2 \ inference.port=1424 ``` 2. Annotate data files using the served reward model. As an example, this can be the Open Assistant train/val files. Then follow the next step to train a SteerLM model based on [SteerLM training user guide](https://docs.nvidia.com/nemo-framework/user-guide/latest/modelalignment/steerlm.html#step-5-train-the-attribute-conditioned-sft-model) . ``` python /opt/NeMo-Aligner/examples/nlp/data/steerlm/preprocess_openassistant_data.py --output_directory=data/oasst python /opt/NeMo-Aligner/examples/nlp/data/steerlm/attribute_annotate.py \ --input-file=data/oasst/train.jsonl \ --output-file=data/oasst/train_labeled.jsonl \ --port=1424 ``` 3. Alternatively, this can be any conversational data file (in .jsonl) in the following format, where each line looks like ``` { "conversations": [ {"value": , "from": "User", "label": None}, {"value": , "from": "Assistant", "label": }, {"value": , "from": "User", "label": None}, {"value": , "from": "Assistant", "label": }, ], "mask": "User" } ``` Ideally, each `````` refers to the ground truth label for the assistant turn but if they are not available, we can also use ```helpfulness:4,correctness:4,coherence:4,complexity:2,verbosity:2``` (i.e. defaulting to moderate complexity and verbosity, adjust if needed. or simply ```helpfulness:-1```. It must not be ```None``` or an empty string. ### Model Architecture: Nemotron-4-340B-Reward is extended from Nemotron-4-340B-Base with an additional linear layer. It was trained with a global batch-size of 128. **Architecture Type:** Transformer Decoder (auto-regressive language model) ### Intended use Nemotron-4-340B-Reward is a pretrained Reward Model intended for use in English Synthetic Data Generation and English Reinforcement Learning from AI Feedback (RLAIF). ### Dataset & Training Nemotron-4-340B-Reward was trained for 2 epochs using the NVIDIA [HelpSteer2](https://arxiv.org/abs/2406.08673) data. The HelpSteer2 dataset is a permissively licensed preference dataset (CC-by-4.0) with ten thousand English response pairs and can be found [here](https://huggingface.co/datasets/nvidia/HelpSteer2). ### Evaluation Results #### Reward Bench Primary Dataset Evaluated using RewardBench - as introduced in the paper [RewardBench: Evaluating Reward Models for Language Modeling](https://arxiv.org/abs/2403.13787). | Overall | Chat | Chat-Hard | Safety | Reasoning | | --------- | ------- | -------------- | --------- | -------------- | | 92.0 | 95.8 | 87.1 | 91.5 | 93.7 | ### Limitations This model was trained using an English dataset, and as such its use is optimized for English language use cases. In order to extend this model to other language domains, fine-tuning will be required. ### Ethical Considerations: NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal model team to ensure this model meets requirements for the relevant industry and use case and addresses unforeseen product misuse. For more detailed information on ethical considerations for this model, please see the Model Card++ Explainability, Bias, Safety & Security, and Privacy Subcards [here](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/nemo/models/nemotron-4-340b-reward). Please report security vulnerabilities or NVIDIA AI Concerns [here](https://www.nvidia.com/en-us/support/submit-security-vulnerability/). ### Citation If you find this model useful, please cite the following works ```bibtex @misc{wang2024helpsteer2, title={HelpSteer2: Open-source dataset for training top-performing reward models}, author={Zhilin Wang and Yi Dong and Olivier Delalleau and Jiaqi Zeng and Gerald Shen and Daniel Egert and Jimmy J. Zhang and Makesh Narsimhan Sreedhar and Oleksii Kuchaiev}, year={2024}, eprint={2406.08673}, archivePrefix={arXiv}, primaryClass={id='cs.CL' full_name='Computation and Language' is_active=True alt_name='cmp-lg' in_archive='cs' is_general=False description='Covers natural language processing. Roughly includes material in ACM Subject Class I.2.7. Note that work on artificial languages (programming languages, logics, formal systems) that does not explicitly address natural-language issues broadly construed (natural-language processing, computational linguistics, speech, text retrieval, etc.) is not appropriate for this area.'} } ```42:T42c,|Field:|Response:| |:---:|:---:| |Intended Application(s) & Domain(s):|Large Language Model Development| |Model Type:|Generative Pre-Trained Transformer (GPT)| |Intended Users:|This model is intended for developers and researchers building LLMs.| |Output:|Scalar Values (List of 9 Floats)| |Describe how the model works:|The network architecture of this model is Nemotron-4 Reward. The decoder in Nemotron-4 generates a list of 5 floating point numbers associated with the 5 HelpSteer2 Dataset (Helpfulness, Correctness, Coherence, Complexity, Verbosity) based on the end-of-response token from the final layer of the model.| |Technical Limitations:|The model was trained on English preference data and has not been tested on non-English use-cases.| |Verified to have met prescribed quality standards?|Yes| |Performance Metrics:|Accuracy, Throughput, and Latency| |Potential Known Risks:|The Model may produce scores that are biased or incorrect based on the provided prompt and the user’s preferences.| |End User License Agreement:|Please see detailed model cards.|43:T472,|Field:|Response:| |:---:|:---:| |Generatable or Reverse engineerable personally-identifiable information?|None| |Personal dataused to create this model?|None Known| |Was consent obtained for any personal data used?|Not Applicable| |How often is dataset reviewed?|Before Release| |Is a mechanism in place to honor data subject right of access or deletion of personal data?|Not Applicable| |If personal data collected for the development of the model, was it collected by NVIDIA?|Not Applicable| |If personal data collected for the development of the model by NVIDIA, do you maintain or have access to disclosures made to data subjects?|Not Applicable| |If personal data collected for the development of this AI model, was it minimized to only what was required?|Not Applicable| |Is there provenance for all datasets used in training?|Yes| |Does data labeling (annotation, metadata) comply with privacy laws?|Not Applicable| |Is data compliant with data subject requests for data correction or removal, if such a request was made?|Not Applicable| |Applicable NVIDIA Privacy Policy|https://www.nvidia.com/en-us/about-nvidia/privacy-policy/|44:T436,{ "id": "id-123", "choices": [ { "index": 0, "message": [ { "content": "helpfulness:1.6171875,correctness:1.6484375,coherence:3.3125,complexity:0.546875,verbosity:0.515625", "role": "assistant" } ], "logprobs": { "content": [ { "token": "helpfulness", "logprob": 1.6171875, "top_logprobs": [] }, { "token": "correctness", "logprob": 1.6484375, "top_logprobs": [] }, { "token": "coherence", "logprob": 3.3125, "top_logprobs": [] }, { "token": "complexity", "logprob": 0.546875, "top_logprobs": [] }, { "token": "verbosity", "logprob": 0.515625, "top_logprobs": [] } ] }, "finish_reason": "length" } ], "usage": { "completion_tokens": 1, "prompt_tokens": 78, "total_tokens": 79 } } 45:T442,{ "id": "id-123", "choices": [ { "index": 0, "message": [ { "content": "helpfulness:2.09375,correctness:2.046875,coherence:3.546875,complexity:0.373046875,verbosity:0.287109375", "role": "assistant" } ], "logprobs": { "content": [ { "token": "helpfulness", "logprob": 2.09375, "top_logprobs": [] }, { "token": "correctness", "logprob": 2.046875, "top_logprobs": [] }, { "token": "coherence", "logprob": 3.546875, "top_logprobs": [] }, { "token": "complexity", "logprob": 0.373046875, "top_logprobs": [] }, { "token": "verbosity", "logprob": 0.287109375, "top_logprobs": [] } ] }, "finish_reason": "length" } ], "usage": { "completion_tokens": 1, "prompt_tokens": 103, "total_tokens": 104 } } 46:T440,{ "id": "id-123", "choices": [ { "index": 0, "message": [ { "content": "helpfulness:0.2578125,correctness:0.13671875,coherence:2.640625,complexity:0.328125,verbosity:0.04296875", "role": "assistant" } ], "logprobs": { "content": [ { "token": "helpfulness", "logprob": 0.2578125, "top_logprobs": [] }, { "token": "correctness", "logprob": 0.13671875, "top_logprobs": [] }, { "token": "coherence", "logprob": 2.640625, "top_logprobs": [] }, { "token": "complexity", "logprob": 0.328125, "top_logprobs": [] }, { "token": "verbosity", "logprob": 0.04296875, "top_logprobs": [] } ] }, "finish_reason": "length" } ], "usage": { "completion_tokens": 1, "prompt_tokens": 67, "total_tokens": 68 } } 39:[["$","div",".3",{"className":"relative","children":[["$","div",null,{"className":"relative flex min-h-[194px] w-full flex-col overflow-hidden p-md sm:p-ml bg-manitoulinLightBlack","children":[["$","$L3c",null,{"alt":"","className":"object-cover object-center !left-1/2 z-10 !w-1/2 brightness-20 lg:brightness-75","fill":true,"priority":true,"src":"https://assets.ngc.nvidia.com/products/api-catalog/images/nemotron-4-340b-reward.jpg","style":{"maskImage":"linear-gradient(to left, rgba(0,0,0,1), rgba(0,0,0,0))"}}],["$","div",null,{"className":"z-30 mx-auto flex w-full max-w-screen-xl min-w-0 flex-1 flex-col items-start justify-between gap-md lg:flex-row lg:gap-ml","children":["$","section",null,{"aria-describedby":"resource-description","aria-labelledby":"resource-name","className":"flex w-full min-w-0 flex-col lg:flex-1","children":["$","div",null,{"className":"flex flex-col gap-1.5 whitespace-pre-wrap","children":[["$","$L3d",null,{"className":"w-fit text-ml text-n300 lowercase hover:text-n050","href":"/nvidia","children":"NVIDIA"}],["$","div",null,{"className":"flex flex-col items-start gap-3 lg:flex-row lg:items-center","children":[["$","h1",null,{"className":"max-w-full text-lg leading-heading font-medium break-words text-primary","data-testid":"resource-name","id":"resource-name","children":"nemotron-4-340b-reward"}],[["$","span","DEPRECATION",{"className":"nv-badge nv-badge--kind-solid nv-badge--color-red h-[18px] py-0","data-testid":"nv-badge","ref":"$undefined","children":["$undefined","Deprecated"]}],["$","span","PREVIEW",{"className":"nv-badge nv-badge--kind-solid nv-badge--color-purple h-[18px] py-0","data-testid":"nv-badge","ref":"$undefined","children":["$undefined","Free Endpoint"]}]],"$undefined"]}],["$","p",null,{"className":"max-w-[85ch] text-ms text-primary","id":"resource-description","children":"Grades responses on five attributes helpfulness, correctness, coherence, complexity and verbosity."}],["$","div",null,{"className":"flex w-full flex-col items-start gap-md lg:flex-row lg:items-end lg:justify-between","children":[["$","div",null,{"className":"flex w-full min-w-0 flex-wrap gap-sm lg:flex-1","children":["$","$L3e",null,{"artifactType":"ENDPOINT","density":"standard","labels":["Reward Model","Synthetic Data Generation","Text-to-text","nimType:endpoint:nim_type_preview","usecase:endpoint:usecase_sdg","usecase:endpoint:usecase_text_gen"],"searchRoutePath":"/models"}]}],["$","div",null,{"className":"z-30 flex w-full flex-row justify-start gap-ms sm:justify-end lg:w-auto lg:shrink-0","children":["$undefined",["$","$L3f",null,{"cta":{"text":"Run Anywhere - Notify Me","url":"https://www.nvidia.com/en-us/ai/nim-notifyme/"},"isNIMAccessRestricted":"$undefined","isNIMAvailable":false}]]}]]}]]}]}]}],["$","div",null,{"className":"pointer-events-none absolute right-0 bottom-0 left-0 z-20 h-30 bg-gradient-to-t to-transparent from-manitoulinLightBlack via-manitoulinLightBlack/70"}]]}],["$","div",null,{"className":"px-4 mx-auto flex max-w-screen-xl flex-col pt-0 lg:px-0","data-body-background":"light-black","children":["$","section",null,{"className":"flex w-full flex-col justify-between rounded-4 border-manitoulinBorderColor bg-manitoulinDarkBlack sm:rounded-6 lg:rounded-8 p-ms lg:p-lg","children":["$","$L40",null,{"artifact":{"artifactType":"ENDPOINT","name":"nemotron-4-340b-reward","displayName":"nemotron-4-340b-reward","publisher":"nvidia","shortDescription":"Grades responses on five attributes helpfulness, correctness, coherence, complexity and verbosity.","logo":"https://assets.ngc.nvidia.com/products/api-catalog/images/nemotron-4-340b-reward.jpg","labels":"$39:0:props:children:0:props:children:1:props:children:props:children:props:children:3:props:children:0:props:children:props:labels","attributes":[{"key":"DEPRECATION","value":"04/02/2025"},{"key":"PREVIEW","value":"true"}],"updatedDate":"2025-05-07T22:55:30.695Z","bias":"|Field:|Response:|\n|:---:|:---:|\n|Participation considerations from adversely impacted groups (protected classes) in model design and testing:|None|\n|Measures taken to mitigate against unwanted bias:|None|","canGuestDownload":true,"createdDate":"2024-06-18T22:28:35.979Z","description":"$41","explainability":"$42","isPublic":true,"isReadOnly":true,"orgName":"qc69jvmznzxy","privacy":"$43","safetyAndSecurity":"|Field:|Response:|\n|:---:|:---:|\n|Model Application(s):|Reward Modeling|\n|Describe life critical application (if present):|None Known|\n|Use Case Restrictions:|See [NVIDIA Open Model License](https://developer.download.nvidia.com/licenses/nvidia-open-model-license-agreement-june-2024.pdf).|\n|Model and Dataset Restrictions:|The Principle of least privilege (PoLP) is applied limiting access for dataset generation. Restrictions enforce dataset access during training, and dataset license constraints adhered to. Model checkpoints are made available on Hugging Face and NGC, and may become available on cloud providers' model catalog.|"},"endpointName":"nemotron-4-340b-reward","endpointSpec":{"openAPISpec":{"openapi":"3.1.0","info":{"title":"NVIDIA NIM API for nvidia/nemotron-4-340b-reward","description":"The NVIDIA NIM REST API. Please see https://docs.api.nvidia.com/nim/reference/nvidia-nemotron-4-340b-reward for more details.","version":"1.0.0","termsOfService":"https://www.nvidia.com/en-us/data-center/products/nvidia-ai-enterprise/eula/","contact":{"name":"NVIDIA Enterprise Support","url":"https://www.nvidia.com/en-us/support/enterprise/"},"license":{"name":"NVIDIA Open Model License","url":"https://developer.download.nvidia.com/licenses/nvidia-open-model-license-agreement-june-2024.pdf"}},"servers":[{"url":"https://integrate.api.nvidia.com/v1/"}],"tags":[{"name":"NVCF API","description":"Run inference on the model"}],"paths":{"/chat/completions":{"post":{"operationId":"create_chat_completion_v1_chat_completions_post","tags":["Chat"],"summary":"Creates a model response for the given chat conversation.","description":"Given a list of messages comprising a conversation, the model will return a response. Compatible with OpenAI. See https://platform.openai.com/docs/api-reference/chat/create","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ChatRequest"}}},"required":true},"responses":{"200":{"description":"Invocation is fulfilled","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ChatCompletion"}}}},"202":{"description":"Result is pending. Client should poll using the requestId.","content":{"application/json":{"example":{},"schema":{}}},"headers":{"NVCF-REQID":{"description":"requestId required for pooling","schema":{"type":"string","format":"uuid"}},"NVCF-STATUS":{"description":"Invocation status","schema":{"type":"string"}}}},"422":{"description":"Validation failed, provided entity could not be processed.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Errors"},"example":{"type":"urn:nvcf-worker-service:problem-details:unprocessable-entity","title":"Unprocessable Entity","status":422,"detail":"string","instance":"/v2/nvcf/pexec/functions/c53ee0e9-bad9-4e09-b365-52c9d6b71254","requestId":"3fa85f64-5717-4562-b3fc-2c963f66afa6"}}}},"500":{"description":"The invocation ended with an error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Errors"},"example":{"type":"urn:nvcf-worker-service:problem-details:internal-server-error","title":"Internal Server Error","status":500,"detail":"string","instance":"/v2/nvcf/pexec/functions/c53ee0e9-bad9-4e09-b365-52c9d6b71254","requestId":"3fa85f64-5717-4562-b3fc-2c963f66afa6"}}}}},"x-nvai-meta":{"name":"Create chat completion","returns":"Returns a [chat completion](/docs/api-reference/chat/object) object\n","path":"create","examples":[{"name":"Grade the assistant response","requestJson":"{\n \"model\": \"nvidia/nemotron-4-340b-reward\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"I am going to Paris, what should I see?\"\n },\n {\n \"role\": \"assistant\",\n \"content\": \"Ah, Paris, the City of Light! There are so many amazing things to see and do in this beautiful city ...\"\n }\n ]\n}\n","responseJson":"$44"},{"name":"Grade the assistant response in multiturn","requestJson":"{\n \"model\": \"nvidia/nemotron-4-340b-reward\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"I am going to Paris, what should I see?\"\n },\n {\n \"role\": \"assistant\",\n \"content\": \"Ah, Paris, the City of Light! There are so many amazing things to see and do in this beautiful city ...\"\n },\n {\n \"role\": \"user\",\n \"content\": \"What cafes do you recommend?\"\n },\n {\n \"role\": \"assistant\",\n \"content\": \"A very popular cafe in the heart of Paris is ...\"\n }\n ]\n}\n","responseJson":"$45"}],"templates":[{"title":"No Streaming","requestEjs":{"python":"from openai import OpenAI\n\nclient = OpenAI(\n base_url = \"https://integrate.api.nvidia.com/v1\",\n api_key = \"$NVIDIA_API_KEY\"\n)\n\ncompletion = client.chat.completions.create(\n model=\"<%- request.model %>\",\n messages=<%- JSON.stringify(request.messages) %>,\n)\nprint(completion)\n","node.js":"import OpenAI from 'openai';\n\nconst openai = new OpenAI({\n apiKey: '$NVIDIA_API_KEY',\n baseURL: 'https://integrate.api.nvidia.com/v1',\n})\n\nasync function main() {\n const completion = await openai.chat.completions.create({\n model: \"<%- request.model %>\",\n messages: <%- JSON.stringify(request.messages) %>,\n })\n process.stdout.write(JSON.stringify(completion));\n}\n\nmain();","curl":"curl https://integrate.api.nvidia.com/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -H \"Authorization: Bearer $NVIDIA_API_KEY\" \\\n -d '{\n \"model\": \"nvidia/nemotron-4-340b-reward\",\n \"messages\": <%- JSON.stringify(request.messages).replaceAll(\"'\", \"'\\\"'\\\"'\") %> }'\n"},"response":"$46"}]}}}},"security":[{"Token":[]}],"components":{"securitySchemes":{"Token":{"type":"http","scheme":"bearer"}},"schemas":{"Errors":{"properties":{"type":{"type":"string","description":"Error type"},"title":{"type":"string","description":"Error title"},"status":{"type":"integer","description":"Error status code"},"detail":{"type":"string","description":"Detailed information about the error"},"instance":{"type":"string","description":"Function instance used to invoke the request"},"requestId":{"type":"string","format":"uuid","description":"UUID of the request"}},"type":"object","required":["type","title","status","detail","instance","requestId"],"title":"InvokeError"},"ChatCompletion":{"properties":{"id":{"description":"A unique identifier for the completion.","format":"uuid","title":"Id","type":"string"},"choices":{"description":"The list of completion choices the model generated for the input prompt.","items":{"$ref":"#/components/schemas/Choice"},"title":"Choices","type":"array"},"usage":{"allOf":[{"$ref":"#/components/schemas/Usage"}],"description":"Usage statistics for the completion request."}},"required":["id","choices","usage"],"title":"ChatCompletion","type":"object"},"ChatCompletionMessage":{"properties":{"content":{"anyOf":[{"type":"string"},{"type":"null"}],"description":"The contents of the message.","title":"Content"},"role":{"const":"assistant","description":"The role of the author of this message.","enum":["assistant"],"title":"Role","type":"string"}},"required":["content","role"],"title":"ChatCompletionMessage","type":"object"},"ChatRequest":{"additionalProperties":false,"properties":{"model":{"type":"string","title":"Model","default":"nvidia/nemotron-4-340b-reward"},"messages":{"description":"A list of messages comprising the conversation so far that is to be graded. The roles of the messages must be alternating between `user` and `assistant`. The last input message should have role `assistant`.","examples":[[{"content":"I am going to Paris, what should I see?","role":"user"},{"content":"Ah, Paris, the City of Light! There are so many amazing things to see and do in this beautiful city ...","role":"assistant"}]],"items":{"$ref":"#/components/schemas/Message"},"title":"Messages","type":"array"}},"required":["messages"],"title":"ChatRequest","type":"object"},"Choice":{"properties":{"index":{"description":"The index of the choice in the list of choices (always 0).","title":"Index","type":"integer"},"message":{"description":"The score key:value pairs separated by a comma.","examples":[{"content":"helpfulness:0.2578125,correctness:0.13671875,coherence:2.640625,complexity:0.328125,verbosity:0.04296875","role":"assistant"}],"items":{"$ref":"#/components/schemas/ChatCompletionMessage"},"title":"Message","type":"array"},"logprobs":{"allOf":[{"$ref":"#/components/schemas/LogProbs"}],"default":"List of output reward score items."},"finish_reason":{"anyOf":[{"enum":["stop","length"],"type":"string"},{"type":"null"}],"default":null,"description":"The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence, or `length` if the maximum number of tokens specified in the request was reached.","examples":["stop"],"title":"Finish Reason"}},"required":["index","message"],"title":"Choice","type":"object"},"LogProbField":{"properties":{"token":{"default":"Field to be scored in the assistant response to the user prompt","title":"Token","type":"string"},"logprob":{"default":"Score of the 'token' field in the assistant response to the user prompt","title":"Logprob","type":"number"},"top_logprobs":{"default":"List of the most likely tokens and their log probability, at this token position. Always empty","items":{"type":"number"},"title":"Top Logprobs","type":"array"}},"title":"LogProbField","type":"object"},"LogProbs":{"properties":{"content":{"default":"List of output reward score items.","items":{"$ref":"#/components/schemas/LogProbField"},"title":"Content","type":"array"}},"title":"LogProbs","type":"object"},"Message":{"additionalProperties":false,"properties":{"role":{"description":"The role of the message author.","enum":["user","assistant"],"title":"Role","type":"string"},"content":{"description":"The contents of the message.","title":"Content","type":"string"}},"required":["role","content"],"title":"Message","type":"object"},"Usage":{"properties":{"completion_tokens":{"description":"Number of tokens in the generated completion.","examples":[1],"title":"Completion Tokens","type":"integer"},"prompt_tokens":{"description":"Number of tokens in the prompt.","examples":[9],"title":"Prompt Tokens","type":"integer"},"total_tokens":{"description":"Total number of tokens used in the request (prompt + completion).","examples":[10],"title":"Total Tokens","type":"integer"}},"required":["completion_tokens","prompt_tokens","total_tokens"],"title":"Usage","type":"object"}}}},"namespace":"qc69jvmznzxy","updatedDate":"2025-05-07T22:55:31.424Z","nvcfFunctionId":"c53ee0e9-bad9-4e09-b365-52c9d6b71254","createdDate":"2024-06-18T22:28:36.969Z","attributes":{"apiDocsUrl":"https://docs.api.nvidia.com/nim/reference/nvidia-nemotron-4-340b-reward","termsOfUse":"GOVERNING TERMS: The trial service is governed by the NVIDIA API Service Agreement; and the use of this model is governed by the NVIDIA Open Model License.\n","deprecationNotice":"This API will be deprecated on 04/02/2025. It will no longer be supported after 04/02/2025. Please transition to llama-3_1-nemotron-70b-reward to avoid any service interruptions. For more information, visit our API Reference.","showUnavailableBanner":false,"cta":"$39:0:props:children:0:props:children:1:props:children:props:children:props:children:3:props:children:1:props:children:1:props:cta","playground":{"type":"hidden"}},"artifactName":"nemotron-4-340b-reward"},"isLoginRequired":true,"playgroundElement":"$47","publisherName":"nvidia","searchParams":{"model":"nemotron-4-340b-reward"},"tabs":[{"label":"Experience","bare":true,"icon":"play","kind":"default","id":"experience","href":"/nvidia/nemotron-4-340b-reward","aliases":["/nvidia/nemotron-4-340b-reward/experience"]}]}]}]}]]}]] 3a:["$","$L48",null,{"spec":"$39:0:props:children:1:props:children:props:children:props:endpointSpec"}]