v3 update part 2

app.py

@@ -42,7 +42,7 @@ with gr.Blocks() as block:
             
             df = get_df()
             df['Date'] = 'unknown'
-            df2 = v3.get_df()
+            df2 = v3.get_df('Overall-V2')
             min_size2, max_size2 = get_size_range(df2)
 
             with gr.Row():
@@ -159,6 +159,7 @@ with gr.Blocks() as block:
 
         # table 5, audio scores
         with gr.TabItem("🎵 Audio (NEW)", elem_id="tab-audio", id=5):
+            gr.Markdown(v3.TABLE_INTRODUCTION_A)
             data_component_a = gr.components.Dataframe(
                 value=v3.rank_models(df2[v3.COLUMN_NAMES_A], 'Audio-Overall'),
                 headers=v3.COLUMN_NAMES_A,
@@ -193,6 +194,7 @@ with gr.Blocks() as block:
 
         # table 7, agent scores
         with gr.TabItem("🤖 Agents (NEW)", elem_id="tab-agents", id=7):
+            gr.Markdown(v3.TABLE_INTRODUCTION_AG)
             data_component_ag = gr.components.Dataframe(
                 value=v3.rank_models(df2[v3.COLUMN_NAMES_AG], 'Agent-Overall'),
                 headers=v3.COLUMN_NAMES_AG,

datasets.py

@@ -36,13 +36,15 @@ DATASETS = {
         "V-MRET": ['QVHighlight', 'Charades-STA', 'MomentSeeker']
         }, 
     "audio": {
-        "A-CLS": ["SpeechCommand", "UrbanSound8k", "ESC50", "NSynth", "CREMA-D"], 
-        "A-RET": ['Clotho', 'SoundDescs', 'TUTSound', 'TUTSound(hard)', 'AVE', 'SpeechCOCO', '?Event']
+        "A-CLS": ["SpeechCommand", "UrbanSound8k", "ESC-50", "NSynth", "CREMA-D"], 
+        "A-RET": ['Clotho', 'SoundDescs', 'TUTSound', 'TUTSound(hard)', 'AVE', 'SpeechCOCO']
         },
     "agent": {
-        "ToolRET": ['toolde/web/apibank', 'toolde/web/apigen', 'toolde/web/mnms', 'toolde/web/reversechain', 'toolde/web/rotbench', 'toolde/web/t-eval-dialog', 'toolde/web/t-eval-step', 'toolde/web/taskbench-daily', 'toolde/web/toolace', 'toolde/web/toolbench', 'toolde/web/toolemu', 'toolde/web/tooleyes', 'toolde/web/toollens', 'toolde/web/ultratool', 'toolde/web/autotools-food', 'toolde/web/autotools-music', 'toolde/web/autotools-weather', 'toolde/web/restgpt-spotify', 'toolde/web/restgpt-tmdb', 'toolde/code/craft-math-algebra', 'toolde/code/craft-tabmwp', 'toolde/code/craft-vqa', 'toolde/code/gorilla-huggingface', 'toolde/code/gorilla-pytorch', 'toolde/code/gorilla-tensor', 'toolde/code/toolink', 'toolde/customized/appbench', 'toolde/customized/gpt4tools', 'toolde/customized/gta', 'toolde/customized/taskbench-huggingface', 'toolde/customized/taskbench-multimedia', 'toolde/customized/metatool', 'toolde/customized/tool-be-honest', 'toolde/customized/toolalpaca', 'toolde/customized/toolbench-sam'],
+        "Tool": ['apibank', 'apigen', 'mnms', 'reversechain', 'rotbench', 't-eval-dialog', 't-eval-step', 'taskbench-daily', 'toolace', 'toolbench', 'toolemu', 'tooleyes', 'toollens', 'ultratool', 'autotools-food', 'autotools-music', 'autotools-weather', 'restgpt-spotify', 'restgpt-tmdb',
+                  'craft-math-algebra', 'craft-tabmwp', 'craft-vqa', 'gorilla-huggingface', 'gorilla-pytorch', 'gorilla-tensor', 'toolink', 
+                  'appbench', 'gpt4tools', 'gta', 'taskbench-huggingface', 'taskbench-multimedia', 'metatool', 'tool-be-honest', 'toolalpaca', 'toolbench-sam'],
         "GUI": ['GAE-GUIAct_q2t', 'GAE-GUIAct_q2s', 'GAE-GUIAct_s2s', 'GAE-GUIAct_t2s', 'GAE-Mind2Web_q2t', 'GAE-Mind2Web_q2s', 'GAE-Mind2Web_s2s', 'GAE-Mind2Web_t2s'],
-        "MemRET": ["REALTALK", "KnowMeBench", "PeerQA", "DeepPlanning"]
+        "Memory": ["REALTALK", "KnowMeBench", "PeerQA", "DeepPlanning"]
     }
 }
 ALL_DATASETS_SPLITS = {k: sum_lol(list(v.values())) for k, v in DATASETS.items()}

report_score_v3.py

@@ -0,0 +1,250 @@
+"""
+report_score_v3.py: The official script to generate the final scores report (in JSON) for MMEB-V3 leaderboard submission. 
+You should only modify the Configuration section below to add your model's metadata. No other changes are needed.
+An example configuration is provided for your reference. 
+"""
+
+import os
+import json
+from datetime import datetime
+
+# ==============================================================================
+# Configuration
+# ==============================================================================
+
+# ==> Unified list of experiments to process.
+# Fill in the metadata for each experiment. `None` will become `null` in the JSON.
+
+EXAMPLES = [
+    {
+        "path": "vlm2vec_exps/VLM2Vec-Qwen2VL-V2.0-2B/",
+        "metadata": {
+            "model_name": "VLM2Vec-Qwen2VL-V2.0-2B",
+            "model_size": "2", # in B, digits only here. 2 for 2B, 0.4 for 400M, etc.
+            "embedding_dimension": None, # Please fill in
+            "max_length_tokens": None,   # Please fill in
+            "model_release_date": "2025-04-01", # Please adjust this date
+            "score_source": "",           # e.g., "Self-Reported" or "TIGER-Lab"
+            "url": ""                    # e.g., Paper, GitHub, or Hugging Face link
+        }
+    },
+    {
+        "path": "vlm2vec_exps/VLM2Vec-Qwen2VL-V2.1-2B/",
+        "metadata": {
+            "model_name": "VLM2Vec-Qwen2VL-V2.1-2B",
+            "model_size": "2", # in B, digits only here. 2 for 2B, 0.4 for 400M, etc.
+            "embedding_dimension": None, # Please fill in
+            "max_length_tokens": None,   # Please fill in
+            "model_release_date": "2025-05-15", # Please adjust this date
+            "score_source": "",           # e.g., "Self-Reported" or "TIGER-Lab"
+            "url": ""                    # e.g., Paper, GitHub, or Hugging Face link
+        }
+    },
+]
+
+
+# ==============================================================================
+# TODO: Your models' metadata goes here. Please fill in the required fields.
+# ==============================================================================
+
+EXPERIMENTS = [
+    {
+        "path": ...,
+        "metadata": {
+            "model_name": ..., # * Mandatory Field
+            "model_backbone": ...,
+            "model_size": ...,
+            "embedding_dimension": ...,
+            "max_length_tokens": ...,
+            "model_release_date": ...,
+            "data_source": "Self-Reported",
+            "url": ..., 
+            "contact": ...,
+        }
+    },
+    ...
+]
+
+
+# ==============================================================================
+# Main Processing Logic (No changes needed below this line)
+# ==============================================================================
+
+
+# Define the datasets grouped by modality
+modality2dataset = {
+    "audio": [
+        'SpeechCommand', 'UrbanSound8k', 'ESC-50', 'NSynth', 'CREMA-D', 'Clotho', 'SoundDescs', 'TUTSound', 'TUTSound(hard)', 'AVE', 'SpeechCOCO'
+        ], 
+    "image": [
+        "ImageNet-1K", "N24News", "HatefulMemes", "VOC2007", "SUN397", "Place365", "ImageNet-A", "ImageNet-R", "ObjectNet", "Country211",
+        "OK-VQA", "A-OKVQA", "DocVQA", "InfographicsVQA", "ChartQA", "Visual7W", "ScienceQA", "VizWiz", "GQA", "TextVQA",
+        "VisDial", "CIRR", "VisualNews_t2i", "VisualNews_i2t", "MSCOCO_t2i", "MSCOCO_i2t", "NIGHTS", "WebQA", "FashionIQ", "Wiki-SS-NQ", "OVEN", "EDIS",
+        "MSCOCO", "RefCOCO", "RefCOCO-Matching", "Visual7W-Pointing"
+        ],
+    "video": [
+        "K700", "SmthSmthV2", "HMDB51", "UCF101", "Breakfast",
+        "MVBench", "Video-MME", "NExTQA", "EgoSchema", "ActivityNetQA",
+        "DiDeMo", "MSR-VTT", "MSVD", "VATEX", "YouCook2",
+        "QVHighlight", "Charades-STA", "MomentSeeker",
+    ],
+    "visdoc": [
+        "ViDoRe_arxivqa", "ViDoRe_docvqa", "ViDoRe_infovqa", "ViDoRe_tabfquad", "ViDoRe_tatdqa", "ViDoRe_shiftproject",
+        "ViDoRe_syntheticDocQA_artificial_intelligence", "ViDoRe_syntheticDocQA_energy", "ViDoRe_syntheticDocQA_government_reports", "ViDoRe_syntheticDocQA_healthcare_industry",
+        "ViDoRe_esg_reports_human_labeled_v2", "ViDoRe_biomedical_lectures_v2_multilingual", "ViDoRe_economics_reports_v2_multilingual", "ViDoRe_esg_reports_v2_multilingual",
+        "VisRAG_ArxivQA", "VisRAG_ChartQA", "VisRAG_MP-DocVQA", "VisRAG_SlideVQA", "VisRAG_InfoVQA", "VisRAG_PlotQA",
+        "ViDoSeek-page", "ViDoSeek-doc", "MMLongBench-page", "MMLongBench-doc"
+    ], 
+    "text": [
+        'core17-instructions', 'news21-instructions', 'robust04-instructions', 'Bioinformatics', 'Biology', 'IIYi-Clinical', 'MedQA-Diag', 'MedXpertQA-Exam', 'Medical-Sciences', 'PMC-Clinical', 'PMC-Treatment', 'Audience-v1', 'Clarity-v1', 'Format-v1', 'Language-v1', 'Length-v1', 'Source-v1', 'aops', 'biology', 'earth_science', 'economics', 'leetcode', 'pony', 'psychology', 'robotics', 'stackoverflow', 'sustainable_living', 'theoremqa_questions', 'theoremqa_theorems', '2wikimqa', 'narrativeqa', 'needle', 'passkey', 'qmsum', 'summ_screen_fd', 'Books', 'Legal Document', 'Medical Case', 'Movies', 'People', 'NanoArguAna', 'NanoClimateFEVER', 'NanoDBPedia', 'NanoFEVER', 'NanoFiQA2018', 'NanoHotpotQA', 'NanoMSMARCO', 'NanoNFCorpus', 'NanoNQ', 'NanoQuoraRetrieval', 'NanoSCIDOCS', 'NanoSciFact', 'NanoTouche2020'
+        ], 
+    "tool": [
+        'apibank', 'apigen', 'mnms', 'reversechain', 'rotbench', 't-eval-dialog', 't-eval-step', 'taskbench-daily', 'toolace', 'toolbench', 'toolemu', 'tooleyes', 'toollens', 'ultratool', 'autotools-food', 'autotools-music', 'autotools-weather', 'restgpt-spotify', 'restgpt-tmdb', 'craft-math-algebra', 'craft-tabmwp', 'craft-vqa', 'gorilla-huggingface', 'gorilla-pytorch', 'gorilla-tensor', 'toolink', 'appbench', 'gpt4tools', 'gta', 'taskbench-huggingface', 'taskbench-multimedia', 'metatool', 'tool-be-honest', 'toolalpaca', 'toolbench-sam'
+        ], 
+    "gui": [
+        'GAE-GUIAct_q2t', 'GAE-GUIAct_q2s', 'GAE-GUIAct_s2s', 'GAE-GUIAct_t2s', 'GAE-Mind2Web_q2t', 'GAE-Mind2Web_q2s', 'GAE-Mind2Web_s2s', 'GAE-Mind2Web_t2s'
+        ], 
+    "memory": [
+        'REALTALK', 'KnowMeBench', 'PeerQA', 'DeepPlanning'
+        ]
+}
+modality2metric = {
+    "image": "hit@1",
+    "video": "hit@1",
+    "visdoc": "ndcg_linear@5",
+    "audio": "hit@1", 
+    "text": "hit@1",
+    "tool": "hit@1",
+    "gui": "hit@1",
+    "memory": "hit@1"
+}
+modalities = ["image", "video", "visdoc", "audio", "text", "tool", "gui", "memory"] # Process in this order
+
+for experiment in EXPERIMENTS:
+    base_path = experiment['path']
+    experiment_metadata = experiment['metadata']
+    experiment_name_for_log = os.path.basename(base_path.strip('/'))
+
+    current_experiment_scores = {}
+
+    print(f"\nProcessing experiment: {experiment_name_for_log}")
+    print(f"Path: {base_path}")
+
+    for modality in modalities:
+        current_experiment_scores[modality] = {}
+        modality_specific_result_dir = os.path.join(base_path, modality)
+
+        for dataset_name in modality2dataset.get(modality, []):
+            current_experiment_scores[modality][dataset_name] = "FILE_N/A" # Initialize
+
+        if not os.path.isdir(modality_specific_result_dir):
+            print(f"    Directory not found: {modality_specific_result_dir}")
+            for dataset_name in modality2dataset.get(modality, []):
+                current_experiment_scores[modality][dataset_name] = "DIR_N/A"
+            continue
+
+        for filename in os.listdir(modality_specific_result_dir):
+            if filename.endswith("_score.json"):
+                score_file_path = os.path.join(modality_specific_result_dir, filename)
+                dataset_name_from_file = None
+                for known_dataset in modality2dataset.get(modality, []):
+                    if filename == f"{known_dataset}_score.json":
+                        dataset_name_from_file = known_dataset
+                        break
+
+                if dataset_name_from_file:
+                    try:
+                        with open(score_file_path, "r") as f:
+                            score_data = json.load(f)
+                            current_experiment_scores[modality][dataset_name_from_file] = score_data
+                    except json.JSONDecodeError:
+                        print(f"      Error decoding JSON from {score_file_path}")
+                        current_experiment_scores[modality][dataset_name_from_file] = "JSON_ERROR"
+                    except Exception as e:
+                        print(f"      Error reading file {score_file_path}: {e}")
+                        current_experiment_scores[modality][dataset_name_from_file] = "READ_ERROR"
+
+    # --- Construct and Save the Final JSON Report ---
+    final_metadata = experiment_metadata.copy()
+    final_metadata['report_generated_date'] = datetime.now().isoformat()
+    # --- Special handling ---
+    # Special handling for VisDoc datasets with "-fixed" suffix
+    current_experiment_scores['visdoc'] = {(f'{k}-fixed' if k in ['MMLongBench-page', 'ViDoSeek-page'] else k): v 
+                                           for k, v in current_experiment_scores['visdoc'].items()} # temp fix
+    # merge "tool", "gui", and "memory" into "agent"
+    # current_experiment_scores['agent'] = current_experiment_scores.pop('tool', {}) | current_experiment_scores.pop('gui', {}) | current_experiment_scores.pop('memory', {})
+    # -------------------------
+
+    final_output = {
+        "metadata": final_metadata,
+        "metrics": current_experiment_scores
+    }
+
+    output_json_path = os.path.join(base_path, f"{final_metadata['model_name']}.json")
+    try:
+        with open(output_json_path, "w") as f:
+            json.dump(final_output, f, indent=4)
+        print(f"  Report for '{experiment_name_for_log}' saved to: {output_json_path}")
+    except Exception as e:
+        print(f"  Error saving JSON report for '{experiment_name_for_log}' to {output_json_path}: {e}")
+
+
+    # --- Print detailed main scores per dataset for easy copy to spreadsheet ---
+    print(f"\n  --- Detailed Main Scores for Spreadsheet (Experiment: {experiment_name_for_log}) ---")
+    for modality in modalities:
+        main_metric_key = modality2metric[modality]
+        for dataset_name in modality2dataset.get(modality, []):
+            score_to_print_val = "NOT_FOUND_IN_RESULTS"
+            modality_data = current_experiment_scores.get(modality, {})
+            score_info = modality_data.get(dataset_name)
+
+            if isinstance(score_info, dict):
+                metric_value = score_info.get(main_metric_key)
+                if isinstance(metric_value, (int, float)):
+                    score_to_print_val = f"{metric_value:.4f}"
+                else:
+                    score_to_print_val = f"METRIC_KEY_MISSING ({main_metric_key})"
+            elif isinstance(score_info, str):
+                score_to_print_val = score_info
+
+            print(f"{dataset_name}\t{score_to_print_val}")
+        print("")
+
+    # --- Print average scores and missing datasets per modality ---
+    print(f"\n  --- Summary for Experiment: {experiment_name_for_log} ---")
+    for modality in modalities:
+        if modality not in current_experiment_scores:
+            print(f"    Modality '{modality.upper()}' not processed.")
+            continue
+        main_metric_key = modality2metric[modality]
+        modality_data = current_experiment_scores[modality]
+        collected_metric_values = []
+        datasets_missing_score_file = []
+        datasets_file_found_metric_missing = []
+
+        for dataset_name in modality2dataset.get(modality, []):
+            score_info = modality_data.get(dataset_name)
+            if isinstance(score_info, dict):
+                metric_value = score_info.get(main_metric_key)
+                if isinstance(metric_value, (int, float)):
+                    collected_metric_values.append(metric_value)
+                else:
+                    datasets_file_found_metric_missing.append(f"{dataset_name} (metric '{main_metric_key}' missing/invalid)")
+            else:
+                datasets_missing_score_file.append(f"{dataset_name} (status: {score_info if score_info else 'Not Processed'})")
+
+        if collected_metric_values:
+            average_score = sum(collected_metric_values) / len(collected_metric_values)
+            print(f"      Average of {modality.upper()}\t- {main_metric_key}:\t{average_score:.4f} (from {len(collected_metric_values)} datasets)")
+        else:
+            print(f"      Average of {modality.upper()}\t-  {main_metric_key}:\tN/A (no valid scores found)")
+
+        if datasets_missing_score_file:
+            print(f"      Datasets with missing/errored score files:")
+            for ds_status in datasets_missing_score_file: print(f"        - {ds_status}")
+        if datasets_file_found_metric_missing:
+            print(f"      Score files found but main metric ('{main_metric_key}') missing/invalid:")
+            for ds_status in datasets_file_found_metric_missing: print(f"        - {ds_status}")
+
+
+print("\nProcessing complete.")

utils.py

@@ -26,13 +26,14 @@ LEADERBOARD_INTRODUCTION = """
 - four video-based tasks: Video Retrieval, Moment Retrieval, Video Classification, and Video Question Answering 
 - one task focused on visual documents: Visual Document Retrieval. 
 
-**MMEB-V3** extends MMEB-V2 toward a fuller modality setting by adding three major new evaluation categories:
+**MMEB-V3** further extends to a fuller modality setting by adding three major new evaluation categories:
 - Audio Tasks: audio classification, cross-modal audio retrieval, and audio temporal grounding.
 - Text Retrieval: instruction-following retrieval, reasoning retrieval, long-context retrieval, multi-condition retrieval, and general text retrieval.
 - Agent Tasks: tool retrieval, GUI control, and agent memory retrieval.
 
 **🔥 What's NEW:**
 <ul>
+<li>[2026-07] MMEB-V3 Leaderboard is now live! </li>
 <li>[2026-07] MMEB-V3 is accepted to COLM 2026!</li>
 <li>[2026-04] MMEB-V3 released!</li>
 <li>[2025-06] MMEB-V2 released!</li>
@@ -91,8 +92,14 @@ SUBMIT_INTRODUCTION = """# Submit on MMEB Leaderboard Introduction \n
     - You can directly use the Hugging Face Space's web UI to upload your files:
         - Go to the [scores folder](https://huggingface.co/spaces/TIGER-Lab/MMEB-Leaderboard/upload/main/scores)
         - Select "Upload file" and upload your JSON files.
-    - Submit a pull request. We will then review and update the leaderboard accordingly. \n\n
-## Please double check your score sheet have the following JSON format ⬇️: \n
+    - Submit a pull request and leave any comments you have. We will then review and update the leaderboard accordingly.\n
+    - To delete or modify your submission, submit a new pull request with the updated file.\n\n
+
+## 🐞 Bug reporting and feedback
+If you encounter any issues or have suggestions for improvement, please report them in Discussion or open an issue on our [GitHub repository](https://github.com/TIGER-AI-Lab/VLM2Vec/issues).
+If you cannot reach us via above methods, please email us at **m7su@uwaterloo.ca**. 
+
+## Appendix 1: Please double check your score sheet have the following JSON format ⬇️: \n
 ```json
 {
     "metadata": {
@@ -100,7 +107,7 @@ SUBMIT_INTRODUCTION = """# Submit on MMEB Leaderboard Introduction \n
         "url": "<Model URL>" or null,
         "model_size": <Model Size> or null,
         "data_source": "Self-Reported",
-        "contact": xxx@gmail.com
+        "contact": xxx@xxxxx.com
         ... ...
     },
     "metrics": {
@@ -123,11 +130,9 @@ SUBMIT_INTRODUCTION = """# Submit on MMEB Leaderboard Introduction \n
 }
 ```
 ## ⚠️ Special Instructions for submitting to MMEB Image (Previously MMEB-V1) Leaderboard
-We understand that some researchers want to exclusively submit to the Image leaderboard, but unfortunately our current leaderboard system cannot exclude your model from other modalities' leaderboards. 
-To do so, please run the 36 image datasets only and simply ignore the other datasets.
-The leaderboard will automatically assign a 0 to the missing datasets. 
-We might be able to hide your model from the other leaderboards in the next leaderboard updates, but for now your model will be shown on all leaderboards and might have a lower rank. \n
-We highly recommend joining our [discord server](https://discord.gg/njyKubdtry), which provides a convenient way to stay informed with latest updates, or share any feedback you have for improving the leaderboard experience. We appreciate your contributions to the MMEB community!
+We understand that some researchers want to exclusively submit to the Image leaderboard, but unfortunately our current leaderboard cannot exclude your model from other modalities' leaderboards. 
+To do so, run the 36 image datasets only and simply ignore other datasets.
+The leaderboard will automatically assign a 0 to the missing datasets and your model will be shown on all leaderboards, and might have a lower rank. \n
 """
 
 def create_hyperlinked_names(df):

utils_v3.py

@@ -6,7 +6,7 @@ from utils import create_hyperlinked_names, process_model_size
 from datasets import *
 
 BASE_COLS = ['Rank', 'Models', 'Model Size(B)', 'Date']
-BASE_DATA_TITLE_TYPE = ['number', 'markdown', 'str', 'str']
+BASE_DATA_TITLE_TYPE = ['str', 'markdown', 'str', 'str']
 
 OVERALL_COLS = ["Overall", "Overall-V2", "Text-Overall", 'Image-Overall', 'Video-Overall', 'Audio-Overall', 'Visdoc-Overall', "Agent-Overall"]
 COLUMN_NAMES = BASE_COLS + OVERALL_COLS
@@ -43,14 +43,14 @@ COLUMN_NAMES_D = BASE_COLS + TASKS_D
 DATA_TITLE_TYPE_D = BASE_DATA_TITLE_TYPE + \
                     ['number'] * len(TASKS_D)
 
-SUB_TASKS_AG = ['ToolRET', 'GUI', 'MemRET']
+SUB_TASKS_AG = ['Tool', 'GUI', 'Memory']
 TASKS_AG = ['Agent-Overall'] + SUB_TASKS_AG + ALL_DATASETS_SPLITS['agent']
 COLUMN_NAMES_AG = BASE_COLS + TASKS_AG
 DATA_TITLE_TYPE_AG = BASE_DATA_TITLE_TYPE + \
                     ['number'] * len(TASKS_AG)
 
 TABLE_INTRODUCTION = """**MMEB**: Massive MultiModal Embedding Benchmark \n
-                        Models are ranked based on **Overall**"""
+                        Models are ranked based on **Overall-V2**."""
 TABLE_INTRODUCTION_I = """**I-CLS**: Image Classification, **I-QA**: (Image) Visual Question Answering, **I-RET**: Image Retrieval, **I-VG**: (Image) Visual Grounding \n
                         Models are ranked based on **Image-Overall**\n
                         **Models from the old V1 leaderboard are missing detailed scores of each dataset. 
@@ -58,9 +58,13 @@ TABLE_INTRODUCTION_I = """**I-CLS**: Image Classification, **I-QA**: (Image) Vis
                         and provide us the scores sheet with the new format, so that we can make them consistent with the other models' formats.**"""
 TABLE_INTRODUCTION_V = """**V-CLS**: Video Classification, **V-QA**: (Video) Visual Question Answering, **V-RET**: Video Retrieval, **V-MRET**: Video Moment Retrieval \n
                         Models are ranked based on **Video-Overall**"""
+TABLE_INTRODUCTION_A = """**A-CLS**: Audio Classification, **A-RET**: Audio Retrieval \n
+                        Models are ranked based on **Audio-Overall**"""
 TABLE_INTRODUCTION_D = """⚠️ Please re-evaluate your models if you see a 0 on ViDoSeek-page-fixed or MMLongBench-page-fixed datasets. \n
 **VisDoc**: Visual Document Understanding \n
                         Models are ranked based on **Visdoc-Overall**"""
+TABLE_INTRODUCTION_AG = """**Tool**: Tool Retrieval, **GUI**: GUI Control, **Memory**: Agent Memory Retrieval \n
+                        Models are ranked based on **Agent-Overall**"""
 
 LEADERBOARD_INFO = """
 ## Dataset Summary
@@ -99,8 +103,12 @@ def load_data(base_dir=SCORE_BASE_DIR):
             all_data.append(data)
     return all_data
 
-def load_scores(raw_scores=None):
+def load_scores(raw_scores={}):
     """This function loads the raw scores from the user provided scores summary and flattens them into a single dictionary."""
+    # temp fix, will figure out later ===========
+    if any(_ in raw_scores for _ in ['tool', 'gui', 'memory']):
+        raw_scores['agent'] = raw_scores.pop('tool', {}) | raw_scores.pop('gui', {}) | raw_scores.pop('memory', {})
+    # ===========================================
     all_scores = {}
     for modality, datasets_list in DATASETS.items(): # Ex.: ('image', {'I-CLS': [...], 'I-QA': [...]})
         for sub_task, datasets in datasets_list.items(): # Ex.: ('I-CLS', ['VOC2007', 'N24News', ...])
@@ -168,13 +176,18 @@ def print_time(time: str|None):
     except (ValueError, TypeError):
         return 'unknown'
 
+medal_map = {
+    "1": "🏆",
+    "2": "🥈",
+    "3": "🥉"
+}
 def rank_models(df, column='Overall', rank_name='Rank'):
     """Ranks the models based on the specific score."""
     df = df.sort_values(by=column, ascending=False).reset_index(drop=True)
-    df[rank_name] = range(1, len(df) + 1)
+    df[rank_name] = df[column].rank(method='min', ascending=False).astype(int).astype(str).map(lambda x: medal_map.get(x, x))
     return df
 
-def get_df():
+def get_df(rank_column='Overall'):
     """Generates a DataFrame from the loaded data."""
     all_data = load_data()
     rows = [generate_model_row(data) for data in all_data]
@@ -182,7 +195,7 @@ def get_df():
     df['Model Size(B)'] = df['Model Size(B)'].apply(process_model_size)
     df['Date'] = df['Date'].apply(print_time)
     df = create_hyperlinked_names(df)
-    df = rank_models(df, column='Overall')
+    df = rank_models(df, column=rank_column)
     return df
 
 def refresh_data():