update metric

app.py

@@ -29,12 +29,13 @@ from src.display.utils import (
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
-#from src.tools.collections import update_collections
-#from src.tools.plots import (
-#    create_metric_plot_obj,
-#    create_plot_df,
-#    create_scores_df,
-#)
+from src.scripts.update_all_request_files import update_dynamic_files
+from src.tools.collections import update_collections
+from src.tools.plots import (
+    create_metric_plot_obj,
+    create_plot_df,
+    create_scores_df,
+)
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
@@ -58,18 +59,18 @@ def init_space():
     except Exception:
         restart_space()
 
-    #raw_data, original_df = get_leaderboard_df(
-    leaderboard_df = get_leaderboard_df(
+    raw_data, original_df = get_leaderboard_df(
+    #leaderboard_df = get_leaderboard_df(
         results_path=EVAL_RESULTS_PATH,
         requests_path=EVAL_REQUESTS_PATH,
         #dynamic_path=DYNAMIC_INFO_FILE_PATH,
         cols=COLS,
         benchmark_cols=BENCHMARK_COLS
     )
-    #update_collections(original_df.copy())
-    #leaderboard_df = original_df.copy()
+    update_collections(original_df.copy())
+    leaderboard_df = original_df.copy()
 
-    #plot_df = create_plot_df(create_scores_df(raw_data))
+    plot_df = create_plot_df(create_scores_df(raw_data))
 
     (
         finished_eval_queue_df,
@@ -77,11 +78,12 @@ def init_space():
         pending_eval_queue_df,
     ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
-    #return leaderboard_df, original_df, plot_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df
-#leaderboard_df, original_df, plot_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = init_space()
-    return leaderboard_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df
+    return leaderboard_df, original_df, plot_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df
+
+leaderboard_df, original_df, plot_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = init_space()
+    #return leaderboard_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df
 
-leaderboard_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = init_space()
+#leaderboard_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = init_space()
 
 
 # Searching and filtering
@@ -297,8 +299,8 @@ with demo:
 
             # Dummy leaderboard for handling the case when the user uses backspace key
             hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                #value=original_df[COLS],
-                value=leaderboard_df[COLS],
+                value=original_df[COLS],
+                #value=leaderboard_df[COLS],
                 headers=COLS,
                 datatype=TYPES,
                 visible=False,
@@ -351,6 +353,23 @@ with demo:
                     queue=True,
                 )    
 
+        with gr.TabItem("📈 Metrics through time", elem_id="llm-benchmark-tab-table", id=4):
+            with gr.Row():
+                with gr.Column():
+                    chart = create_metric_plot_obj(
+                        plot_df,
+                        [AutoEvalColumn.average.name],
+                        title="Average of Top Scores and Human Baseline Over Time (from last update)",
+                    )
+                    gr.Plot(value=chart, min_width=500)
+                with gr.Column():
+                    chart = create_metric_plot_obj(
+                        plot_df,
+                        BENCHMARK_COLS,
+                        title="Top Scores and Human Baseline Over Time (from last update)",
+                    )
+                    gr.Plot(value=chart, min_width=500)
+
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
@@ -453,5 +472,6 @@ with demo:
 
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=1800)
+scheduler.add_job(update_dynamic_files, "cron", minute=30) # launched every hour on the hour
 scheduler.start()
 demo.queue(default_concurrency_limit=40).launch()

src/about.py

@@ -12,17 +12,16 @@ class Task:
 # ---------------------------------------------------
 class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("cmmmu", "acc", "CMMMU")
-    task1 = Task("cmmu", "acc", "CMMU")
-    task2 = Task("cv_bench", "acc", "CV_Bench")
-    task3 = Task("hallusion_bench", "acc", "Hallusion_Bench")
-    task4 = Task("mmmu", "acc", "MMMU")
-    task5 = Task("mmmu_pro_standard", "acc", "MMMU_Pro_Standard")
-    task6 = Task("mmmu_pro_vision", "acc", "MMMU_Pro_Vision")
-    task7 = Task("ocrbench", "acc", "OCRBench")
-    task8 = Task("math_vision", "acc", "Math_Vision")
-    task9 = Task("cvbench", "acc", "CVBench")
-    task10 = Task("ciibench", "acc", "CIIBench")
+    cmmmu = Task("cmmmu", "acc", "CMMMU")
+    cmmu = Task("cmmu", "acc", "CMMU")
+    cv_bench = Task("cv_bench", "acc", "CV_Bench")
+    hallusion_bench = Task("hallusion_bench", "acc", "Hallusion_Bench")
+    mmmu = Task("mmmu", "acc", "MMMU")
+    mmmu_pro_standard = Task("mmmu_pro_standard", "acc", "MMMU_Pro_Standard")
+    mmmu_pro_vision = Task("mmmu_pro_vision", "acc", "MMMU_Pro_Vision")
+    ocrbench = Task("ocrbench", "acc", "OCRBench")
+    math_vision = Task("math_vision", "acc", "Math_Vision")
+    ciibench = Task("ciibench", "acc", "CIIBench")
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------

src/display/utils.py

@@ -136,3 +136,47 @@ NUMERIC_INTERVALS = {
     "~60": pd.Interval(45, 70, closed="right"),
     "70+": pd.Interval(70, 10000, closed="right"),
 }
+
+# Define the baselines
+baseline_row = {
+    AutoEvalColumn.model.name: "<p>Baseline</p>",
+    AutoEvalColumn.revision.name: "N/A",
+    AutoEvalColumn.precision.name: None,
+    AutoEvalColumn.average.name: 92.75,
+    AutoEvalColumn.merged.name: False,
+    AutoEvalColumn.cmmmu.name: 100,
+    AutoEvalColumn.cmmu.name: 100,
+    AutoEvalColumn.cv_bench.name: 100,
+    AutoEvalColumn.hallusion_bench.name: 100,
+    AutoEvalColumn.mmmu.name: 100,
+    AutoEvalColumn.mmmu_pro_standard.name: 100,
+    AutoEvalColumn.mmmu_pro_vision.name: 100,
+    AutoEvalColumn.ocrbench.name: 100,
+    AutoEvalColumn.math_vision.name: 100,
+    AutoEvalColumn.ciibench.name: 100,
+    AutoEvalColumn.dummy.name: "baseline",
+    AutoEvalColumn.model_type.name: "",
+    AutoEvalColumn.flagged.name: False,
+}
+
+# Define the human baselines
+human_baseline_row = {
+    AutoEvalColumn.model.name: "<p>Human performance</p>",
+    AutoEvalColumn.revision.name: "N/A",
+    AutoEvalColumn.precision.name: None,
+    AutoEvalColumn.average.name: 92.75,
+    AutoEvalColumn.merged.name: False,
+    AutoEvalColumn.cmmmu.name: 100,
+    AutoEvalColumn.cmmu.name: 100,
+    AutoEvalColumn.cv_bench.name: 100,
+    AutoEvalColumn.hallusion_bench.name: 100,
+    AutoEvalColumn.mmmu.name: 100,
+    AutoEvalColumn.mmmu_pro_standard.name: 100,
+    AutoEvalColumn.mmmu_pro_vision.name: 100,
+    AutoEvalColumn.ocrbench.name: 100,
+    AutoEvalColumn.math_vision.name: 100,
+    AutoEvalColumn.ciibench.name: 100,
+    AutoEvalColumn.dummy.name: "human_baseline",
+    AutoEvalColumn.model_type.name: "",
+    AutoEvalColumn.flagged.name: False,
+}

src/envs.py

@@ -6,12 +6,18 @@ from huggingface_hub import HfApi
 # ----------------------------------
 TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
 
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+
+#OWNER = "BAAI/open_flageval_vlm_leaderboard" # Change to your org - don't forget to create a results and request dataset, with the correct format!
 # ----------------------------------
 
-REPO_ID = f"{OWNER}/leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
+#REPO_ID = f"{OWNER}/leaderboard"
+#QUEUE_REPO = f"{OWNER}/requests"
+#RESULTS_REPO = f"{OWNER}/results"
+#DYNAMIC_INFO_REPO = f"{OWNER}/dynamic_model_information"
+REPO_ID = "BAAI/open_flageval_vlm_leaderboard"
+QUEUE_REPO = "open-cn-llm-leaderboard/vlm_requests"
+DYNAMIC_INFO_REPO = "open-cn-llm-leaderboard/vlm_dynamic_model_information"
+RESULTS_REPO = "open-cn-llm-leaderboard/vlm_results"
 
 # If you setup a cache later, just change HF_HOME
 CACHE_PATH=os.getenv("HF_HOME", ".")
@@ -21,5 +27,8 @@ EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
 EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
 EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
 EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
+DYNAMIC_INFO_PATH = os.path.join(CACHE_PATH, "dynamic-info")
+DYNAMIC_INFO_FILE_PATH = os.path.join(DYNAMIC_INFO_PATH, "model_infos.json")
 
+PATH_TO_COLLECTION = "open-cn-llm-leaderboard/flageval-vlm-leaderboard-best-models-677e51cdc44f8123e02cbda1"
 API = HfApi(token=TOKEN)

src/populate.py

@@ -4,7 +4,7 @@ import os
 import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn
+from src.display.utils import AutoEvalColumn, EvalQueueColumn, baseline_row
 from src.leaderboard.read_evals import get_raw_eval_results
 from src.leaderboard.filter_models import filter_models_flags
 
@@ -13,6 +13,7 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     """Creates a dataframe from all the individual experiment results"""
     raw_data = get_raw_eval_results(results_path, requests_path)
     all_data_json = [v.to_dict() for v in raw_data]
+    all_data_json.append(baseline_row)
     filter_models_flags(all_data_json)
 
     df = pd.DataFrame.from_records(all_data_json)
@@ -21,7 +22,7 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
 
     # filter out if any of the benchmarks have not been produced
     df = df[has_no_nan_values(df, benchmark_cols)]
-    return df
+    return raw_data, df
 
 
 def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:

src/scripts/create_request_file.py

@@ -0,0 +1,93 @@
+import json
+import os
+import pprint
+from datetime import datetime, timezone
+
+import click
+from colorama import Fore
+from huggingface_hub import HfApi, snapshot_download
+
+from src.submission.check_validity import get_model_size
+from src.display.utils import ModelType, WeightType
+
+EVAL_REQUESTS_PATH = "eval-queue"
+QUEUE_REPO = "open-cn-llm-leaderboard/vlm_requests"
+
+precisions = ("float16", "bfloat16", "8bit (LLM.int8)", "4bit (QLoRA / FP4)", "GPTQ")
+model_types = [e.name for e in ModelType]
+weight_types = [e.name for e in WeightType]
+
+
+def main():
+    api = HfApi()
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH, repo_type="dataset")
+
+    model_name = click.prompt("Enter model name")
+    revision = click.prompt("Enter revision", default="main")
+    precision = click.prompt("Enter precision", default="float16", type=click.Choice(precisions))
+    model_type = click.prompt("Enter model type", type=click.Choice(model_types))
+    weight_type = click.prompt("Enter weight type", default="Original", type=click.Choice(weight_types))
+    base_model = click.prompt("Enter base model", default="")
+    status = click.prompt("Enter status", default="FINISHED")
+
+    try:
+        model_info = api.model_info(repo_id=model_name, revision=revision)
+    except Exception as e:
+        print(f"{Fore.RED}Could not find model info for {model_name} on the Hub\n{e}{Fore.RESET}")
+        return 1
+
+    model_size = get_model_size(model_info=model_info, precision=precision)
+
+    try:
+        license = model_info.cardData["license"]
+    except Exception:
+        license = "?"
+
+    eval_entry = {
+        "model": model_name,
+        "base_model": base_model,
+        "revision": revision,
+        "private": False,
+        "precision": precision,
+        "weight_type": weight_type,
+        "status": status,
+        "submitted_time": current_time,
+        "model_type": model_type,
+        "likes": model_info.likes,
+        "params": model_size,
+        "license": license,
+    }
+
+    user_name = ""
+    model_path = model_name
+    if "/" in model_name:
+        user_name = model_name.split("/")[0]
+        model_path = model_name.split("/")[1]
+
+    pprint.pprint(eval_entry)
+
+    if click.confirm("Do you want to continue? This request file will be pushed to the hub"):
+        click.echo("continuing...")
+
+        out_dir = f"{EVAL_REQUESTS_PATH}/{user_name}"
+        os.makedirs(out_dir, exist_ok=True)
+        out_path = f"{out_dir}/{model_path}_eval_request_{False}_{precision}_{weight_type}.json"
+
+        with open(out_path, "w") as f:
+            # f.write(json.dumps(eval_entry))
+            json.dump(eval_entry, f, indent=4) 
+
+        api.upload_file(
+            path_or_fileobj=out_path,
+            path_in_repo=out_path.split(f"{EVAL_REQUESTS_PATH}/")[1],
+            repo_id=QUEUE_REPO,
+            repo_type="dataset",
+            commit_message=f"Add {model_name} to eval queue",
+        )
+    else:
+        click.echo("aborting...")
+
+
+if __name__ == "__main__":
+    main()

src/scripts/update_all_request_files.py

@@ -0,0 +1,94 @@
+#from huggingface_hub import ModelFilter, snapshot_download
+from huggingface_hub import snapshot_download
+from huggingface_hub import ModelCard
+
+import json
+import time
+
+from src.submission.check_validity import is_model_on_hub, check_model_card, get_model_tags
+from src.envs import DYNAMIC_INFO_REPO, DYNAMIC_INFO_PATH, DYNAMIC_INFO_FILE_PATH, API, TOKEN
+
+def update_models(file_path, models):
+    """
+    Search through all JSON files in the specified root folder and its subfolders,
+    and update the likes key in JSON dict from value of input dict
+    """
+    with open(file_path, "r") as f:
+        model_infos = json.load(f)
+        for model_id, data in model_infos.items():
+            if model_id not in models:
+                data['still_on_hub'] = False
+                data['likes'] = 0
+                data['downloads'] = 0
+                data['created_at'] = ""
+                continue
+
+            model_cfg = models[model_id]
+            data['likes'] = model_cfg.likes
+            data['downloads'] = model_cfg.downloads
+            data['created_at'] = str(model_cfg.created_at)
+            #data['params'] = get_model_size(model_cfg, data['precision'])
+            data['license'] = model_cfg.card_data.license if model_cfg.card_data is not None else ""
+
+            # Is the model still on the hub?
+            model_name = model_id
+            if model_cfg.card_data is not None and model_cfg.card_data.base_model is not None:
+                model_name = model_cfg.card_data.base_model # for adapters, we look at the parent model
+            still_on_hub, _, _ = is_model_on_hub(
+                model_name=model_name, revision=data.get("revision"), trust_remote_code=True, test_tokenizer=False, token=TOKEN
+            )
+            # If the model doesn't have a model card or a license, we consider it's deleted
+            if still_on_hub:
+                try:
+                    status, _, model_card = check_model_card(model_id)
+                    if status is False:
+                        still_on_hub = False
+                except Exception:
+                    model_card = None
+                    still_on_hub = False
+            data['still_on_hub'] = still_on_hub
+
+            tags = get_model_tags(model_card, model_id) if still_on_hub else []
+
+            data["tags"] = tags
+
+    with open(file_path, 'w') as f:
+        json.dump(model_infos, f, indent=2)
+
+def update_dynamic_files():
+    """ This will only update metadata for models already linked in the repo, not add missing ones.
+    """
+    snapshot_download(
+        repo_id=DYNAMIC_INFO_REPO, local_dir=DYNAMIC_INFO_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
+    )
+
+    print("UPDATE_DYNAMIC: Loaded snapshot")
+    # Get models
+    start = time.time()
+
+    models = list(API.list_models(
+        #filter=ModelFilter(task="text-generation"),
+        task="text-generation",
+        full=False,
+        cardData=True,
+        fetch_config=True,
+    ))
+    id_to_model = {model.id : model for model in models}
+
+    print(f"UPDATE_DYNAMIC: Downloaded list of models in {time.time() - start:.2f} seconds")
+
+    start = time.time()
+
+    update_models(DYNAMIC_INFO_FILE_PATH, id_to_model)
+
+    print(f"UPDATE_DYNAMIC: updated in {time.time() - start:.2f} seconds")
+
+    API.upload_file(
+        path_or_fileobj=DYNAMIC_INFO_FILE_PATH,
+        path_in_repo=DYNAMIC_INFO_FILE_PATH.split("/")[-1],
+        repo_id=DYNAMIC_INFO_REPO,
+        repo_type="dataset",
+        commit_message=f"Daily request file update.",
+    )
+    print(f"UPDATE_DYNAMIC: pushed to hub")
+

src/tools/collections.py

@@ -0,0 +1,83 @@
+import os
+
+import pandas as pd
+from huggingface_hub import add_collection_item, delete_collection_item, get_collection, update_collection_item
+from huggingface_hub.utils._errors import HfHubHTTPError
+from pandas import DataFrame
+
+from src.display.utils import AutoEvalColumn, ModelType
+from src.envs import TOKEN, PATH_TO_COLLECTION
+
+# Specific intervals for the collections
+intervals = {
+    "1B": pd.Interval(0, 1.5, closed="right"),
+    "3B": pd.Interval(2.5, 3.5, closed="neither"),
+    "7B": pd.Interval(6, 8, closed="neither"),
+    "13B": pd.Interval(10, 14, closed="neither"),
+    "30B": pd.Interval(25, 35, closed="neither"),
+    "65B": pd.Interval(60, 70, closed="neither"),
+}
+
+
+def update_collections(df: DataFrame):
+    """This function updates the Open LLM Leaderboard model collection with the latest best models for
+    each size category and type.
+    """
+    collection = get_collection(collection_slug=PATH_TO_COLLECTION, token=TOKEN)
+    params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
+
+    cur_best_models = []
+
+    ix = 0
+    for type in ModelType:
+        if type.value.name == "":
+            continue
+        for size in intervals:
+            # We filter the df to gather the relevant models
+            type_emoji = [t[0] for t in type.value.symbol]
+            filtered_df = df[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
+
+            numeric_interval = pd.IntervalIndex([intervals[size]])
+            mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
+            filtered_df = filtered_df.loc[mask]
+
+            best_models = list(
+                filtered_df.sort_values(AutoEvalColumn.average.name, ascending=False)[AutoEvalColumn.dummy.name]
+            )
+            print(type.value.symbol, size, best_models[:10])
+
+            # We add them one by one to the leaderboard
+            for model in best_models:
+                ix += 1
+                cur_len_collection = len(collection.items)
+                try:
+                    collection = add_collection_item(
+                        PATH_TO_COLLECTION,
+                        item_id=model,
+                        item_type="model",
+                        exists_ok=True,
+                        note=f"Best {type.to_str(' ')} model of around {size} on the leaderboard today!",
+                        token=TOKEN,
+                    )
+                    if (
+                        len(collection.items) > cur_len_collection
+                    ):  # we added an item - we make sure its position is correct
+                        item_object_id = collection.items[-1].item_object_id
+                        update_collection_item(
+                            collection_slug=PATH_TO_COLLECTION, item_object_id=item_object_id, position=ix
+                        )
+                        cur_len_collection = len(collection.items)
+                    cur_best_models.append(model)
+                    break
+                except HfHubHTTPError:
+                    continue
+
+    collection = get_collection(PATH_TO_COLLECTION, token=TOKEN)
+    for item in collection.items:
+        if item.item_id not in cur_best_models:
+            try:
+                delete_collection_item(
+                    collection_slug=PATH_TO_COLLECTION, item_object_id=item.item_object_id, token=TOKEN
+                )
+            except HfHubHTTPError:
+                continue

src/tools/model_backlinks.py

@@ -0,0 +1,1309 @@
+models = [
+    "uni-tianyan/Uni-TianYan",
+    "fangloveskari/ORCA_LLaMA_70B_QLoRA",
+    "garage-bAInd/Platypus2-70B-instruct",
+    "upstage/Llama-2-70b-instruct-v2",
+    "fangloveskari/Platypus_QLoRA_LLaMA_70b",
+    "yeontaek/llama-2-70B-ensemble-v5",
+    "TheBloke/Genz-70b-GPTQ",
+    "TheBloke/Platypus2-70B-Instruct-GPTQ",
+    "psmathur/model_007",
+    "yeontaek/llama-2-70B-ensemble-v4",
+    "psmathur/orca_mini_v3_70b",
+    "ehartford/Samantha-1.11-70b",
+    "MayaPH/GodziLLa2-70B",
+    "psmathur/model_007_v2",
+    "chargoddard/MelangeA-70b",
+    "ehartford/Samantha-1.1-70b",
+    "psmathur/model_009",
+    "upstage/Llama-2-70b-instruct",
+    "yeontaek/llama-2-70B-ensemble-v7",
+    "yeontaek/llama-2-70B-ensemble-v6",
+    "chargoddard/MelangeB-70b",
+    "yeontaek/llama-2-70B-ensemble-v3",
+    "chargoddard/MelangeC-70b",
+    "garage-bAInd/Camel-Platypus2-70B",
+    "yeontaek/llama-2-70B-ensemble-v2",
+    "garage-bAInd/Camel-Platypus2-70B",
+    "migtissera/Synthia-70B-v1.2",
+    "v2ray/LLaMA-2-Wizard-70B-QLoRA",
+    "quantumaikr/llama-2-70b-fb16-orca-chat-10k",
+    "v2ray/LLaMA-2-Wizard-70B-QLoRA",
+    "stabilityai/StableBeluga2",
+    "quantumaikr/llama-2-70b-fb16-guanaco-1k",
+    "garage-bAInd/Camel-Platypus2-70B",
+    "migtissera/Synthia-70B-v1.1",
+    "migtissera/Synthia-70B",
+    "psmathur/model_101",
+    "augtoma/qCammel70",
+    "augtoma/qCammel-70",
+    "augtoma/qCammel-70v1",
+    "augtoma/qCammel-70x",
+    "augtoma/qCammel-70-x",
+    "jondurbin/airoboros-l2-70b-gpt4-1.4.1",
+    "dfurman/llama-2-70b-dolphin-peft",
+    "jondurbin/airoboros-l2-70b-2.1",
+    "TheBloke/llama-2-70b-Guanaco-QLoRA-fp16",
+    "quantumaikr/QuantumLM-llama2-70B-Korean-LoRA",
+    "quantumaikr/quantumairk-llama-2-70B-instruct",
+    "psmathur/model_420",
+    "psmathur/model_51",
+    "garage-bAInd/Camel-Platypus2-70B",
+    "TheBloke/Airoboros-L2-70B-2.1-GPTQ",
+    "OpenAssistant/llama2-70b-oasst-sft-v10",
+    "garage-bAInd/Platypus2-70B",
+    "liuxiang886/llama2-70B-qlora-gpt4",
+    "upstage/llama-65b-instruct",
+    "quantumaikr/llama-2-70b-fb16-korean",
+    "NousResearch/Nous-Hermes-Llama2-70b",
+    "v2ray/LLaMA-2-Jannie-70B-QLoRA",
+    "jondurbin/airoboros-l2-70b-gpt4-m2.0",
+    "jondurbin/airoboros-l2-70b-gpt4-m2.0",
+    "OpenAssistant/llama2-70b-oasst-sft-v10",
+    "yeontaek/llama-2-70B-ensemble-v8",
+    "jondurbin/airoboros-l2-70b-gpt4-2.0",
+    "jarradh/llama2_70b_chat_uncensored",
+    "WizardLM/WizardMath-70B-V1.0",
+    "jordiclive/Llama-2-70b-oasst-1-200",
+    "WizardLM/WizardMath-70B-V1.0",
+    "jondurbin/airoboros-l2-70b-gpt4-2.0",
+    "OpenLemur/lemur-70b-chat-v1",
+    "tiiuae/falcon-180B",
+    "tiiuae/falcon-180B",
+    "stabilityai/StableBeluga1-Delta",
+    "psmathur/model_42_70b",
+    "psmathur/test_42_70b",
+    "TheBloke/fiction.live-Kimiko-V2-70B-fp16",
+    "tiiuae/falcon-180B",
+    "WizardLM/WizardMath-70B-V1.0",
+    "tiiuae/falcon-180B-chat",
+    "jondurbin/airoboros-l2-70b-gpt4-2.0",
+    "ehartford/samantha-1.1-llama-33b",
+    "ajibawa-2023/scarlett-33b",
+    "ddobokki/Llama-2-70b-orca-200k",
+    "TheBloke/gpt4-alpaca-lora_mlp-65B-HF",
+    "tiiuae/falcon-180B-chat",
+    "tiiuae/falcon-180B-chat",
+    "tiiuae/falcon-180B",
+    "TheBloke/Lemur-70B-Chat-v1-GPTQ",
+    "NousResearch/Nous-Puffin-70B",
+    "WizardLM/WizardLM-70B-V1.0",
+    "WizardLM/WizardMath-70B-V1.0",
+    "meta-llama/Llama-2-70b-hf",
+    "TheBloke/Llama-2-70B-fp16",
+    "Weyaxi/llama-2-alpacagpt4-1000step",
+    "WizardLM/WizardLM-70B-V1.0",
+    "simsim314/WizardLM-70B-V1.0-HF",
+    "simsim314/WizardLM-70B-V1.0-HF",
+    "WizardLM/WizardLM-70B-V1.0",
+    "openbmb/UltraLM-65b",
+    "psmathur/model_420_preview",
+    "WizardLM/WizardLM-70B-V1.0",
+    "simsim314/WizardLM-70B-V1.0-HF",
+    "OpenBuddy/openbuddy-llama2-70b-v10.1-bf16",
+    "upstage/llama-30b-instruct-2048",
+    "jondurbin/airoboros-65b-gpt4-1.2",
+    "TheBloke/guanaco-65B-HF",
+    "jondurbin/airoboros-65b-gpt4-1.3",
+    "meta-llama/Llama-2-70b-chat-hf",
+    "ValiantLabs/ShiningValiant",
+    "Faradaylab/Aria-70B",
+    "lilloukas/GPlatty-30B",
+    "TheBloke/VicUnlocked-alpaca-65B-QLoRA-fp16",
+    "jondurbin/airoboros-65b-gpt4-1.4-peft",
+    "jondurbin/airoboros-65b-gpt4-1.4",
+    "jondurbin/airoboros-65b-gpt4-2.0",
+    "TheBloke/WizardLM-70B-V1.0-GPTQ",
+    "TheBloke/WizardLM-70B-V1.0-GPTQ",
+    "ariellee/SuperPlatty-30B",
+    "jondurbin/airoboros-65b-gpt4-1.4",
+    "jondurbin/airoboros-65b-gpt4-2.0",
+    "yeontaek/llama-2-70b-IA3-guanaco",
+    "CalderaAI/30B-Lazarus",
+    "Aspik101/trurl-2-13b-pl-instruct_unload",
+    "ehartford/WizardLM-33B-V1.0-Uncensored",
+    "ehartford/WizardLM-33B-V1.0-Uncensored",
+    "OpenBuddy/openbuddy-llama-65b-v8-bf16",
+    "Aspik101/llama-30b-instruct-2048-PL-lora",
+    "h2oai/h2ogpt-research-oasst1-llama-65b",
+    "Aspik101/llama-30b-instruct-2048-PL-lora",
+    "CalderaAI/30B-Epsilon",
+    "Aspik101/llama-30b-2048-instruct-PL-lora_unload",
+    "jondurbin/airoboros-65b-gpt4-m2.0",
+    "jondurbin/airoboros-65b-gpt4-m2.0",
+    "Aeala/Alpaca-elina-65b",
+    "TheBloke/robin-65b-v2-fp16",
+    "TheBloke/gpt4-alpaca-lora-30b-HF",
+    "TheBloke/Llama-2-70B-chat-GPTQ",
+    "upstage/llama-30b-instruct",
+    "OpenLemur/lemur-70b-v1",
+    "lmsys/vicuna-33b-v1.3",
+    "ausboss/llama-30b-supercot",
+    "ai-business/Luban-13B",
+    "Henk717/airochronos-33B",
+    "lmsys/vicuna-33b-v1.3",
+    "Henk717/airochronos-33B",
+    "bavest/fin-llama-33b-merged",
+    "jondurbin/airoboros-33b-gpt4-1.4",
+    "YeungNLP/firefly-llama-30b",
+    "Aspik101/30B-Lazarus-instruct-PL-lora_unload",
+    "uukuguy/speechless-llama2-luban-orca-platypus-13b",
+    "xxyyy123/test_merge_p_ov1_w0.66_w0.5_n1",
+    "jondurbin/airoboros-33b-gpt4-1.2",
+    "TheBloke/alpaca-lora-65B-HF",
+    "bofenghuang/vigogne-33b-instruct",
+    "yeontaek/llama-2-13B-ensemble-v5",
+    "garage-bAInd/Platypus-30B",
+    "Open-Orca/OpenOrca-Platypus2-13B",
+    "kajdun/viwaai-30b_v4",
+    "lilloukas/Platypus-30B",
+    "Open-Orca/OpenOrca-Platypus2-13B",
+    "Henk717/chronoboros-33B",
+    "jondurbin/airoboros-33b-2.1",
+    "HiTZ/alpaca-lora-65b-en-pt-es-ca",
+    "quantumaikr/QuantumLM-70B-hf",
+    "uukuguy/speechless-llama2-13b",
+    "uukuguy/speechless-llama2-hermes-orca-platypus-13b",
+    "openaccess-ai-collective/manticore-30b-chat-pyg-alpha",
+    "LLMs/WizardLM-30B-V1.0",
+    "TheBloke/WizardLM-30B-fp16",
+    "openaccess-ai-collective/hippogriff-30b-chat",
+    "concedo/Vicuzard-30B-Uncensored",
+    "TFLai/OpenOrca-Platypus2-13B-QLoRA-0.80-epoch",
+    "huggingface/llama-65b",
+    "huggyllama/llama-65b",
+    "gaodrew/gaodrew-llama-30b-instruct-2048-Open-Platypus-100steps",
+    "uukuguy/speechless-llama2-hermes-orca-platypus-wizardlm-13b",
+    "Sao10K/Mythical-Destroyer-V2-L2-13B",
+    "camel-ai/CAMEL-33B-Combined-Data",
+    "dsvv-cair/alpaca-cleaned-llama-30b-bf16",
+    "MetaIX/GPT4-X-Alpasta-30b",
+    "garage-bAInd/Stable-Platypus2-13B",
+    "TFLai/Luban-Platypus2-13B-QLora-0.80-epoch",
+    "TheBloke/OpenOrca-Platypus2-13B-GPTQ",
+    "IkariDev/Athena-tmp",
+    "OpenBuddyEA/openbuddy-llama-30b-v7.1-bf16",
+    "OpenBuddyEA/openbuddy-llama-30b-v7.1-bf16",
+    "Open-Orca/OpenOrcaxOpenChat-Preview2-13B",
+    "psmathur/model_007_13b_v2",
+    "Aspik101/Vicuzard-30B-Uncensored-instruct-PL-lora_unload",
+    "jondurbin/airoboros-33b-gpt4-m2.0",
+    "Sao10K/Mythical-Destroyer-L2-13B",
+    "TheBloke/Wizard-Vicuna-30B-Uncensored-fp16",
+    "ehartford/Wizard-Vicuna-30B-Uncensored",
+    "TFLai/Nova-13B",
+    "TheBloke/robin-33B-v2-fp16",
+    "totally-not-an-llm/PuddleJumper-13b",
+    "Aeala/VicUnlocked-alpaca-30b",
+    "Yhyu13/oasst-rlhf-2-llama-30b-7k-steps-hf",
+    "jondurbin/airoboros-33b-gpt4",
+    "jondurbin/airoboros-33b-gpt4-m2.0",
+    "tiiuae/falcon-40b-instruct",
+    "psmathur/orca_mini_v3_13b",
+    "Aeala/GPT4-x-AlpacaDente-30b",
+    "MayaPH/GodziLLa-30B",
+    "jondurbin/airoboros-33b-gpt4-m2.0",
+    "TFLai/SpeechlessV1-Nova-13B",
+    "yeontaek/llama-2-13B-ensemble-v4",
+    "ajibawa-2023/carl-33b",
+    "jondurbin/airoboros-33b-gpt4-2.0",
+    "TFLai/Stable-Platypus2-13B-QLoRA-0.80-epoch",
+    "jondurbin/airoboros-33b-gpt4-1.3",
+    "TehVenom/oasst-sft-6-llama-33b-xor-MERGED-16bit",
+    "TFLai/OrcaMini-Platypus2-13B-QLoRA-0.80-epoch",
+    "jondurbin/airoboros-33b-gpt4-2.0",
+    "chargoddard/Chronorctypus-Limarobormes-13b",
+    "jondurbin/airoboros-33b-gpt4-1.3",
+    "Open-Orca/OpenOrca-Platypus2-13B",
+    "FelixChao/vicuna-33b-coder",
+    "FelixChao/vicuna-33b-coder",
+    "Gryphe/MythoMix-L2-13b",
+    "Aeala/Enterredaas-33b",
+    "yeontaek/llama-2-13B-ensemble-v1",
+    "TFLai/OpenOrcaPlatypus2-Platypus2-13B-QLora-0.80-epoch",
+    "TFLai/Ensemble5-Platypus2-13B-QLora-0.80-epoch",
+    "yeontaek/llama-2-13B-ensemble-v3",
+    "TFLai/MythoMix-Platypus2-13B-QLoRA-0.80-epoch",
+    "yihan6324/llama2-13b-instructmining-40k-sharegpt",
+    "timdettmers/guanaco-33b-merged",
+    "TFLai/EnsembleV5-Nova-13B",
+    "circulus/Llama-2-13b-orca-v1",
+    "Undi95/ReMM-SLERP-L2-13B",
+    "Gryphe/MythoMax-L2-13b",
+    "stabilityai/StableBeluga-13B",
+    "circulus/Llama-2-13b-orca-v1",
+    "ehartford/WizardLM-30B-Uncensored",
+    "The-Face-Of-Goonery/huginnv1.2",
+    "TheBloke/OpenOrcaxOpenChat-Preview2-13B-GPTQ",
+    "Sao10K/Stheno-L2-13B",
+    "bofenghuang/vigogne-2-13b-instruct",
+    "The-Face-Of-Goonery/Huginn-13b-FP16",
+    "grimpep/L2-MythoMax22b-instruct-Falseblock",
+    "TFLai/Nous-Hermes-Platypus2-13B-QLoRA-0.80-epoch",
+    "yeontaek/Platypus2xOpenOrca-13B-IA3-v4",
+    "yeontaek/Platypus2xOpenOrca-13B-IA3",
+    "yeontaek/Platypus2xOpenOrca-13B-IA3-ensemble",
+    "Open-Orca/LlongOrca-13B-16k",
+    "Sao10K/Stheno-Inverted-L2-13B",
+    "garage-bAInd/Camel-Platypus2-13B",
+    "digitous/Alpacino30b",
+    "NousResearch/Nous-Hermes-Llama2-13b",
+    "yeontaek/Platypus2xOpenOrca-13B-IA3-v3",
+    "TFLai/MythicalDestroyerV2-Platypus2-13B-QLora-0.80-epoch",
+    "TheBloke/VicUnlocked-30B-LoRA-HF",
+    "Undi95/Nous-Hermes-13B-Code",
+    "The-Face-Of-Goonery/Chronos-Beluga-v2-13bfp16",
+    "NousResearch/Nous-Hermes-Llama2-13b",
+    "Monero/WizardLM-Uncensored-SuperCOT-StoryTelling-30b",
+    "TheBloke/Wizard-Vicuna-30B-Uncensored-GPTQ",
+    "Open-Orca/OpenOrcaxOpenChat-Preview2-13B",
+    "Austism/chronos-hermes-13b-v2",
+    "yeontaek/Platypus2xOpenOrca-13B-IA3-v2.1",
+    "yeontaek/Platypus2xOpenOrca-13B-IA3-v2",
+    "Gryphe/MythoLogic-L2-13b",
+    "augtoma/qCammel-13",
+    "YeungNLP/firefly-llama2-13b-v1.2",
+    "Aspik101/StableBeluga-13B-instruct-PL-lora_unload",
+    "andreaskoepf/llama2-13b-megacode2_min100",
+    "rombodawg/LosslessMegaCoder-llama2-13b-mini",
+    "yulan-team/YuLan-Chat-2-13b-fp16",
+    "elinas/chronos-33b",
+    "YeungNLP/firefly-llama2-13b",
+    "Sao10K/Medusa-13b",
+    "OptimalScale/robin-65b-v2-delta",
+    "minlik/chinese-alpaca-33b-merged",
+    "OpenAssistant/llama2-13b-megacode2-oasst",
+    "TheBloke/OpenAssistant-SFT-7-Llama-30B-HF",
+    "Undi95/UndiMix-v1-13b",
+    "ehartford/Samantha-1.11-13b",
+    "beaugogh/Llama2-13b-sharegpt4",
+    "Aeala/GPT4-x-AlpacaDente2-30b",
+    "luffycodes/nash-vicuna-13b-v1dot5-ep2-w-rag-w-simple",
+    "WizardLM/WizardLM-13B-V1.1",
+    "uukuguy/speechless-orca-platypus-coig-lite-2k-0.6e-13b",
+    "huggyllama/llama-30b",
+    "Undi95/ReMM-L2-13B-PIPPA",
+    "Undi95/ReMM-L2-13B",
+    "gaodrew/gaodrew-gorgonzola-13b",
+    "lmsys/vicuna-13b-v1.5",
+    "yeontaek/Platypus2xOpenOrca-13B-LoRa",
+    "Yhyu13/llama-30B-hf-openassitant",
+    "huggingface/llama-30b",
+    "lmsys/vicuna-13b-v1.5",
+    "TFLai/Athena-Platypus2-13B-QLora-0.80-epoch",
+    "TheBloke/dromedary-65b-lora-HF",
+    "yeontaek/llama-2-13b-Beluga-QLoRA",
+    "The-Face-Of-Goonery/Huginn-13b-V4",
+    "The-Face-Of-Goonery/Huginn-13b-v4.5",
+    "The-Face-Of-Goonery/Huginn-v3-13b",
+    "tiiuae/falcon-40b",
+    "WhoTookMyAmogusNickname/NewHope_HF_not_official",
+    "gaodrew/OpenOrca-Platypus2-13B-thera-1250",
+    "SLAM-group/NewHope",
+    "garage-bAInd/Platypus2-13B",
+    "migtissera/Synthia-13B",
+    "elinas/chronos-13b-v2",
+    "mosaicml/mpt-30b-chat",
+    "CHIH-HUNG/llama-2-13b-OpenOrca_5w",
+    "uukuguy/speechless-hermes-coig-lite-13b",
+    "TheBloke/tulu-30B-fp16",
+    "uukuguy/speechless-hermes-coig-lite-13b",
+    "xDAN-AI/xDAN_13b_l2_lora",
+    "lmsys/vicuna-13b-v1.5-16k",
+    "openchat/openchat_v3.1",
+    "CHIH-HUNG/llama-2-13b-dolphin_5w",
+    "Aspik101/vicuna-13b-v1.5-PL-lora_unload",
+    "Undi95/MLewd-L2-13B",
+    "ehartford/minotaur-llama2-13b-qlora",
+    "kajdun/iubaris-13b-v3",
+    "TFLai/Limarp-Platypus2-13B-QLoRA-0.80-epoch",
+    "openchat/openchat_v3.1",
+    "uukuguy/speechless-orca-platypus-coig-lite-4k-0.6e-13b",
+    "ziqingyang/chinese-alpaca-2-13b",
+    "TFLai/Airboros2.1-Platypus2-13B-QLora-0.80-epoch",
+    "yeontaek/llama-2-13b-Guanaco-QLoRA",
+    "lmsys/vicuna-13b-v1.5-16k",
+    "ehartford/based-30b",
+    "kingbri/airolima-chronos-grad-l2-13B",
+    "openchat/openchat_v3.2",
+    "uukuguy/speechless-orca-platypus-coig-lite-4k-0.5e-13b",
+    "yeontaek/Platypus2-13B-LoRa",
+    "kingbri/chronolima-airo-grad-l2-13B",
+    "openchat/openchat_v3.2",
+    "TFLai/PuddleJumper-Platypus2-13B-QLoRA-0.80-epoch",
+    "shareAI/llama2-13b-Chinese-chat",
+    "ehartford/WizardLM-1.0-Uncensored-Llama2-13b",
+    "Aspik101/Redmond-Puffin-13B-instruct-PL-lora_unload",
+    "yeontaek/llama-2-13B-ensemble-v6",
+    "WizardLM/WizardLM-13B-V1.2",
+    "TheBloke/WizardLM-13B-V1.1-GPTQ",
+    "bhenrym14/airophin-13b-pntk-16k-fp16",
+    "ehartford/WizardLM-1.0-Uncensored-Llama2-13b",
+    "Mikael110/llama-2-13b-guanaco-fp16",
+    "yeontaek/airoboros-2.1-llama-2-13B-QLoRa",
+    "CalderaAI/13B-Legerdemain-L2",
+    "grimpep/llama2-22b-wizard_vicuna",
+    "grimpep/llama2-22B-GPLATTY",
+    "bhenrym14/airophin-13b-pntk-16k-fp16",
+    "yeontaek/llama-2-13b-QLoRA",
+    "OpenAssistant/llama2-13b-orca-8k-3319",
+    "TheBloke/WizardLM-13B-V1-1-SuperHOT-8K-fp16",
+    "duliadotio/dulia-13b-8k-alpha",
+    "Undi95/LewdEngine",
+    "OpenBuddy/openbuddy-llama2-13b-v8.1-fp16",
+    "CHIH-HUNG/llama-2-13b-open_orca_20w",
+    "bhenrym14/airoboros-33b-gpt4-1.4.1-lxctx-PI-16384-fp16",
+    "FlagAlpha/Llama2-Chinese-13b-Chat",
+    "LLMs/WizardLM-13B-V1.0",
+    "chansung/gpt4-alpaca-lora-13b-decapoda-1024",
+    "TheBloke/wizardLM-13B-1.0-fp16",
+    "digitous/13B-Chimera",
+    "yeontaek/Platypus2xOpenOrcaxGuanaco-13B-LoRa",
+    "jondurbin/airoboros-l2-13b-2.1",
+    "Monero/WizardLM-30B-Uncensored-Guanaco-SuperCOT-30b",
+    "TheBloke/UltraLM-13B-fp16",
+    "openaccess-ai-collective/minotaur-13b-fixed",
+    "NousResearch/Redmond-Puffin-13B",
+    "KoboldAI/LLaMA2-13B-Holomax",
+    "Lajonbot/WizardLM-13B-V1.2-PL-lora_unload",
+    "yeontaek/Platypus2-13B-LoRa-v2",
+    "TheBloke/airoboros-13B-HF",
+    "jondurbin/airoboros-13b",
+    "jjaaaww/posi_13b",
+    "CoolWP/llama-2-13b-guanaco-fp16",
+    "yeontaek/Platypus2-13B-QLoRa",
+    "h2oai/h2ogpt-research-oig-oasst1-512-30b",
+    "dfurman/llama-2-13b-guanaco-peft",
+    "NousResearch/Redmond-Puffin-13B",
+    "pe-nlp/llama-2-13b-platypus-vicuna-wizard",
+    "CHIH-HUNG/llama-2-13b-dolphin_20w",
+    "NousResearch/Nous-Hermes-13b",
+    "NobodyExistsOnTheInternet/GiftedConvo13bLoraNoEconsE4",
+    "ehartford/Wizard-Vicuna-13B-Uncensored",
+    "TheBloke/Wizard-Vicuna-13B-Uncensored-HF",
+    "openchat/openchat_v3.2_super",
+    "bhenrym14/airophin-v2-13b-PI-8k-fp16",
+    "openaccess-ai-collective/manticore-13b",
+    "The-Face-Of-Goonery/Huginn-22b-Prototype",
+    "jphme/Llama-2-13b-chat-german",
+    "grimpep/llama2-28B-Airo03",
+    "TheBloke/Kimiko-v2-13B-fp16",
+    "FPHam/Free_Sydney_13b_HF",
+    "lmsys/vicuna-13b-v1.3",
+    "FelixChao/llama2-13b-math1.1",
+    "CalderaAI/13B-BlueMethod",
+    "meta-llama/Llama-2-13b-chat-hf",
+    "deepse/CodeUp-Llama-2-13b-chat-hf",
+    "WizardLM/WizardMath-13B-V1.0",
+    "WizardLM/WizardMath-13B-V1.0",
+    "HyperbeeAI/Tulpar-7b-v0",
+    "xxyyy123/test_qkvo_adptor",
+    "xxyyy123/mc_data_30k_from_platpus_orca_7b_10k_v1_lora_qkvo_rank14_v2",
+    "openchat/openchat_v2_w",
+    "FelixChao/llama2-13b-math1.1",
+    "psmathur/orca_mini_v3_7b",
+    "TehVenom/Metharme-13b-Merged",
+    "xxyyy123/10k_v1_lora_qkvo_rank14_v3",
+    "OpenAssistant/llama2-13b-orca-v2-8k-3166",
+    "openaccess-ai-collective/wizard-mega-13b",
+    "jondurbin/airoboros-13b-gpt4-1.4",
+    "jondurbin/airoboros-13b-gpt4-1.4-fp16",
+    "Monero/Manticore-13b-Chat-Pyg-Guanaco",
+    "FelixChao/llama2-13b-math1.2",
+    "chargoddard/platypus-2-22b-relora",
+    "FelixChao/llama2-13b-math1.2",
+    "Gryphe/MythoBoros-13b",
+    "CalderaAI/13B-Ouroboros",
+    "OpenAssistant/llama2-13b-orca-v2-8k-3166",
+    "heegyu/LIMA2-13b-hf",
+    "digitous/13B-HyperMantis",
+    "Gryphe/MythoLogic-13b",
+    "TheBloke/Airoboros-L2-13B-2.1-GPTQ",
+    "chargoddard/platypus2-22b-relora",
+    "openchat/openchat_v2",
+    "yeontaek/Platypus2-13B-IA3",
+    "stabilityai/StableBeluga-7B",
+    "circulus/Llama-2-7b-orca-v1",
+    "budecosystem/genz-13b-v2",
+    "TheBloke/gpt4-x-vicuna-13B-HF",
+    "NobodyExistsOnTheInternet/GiftedConvo13bLoraNoEcons",
+    "zarakiquemparte/zarafusionex-1.1-l2-7b",
+    "Lajonbot/tableBeluga-7B-instruct-pl-lora_unload",
+    "jondurbin/airoboros-13b-gpt4",
+    "gaodrew/gaodrew-gorgonzola-13b",
+    "jondurbin/airoboros-13b-gpt4-1.1",
+    "TheBloke/gpt4-alpaca-lora-13B-HF",
+    "zarakiquemparte/zarablendex-vq-l2-7b",
+    "openaccess-ai-collective/manticore-13b-chat-pyg",
+    "Lajonbot/Llama-2-13b-hf-instruct-pl-lora_unload",
+    "NobodyExistsOnTheInternet/PuffedLIMA13bQLORA",
+    "xxyyy123/10k_v1_lora_qkvo_rank28_v2",
+    "jondurbin/airoboros-l2-13b-gpt4-1.4.1",
+    "dhmeltzer/Llama-2-13b-hf-eli5-wiki-1024_r_64_alpha_16",
+    "NobodyExistsOnTheInternet/PuffedConvo13bLoraE4",
+    "yihan6324/llama2-7b-instructmining-40k-sharegpt",
+    "CHIH-HUNG/llama-2-13b-Open_Platypus_and_ccp_2.6w",
+    "Aeala/GPT4-x-Alpasta-13b",
+    "psmathur/orca_mini_v2_13b",
+    "YeungNLP/firefly-llama-13b",
+    "psmathur/orca_mini_v2_13b",
+    "zarakiquemparte/zarafusionix-l2-7b",
+    "yihan6324/llama2-7b-instructmining-60k-sharegpt",
+    "yihan6324/llama-2-7b-instructmining-60k-sharegpt",
+    "layoric/llama-2-13b-code-alpaca",
+    "bofenghuang/vigogne-13b-instruct",
+    "Lajonbot/vicuna-13b-v1.3-PL-lora_unload",
+    "lvkaokao/llama2-7b-hf-chat-lora-v3",
+    "ehartford/dolphin-llama-13b",
+    "YeungNLP/firefly-llama-13b-v1.2",
+    "TheBloke/Kimiko-13B-fp16",
+    "kevinpro/Vicuna-13B-CoT",
+    "eachadea/vicuna-13b-1.1",
+    "pillowtalks-ai/delta13b",
+    "TheBloke/vicuna-13B-1.1-HF",
+    "TheBloke/Vicuna-13B-CoT-fp16",
+    "lmsys/vicuna-13b-delta-v1.1",
+    "lmsys/vicuna-13b-v1.1",
+    "xxyyy123/20k_v1_lora_qkvo_rank14_v2",
+    "TheBloke/guanaco-13B-HF",
+    "TheBloke/vicuna-13b-v1.3.0-GPTQ",
+    "edor/Stable-Platypus2-mini-7B",
+    "totally-not-an-llm/EverythingLM-13b-V2-16k",
+    "zarakiquemparte/zaraxe-l2-7b",
+    "beaugogh/Llama2-7b-openorca-mc-v2",
+    "TheBloke/Nous-Hermes-13B-SuperHOT-8K-fp16",
+    "quantumaikr/QuantumLM",
+    "jondurbin/airoboros-13b-gpt4-1.2",
+    "TheBloke/robin-13B-v2-fp16",
+    "TFLai/llama-2-13b-4bit-alpaca-gpt4",
+    "yihan6324/llama2-7b-instructmining-orca-40k",
+    "dvruette/oasst-llama-13b-2-epochs",
+    "Open-Orca/LlongOrca-7B-16k",
+    "Aspik101/Nous-Hermes-13b-pl-lora_unload",
+    "ehartford/Samantha-1.11-CodeLlama-34b",
+    "nkpz/llama2-22b-chat-wizard-uncensored",
+    "bofenghuang/vigogne-13b-chat",
+    "beaugogh/Llama2-7b-openorca-mc-v1",
+    "OptimalScale/robin-13b-v2-delta",
+    "pe-nlp/llama-2-13b-vicuna-wizard",
+    "chargoddard/llama2-22b",
+    "gywy/llama2-13b-chinese-v1",
+    "frank098/Wizard-Vicuna-13B-juniper",
+    "IGeniusDev/llama13B-quant8-testv1-openorca-customdataset",
+    "CHIH-HUNG/llama-2-13b-huangyt_Fintune_1_17w-gate_up_down_proj",
+    "eachadea/vicuna-13b",
+    "yihan6324/llama2-7b-instructmining-orca-90k",
+    "chargoddard/llama2-22b-blocktriangular",
+    "luffycodes/mcq-vicuna-13b-v1.5",
+    "Yhyu13/chimera-inst-chat-13b-hf",
+    "luffycodes/mcq-vicuna-13b-v1.5",
+    "chargoddard/ypotryll-22b-epoch2-qlora",
+    "totally-not-an-llm/EverythingLM-13b-16k",
+    "luffycodes/mcq-hal-vicuna-13b-v1.5",
+    "openaccess-ai-collective/minotaur-13b",
+    "IGeniusDev/llama13B-quant8-testv1-openorca-customdataset",
+    "chargoddard/llama2-22b-blocktriangular",
+    "TFLai/Platypus2-13B-QLoRA-0.80-epoch",
+    "meta-llama/Llama-2-13b-hf",
+    "CHIH-HUNG/llama-2-13b-huangyt_FINETUNE2_3w-gate_up_down_proj",
+    "luffycodes/mcq-hal-vicuna-13b-v1.5",
+    "TheBloke/Llama-2-13B-fp16",
+    "TaylorAI/Flash-Llama-13B",
+    "shareAI/bimoGPT-llama2-13b",
+    "wahaha1987/llama_13b_sharegpt94k_fastchat",
+    "openchat/openchat_8192",
+    "CHIH-HUNG/llama-2-13b-huangyt_Fintune_1_17w-q_k_v_o_proj",
+    "dvruette/llama-13b-pretrained-sft-do2",
+    "CHIH-HUNG/llama-2-13b-alpaca-test",
+    "OpenBuddy/openbuddy-llama2-13b-v11.1-bf16",
+    "CHIH-HUNG/llama-2-13b-FINETUNE2_TEST_2.2w",
+    "project-baize/baize-v2-13b",
+    "jondurbin/airoboros-l2-13b-gpt4-m2.0",
+    "yeontaek/Platypus2xOpenOrca-13B-LoRa-v2",
+    "CHIH-HUNG/llama-2-13b-huangyt_FINETUNE2_3w",
+    "xzuyn/Alpacino-SuperCOT-13B",
+    "jondurbin/airoboros-l2-13b-gpt4-2.0",
+    "aiplanet/effi-13b",
+    "clibrain/Llama-2-13b-ft-instruct-es",
+    "CHIH-HUNG/llama-2-13b-huangyt_Fintune_1_17w",
+    "bofenghuang/vigogne-2-7b-instruct",
+    "CHIH-HUNG/llama-2-13b-huangyt_FINETUNE2_3w-q_k_v_o_proj",
+    "bofenghuang/vigogne-2-7b-chat",
+    "aiplanet/effi-13b",
+    "haonan-li/bactrian-x-llama-13b-merged",
+    "beaugogh/Llama2-7b-sharegpt4",
+    "HWERI/Llama2-7b-sharegpt4",
+    "jondurbin/airoboros-13b-gpt4-1.3",
+    "jondurbin/airoboros-c34b-2.1",
+    "junelee/wizard-vicuna-13b",
+    "TheBloke/wizard-vicuna-13B-HF",
+    "Open-Orca/OpenOrca-Preview1-13B",
+    "TheBloke/h2ogpt-oasst1-512-30B-HF",
+    "TheBloke/Llama-2-13B-GPTQ",
+    "camel-ai/CAMEL-13B-Combined-Data",
+    "lmsys/vicuna-7b-v1.5",
+    "lmsys/vicuna-7b-v1.5-16k",
+    "lmsys/vicuna-7b-v1.5",
+    "ausboss/llama-13b-supercot",
+    "TheBloke/tulu-13B-fp16",
+    "NousResearch/Nous-Hermes-llama-2-7b",
+    "jlevin/guanaco-13b-llama-2",
+    "lmsys/vicuna-7b-v1.5-16k",
+    "dvruette/llama-13b-pretrained",
+    "nkpz/llama2-22b-daydreamer-v3",
+    "dvruette/llama-13b-pretrained-dropout",
+    "jondurbin/airoboros-l2-13b-2.1",
+    "LLMs/Stable-Vicuna-13B",
+    "64bits/LexPodLM-13B",
+    "lizhuang144/llama_mirror_13b_v1.0",
+    "TheBloke/stable-vicuna-13B-HF",
+    "zarakiquemparte/zaraxls-l2-7b",
+    "TheBloke/Llama-2-13B-GPTQ",
+    "Kiddyz/testlm-3",
+    "migtissera/Synthia-7B",
+    "zarakiquemparte/zarablend-l2-7b",
+    "mosaicml/mpt-30b-instruct",
+    "PocketDoc/Dans-PileOfSets-Mk1-llama-13b-merged",
+    "vonjack/Qwen-LLaMAfied-HFTok-7B-Chat",
+    "l3utterfly/llama2-7b-layla",
+    "Lajonbot/vicuna-7b-v1.5-PL-lora_unload",
+    "heegyu/LIMA-13b-hf",
+    "frank098/WizardLM_13B_juniper",
+    "ashercn97/manatee-7b",
+    "chavinlo/gpt4-x-alpaca",
+    "PocketDoc/Dans-PersonalityEngine-13b",
+    "ehartford/WizardLM-1.0-Uncensored-CodeLlama-34b",
+    "digitous/Alpacino13b",
+    "edor/Hermes-Platypus2-mini-7B",
+    "lvkaokao/llama2-7b-hf-chat-lora-v2",
+    "Kiddyz/testlm-1-1",
+    "Kiddyz/testlm",
+    "Kiddyz/testlm-1",
+    "Kiddyz/testlm2",
+    "radm/Philosophy-Platypus2-13b",
+    "aiplanet/effi-13b",
+    "Harshvir/Llama-2-7B-physics",
+    "YeungNLP/firefly-ziya-13b",
+    "LinkSoul/Chinese-Llama-2-7b",
+    "PeanutJar/LLaMa-2-PeanutButter_v10-7B",
+    "OpenBuddy/openbuddy-llama2-13b-v11-bf16",
+    "StudentLLM/Alpagasus-2-13B-QLoRA-pipeline",
+    "meta-llama/Llama-2-13b-hf",
+    "WizardLM/WizardCoder-Python-34B-V1.0",
+    "dvruette/llama-13b-pretrained-sft-epoch-1",
+    "camel-ai/CAMEL-13B-Role-Playing-Data",
+    "ziqingyang/chinese-llama-2-13b",
+    "rombodawg/LosslessMegaCoder-llama2-7b-mini",
+    "TheBloke/koala-13B-HF",
+    "lmsys/vicuna-7b-delta-v1.1",
+    "eachadea/vicuna-7b-1.1",
+    "Ejafa/vicuna_7B_vanilla_1.1",
+    "lvkaokao/llama2-7b-hf-chat-lora",
+    "OpenBuddy/openbuddy-atom-13b-v9-bf16",
+    "Norquinal/llama-2-7b-claude-chat-rp",
+    "Danielbrdz/Barcenas-7b",
+    "heegyu/WizardVicuna2-13b-hf",
+    "meta-llama/Llama-2-7b-chat-hf",
+    "PeanutJar/LLaMa-2-PeanutButter_v14-7B",
+    "PeanutJar/LLaMa-2-PeanutButter_v4-7B",
+    "davzoku/cria-llama2-7b-v1.3",
+    "OpenBuddy/openbuddy-atom-13b-v9-bf16",
+    "lvkaokao/llama2-7b-hf-instruction-lora",
+    "Tap-M/Luna-AI-Llama2-Uncensored",
+    "ehartford/Samantha-1.11-7b",
+    "WizardLM/WizardCoder-Python-34B-V1.0",
+    "TheBloke/Manticore-13B-Chat-Pyg-Guanaco-SuperHOT-8K-GPTQ",
+    "Mikael110/llama-2-7b-guanaco-fp16",
+    "garage-bAInd/Platypus2-7B",
+    "PeanutJar/LLaMa-2-PeanutButter_v18_B-7B",
+    "mosaicml/mpt-30b",
+    "garage-bAInd/Platypus2-7B",
+    "huggingface/llama-13b",
+    "dvruette/oasst-llama-13b-1000-steps",
+    "jordiclive/gpt4all-alpaca-oa-codealpaca-lora-13b",
+    "huggyllama/llama-13b",
+    "Voicelab/trurl-2-7b",
+    "TFLai/llama-13b-4bit-alpaca",
+    "gywy/llama2-13b-chinese-v2",
+    "lmsys/longchat-13b-16k",
+    "Aspik101/trurl-2-7b-pl-instruct_unload",
+    "WizardLM/WizardMath-7B-V1.0",
+    "Norquinal/llama-2-7b-claude-chat",
+    "TheTravellingEngineer/llama2-7b-chat-hf-dpo",
+    "HuggingFaceH4/starchat-beta",
+    "joehuangx/spatial-vicuna-7b-v1.5-LoRA",
+    "conceptofmind/LLongMA-2-13b-16k",
+    "tianyil1/denas-llama2",
+    "lmsys/vicuna-7b-v1.3",
+    "conceptofmind/LLongMA-2-13b-16k",
+    "openchat/opencoderplus",
+    "ajibawa-2023/scarlett-7b",
+    "dhmeltzer/llama-7b-SFT_eli5_wiki65k_1024_r_64_alpha_16_merged",
+    "psyche/kollama2-7b-v2",
+    "heegyu/LIMA2-7b-hf",
+    "dhmeltzer/llama-7b-SFT-qlora-eli5-wiki_DPO_ds_RM_top_2_1024_r_64_alpha_16",
+    "abhishek/llama2guanacotest",
+    "jondurbin/airoboros-l2-7b-2.1",
+    "llama-anon/instruct-13b",
+    "FelixChao/vicuna-7B-physics",
+    "Aspik101/Llama-2-7b-hf-instruct-pl-lora_unload",
+    "shibing624/chinese-alpaca-plus-13b-hf",
+    "davzoku/cria-llama2-7b-v1.3_peft",
+    "quantumaikr/llama-2-7b-hf-guanaco-1k",
+    "togethercomputer/Llama-2-7B-32K-Instruct",
+    "sia-ai/llama-2-7b-1-percent-open-orca-1000-steps-v0",
+    "TheTravellingEngineer/llama2-7b-hf-guanaco",
+    "Lajonbot/Llama-2-7b-chat-hf-instruct-pl-lora_unload",
+    "jondurbin/airoboros-l2-7b-gpt4-1.4.1",
+    "wahaha1987/llama_7b_sharegpt94k_fastchat",
+    "FelixChao/vicuna-7B-chemical",
+    "TinyPixel/llama2-7b-oa",
+    "chaoyi-wu/MedLLaMA_13B",
+    "edor/Platypus2-mini-7B",
+    "RoversX/llama-2-7b-hf-small-shards-Samantha-V1-SFT",
+    "venkycs/llama-v2-7b-32kC-Security",
+    "psyche/kollama2-7b",
+    "Fredithefish/Guanaco-7B-Uncensored",
+    "TheTravellingEngineer/llama2-7b-chat-hf-guanaco",
+    "ehartford/WizardLM-13B-Uncensored",
+    "PocketDoc/Dans-CreepingSenseOfDoom",
+    "wenge-research/yayi-7b-llama2",
+    "georgesung/llama2_7b_chat_uncensored",
+    "TinyPixel/llama2-7b-instruct",
+    "quantumaikr/QuantumLM-7B",
+    "xzuyn/MedicWizard-7B",
+    "wenge-research/yayi-7b-llama2",
+    "TinyPixel/lima-test",
+    "elyza/ELYZA-japanese-Llama-2-7b-instruct",
+    "lgaalves/llama-2-7b-hf_open-platypus",
+    "ziqingyang/chinese-alpaca-2-7b",
+    "TehVenom/Pygmalion-Vicuna-1.1-7b",
+    "meta-llama/Llama-2-7b-hf",
+    "bongchoi/test-llama2-7b",
+    "TaylorAI/Flash-Llama-7B",
+    "TheTravellingEngineer/llama2-7b-chat-hf-v2",
+    "TheTravellingEngineer/llama2-7b-chat-hf-v4",
+    "kashif/stack-llama-2",
+    "PeanutJar/LLaMa-2-PeanutButter_v18_A-7B",
+    "ToolBench/ToolLLaMA-7b-LoRA",
+    "Monero/WizardLM-13b-OpenAssistant-Uncensored",
+    "TheTravellingEngineer/llama2-7b-chat-hf-v2",
+    "TheTravellingEngineer/llama2-7b-chat-hf-v4",
+    "mrm8488/llama-2-coder-7b",
+    "elyza/ELYZA-japanese-Llama-2-7b-fast-instruct",
+    "clibrain/Llama-2-7b-ft-instruct-es",
+    "medalpaca/medalpaca-7b",
+    "TheBloke/tulu-7B-fp16",
+    "OpenBuddy/openbuddy-openllama-13b-v7-fp16",
+    "TaylorAI/FLAN-Llama-7B-2_Llama2-7B-Flash_868_full_model",
+    "Aspik101/vicuna-7b-v1.3-instruct-pl-lora_unload",
+    "jondurbin/airoboros-l2-7b-gpt4-2.0",
+    "dhmeltzer/llama-7b-SFT_ds_eli5_1024_r_64_alpha_16_merged",
+    "GOAT-AI/GOAT-7B-Community",
+    "AtomEchoAI/AtomGPT_56k",
+    "julianweng/Llama-2-7b-chat-orcah",
+    "TehVenom/Pygmalion-13b-Merged",
+    "jondurbin/airoboros-7b-gpt4-1.1",
+    "dhmeltzer/llama-7b-SFT_ds_wiki65k_1024_r_64_alpha_16_merged",
+    "bofenghuang/vigogne-7b-chat",
+    "lmsys/longchat-7b-v1.5-32k",
+    "jondurbin/airoboros-l2-7b-gpt4-m2.0",
+    "synapsoft/Llama-2-7b-chat-hf-flan2022-1.2M",
+    "jondurbin/airoboros-7b-gpt4-1.4",
+    "Charlie911/vicuna-7b-v1.5-lora-mctaco",
+    "yihan6324/instructmining-platypus-15k",
+    "meta-llama/Llama-2-7b-hf",
+    "TheTravellingEngineer/llama2-7b-chat-hf-v3",
+    "quantumaikr/KoreanLM-hf",
+    "openthaigpt/openthaigpt-1.0.0-alpha-7b-chat-ckpt-hf",
+    "TheBloke/Llama-2-7B-GPTQ",
+    "TheBloke/Llama-2-7B-GPTQ",
+    "LLMs/AlpacaGPT4-7B-elina",
+    "ehartford/Wizard-Vicuna-7B-Uncensored",
+    "TheBloke/Wizard-Vicuna-7B-Uncensored-HF",
+    "TheTravellingEngineer/llama2-7b-chat-hf-v3",
+    "golaxy/gowizardlm",
+    "ehartford/dolphin-llama2-7b",
+    "CHIH-HUNG/llama-2-7b-dolphin_10w-test",
+    "mncai/chatdoctor",
+    "psyche/kollama2-7b-v3",
+    "jondurbin/airoboros-7b-gpt4",
+    "jondurbin/airoboros-7b",
+    "TheBloke/airoboros-7b-gpt4-fp16",
+    "mosaicml/mpt-7b-8k-chat",
+    "elyza/ELYZA-japanese-Llama-2-7b",
+    "bofenghuang/vigogne-7b-instruct",
+    "jxhong/CAlign-alpaca-7b",
+    "golaxy/goims",
+    "jondurbin/airoboros-7b-gpt4-1.2",
+    "jphme/orca_mini_v2_ger_7b",
+    "psmathur/orca_mini_v2_7b",
+    "notstoic/PygmalionCoT-7b",
+    "golaxy/gogpt2-13b",
+    "golaxy/gogpt2-13b-chat",
+    "togethercomputer/LLaMA-2-7B-32K",
+    "TheBloke/wizardLM-7B-HF",
+    "keyfan/vicuna-chinese-replication-v1.1",
+    "golaxy/gogpt2-7b",
+    "aiplanet/effi-7b",
+    "arver/llama7b-qlora",
+    "titan087/OpenLlama13B-Guanaco",
+    "chavinlo/alpaca-native",
+    "project-baize/baize-healthcare-lora-7B",
+    "AlpinDale/pygmalion-instruct",
+    "openlm-research/open_llama_13b",
+    "jondurbin/airoboros-7b-gpt4-1.3",
+    "elyza/ELYZA-japanese-Llama-2-7b-fast",
+    "jondurbin/airoboros-gpt-3.5-turbo-100k-7b",
+    "uukuguy/speechless-codellama-orca-13b",
+    "bigcode/starcoderplus",
+    "TheBloke/guanaco-7B-HF",
+    "Neko-Institute-of-Science/metharme-7b",
+    "TigerResearch/tigerbot-7b-base",
+    "golaxy/gogpt-7b",
+    "togethercomputer/LLaMA-2-7B-32K",
+    "yhyhy3/open_llama_7b_v2_med_instruct",
+    "ajibawa-2023/carl-7b",
+    "stabilityai/stablelm-base-alpha-7b-v2",
+    "conceptofmind/LLongMA-2-7b-16k",
+    "TehVenom/Pygmalion_AlpacaLora-7b",
+    "jondurbin/airoboros-7b-gpt4-1.4.1-qlora",
+    "wannaphong/openthaigpt-0.1.0-beta-full-model_for_open_llm_leaderboard",
+    "ausboss/llama7b-wizardlm-unfiltered",
+    "project-baize/baize-v2-7b",
+    "LMFlow/Robin-v2",
+    "HanningZhang/Robin-v2",
+    "LMFlow/Robin-7b-v2",
+    "OptimalScale/robin-7b-v2-delta",
+    "uukuguy/speechless-codellama-platypus-13b",
+    "jerryjalapeno/nart-100k-7b",
+    "wenge-research/yayi-13b-llama2",
+    "fireballoon/baichuan-vicuna-chinese-7b",
+    "jlevin/guanaco-unchained-llama-2-7b",
+    "csitfun/llama-7b-logicot",
+    "DevaMalla/llama7b_alpaca_1gpu_bf16",
+    "WeOpenML/PandaLM-Alpaca-7B-v1",
+    "illuin/test-custom-llama",
+    "yeontaek/WizardCoder-Python-13B-LoRa",
+    "ashercn97/giraffe-7b",
+    "mosaicml/mpt-7b-chat",
+    "abhishek/autotrain-llama-alpaca-peft-52508123785",
+    "Neko-Institute-of-Science/pygmalion-7b",
+    "TFLai/llama-7b-4bit-alpaca",
+    "huggingface/llama-7b",
+    "TheBloke/Planner-7B-fp16",
+    "shibing624/chinese-llama-plus-13b-hf",
+    "AGI-inc/lora_moe_7b_baseline",
+    "DevaMalla/llama-base-7b",
+    "AGI-inc/lora_moe_7b",
+    "togethercomputer/GPT-JT-6B-v0",
+    "ehartford/WizardLM-7B-Uncensored",
+    "shibing624/chinese-alpaca-plus-7b-hf",
+    "beomi/llama-2-ko-7b",
+    "mosaicml/mpt-7b-8k-instruct",
+    "Enno-Ai/ennodata-7b",
+    "mosaicml/mpt-7b-instruct",
+    "facebook/opt-iml-max-30b",
+    "WeOpenML/Alpaca-7B-v1",
+    "TheBloke/Project-Baize-v2-7B-GPTQ",
+    "codellama/CodeLlama-13b-Instruct-hf",
+    "TheBloke/CodeLlama-13B-Instruct-fp16",
+    "facebook/galactica-30b",
+    "FreedomIntelligence/phoenix-inst-chat-7b",
+    "openlm-research/open_llama_7b_v2",
+    "GeorgiaTechResearchInstitute/galpaca-30b",
+    "THUDM/chatglm2-6b",
+    "togethercomputer/GPT-JT-6B-v1",
+    "TheBloke/koala-7B-HF",
+    "nathan0/mpt_delta_tuned_model_v3",
+    "nathan0/mpt_delta_tuned_model_v2",
+    "GeorgiaTechResearchInstitute/galpaca-30b",
+    "JosephusCheung/Guanaco",
+    "shareAI/CodeLLaMA-chat-13b-Chinese",
+    "TigerResearch/tigerbot-7b-sft",
+    "Writer/InstructPalmyra-20b",
+    "OpenAssistant/codellama-13b-oasst-sft-v10",
+    "bigscience/bloomz-7b1-mt",
+    "nathan0/mpt_delta_tuned_model_v3",
+    "VMware/open-llama-7b-open-instruct",
+    "baichuan-inc/Baichuan-7B",
+    "anas-awadalla/mpt-7b",
+    "mosaicml/mpt-7b",
+    "bigscience/bloomz-7b1",
+    "ziqingyang/chinese-llama-2-7b",
+    "OpenAssistant/codellama-13b-oasst-sft-v10",
+    "wenge-research/yayi-7b",
+    "tiiuae/falcon-7b",
+    "togethercomputer/RedPajama-INCITE-Instruct-7B-v0.1",
+    "togethercomputer/RedPajama-INCITE-7B-Instruct",
+    "TheBloke/landmark-attention-llama7b-fp16",
+    "togethercomputer/GPT-JT-Moderation-6B",
+    "h2oai/h2ogpt-gm-oasst1-en-1024-20b",
+    "dvruette/gpt-neox-20b-full-precision",
+    "TehVenom/Moderator-Chan_GPT-JT-6b",
+    "dvruette/oasst-gpt-neox-20b-1000-steps",
+    "AlekseyKorshuk/pygmalion-6b-vicuna-chatml",
+    "facebook/opt-66b",
+    "Salesforce/codegen-16B-nl",
+    "Vmware/open-llama-7b-v2-open-instruct",
+    "mosaicml/mpt-7b-storywriter",
+    "acrastt/Marx-3B-V2",
+    "openlm-research/open_llama_7b",
+    "Fredithefish/ReasonixPajama-3B-HF",
+    "togethercomputer/GPT-NeoXT-Chat-Base-20B",
+    "psmathur/orca_mini_13b",
+    "RWKV/rwkv-raven-14b",
+    "h2oai/h2ogpt-oasst1-512-20b",
+    "acrastt/Marx-3B",
+    "klosax/open_llama_13b_600bt_preview",
+    "synapsoft/Llama-2-7b-hf-flan2022-1.2M",
+    "OpenAssistant/oasst-sft-1-pythia-12b",
+    "golaxy/gogpt-7b-bloom",
+    "Writer/palmyra-large",
+    "psmathur/orca_mini_7b",
+    "dvruette/oasst-pythia-12b-6000-steps",
+    "NousResearch/CodeLlama-13b-hf",
+    "codellama/CodeLlama-13b-hf",
+    "h2oai/h2ogpt-gm-oasst1-multilang-1024-20b",
+    "VMware/open-llama-0.7T-7B-open-instruct-v1.1",
+    "dvruette/oasst-pythia-12b-flash-attn-5000-steps",
+    "dvruette/oasst-gpt-neox-20b-3000-steps",
+    "RobbeD/OpenLlama-Platypus-3B",
+    "facebook/opt-30b",
+    "acrastt/Puma-3B",
+    "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5",
+    "dvruette/oasst-pythia-12b-pretrained-sft",
+    "digitous/GPT-R",
+    "acrastt/Griffin-3B",
+    "togethercomputer/RedPajama-INCITE-Base-7B-v0.1",
+    "togethercomputer/RedPajama-INCITE-7B-Base",
+    "CobraMamba/mamba-gpt-3b-v3",
+    "Danielbrdz/CodeBarcenas-7b",
+    "l3utterfly/open-llama-3b-v2-layla",
+    "CobraMamba/mamba-gpt-3b-v2",
+    "OpenAssistant/pythia-12b-sft-v8-7k-steps",
+    "KoboldAI/GPT-NeoX-20B-Erebus",
+    "RobbeD/Orca-Platypus-3B",
+    "h2oai/h2ogpt-gm-oasst1-en-1024-12b",
+    "OpenAssistant/pythia-12b-sft-v8-2.5k-steps",
+    "AlekseyKorshuk/chatml-pyg-v1",
+    "togethercomputer/RedPajama-INCITE-Chat-7B-v0.1",
+    "togethercomputer/RedPajama-INCITE-7B-Chat",
+    "digitous/Javelin-R",
+    "dvruette/oasst-pythia-12b-reference",
+    "EleutherAI/gpt-neox-20b",
+    "KoboldAI/fairseq-dense-13B",
+    "OpenAssistant/pythia-12b-sft-v8-rlhf-2k-steps",
+    "codellama/CodeLlama-7b-Instruct-hf",
+    "digitous/Javelin-GPTJ",
+    "KoboldAI/GPT-NeoX-20B-Skein",
+    "digitous/Javalion-R",
+    "h2oai/h2ogpt-oasst1-512-12b",
+    "acrastt/Bean-3B",
+    "KoboldAI/GPT-J-6B-Skein",
+    "nomic-ai/gpt4all-j",
+    "databricks/dolly-v2-12b",
+    "TehVenom/Dolly_Shygmalion-6b-Dev_V8P2",
+    "databricks/dolly-v2-7b",
+    "Aspik101/WizardVicuna-Uncensored-3B-instruct-PL-lora_unload",
+    "digitous/Adventien-GPTJ",
+    "openlm-research/open_llama_3b_v2",
+    "RWKV/rwkv-4-14b-pile",
+    "Lazycuber/Janemalion-6B",
+    "OpenAssistant/pythia-12b-pre-v8-12.5k-steps",
+    "digitous/Janin-R",
+    "kfkas/Llama-2-ko-7b-Chat",
+    "heegyu/WizardVicuna-Uncensored-3B-0719",
+    "h2oai/h2ogpt-gm-oasst1-en-1024-open-llama-7b-preview-400bt",
+    "TaylorAI/Flash-Llama-3B",
+    "kfkas/Llama-2-ko-7b-Chat",
+    "digitous/Skegma-GPTJ",
+    "digitous/Javalion-GPTJ",
+    "Pirr/pythia-13b-deduped-green_devil",
+    "TehVenom/PPO_Shygmalion-V8p4_Dev-6b",
+    "dvruette/oasst-pythia-6.9b-4000-steps",
+    "heegyu/WizardVicuna-3B-0719",
+    "psmathur/orca_mini_3b",
+    "OpenAssistant/galactica-6.7b-finetuned",
+    "frank098/orca_mini_3b_juniper",
+    "PygmalionAI/pygmalion-6b",
+    "TehVenom/PPO_Pygway-V8p4_Dev-6b",
+    "TFLai/gpt-neox-20b-4bit-alpaca",
+    "Corianas/gpt-j-6B-Dolly",
+    "TehVenom/Dolly_Shygmalion-6b",
+    "digitous/Janin-GPTJ",
+    "TehVenom/GPT-J-Pyg_PPO-6B-Dev-V8p4",
+    "EleutherAI/gpt-j-6b",
+    "KoboldAI/GPT-J-6B-Shinen",
+    "TehVenom/Dolly_Malion-6b",
+    "TehVenom/ChanMalion",
+    "Salesforce/codegen-6B-nl",
+    "Fredithefish/RedPajama-INCITE-Chat-3B-Instruction-Tuning-with-GPT-4",
+    "KoboldAI/GPT-J-6B-Janeway",
+    "togethercomputer/RedPajama-INCITE-Chat-3B-v1",
+    "togethercomputer/Pythia-Chat-Base-7B",
+    "heegyu/RedTulu-Uncensored-3B-0719",
+    "KoboldAI/PPO_Pygway-6b-Mix",
+    "KoboldAI/OPT-13B-Erebus",
+    "KoboldAI/fairseq-dense-6.7B",
+    "EleutherAI/pythia-12b-deduped",
+    "pszemraj/pythia-6.9b-HC3",
+    "Fredithefish/Guanaco-3B-Uncensored-v2",
+    "facebook/opt-13b",
+    "TehVenom/GPT-J-Pyg_PPO-6B",
+    "EleutherAI/pythia-6.9b-deduped",
+    "Devio/test-1400",
+    "Fredithefish/Guanaco-3B-Uncensored",
+    "codellama/CodeLlama-7b-hf",
+    "acrastt/RedPajama-INCITE-Chat-Instruct-3B-V1",
+    "Fredithefish/ScarletPajama-3B-HF",
+    "KoboldAI/OPT-13B-Nerybus-Mix",
+    "YeungNLP/firefly-bloom-7b1",
+    "DanielSc4/RedPajama-INCITE-Chat-3B-v1-RL-LoRA-8bit-test1",
+    "klosax/open_llama_7b_400bt_preview",
+    "KoboldAI/OPT-13B-Nerys-v2",
+    "TehVenom/PPO_Shygmalion-6b",
+    "amazon/LightGPT",
+    "KnutJaegersberg/black_goo_recipe_c",
+    "NousResearch/CodeLlama-7b-hf",
+    "togethercomputer/RedPajama-INCITE-Instruct-3B-v1",
+    "heegyu/WizardVicuna-open-llama-3b-v2",
+    "bigscience/bloom-7b1",
+    "Devio/test-22B",
+    "RWKV/rwkv-raven-7b",
+    "hakurei/instruct-12b",
+    "CobraMamba/mamba-gpt-3b",
+    "KnutJaegersberg/black_goo_recipe_a",
+    "acrastt/OmegLLaMA-3B",
+    "codellama/CodeLlama-7b-Instruct-hf",
+    "h2oai/h2ogpt-oig-oasst1-512-6_9b",
+    "KoboldAI/OPT-6.7B-Erebus",
+    "facebook/opt-6.7b",
+    "KnutJaegersberg/black_goo_recipe_d",
+    "KnutJaegersberg/LLongMA-3b-LIMA",
+    "KnutJaegersberg/black_goo_recipe_b",
+    "KoboldAI/OPT-6.7B-Nerybus-Mix",
+    "health360/Healix-3B",
+    "EleutherAI/pythia-12b",
+    "Fredithefish/RedPajama-INCITE-Chat-3B-ShareGPT-11K",
+    "GeorgiaTechResearchInstitute/galactica-6.7b-evol-instruct-70k",
+    "h2oai/h2ogpt-oig-oasst1-256-6_9b",
+    "ikala/bloom-zh-3b-chat",
+    "Taekyoon/llama2-ko-7b-test",
+    "anhnv125/pygmalion-6b-roleplay",
+    "TehVenom/DiffMerge_Pygmalion_Main-onto-V8P4",
+    "KoboldAI/OPT-6B-nerys-v2",
+    "Lazycuber/pyg-instruct-wizardlm",
+    "Devio/testC",
+    "KoboldAI/OPT-30B-Erebus",
+    "Fredithefish/CrimsonPajama",
+    "togethercomputer/RedPajama-INCITE-Base-3B-v1",
+    "bigscience/bloomz-3b",
+    "conceptofmind/Open-LLongMA-3b",
+    "RWKV/rwkv-4-7b-pile",
+    "openlm-research/open_llama_3b",
+    "ewof/koishi-instruct-3b",
+    "DanielSc4/RedPajama-INCITE-Chat-3B-v1-FT-LoRA-8bit-test1",
+    "cerebras/Cerebras-GPT-13B",
+    "EleutherAI/pythia-6.7b",
+    "aisquared/chopt-2_7b",
+    "Azure99/blossom-v1-3b",
+    "PSanni/Deer-3b",
+    "bertin-project/bertin-gpt-j-6B-alpaca",
+    "OpenBuddy/openbuddy-openllama-3b-v10-bf16",
+    "KoboldAI/fairseq-dense-2.7B",
+    "ehartford/CodeLlama-34b-Instruct-hf",
+    "codellama/CodeLlama-34b-Instruct-hf",
+    "TheBloke/CodeLlama-34B-Instruct-fp16",
+    "h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt-v2",
+    "openlm-research/open_llama_7b_700bt_preview",
+    "NbAiLab/nb-gpt-j-6B-alpaca",
+    "KoboldAI/OPT-2.7B-Erebus",
+    "Writer/camel-5b-hf",
+    "EleutherAI/pythia-2.7b",
+    "facebook/xglm-7.5B",
+    "EleutherAI/pythia-2.8b-deduped",
+    "klosax/open_llama_3b_350bt_preview",
+    "klosax/openllama-3b-350bt",
+    "KoboldAI/OPT-2.7B-Nerybus-Mix",
+    "KoboldAI/GPT-J-6B-Adventure",
+    "cerebras/Cerebras-GPT-6.7B",
+    "TFLai/pythia-2.8b-4bit-alpaca",
+    "facebook/opt-2.7b",
+    "KoboldAI/OPT-2.7B-Nerys-v2",
+    "bigscience/bloom-3b",
+    "Devio/test100",
+    "RWKV/rwkv-raven-3b",
+    "Azure99/blossom-v2-3b",
+    "codellama/CodeLlama-34b-Python-hf",
+    "bhenrym14/airoboros-33b-gpt4-1.4.1-PI-8192-fp16",
+    "EleutherAI/gpt-neo-2.7B",
+    "danielhanchen/open_llama_3b_600bt_preview",
+    "HuggingFaceH4/starchat-alpha",
+    "pythainlp/wangchanglm-7.5B-sft-en-sharded",
+    "beaugogh/pythia-1.4b-deduped-sharegpt",
+    "HWERI/pythia-1.4b-deduped-sharegpt",
+    "OpenAssistant/stablelm-7b-sft-v7-epoch-3",
+    "codellama/CodeLlama-7b-Python-hf",
+    "aisquared/chopt-1_3b",
+    "PygmalionAI/metharme-1.3b",
+    "Linly-AI/Chinese-LLaMA-2-13B-hf",
+    "chargoddard/llama-2-34b-uncode",
+    "RWKV/rwkv-4-3b-pile",
+    "pythainlp/wangchanglm-7.5B-sft-enth",
+    "MBZUAI/LaMini-GPT-1.5B",
+    "Writer/palmyra-base",
+    "KoboldAI/fairseq-dense-1.3B",
+    "EleutherAI/pythia-1.4b-deduped",
+    "MBZUAI/lamini-neo-1.3b",
+    "h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt",
+    "sartmis1/starcoder-finetune-openapi",
+    "MayaPH/opt-flan-iml-6.7b",
+    "facebook/xglm-4.5B",
+    "WizardLM/WizardCoder-15B-V1.0",
+    "facebook/opt-iml-max-1.3b",
+    "stabilityai/stablelm-tuned-alpha-7b",
+    "aisquared/dlite-v2-1_5b",
+    "stabilityai/stablelm-base-alpha-7b",
+    "sartmis1/starcoder-finetune-selfinstruct",
+    "lizhuang144/starcoder_mirror",
+    "bigcode/starcoder",
+    "TheBloke/CodeLlama-34B-Python-fp16",
+    "open-llm-leaderboard/bloomz-1b7-4bit-alpaca-auto-eval-adapter-applied",
+    "ehartford/CodeLlama-34b-Python-hf",
+    "codellama/CodeLlama-7b-Python-hf",
+    "GeorgiaTechResearchInstitute/starcoder-gpteacher-code-instruct",
+    "LoupGarou/WizardCoder-Guanaco-15B-V1.0",
+    "golaxy/gogpt-3b-bloom",
+    "EleutherAI/pythia-1.3b",
+    "codellama/CodeLlama-13b-Python-hf",
+    "hakurei/lotus-12B",
+    "NYTK/PULI-GPTrio",
+    "facebook/opt-1.3b",
+    "TheBloke/CodeLlama-13B-Python-fp16",
+    "codellama/CodeLlama-13b-Python-hf",
+    "RWKV/rwkv-raven-1b5",
+    "PygmalionAI/pygmalion-2.7b",
+    "bigscience/bloom-1b7",
+    "gpt2-xl",
+    "LoupGarou/WizardCoder-Guanaco-15B-V1.1",
+    "RWKV/rwkv-4-1b5-pile",
+    "codellama/CodeLlama-34b-hf",
+    "NousResearch/CodeLlama-34b-hf",
+    "rinna/bilingual-gpt-neox-4b-8k",
+    "lxe/Cerebras-GPT-2.7B-Alpaca-SP",
+    "cerebras/Cerebras-GPT-2.7B",
+    "jzjiao/opt-1.3b-rlhf",
+    "EleutherAI/gpt-neo-1.3B",
+    "aisquared/dlite-v1-1_5b",
+    "Corianas/Quokka_2.7b",
+    "MrNJK/gpt2-xl-sft",
+    "facebook/galactica-1.3b",
+    "aisquared/dlite-v2-774m",
+    "EleutherAI/pythia-1b-deduped",
+    "Kunhao/pile-7b-250b-tokens",
+    "w601sxs/b1ade-1b",
+    "rinna/bilingual-gpt-neox-4b",
+    "shaohang/SparseOPT-1.3B",
+    "shaohang/Sparse0.5_OPT-1.3",
+    "EleutherAI/polyglot-ko-12.8b",
+    "Salesforce/codegen-6B-multi",
+    "bigscience/bloom-1b1",
+    "TFLai/gpt-neo-1.3B-4bit-alpaca",
+    "FabbriSimo01/Bloom_1b_Quantized",
+    "MBZUAI/LaMini-GPT-774M",
+    "Locutusque/gpt2-large-conversational",
+    "Devio/test-3b",
+    "stabilityai/stablelm-tuned-alpha-3b",
+    "PygmalionAI/pygmalion-1.3b",
+    "KoboldAI/fairseq-dense-355M",
+    "Rachneet/gpt2-xl-alpaca",
+    "gpt2-large",
+    "Mikivis/gpt2-large-lora-sft",
+    "stabilityai/stablelm-base-alpha-3b",
+    "gpt2-medium",
+    "Kunhao/pile-7b",
+    "aisquared/dlite-v1-774m",
+    "aisquared/dlite-v2-355m",
+    "YeungNLP/firefly-bloom-2b6-v2",
+    "KnutJaegersberg/gpt-2-xl-EvolInstruct",
+    "KnutJaegersberg/galactica-orca-wizardlm-1.3b",
+    "cerebras/Cerebras-GPT-1.3B",
+    "FabbriSimo01/Cerebras_1.3b_Quantized",
+    "facebook/xglm-1.7B",
+    "EleutherAI/pythia-410m-deduped",
+    "TheBloke/GPlatty-30B-SuperHOT-8K-fp16",
+    "DataLinguistic/DataLinguistic-34B-V1.0",
+    "Corianas/Quokka_1.3b",
+    "TheTravellingEngineer/bloom-560m-RLHF-v2",
+    "Corianas/1.3b",
+    "RWKV/rwkv-4-430m-pile",
+    "porkorbeef/Llama-2-13b-sf",
+    "xhyi/PT_GPTNEO350_ATG",
+    "TheBloke/Wizard-Vicuna-13B-Uncensored-GPTQ",
+    "bigscience/bloomz-560m",
+    "TheBloke/medalpaca-13B-GPTQ-4bit",
+    "TheBloke/Vicuna-33B-1-3-SuperHOT-8K-fp16",
+    "aisquared/dlite-v1-355m",
+    "uukuguy/speechless-codellama-orca-airoboros-13b-0.10e",
+    "yhyhy3/med-orca-instruct-33b",
+    "TheBloke/Wizard-Vicuna-30B-Superhot-8K-fp16",
+    "TheTravellingEngineer/bloom-1b1-RLHF",
+    "MBZUAI/lamini-cerebras-1.3b",
+    "IDEA-CCNL/Ziya-LLaMA-13B-Pretrain-v1",
+    "TheBloke/WizardLM-7B-uncensored-GPTQ",
+    "TheBloke/EverythingLM-13B-16K-GPTQ",
+    "quantumaikr/open_llama_7b_hf",
+    "TheBloke/chronos-wizardlm-uc-scot-st-13B-GPTQ",
+    "TheBloke/WizardLM-30B-Uncensored-GPTQ",
+    "IDEA-CCNL/Ziya-LLaMA-13B-v1",
+    "Phind/Phind-CodeLlama-34B-v1",
+    "robowaifudev/megatron-gpt2-345m",
+    "MayaPH/GodziLLa-30B-instruct",
+    "TheBloke/CAMEL-33B-Combined-Data-SuperHOT-8K-fp16",
+    "uukuguy/speechless-codellama-orca-platypus-13b-0.10e",
+    "doas/test2",
+    "BreadAi/PM_modelV2",
+    "bigcode/santacoder",
+    "TheBloke/wizard-vicuna-13B-GPTQ",
+    "porkorbeef/Llama-2-13b",
+    "TehVenom/DiffMerge-DollyGPT-Pygmalion",
+    "PygmalionAI/pygmalion-350m",
+    "TheBloke/orca_mini_v3_7B-GPTQ",
+    "TheBloke/WizardLM-Uncensored-SuperCOT-StoryTelling-30B-GPTQ",
+    "TheBloke/WizardLM-30B-GPTQ",
+    "bigscience/bloom-560m",
+    "TFLai/gpt2-turkish-uncased",
+    "TheBloke/guanaco-33B-GPTQ",
+    "TheBloke/openchat_v2_openorca_preview-GPTQ",
+    "porkorbeef/Llama-2-13b-public",
+    "TheBloke/LongChat-13B-GPTQ",
+    "yhyhy3/med-orca-instruct-33b",
+    "TheBloke/airoboros-33B-gpt4-1-4-SuperHOT-8K-fp16",
+    "TheBloke/Chinese-Alpaca-33B-SuperHOT-8K-fp16",
+    "MayaPH/FinOPT-Franklin",
+    "TheBloke/WizardLM-33B-V1.0-Uncensored-GPTQ",
+    "TheBloke/Project-Baize-v2-13B-GPTQ",
+    "malhajar/Platypus2-70B-instruct-4bit-gptq",
+    "KoboldAI/OPT-350M-Erebus",
+    "rishiraj/bloom-560m-guanaco",
+    "Panchovix/WizardLM-33B-V1.0-Uncensored-SuperHOT-8k",
+    "doas/test5",
+    "vicgalle/alpaca-7b",
+    "beomi/KoAlpaca-Polyglot-5.8B",
+    "Phind/Phind-CodeLlama-34B-Python-v1",
+    "timdettmers/guanaco-65b-merged",
+    "TheBloke/wizard-mega-13B-GPTQ",
+    "MayaPH/GodziLLa-30B-plus",
+    "TheBloke/Platypus-30B-SuperHOT-8K-fp16",
+    "facebook/opt-350m",
+    "KoboldAI/OPT-350M-Nerys-v2",
+    "TheBloke/robin-33B-v2-GPTQ",
+    "jaspercatapang/Echidna-30B",
+    "TheBloke/llama-30b-supercot-SuperHOT-8K-fp16",
+    "marcchew/test1",
+    "Harshvir/LaMini-Neo-1.3B-Mental-Health_lora",
+    "golaxy/gogpt-560m",
+    "TheBloke/orca_mini_13B-GPTQ",
+    "Panchovix/airoboros-33b-gpt4-1.2-SuperHOT-8k",
+    "Aspik101/tulu-7b-instruct-pl-lora_unload",
+    "Phind/Phind-CodeLlama-34B-v2",
+    "BreadAi/MusePy-1-2",
+    "cerebras/Cerebras-GPT-590M",
+    "microsoft/CodeGPT-small-py",
+    "victor123/WizardLM-13B-1.0",
+    "OptimalScale/robin-65b-v2-delta",
+    "voidful/changpt-bart",
+    "FabbriSimo01/GPT_Large_Quantized",
+    "MayaPH/FinOPT-Lincoln",
+    "KoboldAI/fairseq-dense-125M",
+    "SebastianSchramm/Cerebras-GPT-111M-instruction",
+    "TheTravellingEngineer/bloom-560m-RLHF",
+    "breadlicker45/dough-instruct-base-001",
+    "WizardLM/WizardLM-30B-V1.0",
+    "WizardLM/WizardLM-30B-V1.0",
+    "WizardLM/WizardLM-30B-V1.0",
+    "TaylorAI/Flash-Llama-30M-20001",
+    "porkorbeef/Llama-2-13b-12_153950",
+    "huggingtweets/bladeecity-jerma985",
+    "KnutJaegersberg/megatron-GPT-2-345m-EvolInstruct",
+    "bhenrym14/airoboros-33b-gpt4-1.4.1-lxctx-PI-16384-fp16",
+    "microsoft/DialoGPT-small",
+    "Corianas/590m",
+    "facebook/xglm-564M",
+    "EleutherAI/gpt-neo-125m",
+    "EleutherAI/pythia-160m-deduped",
+    "klosax/pythia-160m-deduped-step92k-193bt",
+    "MBZUAI/lamini-neo-125m",
+    "bigcode/tiny_starcoder_py",
+    "concedo/OPT-19M-ChatSalad",
+    "anton-l/gpt-j-tiny-random",
+    "grantprice/Cerebras-GPT-590M-finetuned-DND",
+    "deepnight-research/zsc-text",
+    "WangZeJun/bloom-820m-chat",
+    "cerebras/Cerebras-GPT-256M",
+    "ai-forever/rugpt3large_based_on_gpt2",
+    "alibidaran/medical_transcription_generator",
+    "Deci/DeciCoder-1b",
+    "microsoft/DialoGPT-medium",
+    "ogimgio/gpt-neo-125m-neurallinguisticpioneers",
+    "open-llm-leaderboard/bloom-560m-4bit-alpaca-auto-eval-adapter-applied",
+    "BreadAi/gpt-YA-1-1_160M",
+    "microsoft/DialoGPT-large",
+    "facebook/opt-125m",
+    "huggingtweets/jerma985",
+    "Locutusque/gpt2-conversational-or-qa",
+    "concedo/Pythia-70M-ChatSalad",
+    "roneneldan/TinyStories-1M",
+    "BreadAi/DiscordPy",
+    "bigcode/gpt_bigcode-santacoder",
+    "Tincando/fiction_story_generator",
+    "klosax/pythia-70m-deduped-step44k-92bt",
+    "Quake24/easyTermsSummerizer",
+    "BreadAi/gpt-YA-1-1_70M",
+    "EleutherAI/pythia-160m",
+    "euclaise/gpt-neox-122m-minipile-digits",
+    "MBZUAI/lamini-cerebras-590m",
+    "nicholasKluge/Aira-124M",
+    "MayaPH/FinOPT-Washington",
+    "cyberagent/open-calm-large",
+    "BreadAi/StoryPy",
+    "EleutherAI/pythia-70m",
+    "BreadAi/gpt-Youtube",
+    "roneneldan/TinyStories-33M",
+    "EleutherAI/pythia-70m-deduped",
+    "lgaalves/gpt2_guanaco-dolly-platypus",
+    "Corianas/Quokka_590m",
+    "lgaalves/gpt2_platypus-dolly-guanaco",
+    "cyberagent/open-calm-7b",
+    "RWKV/rwkv-4-169m-pile",
+    "gpt2",
+    "roneneldan/TinyStories-28M",
+    "lgaalves/gpt2_open-platypus",
+    "gpt2",
+    "SaylorTwift/gpt2_test",
+    "roneneldan/TinyStories-3M",
+    "nthngdy/pythia-owt2-70m-50k",
+    "Corianas/256_5epoch",
+    "roneneldan/TinyStories-8M",
+    "lgaalves/gpt2-dolly",
+    "nthngdy/pythia-owt2-70m-100k",
+    "aisquared/dlite-v2-124m",
+    "mncai/SGPT-1.3B-insurance-epoch10",
+    "huggingtweets/gladosystem",
+    "abhiramtirumala/DialoGPT-sarcastic-medium",
+    "MBZUAI/lamini-cerebras-256m",
+    "cerebras/Cerebras-GPT-111M",
+    "uberkie/metharme-1.3b-finetuned",
+    "MBZUAI/lamini-cerebras-111m",
+    "psyche/kogpt",
+    "Corianas/Quokka_256m",
+    "vicgalle/gpt2-alpaca-gpt4",
+    "aisquared/dlite-v1-124m",
+    "Mikivis/xuanxuan",
+    "MBZUAI/LaMini-GPT-124M",
+    "vicgalle/gpt2-alpaca",
+    "huashiyiqike/testmodel",
+    "Corianas/111m",
+    "baseline",
+]

src/tools/plots.py

@@ -0,0 +1,155 @@
+import pandas as pd
+import numpy as np
+import plotly.express as px
+from plotly.graph_objs import Figure
+
+from src.leaderboard.filter_models import FLAGGED_MODELS
+from src.display.utils import human_baseline_row as HUMAN_BASELINE, AutoEvalColumn, BENCHMARK_COLS
+from src.about import Tasks, Task
+from src.leaderboard.read_evals import EvalResult
+
+
+
+def create_scores_df(raw_data: list[EvalResult]) -> pd.DataFrame:
+    """
+    Generates a DataFrame containing the maximum scores until each date.
+
+    :param results_df: A DataFrame containing result information including metric scores and dates.
+    :return: A new DataFrame containing the maximum scores until each date for every metric.
+    """
+    # Step 1: Ensure 'date' is in datetime format and sort the DataFrame by it
+    results_df = pd.DataFrame(raw_data)
+    #results_df["date"] = pd.to_datetime(results_df["date"], format="mixed", utc=True)
+    results_df.sort_values(by="date", inplace=True)
+
+    # Step 2: Initialize the scores dictionary
+    scores = {k: [] for k in BENCHMARK_COLS + [AutoEvalColumn.average.name]}
+
+    # Step 3: Iterate over the rows of the DataFrame and update the scores dictionary
+    for task in [t.value for t in Tasks] + [Task("Average", "avg", AutoEvalColumn.average.name)]:
+        current_max = 0
+        last_date = ""
+        column = task.col_name
+        for _, row in results_df.iterrows():
+            current_model = row["full_model"]
+            if current_model in FLAGGED_MODELS:
+                continue
+
+            current_date = row["date"]
+            if task.benchmark == "Average":
+                current_score = np.mean(list(row["results"].values()))
+            else:
+                current_score = row["results"][task.benchmark]
+
+            if current_score > current_max:
+                if current_date == last_date and len(scores[column]) > 0:
+                    scores[column][-1] = {"model": current_model, "date": current_date, "score": current_score}
+                else:
+                    scores[column].append({"model": current_model, "date": current_date, "score": current_score})
+                current_max = current_score
+                last_date = current_date
+
+    # Step 4: Return all dictionaries as DataFrames
+    return {k: pd.DataFrame(v) for k, v in scores.items()}
+
+
+def create_plot_df(scores_df: dict[str: pd.DataFrame]) -> pd.DataFrame:
+    """
+    Transforms the scores DataFrame into a new format suitable for plotting.
+
+    :param scores_df: A DataFrame containing metric scores and dates.
+    :return: A new DataFrame reshaped for plotting purposes.
+    """
+    # Initialize the list to store DataFrames
+    dfs = []
+
+    # Iterate over the cols and create a new DataFrame for each column
+    for col in BENCHMARK_COLS + [AutoEvalColumn.average.name]:
+        d = scores_df[col].reset_index(drop=True)
+        d["task"] = col
+        dfs.append(d)
+
+    # Concatenate all the created DataFrames
+    concat_df = pd.concat(dfs, ignore_index=True)
+
+    # Sort values by 'date'
+    concat_df.sort_values(by="date", inplace=True)
+    concat_df.reset_index(drop=True, inplace=True)
+    return concat_df
+
+
+def create_metric_plot_obj(
+    df: pd.DataFrame, metrics: list[str], title: str
+) -> Figure:
+    """
+    Create a Plotly figure object with lines representing different metrics
+    and horizontal dotted lines representing human baselines.
+
+    :param df: The DataFrame containing the metric values, names, and dates.
+    :param metrics: A list of strings representing the names of the metrics
+                    to be included in the plot.
+    :param title: A string representing the title of the plot.
+    :return: A Plotly figure object with lines representing metrics and
+             horizontal dotted lines representing human baselines.
+    """
+
+    # Filter the DataFrame based on the specified metrics
+    df = df[df["task"].isin(metrics)]
+
+    # Filter the human baselines based on the specified metrics
+    filtered_human_baselines = {k: v for k, v in HUMAN_BASELINE.items() if k in metrics}
+
+    # Create a line figure using plotly express with specified markers and custom data
+    fig = px.line(
+        df,
+        x="date",
+        y="score",
+        color="task",
+        markers=True,
+        custom_data=["task", "score", "model"],
+        title=title,
+    )
+
+    # Update hovertemplate for better hover interaction experience
+    fig.update_traces(
+        hovertemplate="<br>".join(
+            [
+                "Model Name: %{customdata[2]}",
+                "Metric Name: %{customdata[0]}",
+                "Date: %{x}",
+                "Metric Value: %{y}",
+            ]
+        )
+    )
+
+    # Update the range of the y-axis
+    fig.update_layout(yaxis_range=[0, 100])
+
+    # Create a dictionary to hold the color mapping for each metric
+    metric_color_mapping = {}
+
+    # Map each metric name to its color in the figure
+    for trace in fig.data:
+        metric_color_mapping[trace.name] = trace.line.color
+
+    # Iterate over filtered human baselines and add horizontal lines to the figure
+    #for metric, value in filtered_human_baselines.items():
+    #    color = metric_color_mapping.get(metric, "blue")  # Retrieve color from mapping; default to blue if not found
+    #    location = "top left" if metric == "HellaSwag" else "bottom left"  # Set annotation position
+    #    # Add horizontal line with matched color and positioned annotation
+    #    fig.add_hline(
+    #        y=value,
+    #        line_dash="dot",
+    #        annotation_text=f"{metric} human baseline",
+    #        annotation_position=location,
+    #        annotation_font_size=10,
+    #        annotation_font_color=color,
+    #        line_color=color,
+    #    )
+
+    return fig
+
+
+# Example Usage:
+# human_baselines dictionary is defined.
+# chart = create_metric_plot_obj(scores_df, ["ARC", "HellaSwag", "MMLU", "TruthfulQA"], human_baselines, "Graph Title")