Add Magistral and Qwen3-Coder hehe

results/aggregated_scores_icarus.csv

@@ -1,29 +1,31 @@
 Model,Agg S2R,Agg MC,Agg VerilogEval S2R,Agg VerilogEval MC,Agg RTLLM,Agg VeriGen
-DeepSeek R1-0528,76.79,77.79,78.84,79.65,70.04,71.64
-DeepSeek R1,75.53,76.24,77.67,77.55,68.49,71.92
-Llama 3.1 405B,53.23,55.25,56.55,54.35,42.26,58.23
-Qwen3 235B A22B,69.16,66.42,74.83,68.36,50.48,60.01
-Qwen2.5 72B,49.36,49.97,50.22,50.74,46.51,47.42
-Llama 3.(1-3) 70B,39.48,46.29,39.47,40.83,39.53,64.29
-QwQ 32B,62.6,41.92,65.02,38.68,54.6,52.6
-Qwen2.5 32B,50.39,41.94,50.86,41.01,48.86,45.0
-StarChat2 15B v0.1,38.76,41.72,36.68,35.58,45.61,61.96
-DeepSeek R1 Distill Qwen 14B,23.14,24.94,24.94,24.3,17.22,27.07
-CodeLlama 70B,33.04,35.59,32.2,32.27,35.81,46.56
-DeepSeek Coder 33B,27.03,39.59,22.65,37.64,41.47,46.01
-QwenCoder 2.5 32B,44.02,46.49,43.68,44.05,45.15,54.52
-DeepCoder 14B,26.4,32.92,27.32,33.18,23.38,32.07
-QwenCoder 2.5 14B,37.69,41.98,35.32,40.26,45.5,47.64
-SeedCoder 8B,50.89,36.79,51.71,36.52,48.2,37.66
-SeedCoder 8B Reasoning,43.75,49.83,47.85,46.76,30.22,59.97
-OpenCoder 8B,30.06,36.95,26.8,34.57,40.83,44.8
-QwenCoder 2.5 7B,14.15,34.5,6.57,33.0,39.16,39.46
-DeepSeek Coder 6.7B,31.87,30.9,28.88,28.99,41.75,37.19
-CodeV R1 Distill Qwen 7B,36.12,30.66,32.35,20.56,48.57,63.98
-HaVen-CodeQwen,43.58,50.68,44.67,47.23,39.98,62.04
-CodeV-QW-7B,20.37,52.85,18.82,50.64,25.48,60.14
-RTLCoder Mistral,21.82,32.19,23.71,26.34,15.58,51.5
-RTLCoder DeepSeek,37.22,40.19,38.33,36.13,33.58,53.57
-OriGen,52.88,55.97,53.0,50.0,52.47,75.66
-CodeV-CL-7B,14.73,35.1,12.71,30.93,21.38,48.85
-CodeV-DS-6.7B,19.62,49.29,14.8,46.08,35.52,59.87
+DeepSeek R1-0528,76.79,78.86,78.84,79.65,70.04,71.64
+DeepSeek R1,75.53,76.99,77.67,77.55,68.49,71.92
+Qwen3 Coder 480B A35B,60.56,57.84,62.98,56.13,52.56,73.45
+Llama 3.1 405B,53.23,54.73,56.55,54.35,42.26,58.23
+Qwen3 236B A22B,69.16,67.53,74.83,68.36,50.48,60.01
+Qwen2.5 72B,49.36,50.41,50.22,50.74,46.51,47.42
+Llama 3.(1-3) 70B,39.48,43.15,39.47,40.83,39.53,64.29
+QwQ 32B,62.6,40.06,65.02,38.68,54.6,52.6
+Qwen2.5 32B,50.39,41.4,50.86,41.01,48.86,45.0
+Magistral Small 2506,40.82,22.62,40.96,19.96,40.37,46.88
+StarChat2 15B v0.1,38.76,38.19,36.68,35.58,45.61,61.96
+DeepSeek R1 Distill Qwen 14B,23.14,24.57,24.94,24.3,17.22,27.07
+CodeLlama 70B,33.04,33.68,32.2,32.27,35.81,46.56
+DeepSeek Coder 33B,27.03,38.47,22.65,37.64,41.47,46.01
+QwenCoder 2.5 32B,44.02,45.08,43.68,44.05,45.15,54.52
+DeepCoder 14B,26.4,33.07,27.32,33.18,23.38,32.07
+QwenCoder 2.5 14B,37.69,40.99,35.32,40.26,45.5,47.64
+SeedCoder 8B,50.89,36.63,51.71,36.52,48.2,37.66
+SeedCoder 8B Reasoning,43.75,48.07,47.85,46.76,30.22,59.97
+OpenCoder 8B,30.06,35.58,26.8,34.57,40.83,44.8
+QwenCoder 2.5 7B,14.15,33.64,6.57,33.0,39.16,39.46
+DeepSeek Coder 6.7B,31.87,29.8,28.88,28.99,41.75,37.19
+CodeV R1 Distill Qwen 7B,36.12,24.85,32.35,20.56,48.57,63.98
+HaVen-CodeQwen,43.58,48.69,44.67,47.23,39.98,62.04
+CodeV-QW-7B,20.37,51.58,18.82,50.64,25.48,60.14
+RTLCoder Mistral,21.82,28.83,23.71,26.34,15.58,51.5
+RTLCoder DeepSeek,37.22,37.85,38.33,36.13,33.58,53.57
+OriGen,52.88,52.54,53.0,50.0,52.47,75.66
+CodeV-CL-7B,14.73,32.7,12.71,30.93,21.38,48.85
+CodeV-DS-6.7B,19.62,47.44,14.8,46.08,35.52,59.87

results/aggregated_scores_verilator.csv

@@ -1,29 +1,31 @@
 Model,Agg S2R,Agg MC,Agg VerilogEval S2R,Agg VerilogEval MC,Agg RTLLM,Agg VeriGen
-DeepSeek R1-0528,75.83,77.12,77.64,78.78,69.85,71.64
-DeepSeek R1,75.78,75.4,78.04,76.42,68.31,72.05
-Llama 3.1 405B,52.08,55.01,54.39,55.1,44.48,54.7
-Qwen3 236B A22B,69.17,65.8,74.24,67.55,52.44,60.01
-Qwen2.5 72B,51.72,52.09,52.7,52.44,48.49,50.95
-Llama 3.(1-3) 70B,40.06,47.44,39.84,42.69,40.79,63.12
-QwQ 32B,63.76,44.52,66.09,38.64,56.06,63.91
-Qwen2.5 32B,53.2,44.35,54.23,43.05,49.81,48.64
-StarChat2 15B v0.1,40.2,41.91,37.97,35.85,47.54,61.91
-DeepSeek R1 Distill Qwen 14B,22.93,25.34,24.44,24.43,17.96,28.33
-CodeLlama 70B,34.55,38.02,33.58,33.89,37.75,51.62
-DeepSeek Coder 33B,27.93,41.16,23.14,38.14,43.71,51.12
-QwenCoder 2.5 32B,45.72,48.38,45.31,46.49,47.07,54.62
-DeepCoder 14B,27.06,33.64,27.87,33.39,24.38,34.45
-QwenCoder 2.5 14B,39.39,43.55,36.86,41.54,47.73,50.17
-SeedCoder 8B,52.04,38.31,52.58,37.24,50.25,41.84
-SeedCoder 8B Reasoning,43.41,50.73,47.02,47.51,31.51,61.35
-OpenCoder 8B,30.45,37.27,26.96,35.34,41.96,43.63
-QwenCoder 2.5 7B,14.91,37.2,7.08,34.02,40.74,47.7
-DeepSeek Coder 6.7B,34.14,31.2,30.46,29.39,46.28,37.19
-CodeV R1 Distill Qwen 7B,37.26,30.97,33.33,21.67,50.21,61.63
-HaVen-CodeQwen,44.57,52.4,45.09,48.07,42.87,66.7
-CodeV-QW-7B,21.69,52.12,19.85,50.51,27.77,57.45
-RTLCoder Mistral,22.64,33.32,24.75,26.44,15.68,56.02
-RTLCoder DeepSeek,38.48,39.89,39.19,35.74,36.14,53.57
-OriGen,52.85,55.3,52.72,49.84,53.29,73.3
-CodeV-CL-7B,15.32,35.2,13.21,31.06,22.28,48.85
-CodeV-DS-6.7B,19.28,49.43,14.23,45.91,35.94,61.05
+DeepSeek R1-0528,75.83,78.07,77.64,78.78,69.85,71.64
+DeepSeek R1,75.78,75.99,78.04,76.42,68.31,72.05
+Qwen3 Coder 480B A35B,61.46,58.18,64.01,56.5,53.04,73.45
+Llama 3.1 405B,52.08,55.06,54.39,55.1,44.48,54.7
+Qwen3 236B A22B,69.17,66.8,74.24,67.55,52.44,60.01
+Qwen2.5 72B,51.72,52.29,52.7,52.44,48.49,50.95
+Llama 3.(1-3) 70B,40.06,44.71,39.84,42.69,40.79,63.12
+QwQ 32B,63.76,41.14,66.09,38.64,56.06,63.91
+Qwen2.5 32B,53.2,43.6,54.23,43.05,49.81,48.64
+Magistral Small 2506,41.02,23.48,41.25,20.36,40.27,51.89
+StarChat2 15B v0.1,40.2,38.43,37.97,35.85,47.54,61.91
+DeepSeek R1 Distill Qwen 14B,22.93,24.82,24.44,24.43,17.96,28.33
+CodeLlama 70B,34.55,35.64,33.58,33.89,37.75,51.62
+DeepSeek Coder 33B,27.93,39.42,23.14,38.14,43.71,51.12
+QwenCoder 2.5 32B,45.72,47.29,45.31,46.49,47.07,54.62
+DeepCoder 14B,27.06,33.49,27.87,33.39,24.38,34.45
+QwenCoder 2.5 14B,39.39,42.39,36.86,41.54,47.73,50.17
+SeedCoder 8B,52.04,37.69,52.58,37.24,50.25,41.84
+SeedCoder 8B Reasoning,43.41,48.88,47.02,47.51,31.51,61.35
+OpenCoder 8B,30.45,36.16,26.96,35.34,41.96,43.63
+QwenCoder 2.5 7B,14.91,35.37,7.08,34.02,40.74,47.7
+"DeepSeek Coder 6,7B",34.14,30.16,30.46,29.39,46.28,37.19
+CodeV R1 Distill Qwen 7B,37.26,25.62,33.33,21.67,50.21,61.63
+HaVen-CodeQwen,44.57,49.91,45.09,48.07,42.87,66.7
+CodeV-QW-7B,21.69,51.2,19.85,50.51,27.77,57.45
+RTLCoder Mistral,22.64,29.36,24.75,26.44,15.68,56.02
+RTLCoder DeepSeek,38.48,37.5,39.19,35.74,36.14,53.57
+OriGen,52.85,52.16,52.72,49.84,53.29,73.3
+CodeV-CL-7B,15.32,32.82,13.21,31.06,22.28,48.85
+CodeV-DS-6.7B,19.28,47.41,14.23,45.91,35.94,61.05

results/compute_agg_results.py

@@ -1,4 +1,6 @@
 import csv
+import os
+import sys
 
 
 def agg_S2R_metrics(verilog_eval_rtl, rtllm):
@@ -26,70 +28,82 @@ def avg_ppa(metric):
     return round(result, 2)
 
 
-#######################################
-
-RESULTS_CSV = "results.csv"
-AGG_CSV = "aggregated_scores.csv"
-
-agg_csv = []
-first_row_agg_csv = [
-    "Model",
-    "Agg S2R",
-    "Agg MC",
-    "Agg VerilogEval S2R",
-    "Agg VerilogEval MC",
-    "Agg RTLLM",
-    "Agg VeriGen",
-]
-agg_csv.append(first_row_agg_csv)
-
-with open(RESULTS_CSV, newline="", encoding="utf-8") as csvfile:
-    reader = csv.reader(csvfile)
-    goals_row = next(reader)[1:]  # Skip the first column (model name)
-    benchmarks_row = next(reader)[1:]
-
-    for model_metrics in reader:
-        model = model_metrics[0]
-        # Create a dict of dicts with keys benchmark, goal
-        results = {}
-        for i, metric in enumerate(model_metrics[1:]):
-            benchmark = benchmarks_row[i]
-            goal = goals_row[i]
-
-            if benchmark not in results:
-                results[benchmark] = {}
-
-            results[benchmark][goal] = metric
-
-        # Aggregate metrics
-        agg_csv_row = []
-        # ['Model','Agg S2R','Agg MC','Agg VerilogEval S2R','Agg VerilogEval MC','Agg RTLLM','Agg VeriGen']
-        for col in first_row_agg_csv:
-            if col == "Model":
-                agg_csv_row.append(model)
-            elif col == "Agg S2R":
-                agg_csv_row.append(
-                    agg_S2R_metrics(
-                        avg_ppa(results["VerilogEval S2R"]), avg_ppa(results["RTLLM"])
-                    )
-                )
-            elif col == "Agg MC":
-                agg_csv_row.append(
-                    agg_S2R_metrics(
-                        avg_ppa(results["VerilogEval MC"]), avg_ppa(results["VeriGen"])
-                    )
-                )
-            elif col == "Agg VerilogEval S2R":
-                agg_csv_row.append(avg_ppa(results["VerilogEval S2R"]))
-            elif col == "Agg VerilogEval MC":
-                agg_csv_row.append(avg_ppa(results["VerilogEval MC"]))
-            elif col == "Agg RTLLM":
-                agg_csv_row.append(avg_ppa(results["RTLLM"]))
-            elif col == "Agg VeriGen":
-                agg_csv_row.append(avg_ppa(results["VeriGen"]))
-
-        agg_csv.append(agg_csv_row)
-
-with open(AGG_CSV, "w", newline="") as csvfile:
-    writer = csv.writer(csvfile)
-    writer.writerows(agg_csv)
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print(f"Usage: python {sys.argv[0]} <path_to_input_csv>")
+        sys.exit(1)
+
+    input_csv_path = sys.argv[1]
+
+    if not os.path.exists(input_csv_path):
+        print(f"Error: File not found at {input_csv_path}")
+        sys.exit(1)
+
+    dir_name = os.path.dirname(input_csv_path)
+    base_name = os.path.basename(input_csv_path)
+    output_base_name = base_name.replace("results", "aggregated_scores")
+    output_csv_path = os.path.join(dir_name, output_base_name)
+
+    print(f"Processing {input_csv_path}...")
+
+    agg_csv_data = []
+    header = [
+        "Model",
+        "Agg S2R",
+        "Agg MC",
+        "Agg VerilogEval S2R",
+        "Agg VerilogEval MC",
+        "Agg RTLLM",
+        "Agg VeriGen",
+    ]
+    agg_csv_data.append(header)
+
+    with open(input_csv_path, newline="", encoding="utf-8") as csvfile:
+        reader = csv.reader(csvfile)
+        goals_row = next(reader)[1:]  # Skip the first column (model name)
+        benchmarks_row = next(reader)[1:]
+
+        for model_metrics in reader:
+            if not model_metrics or not model_metrics[0].strip():
+                continue  # Skip empty lines
+
+            model = model_metrics[0]
+            # Create a dict of dicts with keys benchmark, goal
+            results = {}
+            for i, metric in enumerate(model_metrics[1:]):
+                benchmark = benchmarks_row[i]
+                goal = goals_row[i]
+
+                if benchmark not in results:
+                    results[benchmark] = {}
+
+                results[benchmark][goal] = metric
+
+            # Agg metrics
+            agg_csv_row = [model]
+            agg_verilog_eval_s2r = avg_ppa(results.get("VerilogEval S2R", {}))
+            agg_rtllm = avg_ppa(results.get("RTLLM", {}))
+            agg_s2r = agg_S2R_metrics(agg_verilog_eval_s2r, agg_rtllm)
+
+            agg_verilog_eval_mc = avg_ppa(results.get("VerilogEval MC", {}))
+            agg_verigen = avg_ppa(results.get("VeriGen", {}))
+            agg_mc = agg_MC_metrics(agg_verilog_eval_mc, agg_verigen)
+
+            # Append all calculated values in the correct order
+            agg_csv_row.extend(
+                [
+                    agg_s2r,
+                    agg_mc,
+                    agg_verilog_eval_s2r,
+                    agg_verilog_eval_mc,
+                    agg_rtllm,
+                    agg_verigen,
+                ]
+            )
+            agg_csv_data.append(agg_csv_row)
+
+    with open(output_csv_path, "w", newline="", encoding="utf-8") as csvfile:
+        writer = csv.writer(csvfile)
+        writer.writerows(agg_csv_data)
+
+    print(f"Done")

results/parse.py

@@ -2,6 +2,7 @@ import csv
 import json
 import locale
 import os
+import sys
 from typing import Dict, Union
 
 import pandas as pd
@@ -160,6 +161,18 @@ model_details = {
         "V1",
     ),
     "OriGen": ("https://huggingface.co/henryen/OriGen", 6.74, "RTL-Specific", "V1"),
+    "Qwen3 Coder 480B A35B": (
+        "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        480,
+        "Coding",
+        "V1",
+    ),
+    "Magistral Small 2506": (
+        "https://huggingface.co/mistralai/Magistral-Small-2506",
+        23.6,
+        "General",
+        "V1",
+    ),
 }
 
 
@@ -178,7 +191,7 @@ def get_headers(reader, agg=False) -> Union[list, list]:
 
 def get_model_params_and_url(model) -> Union[str, str, float, str]:
     if model not in model_details:
-        return "-", "-", "-"
+        return "-", 0.0, "Unknown", "Unknown"
     url = model_details[model][0]
     params = model_details[model][1]
     type = model_details[model][2]
@@ -193,12 +206,15 @@ def parse_results(csv_path: str) -> list[dict]:
     """
     dataset = []
     models = []
-    with open(os.path.join("results", csv_path), newline="") as csvfile:
+    with open(csv_path, newline="") as csvfile:
         reader = csv.reader(csvfile, delimiter=",")
         metrics, benchs = get_headers(reader)
         for i, row in enumerate(reader):
+            if not row or all(not cell.strip() for cell in row):
+                continue
             model = row[0]
-            print(model)
+            if not model:
+                continue
             url, params, type, release = get_model_params_and_url(model)
             models.append(model)
             row = row[1:]
@@ -229,8 +245,8 @@ def parse_agg(csv_path: str = "results/aggregated_scores_icarus.csv") -> pd.Data
     return pd.read_csv(csv_path)
 
 
-def writeJson(data: list):
-    with open("results/results.json", "w") as f:
+def writeJson(data: list, path: str):
+    with open(path, "w") as f:
         json.dump(data, f, indent=4, ensure_ascii=False)
     print("Done")
 
@@ -266,6 +282,18 @@ def read_data(
 
 
 if __name__ == "__main__":
-    csv_path = "./results.csv"
-    d = parse_results(csv_path)
-    writeJson(d)
+    if len(sys.argv) < 2:
+        print("Usage: python results/parse.py <path_to_input_csv>")
+        sys.exit(1)
+
+    csv_path = sys.argv[1]
+
+    if not os.path.exists(csv_path):
+        print(f"Error: File not found at {csv_path}")
+        sys.exit(1)
+
+    json_path = os.path.splitext(csv_path)[0] + ".json"
+
+    print(f"Parsing {csv_path}...")
+    parsed_data = parse_results(csv_path)
+    writeJson(parsed_data, json_path)

results/results_icarus.json

@@ -499,6 +499,256 @@
         "Params": 685,
         "Release": "V1"
     },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Syntax (STX)",
+        "Result": 96.41,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "RTLLM",
+        "Task": "Syntax (STX)",
+        "Result": 86.38,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Functionality (FNC)",
+        "Result": 65.38,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "RTLLM",
+        "Task": "Functionality (FNC)",
+        "Result": 59.57,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Synthesis (SYN)",
+        "Result": 64.74,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "RTLLM",
+        "Task": "Synthesis (SYN)",
+        "Result": 53.62,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Power",
+        "Result": 64.09,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "RTLLM",
+        "Task": "Power",
+        "Result": 52.85,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Performance",
+        "Result": 61.91,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "RTLLM",
+        "Task": "Performance",
+        "Result": 53.03,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Area",
+        "Result": 62.93,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "RTLLM",
+        "Task": "Area",
+        "Result": 51.81,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "RTL-Repo",
+        "Task": "Exact Matching (EM)",
+        "Result": 38.88,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Syntax (STX)",
+        "Result": 84.1,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VeriGen",
+        "Task": "Syntax (STX)",
+        "Result": 100.0,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Functionality (FNC)",
+        "Result": 58.85,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VeriGen",
+        "Task": "Functionality (FNC)",
+        "Result": 75.29,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Synthesis (SYN)",
+        "Result": 58.46,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VeriGen",
+        "Task": "Synthesis (SYN)",
+        "Result": 75.29,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Power",
+        "Result": 57.05,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VeriGen",
+        "Task": "Power",
+        "Result": 69.63,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Performance",
+        "Result": 55.54,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VeriGen",
+        "Task": "Performance",
+        "Result": 73.95,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Area",
+        "Result": 55.79,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VeriGen",
+        "Task": "Area",
+        "Result": 76.76,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
     {
         "Model": "Llama 3.1 405B",
         "Model Type": "General",
@@ -1999,6 +2249,256 @@
         "Params": 32.5,
         "Release": "V1"
     },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Syntax (STX)",
+        "Result": 63.85,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V2"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "RTLLM",
+        "Task": "Syntax (STX)",
+        "Result": 71.91,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V2"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Functionality (FNC)",
+        "Result": 42.95,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "RTLLM",
+        "Task": "Functionality (FNC)",
+        "Result": 43.4,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Synthesis (SYN)",
+        "Result": 42.44,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "RTLLM",
+        "Task": "Synthesis (SYN)",
+        "Result": 37.45,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Power",
+        "Result": 41.39,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "RTLLM",
+        "Task": "Power",
+        "Result": 43.36,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Performance",
+        "Result": 40.09,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "RTLLM",
+        "Task": "Performance",
+        "Result": 37.44,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Area",
+        "Result": 41.39,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "RTLLM",
+        "Task": "Area",
+        "Result": 40.32,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "RTL-Repo",
+        "Task": "Exact Matching (EM)",
+        "Result": -1.0,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Syntax (STX)",
+        "Result": 31.28,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VeriGen",
+        "Task": "Syntax (STX)",
+        "Result": 68.24,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Functionality (FNC)",
+        "Result": 20.9,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VeriGen",
+        "Task": "Functionality (FNC)",
+        "Result": 50.59,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Synthesis (SYN)",
+        "Result": 20.77,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VeriGen",
+        "Task": "Synthesis (SYN)",
+        "Result": 49.41,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Power",
+        "Result": 20.11,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VeriGen",
+        "Task": "Power",
+        "Result": 41.59,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Performance",
+        "Result": 19.61,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VeriGen",
+        "Task": "Performance",
+        "Result": 48.97,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Area",
+        "Result": 20.16,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VeriGen",
+        "Task": "Area",
+        "Result": 50.07,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
     {
         "Model": "StarChat2 15B v0.1",
         "Model Type": "General",
@@ -6999,4 +7499,4 @@
         "Params": 6.74,
         "Release": "V1"
     }
-]
+]

results/results_v3_mlcad_icarus.csv

@@ -0,0 +1,63 @@
+,Syntax (STX),Syntax (STX),Functionality (FNC),Functionality (FNC),Synthesis (SYN),Synthesis (SYN),Power,Power,Performance,Performance,Area,Area,EM,Syntax (STX),Syntax (STX),Functionality (FNC),Functionality (FNC),Synthesis (SYN),Synthesis (SYN),Power,Power,Performance,Performance,Area,Area
+,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,RTL-Repo,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen
+DeepSeek R1-0528,96.150000000000006,86.120000000000005,81.540000000000006,64.900000000000006,81.280000000000001,64.489999999999995,79.150000000000006,73.329999999999998,78.090000000000003,65.640000000000001,79.280000000000001,71.150000000000006,-1,95.379999999999995,91.760000000000005,81.540000000000006,74.120000000000005,81.409999999999997,74.120000000000005,80.269999999999996,63.859999999999999,78.689999999999998,74.049999999999997,79.989999999999995,77.010000000000005
+DeepSeek R1,97.180000000000007,89.799999999999997,79.739999999999995,65.709999999999994,79.620000000000005,63.270000000000003,78.329999999999998,71.340000000000003,76.489999999999995,64.060000000000002,78.189999999999998,70.079999999999998,-1,97.439999999999998,96.469999999999999,79.489999999999995,74.120000000000005,79.489999999999995,74.120000000000005,78.269999999999996,64.340000000000003,76.430000000000007,74.290000000000006,77.959999999999994,77.120000000000005
+Qwen3 Coder 480B A35B,96.409999999999997,86.379999999999995,65.379999999999995,59.57,64.739999999999995,53.619999999999997,64.090000000000003,52.850000000000001,61.909999999999997,53.030000000000001,62.93,51.810000000000002,38.880000000000003,84.099999999999994,100,58.850000000000001,75.290000000000006,58.460000000000001,75.290000000000006,57.049999999999997,69.629999999999995,55.539999999999999,73.950000000000003,55.789999999999999,76.760000000000005
+Llama 3.1 405B,87.439999999999998,77.140000000000001,58.969999999999999,45.710000000000001,58.850000000000001,41.630000000000003,57.579999999999998,50.880000000000003,55.93,32.439999999999998,56.130000000000003,43.450000000000003,34.619999999999997,88.590000000000003,95.290000000000006,56.149999999999999,58.82,55.899999999999999,58.82,55.130000000000003,55.100000000000001,53.450000000000003,58.399999999999999,54.479999999999997,61.200000000000003
+Qwen3 236B A22B,91.280000000000001,73.879999999999995,76.920000000000002,51.43,76.790000000000006,48.57,75.25,54.609999999999999,73.560000000000002,46.369999999999997,75.670000000000002,50.469999999999999,41.939999999999998,82.180000000000007,87.060000000000002,69.620000000000005,62.350000000000001,69.620000000000005,62.350000000000001,69.040000000000006,54.630000000000003,66.890000000000001,62.630000000000003,69.150000000000006,62.780000000000001
+Qwen2.5 72B,82.180000000000007,79.590000000000003,52.439999999999998,45.310000000000002,51.920000000000002,44.079999999999998,51.829999999999998,46.469999999999999,48.75,45.399999999999999,50.090000000000003,47.649999999999999,37.439999999999998,80.900000000000006,84.709999999999994,52.950000000000003,47.060000000000002,52.689999999999998,47.060000000000002,51.659999999999997,47.590000000000003,49.369999999999997,46.960000000000001,51.18,47.700000000000003
+Llama 3.(1-3) 70B,66.150000000000006,73.879999999999995,40.640000000000001,42.450000000000003,40.640000000000001,39.18,40.460000000000001,40.810000000000002,38.079999999999998,38.140000000000001,39.859999999999999,39.649999999999999,28.719999999999999,84.739999999999995,89.409999999999997,41.670000000000002,65.879999999999995,41.670000000000002,64.709999999999994,41.380000000000003,63.469999999999999,39.75,64.689999999999998,41.359999999999999,64.709999999999994
+QwQ 32B,87.950000000000003,82.450000000000003,66.409999999999997,56.729999999999997,66.409999999999997,52.240000000000002,66.150000000000006,55.829999999999998,63.799999999999997,51.909999999999997,65.120000000000005,56.07,-1,58.969999999999999,68.239999999999995,40,52.939999999999998,39.619999999999997,52.939999999999998,39.399999999999999,51.469999999999999,37.530000000000001,52.93,39.100000000000001,53.390000000000001
+Qwen2.5 32B,88.590000000000003,84.079999999999998,52.560000000000002,50.200000000000003,52.18,46.119999999999997,52.32,49.729999999999997,49.43,46.43,50.82,50.43,28.93,93.209999999999994,85.879999999999995,41.539999999999999,45.880000000000003,41.539999999999999,45.880000000000003,41.310000000000002,43.560000000000002,40.479999999999997,46.079999999999998,41.229999999999997,45.369999999999997
+Magistral Small 2506,63.850000000000001,71.909999999999997,42.950000000000003,43.399999999999999,42.439999999999998,37.450000000000003,41.390000000000001,43.359999999999999,40.090000000000003,37.439999999999998,41.390000000000001,40.32,-1,31.280000000000001,68.239999999999995,20.899999999999999,50.590000000000003,20.77,49.409999999999997,20.109999999999999,41.590000000000003,19.609999999999999,48.969999999999999,20.16,50.07
+StarChat2 15B v0.1,88.459999999999994,84.900000000000006,37.950000000000003,44.490000000000002,37.950000000000003,44.079999999999998,37.560000000000002,46.950000000000003,35.299999999999997,43.219999999999999,37.189999999999998,46.649999999999999,13.42,79.739999999999995,92.939999999999998,36.409999999999997,63.530000000000001,36.030000000000001,63.530000000000001,36.079999999999998,58.060000000000002,34.909999999999997,63.259999999999998,35.759999999999998,64.560000000000002
+DeepSeek R1 Distill Qwen 14B,42.18,34.689999999999998,25.510000000000002,18.370000000000001,25.510000000000002,16.329999999999998,25.359999999999999,17.859999999999999,24.190000000000001,16.48,25.27,17.329999999999998,-1,45,44.710000000000001,25.640000000000001,28.239999999999998,25.260000000000002,28.239999999999998,24.789999999999999,24.710000000000001,23.48,28.140000000000001,24.629999999999999,28.350000000000001
+CodeLlama 70B,67.049999999999997,69.799999999999997,33.079999999999998,36.329999999999998,33.079999999999998,34.289999999999999,32.689999999999998,37.189999999999998,31.460000000000001,34.289999999999999,32.439999999999998,35.950000000000003,24.329999999999998,90.769999999999996,88.239999999999995,33.329999999999998,47.060000000000002,33.329999999999998,47.060000000000002,33.020000000000003,45.799999999999997,30.800000000000001,46.909999999999997,32.990000000000002,46.979999999999997
+DeepSeek Coder 33B,62.82,83.670000000000002,23.329999999999998,42.450000000000003,23.079999999999998,42.039999999999999,22.859999999999999,42.289999999999999,22.809999999999999,39.420000000000002,22.289999999999999,42.710000000000001,24.579999999999998,75.260000000000005,88.239999999999995,39.619999999999997,45.880000000000003,39.359999999999999,45.880000000000003,38.229999999999997,46.259999999999998,36.789999999999999,45.609999999999999,37.899999999999999,46.170000000000002
+QwenCoder 2.5 32B,87.180000000000007,77.959999999999994,45,43.270000000000003,44.869999999999997,43.270000000000003,44.25,46.82,43.030000000000001,43.200000000000003,43.759999999999998,45.420000000000002,31.07,83.719999999999999,87.060000000000002,45.640000000000001,54.119999999999997,45.130000000000003,54.119999999999997,44.590000000000003,54.549999999999997,43.009999999999998,54.009999999999998,44.549999999999997,55.009999999999998
+DeepCoder 14B,43.850000000000001,39.590000000000003,28.079999999999998,23.670000000000002,28.079999999999998,22.039999999999999,27.940000000000001,25,26.260000000000002,22,27.77,23.149999999999999,-1,61.920000000000002,48.240000000000002,34.100000000000001,32.939999999999998,33.719999999999999,32.939999999999998,33.700000000000003,30.469999999999999,32.170000000000002,32.840000000000003,33.670000000000002,32.909999999999997
+QwenCoder 2.5 14B,78.969999999999999,81.629999999999995,37.82,46.119999999999997,37.439999999999998,45.310000000000002,35.939999999999998,45.82,34.829999999999998,44.640000000000001,35.18,46.049999999999997,37.530000000000001,80,83.530000000000001,41.670000000000002,48.240000000000002,41.149999999999999,48.240000000000002,40.740000000000002,47.090000000000003,39.200000000000003,48.289999999999999,40.829999999999998,47.539999999999999
+SeedCoder 8B,91.409999999999997,85.310000000000002,53.460000000000001,47.350000000000001,53.329999999999998,46.530000000000001,52.859999999999999,49.420000000000002,50.619999999999997,45.600000000000001,51.649999999999999,49.590000000000003,28.23,77.439999999999998,94.120000000000005,37.310000000000002,42.350000000000001,37.310000000000002,38.82,37.32,35.289999999999999,35.350000000000001,38.689999999999998,36.890000000000001,38.990000000000002
+SeedCoder 8B Reasoning,67.819999999999993,53.469999999999999,49.229999999999997,30.199999999999999,49.229999999999997,29.390000000000001,48.920000000000002,32.039999999999999,46.759999999999998,28.640000000000001,47.869999999999997,29.989999999999998,-1,83.329999999999998,78.819999999999993,48.210000000000001,62.350000000000001,48.079999999999998,62.350000000000001,47.780000000000001,53.479999999999997,45.439999999999998,61.810000000000002,47.060000000000002,64.620000000000005
+OpenCoder 8B,78.209999999999994,75.920000000000002,28.460000000000001,42.859999999999999,27.82,40.82,27.34,41.359999999999999,25.949999999999999,39.770000000000003,27.109999999999999,41.359999999999999,16.170000000000002,80,95.290000000000006,35.640000000000001,45.880000000000003,35.380000000000003,45.880000000000003,35.119999999999997,42.399999999999999,33.469999999999999,45.75,35.130000000000003,46.259999999999998
+QwenCoder 2.5 7B,20.129999999999999,76.329999999999998,6.9199999999999999,38.780000000000001,6.6699999999999999,37.140000000000001,6.5099999999999998,40.649999999999999,6.6299999999999999,37.25,6.5599999999999996,39.579999999999998,28.329999999999998,74.099999999999994,90.590000000000003,33.719999999999999,40,33.719999999999999,40,33.590000000000003,37.729999999999997,31.780000000000001,40.07,33.619999999999997,40.57
+DeepSeek Coder 6.7B,82.049999999999997,78.780000000000001,29.620000000000001,41.219999999999999,29.489999999999998,38.780000000000001,29.510000000000002,42.619999999999997,27.73,39.329999999999998,29.41,43.299999999999997,24.629999999999999,67.180000000000007,84.709999999999994,31.670000000000002,37.649999999999999,29.870000000000001,37.649999999999999,29.780000000000001,36.450000000000003,27.98,37.469999999999999,29.210000000000001,37.659999999999997
+CodeV R1 Distill Qwen 7B,56.920000000000002,73.060000000000002,33.329999999999998,49.799999999999997,33.329999999999998,47.350000000000001,32.579999999999998,49.25,32.009999999999998,47.450000000000003,32.450000000000003,49.009999999999998,-1,92.689999999999998,89.409999999999997,21.280000000000001,65.879999999999995,21.280000000000001,65.879999999999995,21.039999999999999,60.100000000000001,19.59,65.590000000000003,21.050000000000001,66.239999999999995
+HaVen-CodeQwen,93.329999999999998,80.409999999999997,47.310000000000002,42.859999999999999,46.149999999999999,41.219999999999999,45.079999999999998,40.590000000000003,44.259999999999998,38.829999999999998,44.68,40.530000000000001,25.140000000000001,93.590000000000003,100,50.130000000000003,62.350000000000001,49.490000000000002,62.350000000000001,47.549999999999997,61.82,47.049999999999997,62.530000000000001,47.090000000000003,61.759999999999998
+CodeV-QW-7B,45.380000000000003,68.159999999999997,19.620000000000001,34.289999999999999,18.969999999999999,26.530000000000001,18.91,28.140000000000001,18.710000000000001,21.800000000000001,18.850000000000001,26.5,20.940000000000001,93.329999999999998,100,52.310000000000002,60,51.539999999999999,60,51.689999999999998,59.899999999999999,48.789999999999999,59.950000000000003,51.450000000000003,60.579999999999998
+RTLCoder Mistral,54.869999999999997,32.240000000000002,24.620000000000001,16.329999999999998,24.620000000000001,15.92,24.280000000000001,16.030000000000001,22.780000000000001,14.710000000000001,24.059999999999999,16,14.77,60.509999999999998,85.879999999999995,27.050000000000001,51.759999999999998,27.050000000000001,51.759999999999998,26.940000000000001,49.850000000000001,25.219999999999999,51.939999999999998,26.870000000000001,52.719999999999999
+RTLCoder DeepSeek,84.620000000000005,73.060000000000002,39.490000000000002,37.140000000000001,39.490000000000002,34.689999999999998,38.909999999999997,34.299999999999997,37.520000000000003,32.759999999999998,38.549999999999997,33.689999999999998,19.350000000000001,77.310000000000002,85.879999999999995,36.920000000000002,55.289999999999999,36.789999999999999,55.289999999999999,36.939999999999998,50.789999999999999,34.840000000000003,55.219999999999999,36.619999999999997,54.689999999999998
+OriGen,96.150000000000006,81.629999999999995,54.229999999999997,50.609999999999999,54.229999999999997,50.609999999999999,54.289999999999999,53.100000000000001,51.57,50.859999999999999,53.149999999999999,53.439999999999998,17.07,92.439999999999998,98.819999999999993,50.770000000000003,76.469999999999999,50.770000000000003,76.469999999999999,50.950000000000003,71.659999999999997,48.530000000000001,76.609999999999999,50.509999999999998,78.700000000000003
+CodeV-CL-7B,32.18,48.159999999999997,13.08,24.489999999999998,12.949999999999999,21.629999999999999,12.800000000000001,22.25,12.51,20.59,12.82,21.289999999999999,12.27,92.049999999999997,98.819999999999993,31.789999999999999,49.409999999999997,31.789999999999999,49.409999999999997,31.739999999999998,48.130000000000003,29.449999999999999,49.340000000000003,31.609999999999999,49.079999999999998
+CodeV-DS-6.7B,33.590000000000003,67.349999999999994,15,38.780000000000001,15,37.140000000000001,15.1,35.560000000000002,14.460000000000001,35.130000000000003,14.85,35.880000000000003,21.260000000000002,95.510000000000005,100,47.049999999999997,61.18,47.049999999999997,60,47.369999999999997,59.880000000000003,44.350000000000001,59.960000000000001,46.520000000000003,59.770000000000003
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,

results/results_v3_mlcad_verilator.csv

@@ -0,0 +1,32 @@
+,Syntax (STX),Syntax (STX),Functionality (FNC),Functionality (FNC),Synthesis (SYN),Synthesis (SYN),Power,Power,Performance,Performance,Area,Area,EM,Syntax (STX),Syntax (STX),Functionality (FNC),Functionality (FNC),Synthesis (SYN),Synthesis (SYN),Power,Power,Performance,Performance,Area,Area
+,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,RTL-Repo,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen
+DeepSeek R1-0528,96.540000000000006,89.790000000000006,79.099999999999994,67.659999999999997,78.969999999999999,64.260000000000005,78.439999999999998,73.319999999999993,76.290000000000006,65.379999999999995,78.200000000000003,70.859999999999999,-1,94.739999999999995,98.819999999999993,80,80,79.870000000000005,74.120000000000005,79.659999999999997,63.859999999999999,77.280000000000001,74.049999999999997,79.409999999999997,77.010000000000005
+DeepSeek R1,97.689999999999998,93.189999999999998,79.359999999999999,67.659999999999997,79.099999999999994,62.979999999999997,79.010000000000005,71.379999999999995,76.489999999999995,63.740000000000002,78.629999999999995,69.819999999999993,-1,97.439999999999998,95.290000000000006,77.819999999999993,74.120000000000005,77.689999999999998,74.120000000000005,77.299999999999997,64.579999999999998,75.030000000000001,74.420000000000002,76.939999999999998,77.150000000000006
+Qwen3 Coder 480B A35B,95.900000000000006,89.359999999999999,66.030000000000001,61.280000000000001,65.260000000000005,54.039999999999999,65.329999999999998,53.43,62.450000000000003,53.43,64.239999999999995,52.259999999999998,38.880000000000003,83.969999999999999,100,58.719999999999999,75.290000000000006,58.210000000000001,75.290000000000006,57.579999999999998,69.629999999999995,55.420000000000002,73.950000000000003,56.5,76.760000000000005
+Llama 3.1 405B,88.209999999999994,83.400000000000006,56.789999999999999,50.210000000000001,56.409999999999997,43.829999999999998,55.159999999999997,53.869999999999997,54.009999999999998,33.829999999999998,53.990000000000002,45.729999999999997,34.619999999999997,90.510000000000005,95.290000000000006,57.689999999999998,60,56.670000000000002,55.289999999999999,55.950000000000003,51.57,54.090000000000003,54.869999999999997,55.259999999999998,57.670000000000002
+Qwen3 236B A22B,93.969999999999999,78.719999999999999,75.769999999999996,56.170000000000002,75.379999999999995,50.210000000000001,75,57.009999999999998,72.939999999999998,47.850000000000001,74.780000000000001,52.450000000000003,41.939999999999998,83.719999999999999,88.239999999999995,68.969999999999999,62.350000000000001,68.459999999999994,62.350000000000001,68.299999999999997,54.630000000000003,65.989999999999995,62.630000000000003,68.349999999999994,62.780000000000001
+Qwen2.5 72B,83.849999999999994,82.980000000000004,55,48.509999999999998,54.490000000000002,45.960000000000001,53.969999999999999,48.450000000000003,51.299999999999997,47.340000000000003,52.840000000000003,49.68,37.439999999999998,83.079999999999998,87.060000000000002,54.740000000000002,50.590000000000003,54.359999999999999,50.590000000000003,53.359999999999999,50.049999999999997,51.200000000000003,50.560000000000002,52.759999999999998,52.229999999999997
+Llama 3.(1-3) 70B,68.329999999999998,79.150000000000006,40.899999999999999,45.960000000000001,40.899999999999999,40.43,40.770000000000003,42.119999999999997,38.329999999999998,39.340000000000003,40.420000000000002,40.920000000000002,28.719999999999999,86.030000000000001,87.060000000000002,43.719999999999999,63.530000000000001,43.590000000000003,63.530000000000001,43.549999999999997,62.289999999999999,41.57,63.520000000000003,42.939999999999998,63.539999999999999
+QwQ 32B,89.230000000000004,85.959999999999994,67.560000000000002,58.299999999999997,67.180000000000007,53.619999999999997,67.040000000000006,57.289999999999999,64.780000000000001,53.340000000000003,66.439999999999998,57.560000000000002,-1,59.619999999999997,87.060000000000002,40.130000000000003,65.879999999999995,39.490000000000002,64.709999999999994,39.259999999999998,60.950000000000003,37.490000000000002,64.480000000000004,39.170000000000002,66.310000000000002
+Qwen2.5 32B,90.900000000000006,87.659999999999997,55.899999999999999,50.210000000000001,55.259999999999998,46.810000000000002,55.560000000000002,50.659999999999997,53.100000000000001,47.409999999999997,54.020000000000003,51.369999999999997,28.93,95.510000000000005,85.879999999999995,45,49.409999999999997,43.719999999999999,49.409999999999997,43.719999999999999,46.409999999999997,42.490000000000002,49.890000000000001,42.950000000000003,49.609999999999999
+Magistral Small 2506,65.129999999999995,74.040000000000006,42.950000000000003,44.259999999999998,42.310000000000002,37.450000000000003,41.810000000000002,43.100000000000001,40.189999999999998,37.369999999999997,41.75,40.350000000000001,-1,31.789999999999999,70.590000000000003,21.030000000000001,55.289999999999999,20.77,54.119999999999997,20.329999999999998,46.600000000000001,20.079999999999998,54.020000000000003,20.66,55.060000000000002
+StarChat2 15B v0.1,88.719999999999999,89.359999999999999,38.850000000000001,48.509999999999998,38.850000000000001,45.960000000000001,38.920000000000002,48.950000000000003,36.18,45.049999999999997,38.810000000000002,48.630000000000003,13.42,81.409999999999997,91.760000000000005,36.539999999999999,63.530000000000001,36.280000000000001,63.530000000000001,36.189999999999998,57.509999999999998,35.109999999999999,63.229999999999997,36.240000000000002,65
+DeepSeek R1 Distill Qwen 14B,42.82,36.600000000000001,25.260000000000002,20.43,25,17.02,24.809999999999999,18.620000000000001,23.68,17.18,24.829999999999998,18.07,-1,45.640000000000001,44.710000000000001,25.379999999999999,29.41,25.260000000000002,29.41,24.98,25.920000000000002,23.609999999999999,29.43,24.690000000000001,29.649999999999999
+CodeLlama 70B,67.310000000000002,77.450000000000003,34.740000000000002,38.719999999999999,34.490000000000002,36.170000000000002,34.030000000000001,39.18,32.880000000000003,36.140000000000001,33.829999999999998,37.93,24.329999999999998,92.560000000000002,88.239999999999995,35.770000000000003,52.939999999999998,35.380000000000003,51.759999999999998,34.810000000000002,50.619999999999997,32.340000000000003,51.890000000000001,34.520000000000003,52.359999999999999
+DeepSeek Coder 33B,64.489999999999995,88.939999999999998,23.59,46.380000000000003,23.59,44.259999999999998,23.370000000000001,44.539999999999999,23.32,41.539999999999999,22.739999999999998,45.039999999999999,24.579999999999998,76.540000000000006,88.239999999999995,40.130000000000003,50.590000000000003,39.869999999999997,50.590000000000003,38.700000000000003,51.5,37.329999999999998,50.219999999999999,38.399999999999999,51.630000000000003
+QwenCoder 2.5 32B,90.260000000000005,81.280000000000001,46.539999999999999,47.659999999999997,46.409999999999997,45.109999999999999,45.950000000000003,48.810000000000002,44.420000000000002,45.039999999999999,45.57,47.359999999999999,31.07,86.150000000000006,85.879999999999995,48.210000000000001,54.119999999999997,47.560000000000002,54.119999999999997,47.350000000000001,54.590000000000003,45.350000000000001,54.119999999999997,46.780000000000001,55.140000000000001
+DeepCoder 14B,45.130000000000003,42.549999999999997,28.719999999999999,25.530000000000001,28.719999999999999,22.98,28.579999999999998,26.059999999999999,26.859999999999999,22.940000000000001,28.16,24.140000000000001,-1,63.460000000000001,49.409999999999997,34.359999999999999,37.649999999999999,34.100000000000001,35.289999999999999,34.170000000000002,32.789999999999999,32.579999999999998,35.289999999999999,33.409999999999997,35.280000000000001
+QwenCoder 2.5 14B,80.900000000000006,82.980000000000004,39.869999999999997,50.640000000000001,38.969999999999999,47.659999999999997,37.490000000000002,47.82,36.369999999999997,46.979999999999997,36.729999999999997,48.399999999999999,37.530000000000001,81.409999999999997,84.709999999999994,43.079999999999998,50.590000000000003,42.439999999999998,50.590000000000003,42.020000000000003,49.509999999999998,40.479999999999997,50.859999999999999,42.109999999999999,50.149999999999999
+SeedCoder 8B,96.409999999999997,89.359999999999999,54.359999999999999,51.490000000000002,54.229999999999997,48.509999999999998,53.68,51.520000000000003,51.5,47.539999999999999,52.560000000000002,51.700000000000003,28.23,78.209999999999994,91.760000000000005,38.210000000000001,47.060000000000002,38.079999999999998,43.530000000000001,38.100000000000001,37.890000000000001,36.090000000000003,43.770000000000003,37.539999999999999,43.869999999999997
+SeedCoder 8B Reasoning,68.719999999999999,56.170000000000002,48.079999999999998,34.039999999999999,48.079999999999998,30.640000000000001,48.270000000000003,33.409999999999997,45.780000000000001,29.859999999999999,47,31.27,-1,84.489999999999995,80,48.969999999999999,63.530000000000001,48.850000000000001,63.530000000000001,48.68,54.979999999999997,46.32,62.969999999999999,47.530000000000001,66.099999999999994
+OpenCoder 8B,80,80,27.949999999999999,43.829999999999998,27.949999999999999,42.130000000000003,27.52,42.439999999999998,26.07,40.979999999999997,27.280000000000001,42.460000000000001,16.170000000000002,81.030000000000001,94.120000000000005,36.789999999999999,49.409999999999997,36.149999999999999,44.710000000000001,35.890000000000001,41.219999999999999,34.240000000000002,44.579999999999998,35.899999999999999,45.079999999999998
+QwenCoder 2.5 7B,19.23,83.400000000000006,7.1799999999999997,40.850000000000001,7.1799999999999997,38.719999999999999,7.0199999999999996,42.310000000000002,7.1500000000000004,38.710000000000001,7.0800000000000001,41.200000000000003,28.329999999999998,76.150000000000006,90.590000000000003,35,48.240000000000002,34.740000000000002,48.240000000000002,34.619999999999997,44.93,32.799999999999997,48.399999999999999,34.649999999999999,49.759999999999998
+"DeepSeek Coder 6,7B",86.030000000000001,89.359999999999999,31.280000000000001,45.960000000000001,31.149999999999999,42.979999999999997,31.16,47.369999999999997,29.149999999999999,43.299999999999997,31.079999999999998,48.170000000000002,24.629999999999999,69.099999999999994,83.530000000000001,31.670000000000002,37.649999999999999,30.129999999999999,37.649999999999999,30.190000000000001,36.450000000000003,28.16,37.469999999999999,29.82,37.659999999999997
+CodeV R1 Distill Qwen 7B,56.409999999999997,79.150000000000006,33.969999999999999,50.640000000000001,33.850000000000001,48.939999999999998,33.799999999999997,50.920000000000002,32.490000000000002,49.039999999999999,33.710000000000001,50.670000000000002,-1,94.230000000000004,87.060000000000002,22.18,63.530000000000001,22.18,63.530000000000001,22.210000000000001,57.75,20.73,63.240000000000002,22.079999999999998,63.890000000000001
+HaVen-CodeQwen,93.969999999999999,84.260000000000005,46.789999999999999,47.229999999999997,46.280000000000001,44.259999999999998,45.469999999999999,42.270000000000003,44.289999999999999,42.07,45.5,44.270000000000003,25.140000000000001,95,95.290000000000006,50,67.060000000000002,50,67.060000000000002,48.509999999999998,65.700000000000003,47.560000000000002,67.609999999999999,48.140000000000001,66.799999999999997
+CodeV-QW-7B,45.259999999999998,71.489999999999995,20.640000000000001,36.170000000000002,20,29.359999999999999,19.940000000000001,30.309999999999999,19.739999999999998,24.52,19.879999999999999,28.48,20.940000000000001,92.689999999999998,96.469999999999999,51.539999999999999,57.649999999999999,51.409999999999997,57.649999999999999,51.520000000000003,56.439999999999998,48.710000000000001,57.560000000000002,51.310000000000002,58.350000000000001
+RTLCoder Mistral,56.030000000000001,35.32,25.379999999999999,17.02,25.379999999999999,16.170000000000002,25.52,15.859999999999999,23.18,15.34,25.539999999999999,15.84,14.77,62.310000000000002,87.060000000000002,27.309999999999999,56.469999999999999,27.18,56.469999999999999,26.93,52.350000000000001,25.43,56.549999999999997,26.949999999999999,59.170000000000002
+RTLCoder DeepSeek,84.739999999999995,76.170000000000002,40.509999999999998,42.130000000000003,40.380000000000003,37.450000000000003,39.729999999999997,36.75,38.439999999999998,35.289999999999999,39.399999999999999,36.390000000000001,19.350000000000001,79.230000000000004,85.879999999999995,36.409999999999997,55.289999999999999,36.409999999999997,55.289999999999999,36.490000000000002,50.789999999999999,34.509999999999998,55.219999999999999,36.210000000000001,54.689999999999998
+OriGen,96.790000000000006,85.109999999999999,54.359999999999999,53.189999999999998,53.850000000000001,51.060000000000002,54.079999999999998,54.170000000000002,51.07,51.310000000000002,53.009999999999998,54.390000000000001,17.07,92.560000000000002,96.469999999999999,50.509999999999998,74.120000000000005,50.509999999999998,74.120000000000005,50.829999999999998,69.299999999999997,48.280000000000001,74.260000000000005,50.409999999999997,76.349999999999994
+CodeV-CL-7B,32.82,54.469999999999999,13.460000000000001,29.359999999999999,13.33,22.550000000000001,13.19,23.190000000000001,13.26,21.469999999999999,13.19,22.190000000000001,12.27,92.180000000000007,98.819999999999993,31.789999999999999,49.409999999999997,31.789999999999999,49.409999999999997,31.739999999999998,48.130000000000003,29.829999999999998,49.340000000000003,31.609999999999999,49.079999999999998
+CodeV-DS-6.7B,34.359999999999999,69.790000000000006,14.74,41.280000000000001,14.359999999999999,37.869999999999997,14.380000000000001,35.890000000000001,14.130000000000001,35.740000000000002,14.19,36.18,21.260000000000002,95.769999999999996,100,46.670000000000002,61.18,46.670000000000002,61.18,47.07,61.060000000000002,44.189999999999998,61.130000000000003,46.469999999999999,60.950000000000003

results/results_verilator.json

@@ -499,6 +499,256 @@
         "Params": 685,
         "Release": "V1"
     },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Syntax (STX)",
+        "Result": 95.9,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "RTLLM",
+        "Task": "Syntax (STX)",
+        "Result": 89.36,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Functionality (FNC)",
+        "Result": 66.03,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "RTLLM",
+        "Task": "Functionality (FNC)",
+        "Result": 61.28,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Synthesis (SYN)",
+        "Result": 65.26,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "RTLLM",
+        "Task": "Synthesis (SYN)",
+        "Result": 54.04,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Power",
+        "Result": 65.33,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "RTLLM",
+        "Task": "Power",
+        "Result": 53.43,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Performance",
+        "Result": 62.45,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "RTLLM",
+        "Task": "Performance",
+        "Result": 53.43,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Area",
+        "Result": 64.24,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "RTLLM",
+        "Task": "Area",
+        "Result": 52.26,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "RTL-Repo",
+        "Task": "Exact Matching (EM)",
+        "Result": 38.88,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Syntax (STX)",
+        "Result": 83.97,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VeriGen",
+        "Task": "Syntax (STX)",
+        "Result": 100.0,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Functionality (FNC)",
+        "Result": 58.72,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VeriGen",
+        "Task": "Functionality (FNC)",
+        "Result": 75.29,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Synthesis (SYN)",
+        "Result": 58.21,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VeriGen",
+        "Task": "Synthesis (SYN)",
+        "Result": 75.29,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Power",
+        "Result": 57.58,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VeriGen",
+        "Task": "Power",
+        "Result": 69.63,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Performance",
+        "Result": 55.42,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VeriGen",
+        "Task": "Performance",
+        "Result": 73.95,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Area",
+        "Result": 56.5,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
+    {
+        "Model": "Qwen3 Coder 480B A35B",
+        "Model Type": "Coding",
+        "Benchmark": "VeriGen",
+        "Task": "Area",
+        "Result": 76.76,
+        "Model URL": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Params": 480,
+        "Release": "V1"
+    },
     {
         "Model": "Llama 3.1 405B",
         "Model Type": "General",
@@ -1999,6 +2249,256 @@
         "Params": 32.5,
         "Release": "V1"
     },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Syntax (STX)",
+        "Result": 65.13,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "RTLLM",
+        "Task": "Syntax (STX)",
+        "Result": 74.04,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Functionality (FNC)",
+        "Result": 42.95,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "RTLLM",
+        "Task": "Functionality (FNC)",
+        "Result": 44.26,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Synthesis (SYN)",
+        "Result": 42.31,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "RTLLM",
+        "Task": "Synthesis (SYN)",
+        "Result": 37.45,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Power",
+        "Result": 41.81,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "RTLLM",
+        "Task": "Power",
+        "Result": 43.1,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Performance",
+        "Result": 40.19,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "RTLLM",
+        "Task": "Performance",
+        "Result": 37.37,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Area",
+        "Result": 41.75,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "RTLLM",
+        "Task": "Area",
+        "Result": 40.35,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "RTL-Repo",
+        "Task": "Exact Matching (EM)",
+        "Result": -1.0,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Syntax (STX)",
+        "Result": 31.79,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VeriGen",
+        "Task": "Syntax (STX)",
+        "Result": 70.59,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Functionality (FNC)",
+        "Result": 21.03,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VeriGen",
+        "Task": "Functionality (FNC)",
+        "Result": 55.29,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Synthesis (SYN)",
+        "Result": 20.77,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VeriGen",
+        "Task": "Synthesis (SYN)",
+        "Result": 54.12,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Power",
+        "Result": 20.33,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VeriGen",
+        "Task": "Power",
+        "Result": 46.6,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Performance",
+        "Result": 20.08,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VeriGen",
+        "Task": "Performance",
+        "Result": 54.02,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Area",
+        "Result": 20.66,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
+    {
+        "Model": "Magistral Small 2506",
+        "Model Type": "General",
+        "Benchmark": "VeriGen",
+        "Task": "Area",
+        "Result": 55.06,
+        "Model URL": "https://huggingface.co/mistralai/Magistral-Small-2506",
+        "Params": 23.6,
+        "Release": "V1"
+    },
     {
         "Model": "StarChat2 15B v0.1",
         "Model Type": "General",
@@ -4750,254 +5250,254 @@
         "Release": "V1"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "VerilogEval S2R",
         "Task": "Syntax (STX)",
         "Result": 86.03,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "RTLLM",
         "Task": "Syntax (STX)",
         "Result": 89.36,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "VerilogEval S2R",
         "Task": "Functionality (FNC)",
         "Result": 31.28,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "RTLLM",
         "Task": "Functionality (FNC)",
         "Result": 45.96,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "VerilogEval S2R",
         "Task": "Synthesis (SYN)",
         "Result": 31.15,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "RTLLM",
         "Task": "Synthesis (SYN)",
         "Result": 42.98,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "VerilogEval S2R",
         "Task": "Power",
         "Result": 31.16,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "RTLLM",
         "Task": "Power",
         "Result": 47.37,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "VerilogEval S2R",
         "Task": "Performance",
         "Result": 29.15,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "RTLLM",
         "Task": "Performance",
         "Result": 43.3,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "VerilogEval S2R",
         "Task": "Area",
         "Result": 31.08,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "RTLLM",
         "Task": "Area",
         "Result": 48.17,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "RTL-Repo",
         "Task": "Exact Matching (EM)",
         "Result": 24.63,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "VerilogEval MC",
         "Task": "Syntax (STX)",
         "Result": 69.1,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "VeriGen",
         "Task": "Syntax (STX)",
         "Result": 83.53,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "VerilogEval MC",
         "Task": "Functionality (FNC)",
         "Result": 31.67,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "VeriGen",
         "Task": "Functionality (FNC)",
         "Result": 37.65,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "VerilogEval MC",
         "Task": "Synthesis (SYN)",
         "Result": 30.13,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "VeriGen",
         "Task": "Synthesis (SYN)",
         "Result": 37.65,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "VerilogEval MC",
         "Task": "Power",
         "Result": 30.19,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "VeriGen",
         "Task": "Power",
         "Result": 36.45,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "VerilogEval MC",
         "Task": "Performance",
         "Result": 28.16,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "VeriGen",
         "Task": "Performance",
         "Result": 37.47,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "VerilogEval MC",
         "Task": "Area",
         "Result": 29.82,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
-        "Model": "DeepSeek Coder 6.7B",
-        "Model Type": "Coding",
+        "Model": "DeepSeek Coder 6,7B",
+        "Model Type": "Unknown",
         "Benchmark": "VeriGen",
         "Task": "Area",
         "Result": 37.66,
-        "Model URL": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
-        "Params": 6.74,
-        "Release": "V1"
+        "Model URL": "-",
+        "Params": 0.0,
+        "Release": "Unknown"
     },
     {
         "Model": "CodeV R1 Distill Qwen 7B",

results/v2/aggregated_scores_icarus.csv

@@ -0,0 +1,29 @@
+Model,Agg S2R,Agg MC,Agg VerilogEval S2R,Agg VerilogEval MC,Agg RTLLM,Agg VeriGen
+DeepSeek R1-0528,76.79,77.79,78.84,79.65,70.04,71.64
+DeepSeek R1,75.53,76.24,77.67,77.55,68.49,71.92
+Llama 3.1 405B,53.23,55.25,56.55,54.35,42.26,58.23
+Qwen3 235B A22B,69.16,66.42,74.83,68.36,50.48,60.01
+Qwen2.5 72B,49.36,49.97,50.22,50.74,46.51,47.42
+Llama 3.(1-3) 70B,39.48,46.29,39.47,40.83,39.53,64.29
+QwQ 32B,62.6,41.92,65.02,38.68,54.6,52.6
+Qwen2.5 32B,50.39,41.94,50.86,41.01,48.86,45.0
+StarChat2 15B v0.1,38.76,41.72,36.68,35.58,45.61,61.96
+DeepSeek R1 Distill Qwen 14B,23.14,24.94,24.94,24.3,17.22,27.07
+CodeLlama 70B,33.04,35.59,32.2,32.27,35.81,46.56
+DeepSeek Coder 33B,27.03,39.59,22.65,37.64,41.47,46.01
+QwenCoder 2.5 32B,44.02,46.49,43.68,44.05,45.15,54.52
+DeepCoder 14B,26.4,32.92,27.32,33.18,23.38,32.07
+QwenCoder 2.5 14B,37.69,41.98,35.32,40.26,45.5,47.64
+SeedCoder 8B,50.89,36.79,51.71,36.52,48.2,37.66
+SeedCoder 8B Reasoning,43.75,49.83,47.85,46.76,30.22,59.97
+OpenCoder 8B,30.06,36.95,26.8,34.57,40.83,44.8
+QwenCoder 2.5 7B,14.15,34.5,6.57,33.0,39.16,39.46
+DeepSeek Coder 6.7B,31.87,30.9,28.88,28.99,41.75,37.19
+CodeV R1 Distill Qwen 7B,36.12,30.66,32.35,20.56,48.57,63.98
+HaVen-CodeQwen,43.58,50.68,44.67,47.23,39.98,62.04
+CodeV-QW-7B,20.37,52.85,18.82,50.64,25.48,60.14
+RTLCoder Mistral,21.82,32.19,23.71,26.34,15.58,51.5
+RTLCoder DeepSeek,37.22,40.19,38.33,36.13,33.58,53.57
+OriGen,52.88,55.97,53.0,50.0,52.47,75.66
+CodeV-CL-7B,14.73,35.1,12.71,30.93,21.38,48.85
+CodeV-DS-6.7B,19.62,49.29,14.8,46.08,35.52,59.87

results/v2/aggregated_scores_verilator.csv

@@ -0,0 +1,29 @@
+Model,Agg S2R,Agg MC,Agg VerilogEval S2R,Agg VerilogEval MC,Agg RTLLM,Agg VeriGen
+DeepSeek R1-0528,75.83,77.12,77.64,78.78,69.85,71.64
+DeepSeek R1,75.78,75.4,78.04,76.42,68.31,72.05
+Llama 3.1 405B,52.08,55.01,54.39,55.1,44.48,54.7
+Qwen3 236B A22B,69.17,65.8,74.24,67.55,52.44,60.01
+Qwen2.5 72B,51.72,52.09,52.7,52.44,48.49,50.95
+Llama 3.(1-3) 70B,40.06,47.44,39.84,42.69,40.79,63.12
+QwQ 32B,63.76,44.52,66.09,38.64,56.06,63.91
+Qwen2.5 32B,53.2,44.35,54.23,43.05,49.81,48.64
+StarChat2 15B v0.1,40.2,41.91,37.97,35.85,47.54,61.91
+DeepSeek R1 Distill Qwen 14B,22.93,25.34,24.44,24.43,17.96,28.33
+CodeLlama 70B,34.55,38.02,33.58,33.89,37.75,51.62
+DeepSeek Coder 33B,27.93,41.16,23.14,38.14,43.71,51.12
+QwenCoder 2.5 32B,45.72,48.38,45.31,46.49,47.07,54.62
+DeepCoder 14B,27.06,33.64,27.87,33.39,24.38,34.45
+QwenCoder 2.5 14B,39.39,43.55,36.86,41.54,47.73,50.17
+SeedCoder 8B,52.04,38.31,52.58,37.24,50.25,41.84
+SeedCoder 8B Reasoning,43.41,50.73,47.02,47.51,31.51,61.35
+OpenCoder 8B,30.45,37.27,26.96,35.34,41.96,43.63
+QwenCoder 2.5 7B,14.91,37.2,7.08,34.02,40.74,47.7
+DeepSeek Coder 6.7B,34.14,31.2,30.46,29.39,46.28,37.19
+CodeV R1 Distill Qwen 7B,37.26,30.97,33.33,21.67,50.21,61.63
+HaVen-CodeQwen,44.57,52.4,45.09,48.07,42.87,66.7
+CodeV-QW-7B,21.69,52.12,19.85,50.51,27.77,57.45
+RTLCoder Mistral,22.64,33.32,24.75,26.44,15.68,56.02
+RTLCoder DeepSeek,38.48,39.89,39.19,35.74,36.14,53.57
+OriGen,52.85,55.3,52.72,49.84,53.29,73.3
+CodeV-CL-7B,15.32,35.2,13.21,31.06,22.28,48.85
+CodeV-DS-6.7B,19.28,49.43,14.23,45.91,35.94,61.05