From 7f060dc08c29a4afdf2b0eecb3d0b216c190b966 Mon Sep 17 00:00:00 2001 From: Yifeng He Date: Sat, 21 Feb 2026 22:11:51 -0800 Subject: [PATCH 1/6] chore: add vanilla ModernBERT-base as downstream baseline --- .../clone-bcb_modernbert.sh | 14 ++++++++++++++ .../clone-codenet_C++1400_modernbert.sh | 14 ++++++++++++++ .../clone-codenet_Java250_modernbert.sh | 14 ++++++++++++++ .../clone-codenet_Python800_modernbert.sh | 14 ++++++++++++++ .../clone-poj104_modernbert.sh | 14 ++++++++++++++ .../cls-codenet_C++1400_modernbert.sh | 14 ++++++++++++++ .../cls-codenet_Java250_modernbert.sh | 14 ++++++++++++++ .../cls-codenet_Python800_modernbert.sh | 14 ++++++++++++++ .../cls-poj104_modernbert.sh | 14 ++++++++++++++ .../Code-translation/translation_modernbert.sh | 14 ++++++++++++++ .../Defect-detection/defect_modernbert.sh | 14 ++++++++++++++ experiments_downstream/gen_all.py | 6 ++++++ 12 files changed, 160 insertions(+) create mode 100755 experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_modernbert.sh create mode 100755 experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_modernbert.sh create mode 100755 experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_modernbert.sh create mode 100755 experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_modernbert.sh create mode 100755 experiments_downstream/Clone-detection-POJ104/clone-poj104_modernbert.sh create mode 100755 experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_modernbert.sh create mode 100755 experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_modernbert.sh create mode 100755 experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_modernbert.sh create mode 100755 experiments_downstream/Code-classification-POJ104/cls-poj104_modernbert.sh create mode 100755 experiments_downstream/Code-translation/translation_modernbert.sh create mode 100755 experiments_downstream/Defect-detection/defect_modernbert.sh diff --git a/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_modernbert.sh b/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_modernbert.sh new file mode 100755 index 0000000..5e3ea1f --- /dev/null +++ b/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_modernbert.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-BigCloneBench with modernbert +set -euo pipefail + +# Parse CUDA device argument (default: 0) +CUDA_DEVICE="${1:-0}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +export CUDA_VISIBLE_DEVICES="$CUDA_DEVICE" + +cd "$ROOT_DIR/downstream/Clone-detection-BigCloneBench" +./run.sh answerdotai/ModernBERT-base "$ROOT_DIR/results/modernbert/Clone-detection-BigCloneBench" diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_modernbert.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_modernbert.sh new file mode 100755 index 0000000..87513f3 --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_modernbert.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (C++1400) with modernbert +set -euo pipefail + +# Parse CUDA device argument (default: 0) +CUDA_DEVICE="${1:-0}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +export CUDA_VISIBLE_DEVICES="$CUDA_DEVICE" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh answerdotai/ModernBERT-base "$ROOT_DIR/results/modernbert/Clone-detection-CodeNet" C++1400 modernbert answerdotai/ModernBERT-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_modernbert.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_modernbert.sh new file mode 100755 index 0000000..dd7937b --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_modernbert.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (Java250) with modernbert +set -euo pipefail + +# Parse CUDA device argument (default: 0) +CUDA_DEVICE="${1:-0}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +export CUDA_VISIBLE_DEVICES="$CUDA_DEVICE" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh answerdotai/ModernBERT-base "$ROOT_DIR/results/modernbert/Clone-detection-CodeNet" Java250 modernbert answerdotai/ModernBERT-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_modernbert.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_modernbert.sh new file mode 100755 index 0000000..a01ed4f --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_modernbert.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (Python800) with modernbert +set -euo pipefail + +# Parse CUDA device argument (default: 0) +CUDA_DEVICE="${1:-0}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +export CUDA_VISIBLE_DEVICES="$CUDA_DEVICE" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh answerdotai/ModernBERT-base "$ROOT_DIR/results/modernbert/Clone-detection-CodeNet" Python800 modernbert answerdotai/ModernBERT-base diff --git a/experiments_downstream/Clone-detection-POJ104/clone-poj104_modernbert.sh b/experiments_downstream/Clone-detection-POJ104/clone-poj104_modernbert.sh new file mode 100755 index 0000000..f6ba7fe --- /dev/null +++ b/experiments_downstream/Clone-detection-POJ104/clone-poj104_modernbert.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-POJ104 with modernbert +set -euo pipefail + +# Parse CUDA device argument (default: 0) +CUDA_DEVICE="${1:-0}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +export CUDA_VISIBLE_DEVICES="$CUDA_DEVICE" + +cd "$ROOT_DIR/downstream/Clone-detection-POJ104" +./run.sh answerdotai/ModernBERT-base "$ROOT_DIR/results/modernbert/Clone-detection-POJ104" modernbert answerdotai/ModernBERT-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_modernbert.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_modernbert.sh new file mode 100755 index 0000000..b29f8cf --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_modernbert.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (C++1400) with modernbert +set -euo pipefail + +# Parse CUDA device argument (default: 0) +CUDA_DEVICE="${1:-0}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +export CUDA_VISIBLE_DEVICES="$CUDA_DEVICE" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh answerdotai/ModernBERT-base "$ROOT_DIR/results/modernbert/Code-classification-CodeNet" C++1400 modernbert answerdotai/ModernBERT-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_modernbert.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_modernbert.sh new file mode 100755 index 0000000..3c4b7ad --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_modernbert.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (Java250) with modernbert +set -euo pipefail + +# Parse CUDA device argument (default: 0) +CUDA_DEVICE="${1:-0}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +export CUDA_VISIBLE_DEVICES="$CUDA_DEVICE" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh answerdotai/ModernBERT-base "$ROOT_DIR/results/modernbert/Code-classification-CodeNet" Java250 modernbert answerdotai/ModernBERT-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_modernbert.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_modernbert.sh new file mode 100755 index 0000000..149ff32 --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_modernbert.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (Python800) with modernbert +set -euo pipefail + +# Parse CUDA device argument (default: 0) +CUDA_DEVICE="${1:-0}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +export CUDA_VISIBLE_DEVICES="$CUDA_DEVICE" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh answerdotai/ModernBERT-base "$ROOT_DIR/results/modernbert/Code-classification-CodeNet" Python800 modernbert answerdotai/ModernBERT-base diff --git a/experiments_downstream/Code-classification-POJ104/cls-poj104_modernbert.sh b/experiments_downstream/Code-classification-POJ104/cls-poj104_modernbert.sh new file mode 100755 index 0000000..f831784 --- /dev/null +++ b/experiments_downstream/Code-classification-POJ104/cls-poj104_modernbert.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-POJ104 with modernbert +set -euo pipefail + +# Parse CUDA device argument (default: 0) +CUDA_DEVICE="${1:-0}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +export CUDA_VISIBLE_DEVICES="$CUDA_DEVICE" + +cd "$ROOT_DIR/downstream/Code-classification-POJ104" +./run.sh answerdotai/ModernBERT-base "$ROOT_DIR/results/modernbert/Code-classification-POJ104" "" modernbert answerdotai/ModernBERT-base diff --git a/experiments_downstream/Code-translation/translation_modernbert.sh b/experiments_downstream/Code-translation/translation_modernbert.sh new file mode 100755 index 0000000..4f1d359 --- /dev/null +++ b/experiments_downstream/Code-translation/translation_modernbert.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Downstream evaluation: Code-translation with modernbert +set -euo pipefail + +# Parse CUDA device argument (default: 0) +CUDA_DEVICE="${1:-0}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +export CUDA_VISIBLE_DEVICES="$CUDA_DEVICE" + +cd "$ROOT_DIR/downstream/Code-translation" +./run.sh answerdotai/ModernBERT-base "$ROOT_DIR/results/modernbert/Code-translation" diff --git a/experiments_downstream/Defect-detection/defect_modernbert.sh b/experiments_downstream/Defect-detection/defect_modernbert.sh new file mode 100755 index 0000000..a7481ca --- /dev/null +++ b/experiments_downstream/Defect-detection/defect_modernbert.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Downstream evaluation: Defect-detection with modernbert +set -euo pipefail + +# Parse CUDA device argument (default: 0) +CUDA_DEVICE="${1:-0}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +export CUDA_VISIBLE_DEVICES="$CUDA_DEVICE" + +cd "$ROOT_DIR/downstream/Defect-detection" +./run.sh answerdotai/ModernBERT-base "$ROOT_DIR/results/modernbert/Defect-detection" diff --git a/experiments_downstream/gen_all.py b/experiments_downstream/gen_all.py index 4471370..b3ac65e 100644 --- a/experiments_downstream/gen_all.py +++ b/experiments_downstream/gen_all.py @@ -28,6 +28,12 @@ "microsoft/graphcodebert-base", "roberta", ), + ( + "modernbert", + "answerdotai/ModernBERT-base", + "answerdotai/ModernBERT-base", + "modernbert", + ), # Our trained models ( "inv-codebert", From e94acba3c6ca58dc8ada45c98c7357a809eb1183 Mon Sep 17 00:00:00 2001 From: Yifeng He Date: Sat, 21 Feb 2026 23:55:44 -0800 Subject: [PATCH 2/6] fix: update CodeNet download URL after IBM DAX CDN retirement --- downstream/Clone-detection-CodeNet/dataset/download.py | 2 +- downstream/Code-classification-CodeNet/dataset/download.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/downstream/Clone-detection-CodeNet/dataset/download.py b/downstream/Clone-detection-CodeNet/dataset/download.py index f9c8ac3..9a48b7d 100644 --- a/downstream/Clone-detection-CodeNet/dataset/download.py +++ b/downstream/Clone-detection-CodeNet/dataset/download.py @@ -16,7 +16,7 @@ def get_tar_name(subset: str) -> str: def get_url(subset: str) -> str: tar_name = get_tar_name(subset) - url = f"https://dax-cdn.cdn.appdomain.cloud/dax-project-codenet/1.0.0/{tar_name}" + url = f"https://codait-cos-dax.s3.us.cloud-object-storage.appdomain.cloud/dax-project-codenet/1.0.0/{tar_name}" return url diff --git a/downstream/Code-classification-CodeNet/dataset/download.py b/downstream/Code-classification-CodeNet/dataset/download.py index f9c8ac3..9a48b7d 100644 --- a/downstream/Code-classification-CodeNet/dataset/download.py +++ b/downstream/Code-classification-CodeNet/dataset/download.py @@ -16,7 +16,7 @@ def get_tar_name(subset: str) -> str: def get_url(subset: str) -> str: tar_name = get_tar_name(subset) - url = f"https://dax-cdn.cdn.appdomain.cloud/dax-project-codenet/1.0.0/{tar_name}" + url = f"https://codait-cos-dax.s3.us.cloud-object-storage.appdomain.cloud/dax-project-codenet/1.0.0/{tar_name}" return url From 05d36d6c9f24cc3d7d175d272d537ee24802d706 Mon Sep 17 00:00:00 2001 From: Yifeng He Date: Sun, 22 Feb 2026 14:22:09 -0800 Subject: [PATCH 3/6] fix: correct `ouput_file_path` typo in CodeNet dataset scripts --- downstream/Clone-detection-CodeNet/dataset/get_codenet.py | 2 +- .../dataset/augment_Python800.py | 6 +++--- .../Code-classification-CodeNet/dataset/get_codenet.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/downstream/Clone-detection-CodeNet/dataset/get_codenet.py b/downstream/Clone-detection-CodeNet/dataset/get_codenet.py index 6f0ffa8..dad5919 100644 --- a/downstream/Clone-detection-CodeNet/dataset/get_codenet.py +++ b/downstream/Clone-detection-CodeNet/dataset/get_codenet.py @@ -21,7 +21,7 @@ def process(subset: str): case "Python800": augment_Python800( input_file_path=test_file_path, - ouput_file_path=aug_file_path, + output_file_path=aug_file_path, ) case "Java250": augment_Java250( diff --git a/downstream/Code-classification-CodeNet/dataset/augment_Python800.py b/downstream/Code-classification-CodeNet/dataset/augment_Python800.py index 51b4d38..fa614bd 100644 --- a/downstream/Code-classification-CodeNet/dataset/augment_Python800.py +++ b/downstream/Code-classification-CodeNet/dataset/augment_Python800.py @@ -24,7 +24,7 @@ def validate_jsonl_path(path: str): def main( input_file_path: str, - ouput_file_path: str, + output_file_path: str, nproc: int = cpu_count(), ): validate_jsonl_path(input_file_path) @@ -36,11 +36,11 @@ def main( augmented_dataset = pool.map(augment_accumulatively, all_test_json) augmented_jsonl = pool.map(lambda j: json.dumps(asdict(j)), augmented_dataset) - with open(ouput_file_path, "w", encoding=JSON_ENCODING) as f: + with open(output_file_path, "w", encoding=JSON_ENCODING) as f: for aj in augmented_jsonl: f.write(aj + "\n") - print(f"Successfully added transformed data to {ouput_file_path}") + print(f"Successfully added transformed data to {output_file_path}") if __name__ == "__main__": diff --git a/downstream/Code-classification-CodeNet/dataset/get_codenet.py b/downstream/Code-classification-CodeNet/dataset/get_codenet.py index 6f0ffa8..dad5919 100644 --- a/downstream/Code-classification-CodeNet/dataset/get_codenet.py +++ b/downstream/Code-classification-CodeNet/dataset/get_codenet.py @@ -21,7 +21,7 @@ def process(subset: str): case "Python800": augment_Python800( input_file_path=test_file_path, - ouput_file_path=aug_file_path, + output_file_path=aug_file_path, ) case "Java250": augment_Java250( From b7748c830d142d316d977f4e9e0587dad5448a2e Mon Sep 17 00:00:00 2001 From: Yifeng He Date: Mon, 23 Feb 2026 13:51:36 -0800 Subject: [PATCH 4/6] chore: add CodeSage-small as downstream baseline Register codesage/codesage-small (130M params) for code classification and clone detection tasks. CodeSage uses a custom HuggingFace architecture requiring trust_remote_code, and its StarCoder-based tokenizer needs cls/sep/pad token fallbacks. Clone detection uses mean pooling (same as ModernBERT). --- .../Clone-detection-CodeNet/code/model.py | 2 +- .../Clone-detection-CodeNet/code/run.py | 16 +++++++++- .../Clone-detection-POJ104/code/model.py | 2 +- downstream/Clone-detection-POJ104/code/run.py | 16 +++++++++- .../Code-classification-CodeNet/code/run.py | 22 ++++++++++++-- .../Code-classification-POJ104/code/run.py | 29 ++++++++++++++++--- experiments_downstream/run_all_downstream.py | 5 ++++ 7 files changed, 81 insertions(+), 11 deletions(-) diff --git a/downstream/Clone-detection-CodeNet/code/model.py b/downstream/Clone-detection-CodeNet/code/model.py index a540c3f..d053bb9 100644 --- a/downstream/Clone-detection-CodeNet/code/model.py +++ b/downstream/Clone-detection-CodeNet/code/model.py @@ -25,7 +25,7 @@ def forward(self, input_ids=None, p_input_ids=None, n_input_ids=None, labels=Non attention_mask = input_ids.ne(self.tokenizer.pad_token_id) encoder_outputs = self.encoder(input_ids, attention_mask=attention_mask) - if hasattr(self.args, "model_type") and self.args.model_type == "modernbert": + if hasattr(self.args, "model_type") and self.args.model_type in ("modernbert", "codesage"): outputs = self._pool(encoder_outputs[0], attention_mask) elif len(encoder_outputs) > 1: outputs = encoder_outputs[1] diff --git a/downstream/Clone-detection-CodeNet/code/run.py b/downstream/Clone-detection-CodeNet/code/run.py index 6480efc..afeb8a3 100644 --- a/downstream/Clone-detection-CodeNet/code/run.py +++ b/downstream/Clone-detection-CodeNet/code/run.py @@ -81,8 +81,12 @@ "roberta": (RobertaConfig, RobertaModel, RobertaTokenizer), "distilbert": (DistilBertConfig, DistilBertModel, DistilBertTokenizer), "modernbert": (AutoConfig, AutoModel, AutoTokenizer), + "codesage": (AutoConfig, AutoModel, AutoTokenizer), } +# Model types that use custom code hosted on HuggingFace Hub. +_TRUST_REMOTE_CODE_TYPES = {"codesage"} + class InputFeatures(object): """A single training/test features for a example.""" @@ -826,16 +830,25 @@ def main(): ) config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type] + trust_remote = args.model_type in _TRUST_REMOTE_CODE_TYPES config = config_class.from_pretrained( args.config_name if args.config_name else args.model_name_or_path, cache_dir=args.cache_dir if args.cache_dir else None, + trust_remote_code=trust_remote, ) config.num_labels = 1 tokenizer = tokenizer_class.from_pretrained( args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, do_lower_case=args.do_lower_case, cache_dir=args.cache_dir if args.cache_dir else None, - ) + trust_remote_code=trust_remote, + ) + if tokenizer.cls_token is None: + tokenizer.cls_token = tokenizer.eos_token + if tokenizer.sep_token is None: + tokenizer.sep_token = tokenizer.eos_token + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token if args.block_size <= 0: args.block_size = ( tokenizer.max_len_single_sentence @@ -847,6 +860,7 @@ def main(): from_tf=bool(".ckpt" in args.model_name_or_path), config=config, cache_dir=args.cache_dir if args.cache_dir else None, + trust_remote_code=trust_remote, ) else: model = model_class(config) diff --git a/downstream/Clone-detection-POJ104/code/model.py b/downstream/Clone-detection-POJ104/code/model.py index 18df82a..9723508 100644 --- a/downstream/Clone-detection-POJ104/code/model.py +++ b/downstream/Clone-detection-POJ104/code/model.py @@ -24,7 +24,7 @@ def forward(self, input_ids=None, p_input_ids=None, n_input_ids=None, labels=Non attention_mask = input_ids.ne(self.tokenizer.pad_token_id) encoder_outputs = self.encoder(input_ids, attention_mask=attention_mask) - if hasattr(self.args, "model_type") and self.args.model_type == "modernbert": + if hasattr(self.args, "model_type") and self.args.model_type in ("modernbert", "codesage"): outputs = self._pool(encoder_outputs[0], attention_mask) elif len(encoder_outputs) > 1: outputs = encoder_outputs[1] diff --git a/downstream/Clone-detection-POJ104/code/run.py b/downstream/Clone-detection-POJ104/code/run.py index 6480efc..afeb8a3 100644 --- a/downstream/Clone-detection-POJ104/code/run.py +++ b/downstream/Clone-detection-POJ104/code/run.py @@ -81,8 +81,12 @@ "roberta": (RobertaConfig, RobertaModel, RobertaTokenizer), "distilbert": (DistilBertConfig, DistilBertModel, DistilBertTokenizer), "modernbert": (AutoConfig, AutoModel, AutoTokenizer), + "codesage": (AutoConfig, AutoModel, AutoTokenizer), } +# Model types that use custom code hosted on HuggingFace Hub. +_TRUST_REMOTE_CODE_TYPES = {"codesage"} + class InputFeatures(object): """A single training/test features for a example.""" @@ -826,16 +830,25 @@ def main(): ) config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type] + trust_remote = args.model_type in _TRUST_REMOTE_CODE_TYPES config = config_class.from_pretrained( args.config_name if args.config_name else args.model_name_or_path, cache_dir=args.cache_dir if args.cache_dir else None, + trust_remote_code=trust_remote, ) config.num_labels = 1 tokenizer = tokenizer_class.from_pretrained( args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, do_lower_case=args.do_lower_case, cache_dir=args.cache_dir if args.cache_dir else None, - ) + trust_remote_code=trust_remote, + ) + if tokenizer.cls_token is None: + tokenizer.cls_token = tokenizer.eos_token + if tokenizer.sep_token is None: + tokenizer.sep_token = tokenizer.eos_token + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token if args.block_size <= 0: args.block_size = ( tokenizer.max_len_single_sentence @@ -847,6 +860,7 @@ def main(): from_tf=bool(".ckpt" in args.model_name_or_path), config=config, cache_dir=args.cache_dir if args.cache_dir else None, + trust_remote_code=trust_remote, ) else: model = model_class(config) diff --git a/downstream/Code-classification-CodeNet/code/run.py b/downstream/Code-classification-CodeNet/code/run.py index f187b44..8c8f531 100644 --- a/downstream/Code-classification-CodeNet/code/run.py +++ b/downstream/Code-classification-CodeNet/code/run.py @@ -51,8 +51,12 @@ MODEL_CLASSES = { "roberta": (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer), "modernbert": (AutoConfig, AutoModelForSequenceClassification, AutoTokenizer), + "codesage": (AutoConfig, AutoModelForSequenceClassification, AutoTokenizer), } +# Model types that use custom code hosted on HuggingFace Hub. +_TRUST_REMOTE_CODE_TYPES = {"codesage"} + logger = logging.getLogger(__name__) @@ -430,12 +434,24 @@ def main(): set_seed(args.seed) config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type] - config = config_class.from_pretrained(args.model_name_or_path) + trust_remote = args.model_type in _TRUST_REMOTE_CODE_TYPES + config = config_class.from_pretrained( + args.model_name_or_path, trust_remote_code=trust_remote + ) config.num_labels = 104 tokenizer = tokenizer_class.from_pretrained( - args.tokenizer_name if args.tokenizer_name else args.model_name_or_path + args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, + trust_remote_code=trust_remote, + ) + if tokenizer.cls_token is None: + tokenizer.cls_token = tokenizer.eos_token + if tokenizer.sep_token is None: + tokenizer.sep_token = tokenizer.eos_token + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + model = model_class.from_pretrained( + args.model_name_or_path, config=config, trust_remote_code=trust_remote ) - model = model_class.from_pretrained(args.model_name_or_path, config=config) model = Model(model, config, tokenizer, args) diff --git a/downstream/Code-classification-POJ104/code/run.py b/downstream/Code-classification-POJ104/code/run.py index c8c6cb0..1e3ab8f 100644 --- a/downstream/Code-classification-POJ104/code/run.py +++ b/downstream/Code-classification-POJ104/code/run.py @@ -51,15 +51,24 @@ MODEL_CLASSES = { "roberta": (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer), "modernbert": (AutoConfig, AutoModelForSequenceClassification, AutoTokenizer), + "codesage": (AutoConfig, AutoModelForSequenceClassification, AutoTokenizer), } +# Model types that use custom code hosted on HuggingFace Hub. +_TRUST_REMOTE_CODE_TYPES = {"codesage"} + def resolve_model_type(model_type, model_name_or_path): if model_type in MODEL_CLASSES: return model_type candidate = model_name_or_path or model_type or "" name = candidate.lower() - resolved = "modernbert" if "modernbert" in name else "roberta" + if "modernbert" in name: + resolved = "modernbert" + elif "codesage" in name: + resolved = "codesage" + else: + resolved = "roberta" logger.warning( "Unknown model_type '%s'; inferring '%s' from '%s'.", model_type, @@ -448,12 +457,24 @@ def main(): args.model_type = resolve_model_type(args.model_type, args.model_name_or_path) config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type] - config = config_class.from_pretrained(args.model_name_or_path) + trust_remote = args.model_type in _TRUST_REMOTE_CODE_TYPES + config = config_class.from_pretrained( + args.model_name_or_path, trust_remote_code=trust_remote + ) config.num_labels = 104 tokenizer = tokenizer_class.from_pretrained( - args.tokenizer_name if args.tokenizer_name else args.model_name_or_path + args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, + trust_remote_code=trust_remote, + ) + if tokenizer.cls_token is None: + tokenizer.cls_token = tokenizer.eos_token + if tokenizer.sep_token is None: + tokenizer.sep_token = tokenizer.eos_token + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + model = model_class.from_pretrained( + args.model_name_or_path, config=config, trust_remote_code=trust_remote ) - model = model_class.from_pretrained(args.model_name_or_path, config=config) model = Model(model, config, tokenizer, args) diff --git a/experiments_downstream/run_all_downstream.py b/experiments_downstream/run_all_downstream.py index c140460..7e9e61e 100644 --- a/experiments_downstream/run_all_downstream.py +++ b/experiments_downstream/run_all_downstream.py @@ -52,6 +52,11 @@ class RunHandle: "microsoft/graphcodebert-base", "roberta", ), + ("codesage", "supcon"): ModelSpec( + "codesage/codesage-small", + "codesage/codesage-small", + "codesage", + ), # --- InvPT models --- ("inv-codebert", "supcon"): ModelSpec( "./saved_models/InvCodeBERT-supcon/final", From 7ad2ed2299222c8cd9aa2ce12ce5b428fbe628ea Mon Sep 17 00:00:00 2001 From: Yifeng He Date: Mon, 23 Feb 2026 13:55:18 -0800 Subject: [PATCH 5/6] chore: add per-task launcher scripts for CodeSage baseline --- .../clone-codenet_C++1400_codesage.sh | 14 ++++++++++++++ .../clone-codenet_Java250_codesage.sh | 14 ++++++++++++++ .../clone-codenet_Python800_codesage.sh | 14 ++++++++++++++ .../clone-poj104_codesage.sh | 14 ++++++++++++++ .../cls-codenet_C++1400_codesage.sh | 14 ++++++++++++++ .../cls-codenet_Java250_codesage.sh | 14 ++++++++++++++ .../cls-codenet_Python800_codesage.sh | 14 ++++++++++++++ .../cls-poj104_codesage.sh | 14 ++++++++++++++ 8 files changed, 112 insertions(+) create mode 100755 experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_codesage.sh create mode 100755 experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_codesage.sh create mode 100755 experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_codesage.sh create mode 100755 experiments_downstream/Clone-detection-POJ104/clone-poj104_codesage.sh create mode 100755 experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_codesage.sh create mode 100755 experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_codesage.sh create mode 100755 experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_codesage.sh create mode 100755 experiments_downstream/Code-classification-POJ104/cls-poj104_codesage.sh diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_codesage.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_codesage.sh new file mode 100755 index 0000000..54a3df3 --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_codesage.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (C++1400) with codesage +set -euo pipefail + +# Parse CUDA device argument (default: 0) +CUDA_DEVICE="${1:-0}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +export CUDA_VISIBLE_DEVICES="$CUDA_DEVICE" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh codesage/codesage-small "$ROOT_DIR/results/codesage/Clone-detection-CodeNet" C++1400 codesage codesage/codesage-small diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_codesage.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_codesage.sh new file mode 100755 index 0000000..764d4d6 --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_codesage.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (Java250) with codesage +set -euo pipefail + +# Parse CUDA device argument (default: 0) +CUDA_DEVICE="${1:-0}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +export CUDA_VISIBLE_DEVICES="$CUDA_DEVICE" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh codesage/codesage-small "$ROOT_DIR/results/codesage/Clone-detection-CodeNet" Java250 codesage codesage/codesage-small diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_codesage.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_codesage.sh new file mode 100755 index 0000000..61da1d9 --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_codesage.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (Python800) with codesage +set -euo pipefail + +# Parse CUDA device argument (default: 0) +CUDA_DEVICE="${1:-0}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +export CUDA_VISIBLE_DEVICES="$CUDA_DEVICE" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh codesage/codesage-small "$ROOT_DIR/results/codesage/Clone-detection-CodeNet" Python800 codesage codesage/codesage-small diff --git a/experiments_downstream/Clone-detection-POJ104/clone-poj104_codesage.sh b/experiments_downstream/Clone-detection-POJ104/clone-poj104_codesage.sh new file mode 100755 index 0000000..adb9954 --- /dev/null +++ b/experiments_downstream/Clone-detection-POJ104/clone-poj104_codesage.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-POJ104 with codesage +set -euo pipefail + +# Parse CUDA device argument (default: 0) +CUDA_DEVICE="${1:-0}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +export CUDA_VISIBLE_DEVICES="$CUDA_DEVICE" + +cd "$ROOT_DIR/downstream/Clone-detection-POJ104" +./run.sh codesage/codesage-small "$ROOT_DIR/results/codesage/Clone-detection-POJ104" codesage codesage/codesage-small diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_codesage.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_codesage.sh new file mode 100755 index 0000000..fb0732e --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_codesage.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (C++1400) with codesage +set -euo pipefail + +# Parse CUDA device argument (default: 0) +CUDA_DEVICE="${1:-0}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +export CUDA_VISIBLE_DEVICES="$CUDA_DEVICE" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh codesage/codesage-small "$ROOT_DIR/results/codesage/Code-classification-CodeNet" C++1400 codesage codesage/codesage-small diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_codesage.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_codesage.sh new file mode 100755 index 0000000..e046aa5 --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_codesage.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (Java250) with codesage +set -euo pipefail + +# Parse CUDA device argument (default: 0) +CUDA_DEVICE="${1:-0}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +export CUDA_VISIBLE_DEVICES="$CUDA_DEVICE" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh codesage/codesage-small "$ROOT_DIR/results/codesage/Code-classification-CodeNet" Java250 codesage codesage/codesage-small diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_codesage.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_codesage.sh new file mode 100755 index 0000000..986389b --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_codesage.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (Python800) with codesage +set -euo pipefail + +# Parse CUDA device argument (default: 0) +CUDA_DEVICE="${1:-0}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +export CUDA_VISIBLE_DEVICES="$CUDA_DEVICE" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh codesage/codesage-small "$ROOT_DIR/results/codesage/Code-classification-CodeNet" Python800 codesage codesage/codesage-small diff --git a/experiments_downstream/Code-classification-POJ104/cls-poj104_codesage.sh b/experiments_downstream/Code-classification-POJ104/cls-poj104_codesage.sh new file mode 100755 index 0000000..c51657d --- /dev/null +++ b/experiments_downstream/Code-classification-POJ104/cls-poj104_codesage.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-POJ104 with codesage +set -euo pipefail + +# Parse CUDA device argument (default: 0) +CUDA_DEVICE="${1:-0}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +export CUDA_VISIBLE_DEVICES="$CUDA_DEVICE" + +cd "$ROOT_DIR/downstream/Code-classification-POJ104" +./run.sh codesage/codesage-small "$ROOT_DIR/results/codesage/Code-classification-POJ104" "" codesage codesage/codesage-small From 5f42687c12c0903b3541e0d74646bda0542cb510 Mon Sep 17 00:00:00 2001 From: Yifeng He Date: Mon, 23 Feb 2026 14:04:47 -0800 Subject: [PATCH 6/6] fix: patch Conv1D import for CodeSage compat with transformers >=4.45 Conv1D was moved from transformers.modeling_utils to transformers.pytorch_utils. CodeSage's remote model code still imports from the old location, causing an ImportError. Add a monkey-patch in all downstream run.py files to restore the symbol before model loading. --- downstream/Clone-detection-CodeNet/code/run.py | 6 ++++++ downstream/Clone-detection-POJ104/code/run.py | 6 ++++++ downstream/Code-classification-CodeNet/code/run.py | 6 ++++++ downstream/Code-classification-POJ104/code/run.py | 6 ++++++ 4 files changed, 24 insertions(+) diff --git a/downstream/Clone-detection-CodeNet/code/run.py b/downstream/Clone-detection-CodeNet/code/run.py index afeb8a3..6391fe1 100644 --- a/downstream/Clone-detection-CodeNet/code/run.py +++ b/downstream/Clone-detection-CodeNet/code/run.py @@ -22,6 +22,12 @@ from __future__ import absolute_import, division, print_function +# Compat: Conv1D moved from modeling_utils to pytorch_utils in transformers >=4.45 +import transformers.modeling_utils as _mu +if not hasattr(_mu, "Conv1D"): + from transformers.pytorch_utils import Conv1D as _Conv1D + _mu.Conv1D = _Conv1D + import argparse import json import logging diff --git a/downstream/Clone-detection-POJ104/code/run.py b/downstream/Clone-detection-POJ104/code/run.py index afeb8a3..6391fe1 100644 --- a/downstream/Clone-detection-POJ104/code/run.py +++ b/downstream/Clone-detection-POJ104/code/run.py @@ -22,6 +22,12 @@ from __future__ import absolute_import, division, print_function +# Compat: Conv1D moved from modeling_utils to pytorch_utils in transformers >=4.45 +import transformers.modeling_utils as _mu +if not hasattr(_mu, "Conv1D"): + from transformers.pytorch_utils import Conv1D as _Conv1D + _mu.Conv1D = _Conv1D + import argparse import json import logging diff --git a/downstream/Code-classification-CodeNet/code/run.py b/downstream/Code-classification-CodeNet/code/run.py index 8c8f531..cb6ea21 100644 --- a/downstream/Code-classification-CodeNet/code/run.py +++ b/downstream/Code-classification-CodeNet/code/run.py @@ -21,6 +21,12 @@ from __future__ import absolute_import, division, print_function +# Compat: Conv1D moved from modeling_utils to pytorch_utils in transformers >=4.45 +import transformers.modeling_utils as _mu +if not hasattr(_mu, "Conv1D"): + from transformers.pytorch_utils import Conv1D as _Conv1D + _mu.Conv1D = _Conv1D + import argparse import json import logging diff --git a/downstream/Code-classification-POJ104/code/run.py b/downstream/Code-classification-POJ104/code/run.py index 1e3ab8f..a0e6295 100644 --- a/downstream/Code-classification-POJ104/code/run.py +++ b/downstream/Code-classification-POJ104/code/run.py @@ -21,6 +21,12 @@ from __future__ import absolute_import, division, print_function +# Compat: Conv1D moved from modeling_utils to pytorch_utils in transformers >=4.45 +import transformers.modeling_utils as _mu +if not hasattr(_mu, "Conv1D"): + from transformers.pytorch_utils import Conv1D as _Conv1D + _mu.Conv1D = _Conv1D + import argparse import json import logging