Code in benchmarking/examples/fine_tuning_transformer_glue

Selecting pre-trained transformer model from Hugging Face zoo and fine-tuning it to a GLUE task. This is in fact a whole family of benchmarks:

• f"finetune_transformer_glue_{dataset}": Tune number of hyperparameters for fixed pre-trained model, selected by --model_type

• f"finetune_transformer_glue_modsel_{dataset}": Tune the same hyperparameters and select the best pre-trained model from a list of 9 choices

Here, dataset selects the GLUE document classification task (values are “cola”, “mnli”, “mrpc”, “qnli”, “qqp”, “rte”, “sst2”, “stsb”, “wnli”).

benchmarking/examples/fine_tuning_transformer_glue/baselines.py

from syne_tune.experiments.default_baselines import (
    BayesianOptimization,
    MOBSTER,
)


class Methods:
    BO = "BO"
    MOBSTER = "MOBSTER"


methods = {
    Methods.BO: lambda method_arguments: BayesianOptimization(method_arguments),
    Methods.MOBSTER: lambda method_arguments: MOBSTER(method_arguments),
}

benchmarking/examples/fine_tuning_transformer_glue/hpo_main.py

from typing import Dict, Any

from benchmarking.examples.fine_tuning_transformer_glue.baselines import methods
from benchmarking.benchmark_definitions import (
    real_benchmark_definitions as benchmark_definitions,
)
from benchmarking.benchmark_definitions.finetune_transformer_glue import (
    PRETRAINED_MODELS,
    MAX_RESOURCE_ATTR,
    MODEL_TYPE_ATTR,
)
from syne_tune.config_space import Domain
from syne_tune.experiments.launchers.hpo_main_local import main


extra_args = [
    dict(
        name="num_train_epochs",
        type=int,
        default=3,
        help="Maximum number of training epochs",
    ),
    dict(
        name="model_type",
        type=str,
        default="bert-base-cased",
        choices=tuple(PRETRAINED_MODELS),
        help="Pre-trained model to start fine-tuning from",
    ),
]


def map_method_args(args, method: str, method_kwargs: Dict[str, Any]) -> Dict[str, Any]:
    # We need to change ``method_kwargs.config_space``, based on ``extra_args``
    new_method_kwargs = method_kwargs.copy()
    new_config_space = new_method_kwargs["config_space"].copy()
    choose_model = isinstance(["model_name_or_path"], Domain)
    new_config_space[MAX_RESOURCE_ATTR] = args.num_train_epochs
    if not choose_model:
        new_config_space[MODEL_TYPE_ATTR] = args.model_type
    else:
        # Need to change ``points_to_evaluate``
        default_configuration = new_method_kwargs["scheduler_kwargs"][
            "points_to_evaluate"
        ][0]
        new_default_configuration = {
            **default_configuration,
            MODEL_TYPE_ATTR: args.model_type,
        }
        new_method_kwargs["scheduler_kwargs"]["points_to_evaluate"] = [
            new_default_configuration
        ]
    new_method_kwargs["config_space"] = new_config_space
    return new_method_kwargs


if __name__ == "__main__":
    main(methods, benchmark_definitions, extra_args, map_method_args)

benchmarking/examples/fine_tuning_transformer_glue/launch_remote.py

from pathlib import Path

import benchmarking
from benchmarking.benchmark_definitions import (
    real_benchmark_definitions as benchmark_definitions,
)
from benchmarking.examples.fine_tuning_transformer_glue.baselines import methods
from benchmarking.examples.fine_tuning_transformer_glue.hpo_main import extra_args
from syne_tune.experiments.launchers.launch_remote_local import launch_remote


if __name__ == "__main__":
    entry_point = Path(__file__).parent / "hpo_main.py"
    launch_remote(
        entry_point=entry_point,
        methods=methods,
        benchmark_definitions=benchmark_definitions,
        source_dependencies=benchmarking.__path__,
        extra_args=extra_args,
    )

benchmarking/examples/launch_local/requirements-synetune.txt

syne-tune[gpsearchers,aws]
tqdm