Mhof dev erm, only keep erm new feature from mhof_dev

.github/workflows/ci.yml

@@ -2,11 +2,10 @@ name: CI
 
 on:
   push:
-    branches: master
+    branches: mhof_dev_merge
   pull_request:
-    branches: master
+    branches: mhof_dev_merge
   workflow_dispatch:
-
 jobs:
   test:
     name: Run tests

a_reproduce_pacs_diva.yaml

@@ -0,0 +1,24 @@
+te_d: sketch
+tpath: examples/tasks/task_pacs_aug.py
+bs: 32
+model: diva
+trainer: fbopt
+gamma_y: 1.0
+ini_setpoint_ratio: 0.99
+str_diva_multiplier_type: gammad_recon
+coeff_ma_output_state: 0.1
+coeff_ma_setpoint: 0.9
+exp_shoulder_clip: 5
+mu_init: 0.000001
+k_i_gain_ratio: 0.5
+mu_clip: 10
+epos: 1000
+epos_min: 200
+npath: examples/nets/resnet50domainbed.py
+npath_dom: examples/nets/resnet50domainbed.py
+es: 2
+lr: 0.00005
+zx_dim: 0
+zy_dim: 64
+zd_dim: 64
+force_setpoint_change_once: True

domainlab/algos/builder_fbopt_dial.py

@@ -0,0 +1,21 @@
+"""
+builder for feedback optimization of dial
+"""
+from domainlab.algos.builder_diva import NodeAlgoBuilderDIVA
+from domainlab.algos.trainers.train_fbopt_b import TrainerFbOpt
+
+
+class NodeAlgoBuilderFbOptDial(NodeAlgoBuilderDIVA):
+    """
+    builder for feedback optimization for dial
+    """
+
+    def init_business(self, exp):
+        """
+        return trainer, model, observer
+        """
+        trainer_in, model, observer, device = super().init_business(exp)
+        trainer_in.init_business(model, exp.task, observer, device, exp.args)
+        trainer = TrainerFbOpt()
+        trainer.init_business(trainer_in, exp.task, observer, device, exp.args)
+        return trainer, model, observer, device

domainlab/algos/msels/c_msel_setpoint_delay.py

@@ -0,0 +1,54 @@
+"""
+logs the best up-to-event selected model at each event when setpoint shrinks
+"""
+from domainlab.algos.msels.a_model_sel import AMSel
+from domainlab.utils.logger import Logger
+
+
+class MSelSetpointDelay(AMSel):
+    """
+    This class decorate another model selection object, it logs the current
+    selected performance from the decoratee each time the setpoint shrinks
+    """
+
+    def __init__(self, msel, val_threshold = None):
+        super().__init__(val_threshold)
+        # NOTE: super() has to come first always otherwise self.msel will be overwritten to be None
+        self.msel = msel
+        self._oracle_last_setpoint_sel_te_acc = 0.0
+
+    @property
+    def oracle_last_setpoint_sel_te_acc(self):
+        """
+        return the last setpoint best acc
+        """
+        return self._oracle_last_setpoint_sel_te_acc
+
+    def base_update(self, clear_counter=False):
+        """
+        if the best model should be updated
+        currently, clear_counter is set via
+        flag = super().tr_epoch(epoch, self.flag_setpoint_updated)
+        """
+        logger = Logger.get_logger()
+        logger.info(
+            f"setpoint selected current acc {self._oracle_last_setpoint_sel_te_acc}"
+        )
+        if clear_counter:
+            # for the current version of code, clear_counter = flag_setpoint_updated
+            log_message = (
+                f"setpoint msel te acc updated from "
+                # self._oracle_last_setpoint_sel_te_acc start from 0.0, and always saves
+                # the test acc when last setpoint decrease occurs
+                f"{self._oracle_last_setpoint_sel_te_acc} to "
+                # self.sel_model_te_acc defined as a property
+                # in a_msel, which returns self.msel.sel_model_te_acc
+                # is the validation acc based model selection, which
+                # does not take setpoint into account
+                f"{self.sel_model_te_acc}"
+            )
+            logger.info(log_message)
+            self._oracle_last_setpoint_sel_te_acc = self.sel_model_te_acc
+        # let decoratee decide if model should be selected or not
+        flag = self.msel.update(clear_counter)
+        return flag

domainlab/algos/msels/c_msel_val_top_k.py

@@ -0,0 +1,61 @@
+"""
+Model Selection should be decoupled from
+"""
+from domainlab.algos.msels.c_msel_val import MSelValPerf
+from domainlab.utils.logger import Logger
+
+
+class MSelValPerfTopK(MSelValPerf):
+    """
+    1. Model selection using validation performance
+    2. Visitor pattern to trainer
+    """
+
+    def __init__(self, max_es, top_k=2):
+        super().__init__(max_es)  # construct self.tr_obs (observer)
+        self.top_k = top_k
+        self.list_top_k_acc = [0.0 for _ in range(top_k)]
+
+    def update(self, clear_counter=False):
+        """
+        if the best model should be updated
+        """
+        flag_super = super().update(clear_counter)
+        metric_val_current = self.tr_obs.metric_val[self.tr_obs.str_metric4msel]
+        acc_min = min(self.list_top_k_acc)
+        if metric_val_current > acc_min:
+            # overwrite
+            logger = Logger.get_logger()
+            logger.info(
+                f"top k validation acc: {self.list_top_k_acc} \
+                        overwriting/reset  counter"
+            )
+            self.es_c = 0  # restore counter
+            ind = self.list_top_k_acc.index(acc_min)
+            # avoid having identical values
+            if metric_val_current not in self.list_top_k_acc:
+                self.list_top_k_acc[ind] = metric_val_current
+                logger.info(
+                    f"top k validation acc updated: \
+                            {self.list_top_k_acc}"
+                )
+                # overwrite to ensure consistency
+                # issue #569: initially self.list_top_k_acc will be [xx, 0] and it does not matter since 0 will be overwriten by second epoch validation acc.
+                # actually, after epoch 1, most often, sefl._best_val_acc will be the higher value of self.list_top_k_acc will overwriten by min(self.list_top_k_acc)
+                logger.info(
+                    f"top-2 val sel: overwriting best val acc from {self._best_val_acc} to "
+                    f"minimum of {self.list_top_k_acc} which is {min(self.list_top_k_acc)} "
+                    f"to ensure consistency"
+                )
+                self._best_val_acc = min(self.list_top_k_acc)
+            # overwrite test acc, this does not depend on if val top-k acc has been overwritten or not
+            metric_te_current = self.tr_obs.metric_te[self.tr_obs.str_metric4msel]
+            if self._sel_model_te_acc != metric_te_current:
+                # this can only happen if the validation acc has decreased and current val acc is only bigger than min(self.list_top_k_acc} but lower than max(self.list_top_k_acc)
+                logger.info(
+                    f"top-2 val sel: overwriting selected model test acc from "
+                    f"{self._sel_model_te_acc} to {metric_te_current} to ensure consistency"
+                )
+            self._sel_model_te_acc = metric_te_current
+            return True # if metric_val_current > acc_min:
+        return flag_super

domainlab/algos/trainers/args_fbopt.py

@@ -0,0 +1,126 @@
+"""
+feedback opt
+"""
+
+
+def add_args2parser_fbopt(parser):
+    """
+    append hyper-parameters to the main argparser
+    """
+
+    parser.add_argument(
+        "--k_i_gain", type=float, default=0.001, help="PID control gain for integrator"
+    )
+
+    parser.add_argument(
+        "--k_i_gain_ratio",
+        type=float,
+        default=None,
+        help="set k_i_gain to be ratio of \
+                        initial saturation k_i_gain",
+    )
+
+    parser.add_argument(
+        "--mu_clip", type=float, default=1e4, help="maximum value of mu"
+    )
+
+    parser.add_argument(
+        "--mu_min", type=float, default=1e-6, help="minimum value of mu"
+    )
+
+    parser.add_argument(
+        "--mu_init", type=float, default=0.001, help="initial beta for multiplication"
+    )
+
+    parser.add_argument(
+        "--coeff_ma", type=float, default=0.5, help="exponential moving average"
+    )
+
+    parser.add_argument(
+        "--coeff_ma_output_state",
+        type=float,
+        default=0.1,
+        help="state exponential moving average of \
+                        reguarlization loss",
+    )
+
+    parser.add_argument(
+        "--coeff_ma_setpoint",
+        type=float,
+        default=0.9,
+        help="setpoint average coeff for previous setpoint",
+    )
+
+    parser.add_argument(
+        "--exp_shoulder_clip",
+        type=float,
+        default=5,
+        help="clip before exponential operation",
+    )
+
+    parser.add_argument(
+        "--ini_setpoint_ratio",
+        type=float,
+        default=0.99,
+        help="before training start, evaluate reg loss, \
+                        setpoint will be 0.9 of this loss",
+    )
+
+    parser.add_argument(
+        "--force_feedforward",
+        action="store_true",
+        default=False,
+        help="use feedforward scheduler",
+    )
+
+    parser.add_argument(
+        "--force_setpoint_change_once",
+        action="store_true",
+        default=False,
+        help="train until the setpoint changed at least once \
+                        up to maximum epos specified",
+    )
+
+    parser.add_argument(
+        "--no_tensorboard",
+        action="store_true",
+        default=False,
+        help="disable tensorboard",
+    )
+
+    parser.add_argument(
+        "--no_setpoint_update",
+        action="store_true",
+        default=False,
+        help="disable setpoint update",
+    )
+
+    parser.add_argument(
+        "--tr_with_init_mu",
+        action="store_true",
+        default=False,
+        help="disable setpoint update",
+    )
+
+    parser.add_argument(
+        "--overshoot_rewind",
+        type=str,
+        default="yes",
+        help="overshoot_rewind, for benchmark, use yes or no",
+    )
+
+    parser.add_argument(
+        "--setpoint_rewind",
+        type=str,
+        default="no",
+        help="setpoing_rewind, for benchmark, use yes or no",
+    )
+
+    parser.add_argument(
+        "--str_diva_multiplier_type",
+        type=str,
+        default="gammad_recon",
+        help="which penalty to tune",
+    )
+
+    return parser