From cd92491e40c0d73edbb7409c1a68b663207caef9 Mon Sep 17 00:00:00 2001 From: Sniffleupagus Date: Tue, 4 Jul 2023 13:18:18 -0700 Subject: [PATCH] move brain.nn backup into ai_worker before training session instead of backing up on every save. keep the "previous trained" brain --- pwnagotchi/ai/train.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pwnagotchi/ai/train.py b/pwnagotchi/ai/train.py index 2d72e6e6..cba14dfe 100644 --- a/pwnagotchi/ai/train.py +++ b/pwnagotchi/ai/train.py @@ -116,10 +116,7 @@ class AsyncTrainer(object): def _save_ai(self): logging.info("[ai] saving model to %s ..." % self._nn_path) temp = "%s.tmp" % self._nn_path - back = "%s.bak" % self._nn_path self._model.save(temp) - if os.path.isfile(self._nn_path): - os.replace(self._nn_path, back) os.replace(temp, self._nn_path) def on_ai_step(self): @@ -180,7 +177,13 @@ class AsyncTrainer(object): logging.info("[ai] learning for %d epochs ..." % epochs_per_episode) try: self.set_training(True, epochs_per_episode) + # back up brain file before starting new training set + if os.path.isfile(self._nn_path): + back = "%s.bak" % self._nn_path + os.replace(self._nn_path, back) + self._view.set("mode", " ai") self._model.learn(total_timesteps=epochs_per_episode, callback=self.on_ai_training_step) + self._view.set("mode", " AI") except Exception as e: logging.exception("[ai] error while training (%s)", e) finally: