notebooks/intro_to_continual_learning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "intro_to_continual_learning.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.1"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "27fc6efe2c7e43dd8fcb0175eb7493f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0b865be1de3f4e0e9a29cc1bc1bf2710",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8188ae9d25d54c3d828b81acd8835132",
              "IPY_MODEL_00003c41c6554dbb93782acb1457f7b7"
            ]
          }
        },
        "0b865be1de3f4e0e9a29cc1bc1bf2710": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8188ae9d25d54c3d828b81acd8835132": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ba0e3855ff694620ac8671311f7ecba8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 9912422,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 9912422,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9d6dee22059f4294b79073f402091bfc"
          }
        },
        "00003c41c6554dbb93782acb1457f7b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_38584622decb4264a4ce776f26c69859",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 9913344/? [00:08&lt;00:00, 1127003.11it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_28b8fc517dc841d4abf6b63d16cc1ff7"
          }
        },
        "ba0e3855ff694620ac8671311f7ecba8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9d6dee22059f4294b79073f402091bfc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "38584622decb4264a4ce776f26c69859": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "28b8fc517dc841d4abf6b63d16cc1ff7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c3c479acfe7f464985d1a3d0c7040ddd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0a931f98e7b342f9881b9053abd37e56",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_60e38b64954c4f78bbb0479fd0d2225b",
              "IPY_MODEL_ddca9ccecbf24edbae332f679517e1f4"
            ]
          }
        },
        "0a931f98e7b342f9881b9053abd37e56": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "60e38b64954c4f78bbb0479fd0d2225b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8a37c123dec649718e477c2c35faedc1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 28881,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 28881,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b6b5f4257d574836b75259043a712516"
          }
        },
        "ddca9ccecbf24edbae332f679517e1f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d513bcc904f94c5abd45b2779b1e76d9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 29696/? [00:04&lt;00:00, 5990.41it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dac762ec0c5a44a49ac7cfcc27de68a9"
          }
        },
        "8a37c123dec649718e477c2c35faedc1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b6b5f4257d574836b75259043a712516": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d513bcc904f94c5abd45b2779b1e76d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dac762ec0c5a44a49ac7cfcc27de68a9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8c351c0850454644ab322f3aeabe0da2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_fc0b0bfb41034016b70ee6a620badbb4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_31ab7f3ed5734c538d1d8bf95509dc4c",
              "IPY_MODEL_0ff727c0f19945f3b52891cf9c314eeb"
            ]
          }
        },
        "fc0b0bfb41034016b70ee6a620badbb4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "31ab7f3ed5734c538d1d8bf95509dc4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4fbbb297caf0454bbe3ba154a268f619",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1648877,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1648877,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c6a8cd7eda56473b91a60782cad52d98"
          }
        },
        "0ff727c0f19945f3b52891cf9c314eeb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_15a765db42444d419e21c4a87a4c7f9a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1649664/? [00:03&lt;00:00, 514726.49it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c650cf0043064235bb1d7605faf8ca0f"
          }
        },
        "4fbbb297caf0454bbe3ba154a268f619": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c6a8cd7eda56473b91a60782cad52d98": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "15a765db42444d419e21c4a87a4c7f9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c650cf0043064235bb1d7605faf8ca0f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6e73c74b41ac4bc8b09570f6868f1d06": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_69edeb2fb3e74086bc07af4d51bae473",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_031c4959a8d1412d98d201af230237b1",
              "IPY_MODEL_acef702e8dd5445291543f243b67b1ad"
            ]
          }
        },
        "69edeb2fb3e74086bc07af4d51bae473": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "031c4959a8d1412d98d201af230237b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_62ae945d999c44ff8b6a6479e70adb1d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 4542,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 4542,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f726b9e1feb64c7a8f38ce46170b9568"
          }
        },
        "acef702e8dd5445291543f243b67b1ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6054fcd00eeb49f490d1cce4e1a7b446",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 5120/? [00:00&lt;00:00, 13181.30it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f2607229e6954d948d301c88d9c500b6"
          }
        },
        "62ae945d999c44ff8b6a6479e70adb1d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f726b9e1feb64c7a8f38ce46170b9568": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6054fcd00eeb49f490d1cce4e1a7b446": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f2607229e6954d948d301c88d9c500b6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fp9hbokQIxWW"
      },
      "source": [
        "# A Gentle Introduction to Continual Learning in PyTorch\n",
        "\n",
        "In this brief tutorial we will learn the basics of *Continual Learning* using *PyTorch*. We will use the standard MNIST benchmark so that you can swiftly run this notebook from anywhere!\n",
        "\n",
        "This notebook is part of the **[Continual AI Colab](https://github.com/ContinualAI/colab)** is a repository meant for tutorials and demo running on Google Colaboratory. [Continual AI](https://www.continualai.org/) is an open research community on the topic of Continual Learning and AI! Join us today [on slack](https://continualai.herokuapp.com/)! :-D\n",
        "\n",
        "We will start with learning over the standard *MNIST* benchmark, then we will move in the actual continual learning setting  with the *Permuted MNIST* benchmark. Let's have some fun! :-)\n",
        "\n",
        "\n",
        "---\n",
        "** Connecting a local runtime**\n",
        "\n",
        "In case resources are not enough for you (no GPU for example), you can always connect another [local runtime](https://research.google.com/colaboratory/local-runtimes.html) or to a [runtime on a Google Compute Engine instance](https://research.google.com/colaboratory/local-runtimes.html).\n",
        "\n",
        "This notebook has been designed to run fast enough on simple CPUs so you shouldn't find any trouble here, using a free *hosted account*.\n",
        "\n",
        "---\n",
        "\n",
        "\n",
        "**Requisites to run it locally, outside colab (not recommended)**\n",
        "\n",
        "*   Python 3.x\n",
        "*   Jupyter\n",
        "*   PyTorch >= 1.8\n",
        "*   NumPy\n",
        "*   Matplotlib\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z6RUp96FLuMd"
      },
      "source": [
        "## Google Colaboratory\n",
        "\n",
        "First of all, take a moment to look around and discover Google Colab if you haven't before! You can run the commands below to understand how much resources you're using and are still available. Then consider also that you can also connect your Google Drive for additional space or for easily loading your own files.\n",
        "\n",
        "You can always reset the entire VM with \"*Runtime > Reset all runtime*\" in case of difficulty. Make also sure you're using the GPU or TPU in the same  tab (\"*Runtime > Change runtime type*\")."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pPViRmMBqbJ2",
        "outputId": "a6e2b29b-ca3c-43a8-e149-0759397d9093"
      },
      "source": [
        "!free -m\n",
        "!df -h\n",
        "!nvidia-smi"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              total        used        free      shared  buff/cache   available\n",
            "Mem:          12993         542       10367           1        2083       12172\n",
            "Swap:             0           0           0\n",
            "Filesystem      Size  Used Avail Use% Mounted on\n",
            "overlay          69G   39G   30G  57% /\n",
            "tmpfs            64M     0   64M   0% /dev\n",
            "tmpfs           6.4G     0  6.4G   0% /sys/fs/cgroup\n",
            "shm             5.8G     0  5.8G   0% /dev/shm\n",
            "/dev/sda1        75G   41G   35G  54% /opt/bin\n",
            "tmpfs           6.4G   24K  6.4G   1% /var/colab\n",
            "tmpfs           6.4G     0  6.4G   0% /proc/acpi\n",
            "tmpfs           6.4G     0  6.4G   0% /proc/scsi\n",
            "tmpfs           6.4G     0  6.4G   0% /sys/firmware\n",
            "Wed May 12 19:49:53 2021       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 465.19.01    Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla K80           Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   41C    P8    27W / 149W |      0MiB / 11441MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jt_PxOYPmxp_"
      },
      "source": [
        "**Questions to explore:**\n",
        "\n",
        "*   How to connect your Google Drive with Google Colab?\n",
        "*   How to import a new notebook and save it to your GDrive?\n",
        "*   How to use files which are contained in your GDrive?\n",
        "\n",
        "Some tips here: https://medium.com/deep-learning-turkey/google-colab-free-gpu-tutorial-e113627b9f5d\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i74kZQufNv5d"
      },
      "source": [
        "Ok, if you are on Colab PyTorch is already installed! Let's import it and see if it can find the GPU:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hv7FUJ2Wrd_l",
        "outputId": "dbd47da6-0edd-4fd7-997c-14f9cda82eb1"
      },
      "source": [
        "import torch\n",
        "torch.cuda.is_available()"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JuSqVkPnN7iT"
      },
      "source": [
        "That's great, let us import then a few libraries, which we'll be using during this tutorial!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w7AxhUWe68vT"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torchvision.datasets as datasets\n",
        "import torchvision.transforms as transforms\n",
        "import torch.optim as optim\n",
        "import torch.nn.functional as F\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rv89m9nBPXSh"
      },
      "source": [
        "## MNIST: Digits recognition with PyTorch \n",
        "\n",
        "All right, let's start then making sure we all know the basics! Let's recognize the ten handwritten digits learning from 60.000, 28x28 grayscale images.\n",
        "For simplicity let's import a loading script we have already developed inside the **Continual AI Colab** repository:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yKWbcnh474X3",
        "outputId": "d6bece96-9c6e-4e47-c857-763354e1c14e"
      },
      "source": [
        "!git clone https://github.com/ContinualAI/colab.git continualai/colab"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'continualai/colab'...\n",
            "remote: Enumerating objects: 303, done.\u001b[K\n",
            "remote: Counting objects: 100% (183/183), done.\u001b[K\n",
            "remote: Compressing objects: 100% (117/117), done.\u001b[K\n",
            "remote: Total 303 (delta 112), reused 118 (delta 61), pack-reused 120\u001b[K\n",
            "Receiving objects: 100% (303/303), 10.26 MiB | 21.40 MiB/s, done.\n",
            "Resolving deltas: 100% (154/154), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 675,
          "referenced_widgets": [
            "27fc6efe2c7e43dd8fcb0175eb7493f3",
            "0b865be1de3f4e0e9a29cc1bc1bf2710",
            "8188ae9d25d54c3d828b81acd8835132",
            "00003c41c6554dbb93782acb1457f7b7",
            "ba0e3855ff694620ac8671311f7ecba8",
            "9d6dee22059f4294b79073f402091bfc",
            "38584622decb4264a4ce776f26c69859",
            "28b8fc517dc841d4abf6b63d16cc1ff7",
            "c3c479acfe7f464985d1a3d0c7040ddd",
            "0a931f98e7b342f9881b9053abd37e56",
            "60e38b64954c4f78bbb0479fd0d2225b",
            "ddca9ccecbf24edbae332f679517e1f4",
            "8a37c123dec649718e477c2c35faedc1",
            "b6b5f4257d574836b75259043a712516",
            "d513bcc904f94c5abd45b2779b1e76d9",
            "dac762ec0c5a44a49ac7cfcc27de68a9",
            "8c351c0850454644ab322f3aeabe0da2",
            "fc0b0bfb41034016b70ee6a620badbb4",
            "31ab7f3ed5734c538d1d8bf95509dc4c",
            "0ff727c0f19945f3b52891cf9c314eeb",
            "4fbbb297caf0454bbe3ba154a268f619",
            "c6a8cd7eda56473b91a60782cad52d98",
            "15a765db42444d419e21c4a87a4c7f9a",
            "c650cf0043064235bb1d7605faf8ca0f",
            "6e73c74b41ac4bc8b09570f6868f1d06",
            "69edeb2fb3e74086bc07af4d51bae473",
            "031c4959a8d1412d98d201af230237b1",
            "acef702e8dd5445291543f243b67b1ad",
            "62ae945d999c44ff8b6a6479e70adb1d",
            "f726b9e1feb64c7a8f38ce46170b9568",
            "6054fcd00eeb49f490d1cce4e1a7b446",
            "f2607229e6954d948d301c88d9c500b6"
          ]
        },
        "id": "x3BFVukM_y8i",
        "outputId": "2562da01-bbc2-439b-b283-769448f1a2b3"
      },
      "source": [
        "from continualai.colab.scripts import mnist\n",
        "mnist.init()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading train-images-idx3-ubyte.gz...\n",
            "Downloading t10k-images-idx3-ubyte.gz...\n",
            "Downloading train-labels-idx1-ubyte.gz...\n",
            "Downloading t10k-labels-idx1-ubyte.gz...\n",
            "Download complete.\n",
            "Save complete.\n",
            "Downloading https://ossci-datasets.s3.amazonaws.com/mnist/train-images-idx3-ubyte.gz\n",
            "Downloading https://ossci-datasets.s3.amazonaws.com/mnist/train-images-idx3-ubyte.gz to data/mnist/MNIST/raw/train-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "27fc6efe2c7e43dd8fcb0175eb7493f3",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=9912422.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Extracting data/mnist/MNIST/raw/train-images-idx3-ubyte.gz to data/mnist/MNIST/raw\n",
            "\n",
            "Downloading https://ossci-datasets.s3.amazonaws.com/mnist/train-labels-idx1-ubyte.gz\n",
            "Downloading https://ossci-datasets.s3.amazonaws.com/mnist/train-labels-idx1-ubyte.gz to data/mnist/MNIST/raw/train-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c3c479acfe7f464985d1a3d0c7040ddd",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=28881.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Extracting data/mnist/MNIST/raw/train-labels-idx1-ubyte.gz to data/mnist/MNIST/raw\n",
            "\n",
            "Downloading https://ossci-datasets.s3.amazonaws.com/mnist/t10k-images-idx3-ubyte.gz\n",
            "Downloading https://ossci-datasets.s3.amazonaws.com/mnist/t10k-images-idx3-ubyte.gz to data/mnist/MNIST/raw/t10k-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8c351c0850454644ab322f3aeabe0da2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1648877.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Extracting data/mnist/MNIST/raw/t10k-images-idx3-ubyte.gz to data/mnist/MNIST/raw\n",
            "\n",
            "Downloading https://ossci-datasets.s3.amazonaws.com/mnist/t10k-labels-idx1-ubyte.gz\n",
            "Downloading https://ossci-datasets.s3.amazonaws.com/mnist/t10k-labels-idx1-ubyte.gz to data/mnist/MNIST/raw/t10k-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6e73c74b41ac4bc8b09570f6868f1d06",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=4542.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Extracting data/mnist/MNIST/raw/t10k-labels-idx1-ubyte.gz to data/mnist/MNIST/raw\n",
            "\n",
            "Processing...\n",
            "Done!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torchvision/datasets/mnist.py:502: UserWarning: The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at  /pytorch/torch/csrc/utils/tensor_numpy.cpp:143.)\n",
            "  return torch.from_numpy(parsed.astype(m[2], copy=False)).view(*s)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6jIk6-G6AhWi",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c94d8b14-8e60-42c6-c833-03be40e14583"
      },
      "source": [
        "x_train, t_train, x_test, t_test = mnist.load()\n",
        "\n",
        "print(\"x_train dim and type: \", x_train.shape, x_train.dtype)\n",
        "print(\"t_train dim and type: \", t_train.shape, t_train.dtype)\n",
        "print(\"x_test dim and type: \", x_test.shape, x_test.dtype)\n",
        "print(\"t_test dim and type: \", t_test.shape, t_test.dtype)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "x_train dim and type:  (60000, 1, 28, 28) float32\n",
            "t_train dim and type:  (60000,) uint8\n",
            "x_test dim and type:  (10000, 1, 28, 28) float32\n",
            "t_test dim and type:  (10000,) uint8\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XEWG2PmbVvb7"
      },
      "source": [
        "Let's take a look at the actual images!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RyIuYAw8AuO6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 248
        },
        "outputId": "743d131b-c4ef-4c9c-e620-7d126a971385"
      },
      "source": [
        "f, axarr = plt.subplots(2,2)\n",
        "axarr[0,0].imshow(x_train[1, 0], cmap=\"gray\")\n",
        "axarr[0,1].imshow(x_train[2, 0], cmap=\"gray\")\n",
        "axarr[1,0].imshow(x_train[3, 0], cmap=\"gray\")\n",
        "axarr[1,1].imshow(x_train[4, 0], cmap=\"gray\")\n",
        "np.vectorize(lambda ax:ax.axis('off'))(axarr);"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAScAAADnCAYAAABcxZBBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAN30lEQVR4nO3de4hV1RfA8X0nyTRxTDM1QnuNQcVkmWUhjuRgklKikEVW4x8VSRGhEolJYU97gElWOOQDByZrMjMRkzR7qIP2gtIxy1B8YGNlZlkS3v748Vvtte3cruO956xz5vv5a23WfezyzGLvfffZJ5fP5x0AWFORdAcA4N9QnACYRHECYBLFCYBJFCcAJnUolMzlcvyUZ0Q+n88l3Ycs4dq2I+raZuQEwCSKEwCTKE4ATKI4ATCJ4gTAJIoTAJMoTgBMojgBMIniBMAkihMAkyhOAEyiOAEwieIEwKSCpxJk0cCBA1X7vvvuk/iOO+5QuUWLFkk8Z84clfvss8/K0DsA/8fICYBJFCcAJlGcAJiUK/TcuiycFjhgwADVXrNmjWp37dq1qM/55ZdfVLtHjx4n17ETxEmYpZWFa7tchg8fLnFDQ4PK1dTUSLxt27aSfB8nYQJIFYoTAJMyuZXgqquukripqUnlKisrVduf1v76668qd/ToUYnDadzgwYMlDrcV+O9DtgwdOlS1/eti6dKlcXenLAYNGiTxpk2bEusHIycAJlGcAJhEcQJgUmrXnDp37izxFVdcoXKLFy+WuE+fPkV/5vbt21V71qxZEjc2NqrcJ598IvH06dNV7qmnnir6O5Euw4YNU+2qqiqJ07rmVFGhxyjnnXeexP369VO5XC6+HS2MnACYRHECYFJqp3WvvvqqxLfeemtJPjOcHnbp0kXidevWqZw/vK+uri7J98O+8OSKDRs2JNST0gmXPu666y6J/SUS55xraWmJpU/OMXICYBTFCYBJFCcAJqVmzSk8wXLUqFESF/p5M1wrWr58uWo/99xzEu/du1flPv/8c4l//vlnlbvuuuuK+n5kS/izexbU19dH5sLtNXHK3v9pAJlAcQJgkulpnX9Q3OrVq1XOPyQuPDBv5cqVEofbDPzDspzTu7vD4W1ra6vEX375pcodO3ZMYn+K6ZzeksCDENLP3yrSq1evBHtSHuFJHb7w7y5OjJwAmERxAmASxQmASabWnPr376/aU6dOlTicFx84cEDiffv2qdzChQslPnz4sMqtWLGiYLstOnXqpNqTJ0+W+Lbbbjvpz0eybrjhBonDf+u08tfO/FMIQnv27ImjO/+KkRMAkyhOAExKfFrXsWNHif3d2s7p4XT48AH/7vDNmzerXNJD7759+yb6/Sitiy66KDL39ddfx9iT0vH/1sLtEd98843E4d9dnBg5ATCJ4gTAJIoTAJMSX3O6/PLLJfbXmEI33XSTaoenDQBJSPKhkyH/li7nnBs5cqTEEyZMULkRI0ZEfs7MmTMlPnjwYIl6d+IYOQEwieIEwKTEp3UvvPCCxOGhbf7Uzdo0zj90zD+hAO1L9+7d2/S+yy67TOLwuq+trZX4nHPOUblTTz1V4vDug/AgvCNHjkjc3Nyscn/++afEHTroMvDpp58W7HtcGDkBMIniBMAkihMAk2Jfcxo9erRq+6ddhidavvPOO7H0qS38daaw31988UXc3UEZ+Ws34b/1K6+8IvG0adOK/kz/dM1wzemvv/6S+Pfff1e5LVu2SPzaa6+pXHgbl79Ou3//fpXbvXu3xOHtXnE+OLMQRk4ATKI4ATCJ4gTApNjXnML5rb9v44cfflC5119/PZY+RfGPc3n00UcjX7dmzRrVfvjhh8vVJSRg0qRJEu/cuVPlrr322jZ95q5duyR+++23VW7r1q0Sb9y4sU2fH7r77rtVu2fPnhLv2LGjJN9RaoycAJhEcQJgUuK3r/j8LfXOHf/ggnLzp3HO6Qdu+g9bcE7/FPv888+rXPhQBWTHM888k3QX2mT48OGRuaamphh7UjxGTgBMojgBMIniBMAkU2tOSdyu4t8+E64rjR8/XuJly5ap3Lhx48rbMSAmS5cuTboL/4qREwCTKE4ATIp9Whfege23x4wZo3IPPPBAyb//wQcfVO1HHnlE4srKSpVraGiQ2H+IJ4DyY+QEwCSKEwCTKE4ATIp9zSk8SdBv9+7dW+VefPFFicNT/3788UeJBw8erHK33367xP5TLpw7/mkW/t3hq1atUrm5c+ce/x8AZIC/1tu/f3+VK9VJCCeLkRMAkyhOAEwytUP8lFNOUW3/kK9wR/ahQ4ckrqqqKvo71q9fr9pr166VeMaMGUV/DpBm/nJK+DBOK2z2CkC7R3ECYBLFCYBJsa85bdiwQbU3bdok8aBBgyLfF24z6NWrV+Rr/W0GjY2NKleOW2KANLvmmmtUe8GCBcl0JMDICYBJFCcAJsU+rfMfDOCcc2PHjpX4nnvuUTn/AQOFzJ49W7Vffvllib/99tsT7SKQeeHpIBYxcgJgEsUJgEkUJwAm5cJTAlQyl4tOIlb5fN7+IkGKtLdru66uTrX9Uz7mzZuncuHab7lFXduMnACYRHECYBLTupRgWldaXNt2MK0DkCoUJwAmUZwAmERxAmASxQmASRQnACZRnACYRHECYBLFCYBJFCcAJhW8fQUAksLICYBJFCcAJlGcAJhEcQJgEsUJgEkUJwAmUZwAmERxAmASxQmASRQnACZRnACYRHECYBLFCYBJFCcAJlGcAJhEcQJgEsUJgEkUJwAmUZwAmNShUDKXy3HAuBH5fD6XdB+yhGvbjqhrm5ETAJMoTgBMojgBMIniBMAkihMAkyhOAEyiOAEwieIEwCSKEwCTKE4ATKI4ATCJ4gTAJIoTAJMoTgBMojgBMIniBMAkihMAkwqehNneTZ8+XeLHHntM5Soq/qnrw4YNU7l169aVtV9Ae8DICYBJFCcAJjGt89TV1an2Qw89JPGxY8ci35fPc1Y+UGqMnACYRHECYBLFCYBJrDl5+vXrp9qnnXZaQj0B/ufqq69W7QkTJkhcU1Ojcpdccknk50yZMkW19+7dK/GQIUNUbvHixRI3NzcX39kSY+QEwCSKEwCTcoV+Bm8Pz5Ovra2VuLGxUeUqKyslbmlpUbnRo0dLvH//fpX7448/StlF51z08+TRNpav7fHjx0s8e/ZslTvzzDMlzuX0JfHBBx+ods+ePSW++OKLI78v/Jw33nhD4ltuueW/O3ySoq5tRk4ATKI4ATCJ4gTApHa3lSD82XT+/PkS+2tMoWeffVa1d+7cWdqOoV3p0OGfP70rr7xS5ebNmydx586dVe7DDz+UeObMmSr38ccfq3bHjh0lXrJkicqNGDEism+bN2+OzMWJkRMAkyhOAExqd9O6O++8U7XPPvvsyNf6P80uWrSoXF1CO+Tv9K6vr4983erVq1Xb32Zw6NChgt/hv7bQNG737t2qvXDhwoKfGxdGTgBMojgBMIniBMCkzN++4m/3d+74W038Ey4PHjyocjfffLPEa9euLUPvisftK6UV97Ud/uw/bdo0icO/wblz50rsP2TDuf9eZ/Jt3bpV4qqqqsjXjRs3TrWXLVtW9HeUArevAEgVihMAkzK5leDcc8+VuKmpqej3zZkzR7WTnsoh3WbMmCGxP41zzrmjR49KvGrVKpXzH6xx5MiRyM8PD0MMtwv07dtX4vDkgccff1ziuKdxxWLkBMAkihMAkyhOAEzK5JrTyJEjJa6uri742vfff1/i8NRB4ER069ZNtSdNmiRxuF3AX2caM2ZM0d9x4YUXStzQ0KByAwcOjHzfm2++qdqzZs0q+juTwsgJgEkUJwAmZWKHeDgsXrBggcSnn366yq1fv161/V3g4e5xS9ghXlrluLbPOuss1fafDRc6//zzJQ4fiDFx4kSJb7zxRpW79NJLJe7SpYvKhX/Lfnvs2LEqt3z58si+xY0d4gBSheIEwCSKEwCTUruVoK23qOzYsUO1La8zIV38W1Kcc661tVVi/wGXzjn3/fffS1xo3Tfkr2OFJxT06dNHtQ8cOCCxpTWmYjFyAmASxQmASRQnACalds3JP1bCP83yvzz99NPl6A5w3Emq/v67d999V+W6d+8u8Xfffady/hEm/p4955z76aefJG5sbFS5cM0pzKcNIycAJlGcAJiUmmndgAEDVLvQQwJ94Sl/27ZtK1mfgEKam5slDrcStNXQoUMlrqmpUblweSPcNpM2jJwAmERxAmASxQmASalZc3rvvfdU+4wzzoh87caNGyWuq6srV5eA2HXq1EnicI0pvA2GrQQAUAYUJwAmpWZa16NHD9UutCvcf9b84cOHy9YnIG7hAzizjJETAJMoTgBMojgBMMn0mtP8+fMlrqgovo6GT1gBsuL6669PuguxYeQEwCSKEwCTTE3rwpMHamtrJQ63DviHyb/00ksqx0MLkFX+wzizjpETAJMoTgBMojgBMMnUmlO3bt1Uu3fv3pGv3bNnj8RTpkwpW58ASz766COJw+01J/KgjzRg5ATAJIoTAJNMTesAFPbVV19JvH37dpULtxlccMEFEre2tpa3Y2XAyAmASRQnACZRnACYZGrNqaWlRbX90wWGDBkSd3cA05588knVrq+vV+0nnnhC4vvvv1/ltmzZUr6OlQgjJwAmUZwAmJQLn3WlkrlcdBKxyufzuaT7kCVZuLa7du2q2kuWLFFt/1SPt956S+UmTpwo8W+//VaG3hUv6tpm5ATAJIoTAJMoTgBMYs0pJVhzKq0sXtvhGpS/leDee+9VuerqaomT3lbAmhOAVKE4ATCJaV1KMK0rLa5tO5jWAUgVihMAkyhOAEwquOYEAElh5ATAJIoTAJMoTgBMojgBMIniBMAkihMAk/4GYiFm5wDOyuwAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 4 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "baEsU4PGXsgS"
      },
      "source": [
        "Good! Let's now set up a few general setting before using torch..."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ztZAPQNXZ4ll"
      },
      "source": [
        "# switch to False to use CPU\n",
        "use_cuda = True\n",
        "\n",
        "use_cuda = use_cuda and torch.cuda.is_available()\n",
        "device = torch.device(\"cuda\" if use_cuda else \"cpu\");\n",
        "torch.manual_seed(1);"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0Ek0mErIac6n"
      },
      "source": [
        "... and define our first conv-net! We will use 3 layers of convolutions and two fully connected layers:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ONMdybG4Be0z"
      },
      "source": [
        "class Net(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(Net, self).__init__()\n",
        "        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)\n",
        "        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)\n",
        "        self.conv2_drop = nn.Dropout2d()\n",
        "        self.fc1 = nn.Linear(320, 50)\n",
        "        self.fc2 = nn.Linear(50, 10)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = F.relu(F.max_pool2d(self.conv1(x), 2))\n",
        "        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))\n",
        "        x = x.view(-1, 320)\n",
        "        x = F.relu(self.fc1(x))\n",
        "        x = F.dropout(x, training=self.training)\n",
        "        x = self.fc2(x)\n",
        "        return x"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p9S6a-MlYAsu"
      },
      "source": [
        "Then we can write the *train* and *test* functions. Note that for simplicity here we are not using PyTorch [Data Loaders](https://pytorch.org/tutorials/beginner/data_loading_tutorial.html) but this is not recommended for efficiency."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HGJJfXhJB-zk"
      },
      "source": [
        "def train(model, device, x_train, t_train, optimizer, epoch):\n",
        "    model.train()\n",
        "    \n",
        "    for start in range(0, len(t_train)-1, 256):\n",
        "      end = start + 256\n",
        "      x, y = torch.from_numpy(x_train[start:end]), torch.from_numpy(t_train[start:end]).long()\n",
        "      x, y = x.to(device), y.to(device)\n",
        "      \n",
        "      optimizer.zero_grad()\n",
        "\n",
        "      output = model(x)\n",
        "      loss = F.cross_entropy(output, y)\n",
        "      loss.backward()\n",
        "      optimizer.step()\n",
        "      #print(loss.item())\n",
        "    print('Train Epoch: {} \\tLoss: {:.6f}'.format(epoch, loss.item()))\n",
        "\n",
        "def test(model, device, x_test, t_test):\n",
        "    model.eval()\n",
        "    test_loss = 0\n",
        "    correct = 0\n",
        "    for start in range(0, len(t_test)-1, 256):\n",
        "      end = start + 256\n",
        "      with torch.no_grad():\n",
        "        x, y = torch.from_numpy(x_test[start:end]), torch.from_numpy(t_test[start:end]).long()\n",
        "        x, y = x.to(device), y.to(device)\n",
        "        output = model(x)\n",
        "        test_loss += F.cross_entropy(output, y).item() # sum up batch loss\n",
        "        pred = output.max(1, keepdim=True)[1] # get the index of the max logit\n",
        "        correct += pred.eq(y.view_as(pred)).sum().item()\n",
        "\n",
        "    test_loss /= len(t_test)\n",
        "    print('Test set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
        "        test_loss, correct, len(t_test),\n",
        "        100. * correct / len(t_test)))\n",
        "    return 100. * correct / len(t_test)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IxIISdDPaqb9"
      },
      "source": [
        "Then we are ready to instantiate our model and start the training!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1cJURe0JCFh8"
      },
      "source": [
        "model = Net().to(device)\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BlhVt8vylpUv",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "16e4e238-c964-4bfd-9d0f-cf069f68fbf4"
      },
      "source": [
        "for epoch in range(1, 3):\n",
        "  train(model, device, x_train, t_train, optimizer, epoch)\n",
        "  test(model, device, x_test, t_test)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train Epoch: 1 \tLoss: 0.760294\n",
            "Test set: Average loss: 0.0013, Accuracy: 9024/10000 (90%)\n",
            "\n",
            "Train Epoch: 2 \tLoss: 0.493347\n",
            "Test set: Average loss: 0.0007, Accuracy: 9435/10000 (94%)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7qwh4T5Va86-"
      },
      "source": [
        "Wow! 94% accuracy in such a short time. \n",
        "\n",
        "**Questions to explore:**\n",
        "\n",
        "*   Can you find a better parametrization to improve the final accuracy?\n",
        "*   Can you change the network architecture to improve the final accuracy?\n",
        "*   Can you achieve the same performances with a smaller architecture?\n",
        "*   What's the difference in accuracy if you change convolutions with fully connected layers?\n",
        "\n",
        "Some tips here: http://rodrigob.github.io/are_we_there_yet/build/classification_datasets_results.html#4d4e495354"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2dn-5gOGq08g"
      },
      "source": [
        "But what if now we want we the same model being able to solve a new task we encounter over time like a permuted version of the same MNIST? Let's define our custom function to permute it!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6Xq_4UvjgXPQ"
      },
      "source": [
        "def permute_mnist(mnist, seed):\n",
        "    \"\"\" Given the training set, permute pixels of each img the same way. \"\"\"\n",
        "\n",
        "    np.random.seed(seed)\n",
        "    print(\"starting permutation...\")\n",
        "    h = w = 28\n",
        "    perm_inds = list(range(h*w))\n",
        "    np.random.shuffle(perm_inds)\n",
        "    # print(perm_inds)\n",
        "    perm_mnist = []\n",
        "    for set in mnist:\n",
        "        num_img = set.shape[0]\n",
        "        flat_set = set.reshape(num_img, w * h)\n",
        "        perm_mnist.append(flat_set[:, perm_inds].reshape(num_img, 1, w, h))\n",
        "    print(\"done.\")\n",
        "    return perm_mnist"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4xG5LFwLgkpu",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6c855633-6efe-4abb-f62d-95b8fa1a82ac"
      },
      "source": [
        "x_train2, x_test2 = permute_mnist([x_train, x_test], 0)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "starting permutation...\n",
            "done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LYBa_Gedh_do",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 183
        },
        "outputId": "8517948f-c804-4819-9a90-be66ba5419d2"
      },
      "source": [
        "f, axarr = plt.subplots(1,2)\n",
        "axarr[0].imshow(x_train[1, 0], cmap=\"gray\")\n",
        "axarr[1].imshow(x_train2[2, 0], cmap=\"gray\")\n",
        "np.vectorize(lambda ax:ax.axis('off'))(axarr);"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAACmCAYAAAB5qlzZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAALAElEQVR4nO3dWUhVfRfH8X2eymYLgkabwMpuzIpCoom6ayCiwSKUuimKIIQKA7MRymgODCNSNEmxiPIigjKKyKBouqqILpqkieYoG3wuXnh532et7bN3++x19tHv5/LXOnv/PZ4WB//DjjU3NzsAABt/JXoAANCW0HQBwBBNFwAM0XQBwBBNFwAM0XQBwFD7lv4xFouxngyham5ujiXivlH4bF+7dk1kEyZMCOVekydPVvMrV66Ecj+vMjIy1Pz+/fvGI4k/t88233QBwBBNFwAM0XQBwFCspW3AUfi7F1q3tvw3Xc2oUaPU/O7du4GuW1NTo+Y5OTkiW758uciOHDmivj4rK0tkd+7c8Tk6b/76S/+O+PDhQ5Glp6eHMgY/+JsuAEQATRcADNF0AcAQTRcADNF0AcAQqxeQUMmweiEtLU1kz549CzyGyspKkeXm5ga+rh/79u0TWX5+vufXl5WViWz+/Plqbffu3UX2+/dvtVZbqVBXV6fWzp49W2QfPnxQa6urq0W2YsUKtTYoVi8AQATQdAHAEE0XAAzRdAHAEBNpSKhkmEhrDZYtW6bm2kRYshkxYoTIHjx4EPi6mZmZIrt3757n1zORBgARQNMFAEM0XQAwRNMFAEM0XQAwxOoFA2PHjhXZ6tWr1dq8vDyRVVRUqLWHDh0S2a1bt3yOLrFYvRB/2tZatwPAgyotLRVZPLbVfvv2TWSdOnUKfF1LrF4AgAig6QKAIZouABii6QKAISbS4kh7MqrjOE59fb3IUlNTA99POzO0V69ega9riYk0G+/fv1fznj17Go/k/505c0bN58yZE8r9LM8wZiINACKApgsAhmi6AGCIpgsAhmi6AGCI1Qt/aPz48SI7deqUWtu/f3+Rub3vnz59EllTU5Naq61UmDhxolqrbQ92u64lVi94U1RUJLKtW7eGcq8BAwaI7Pnz54Gvm52dLbKuXbuqtRcvXhTZ/fv31dobN26IzG1FQmNjo8j69eun1gbF6gUAiACaLgAYoukCgCGaLgAYYiLtf3Tp0kXNx4wZI7Ljx4+LLC0tTX19LCb/nu72vmsTXrt27VJrq6urPd3LcRynsLBQZDt27FBrLbWFiTTtfFvHCe+MW0QDE2kAEAE0XQAwRNMFAEM0XQAwRNMFAEPtEz2AKNGebOo4jrN48WKzMWgrJbp166bWXr58WWRTp05VazMzMwONC968fftWZGGtUli7dq2a7969O5T7WSooKBDZzp07EzASb4qLiz3X8k0XAAzRdAHAEE0XAAzRdAHAUJudSBs7dqzIZs6cqda6ba39J21iy3Ecp66uTmRukx0vXrwQ2e3bt9Xad+/eiWzatGlqrdefAZLbNl6N5fvsNpE2ffp0kfXt21etXblypcjy8/NFlpOTo75+//79Ihs6dKhaqz3hV3s6r+OE94TeoCoqKtQ8Ly9PZOvXr1dr+aYLAIZougBgiKYLAIZougBgiKYLAIZa/SHmWVlZal5fXy+y1NRUz9c9d+6cyNy2C0+ZMkVkbttyjx49KrLXr197HtevX7/U/OvXr57G5Tj6QephSYZDzBcsWCCy2trauI4nbBkZGWq+ZcsWkbmtVNBoKztGjhyp1j548EBkZWVlau2yZcs8j8GPuXPniuz06dOh3ItDzAEgAmi6AGCIpgsAhmi6AGCoVU2kDR8+XGSbNm1SaxctWiSyN2/eqLWNjY0i2759u8hOnjz5b0MMndtEmvZ7rqmpUWuXLFkS1zG1JBkm0sKi/a7atWuXgJG0Hdpnu6qqKpR7MZEGABFA0wUAQzRdADBE0wUAQzRdADCUlIeYd+zYUc21g8FnzJih1n769Elk2kHEjuM4N2/eFFnnzp1bGmJSGDRoUKKHkLTcDjb38+RfP0+Z1laa+Nmui/8Ia6WCH3zTBQBDNF0AMETTBQBDNF0AMJSU24Czs7PV/OrVq56voT0x1e1pvsnEzzbghoYGtXbSpElxHVNL2vI24FevXomsd+/eCRhJcnA7e/fChQsii8KEGduAASACaLoAYIimCwCGaLoAYCgpd6Tt3btXzWMx+Xdrt8mx1jBppnHbEeW2gwqJw6SZP/F4WOWZM2dENmfOnMDX9YNvugBgiKYLAIZougBgiKYLAIZougBgKPKrF2bNmiWyrKwstVbb6nr27Nm4jynK3FYpaO/NnTt3wh4O4kT7vdbW1qq1nLPr7tq1ayIrKChQa7X3cfTo0YHHwDddADBE0wUAQzRdADBE0wUAQ5GfSNMeAJmSkqLWaueTag/0SzZuD+LcvHmz52vU19eLbMOGDX86JBjz88DLZNKhQwc1//HjR6DrLl26VM2Li4s9X2Pnzp0i07YRO46/rcSt8zcJABFF0wUAQzRdADBE0wUAQzRdADAU+dULfnz//l1kjY2NCRjJn9NWKhQWFqq169atE9mzZ8/U2j179ojs8+fPPkeHP+G2NVtbkfDlyxe1tmvXrp7vd/v2bZHFY/tqUE1NTSJzW4kUVHl5uedaPysS3FYpaE8Xd8M3XQAwRNMFAEM0XQAwRNMFAEOtaiItmc7OdTsTWJscczsfVZsAmDdvXrCBQUhNTRXZx48fPb/ezxZePxNmbsKYNBs4cKCaP3361PM1tEmzkpIStXbVqlWer6udkTthwgS1VjtXWnuKuF8XL170XMs3XQAwRNMFAEM0XQAwRNMFAEM0XQAwFNNm8/77j7GY+z8aWbhwochOnDih1mpbYAcPHhz3MfmVn58vso0bN6q1PXr0EFlVVZVam5eXF2xgEdDc3Bx86vgP+Pls9+zZU2Tv378PPAZte3BrPaw8yurq6tR89uzZga7r9tnmNwwAhmi6AGCIpgsAhmi6AGAo8tuAtYk+t8m/vn37iuzgwYNq7bFjx0T29u1btTY7O1tkubm5Ihs1apT6+rS0NJE9efJErT1//rzI3LZKIr5KS0vVfMWKFaHcL+ikWUFBgZprT7H149evXyJr165doGvGg9vEcUVFRaDruk2Y3bt3T2SZmZmB7uU4fNMFAFM0XQAwRNMFAEM0XQAwRNMFAEOR3wa8YMECkbltA/bj5cuXInM7mHrYsGGB7tXQ0CCyS5cuqbVFRUWB7pVskmEbsKampkbN3Q6ct6Qd5u/2VOywVmZotP9zffr0Mbt/PLiNV/vZ2AYMABFA0wUAQzRdADBE0wUAQ5GfSNO20NbW1qq148aN83xd7QmgLb0X/6RtGa6urlZr16xZ4/m6bU2UJtIePXqk1qanp4c+Hvy7Dx8+qLn2/6u8vFyttTzDmIk0AIgAmi4AGKLpAoAhmi4AGKLpAoChyK9e0PTr10/NtS2NhYWFaq2f1QsHDhwQ2eHDh0XmNvsNd1FavYDW7+fPnyJr3z6cZzmwegEAIoCmCwCGaLoAYIimCwCGknIiDa0HE2nxp211raqqUmuXLFkisrC2xT5+/FhkQ4YMUWvDGoMlJtIAIAJougBgiKYLAIZougBgiKYLAIZYvYCEYvVC/GlPKrZ8SrG2esJxwluRUFlZKbLc3NxQ7uUHqxcAIAJougBgiKYLAIZougBgiIk0JFSiJtKKiorEZ3vbtm1qreUTZN1kZ2eL7Pr164GvG4WfzauSkhI1X7VqldkYmpqa1DwlJUVkTKQBQATQdAHAEE0XAAzRdAHAEE0XAAyxegEJxTZgb1i9kHxYvQAAEUDTBQBDNF0AMETTBQBDLU6kAQDii2+6AGCIpgsAhmi6AGCIpgsAhmi6AGCIpgsAhv4GzmwpIfcmmzcAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "46wHcbNAchH-"
      },
      "source": [
        "Amazing. Now let's see how our pre-trained model is working on both the original and the permuted MNIST dataset:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Sxusb8s3itli",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ca3a49c2-d114-4a26-d559-27d643f34a4a"
      },
      "source": [
        "print(\"Testing on the first task:\")\n",
        "test(model, device, x_test, t_test)\n",
        "\n",
        "print(\"Testing on the second task:\")\n",
        "test(model, device, x_test2, t_test);"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Testing on the first task:\n",
            "Test set: Average loss: 0.0007, Accuracy: 9435/10000 (94%)\n",
            "\n",
            "Testing on the second task:\n",
            "Test set: Average loss: 0.0108, Accuracy: 1031/10000 (10%)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0pHMg4G_dHFY"
      },
      "source": [
        "Mmmh... that's pretty bad, our model cannot generalize to this apparently very different new task! Well, we can just finetune our model using the new permuted training set!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J5PtR8Gqib00",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d05c4582-8ff3-41ee-88e1-5e006527639b"
      },
      "source": [
        "for epoch in range(1, 3):\n",
        "  train(model, device, x_train2, t_train, optimizer, epoch)\n",
        "  test(model, device, x_test2, t_test)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train Epoch: 1 \tLoss: 1.486630\n",
            "Test set: Average loss: 0.0032, Accuracy: 7414/10000 (74%)\n",
            "\n",
            "Train Epoch: 2 \tLoss: 1.154427\n",
            "Test set: Average loss: 0.0022, Accuracy: 8286/10000 (83%)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ML7Evzb9jPAZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e21471ac-d265-4ce1-f3bb-2f5a4eb0c1bf"
      },
      "source": [
        "print(\"Testing on the first task:\")\n",
        "test(model, device, x_test, t_test)\n",
        "\n",
        "print(\"Testing on the second task:\")\n",
        "test(model, device, x_test2, t_test);"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Testing on the first task:\n",
            "Test set: Average loss: 0.0203, Accuracy: 2473/10000 (25%)\n",
            "\n",
            "Testing on the second task:\n",
            "Test set: Average loss: 0.0022, Accuracy: 8286/10000 (83%)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0UZ6FBuHdm7N"
      },
      "source": [
        "This is very annoying! Now we are not able to solve the original MNIST task anymore! :-( This is the phenomenon known in literature as **Catastrophic Forgetting**! In the following section we well compare three different strategies for learning continually (and trying to not forget!)\n",
        "\n",
        "**Questions to explore:**\n",
        "\n",
        "*   When the permuted MNIST benchmark has been firstly introduced? \n",
        "*   Can simple Dropout and Regularization techniques reduce forgetting?\n",
        "*   In the permuted MNIST task, do convolutions still help increasing the accuracy?\n",
        "\n",
        "Some tips here: https://papers.nips.cc/paper/5059-compete-to-compute"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9rUgLpUakTy6"
      },
      "source": [
        "## CL Strategies\n",
        "\n",
        "Let us now focus on some strategies for reducing catastrofic forgetting, one of the principal problems when learning continuously. in this section we will take a look at three different strategies:\n",
        "\n",
        "1.   Naive\n",
        "2.   Rehearsal\n",
        "3.   Elastic Weight Consolidation (EWC)\n",
        "\n",
        "and run it on a 3-tasks Permuted MNIST. Finally we will plot our results for comparison. For a more comprehensive overview on recent CL strategies for deep learning take a look at the recent paper \"[Continuous Learning in Single-Incremental-Task Scenarios](https://arxiv.org/abs/1806.08568)\".\n",
        "\n",
        "Let's start by defining our 3 tasks with the function we have already introduced before:\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yu0T_V24joGY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cb416eb1-e1e3-4398-a082-6b1621fc0007"
      },
      "source": [
        "# task 1\n",
        "task_1 = [(x_train, t_train), (x_test, t_test)]\n",
        "\n",
        "# task 2\n",
        "x_train2, x_test2 = permute_mnist([x_train, x_test], 1)\n",
        "task_2 = [(x_train2, t_train), (x_test2, t_test)]\n",
        "\n",
        "# task 3\n",
        "x_train3, x_test3 = permute_mnist([x_train, x_test], 2)\n",
        "task_3 = [(x_train3, t_train), (x_test3, t_test)]\n",
        "\n",
        "# task list\n",
        "tasks = [task_1, task_2, task_3]"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "starting permutation...\n",
            "done.\n",
            "starting permutation...\n",
            "done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yUNaukXyiJni"
      },
      "source": [
        "### Naive Strategy\n",
        "\n",
        "The  *Naive* strategy, is the simple idea of continuing the back-prop process on the new batches/tasks. This is very simple, but at the same time very prone to forgetting as we have witnessed before. Let's how it works on three tasks:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gzFEA5F8pI_O"
      },
      "source": [
        "model = Net().to(device)\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dLU6KdIbnLMN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e05b2c3a-6993-4236-e2e0-49817bfe0124"
      },
      "source": [
        "naive_accs = []\n",
        "\n",
        "for id, task in enumerate(tasks):\n",
        "  avg_acc = 0\n",
        "  print(\"Training on task: \", id)\n",
        "  \n",
        "  (x_train, t_train), _ = task\n",
        "  \n",
        "  for epoch in range(1, 2):\n",
        "    train(model, device, x_train, t_train, optimizer, epoch)\n",
        "    \n",
        "  for id_test, task in enumerate(tasks):\n",
        "    print(\"Testing on task: \", id_test)\n",
        "    _, (x_test, t_test) = task\n",
        "    acc = test(model, device, x_test, t_test)\n",
        "    avg_acc = avg_acc + acc \n",
        "  \n",
        "  naive_accs.append(avg_acc / 3)\n",
        "  print(\"Avg acc: \", avg_acc / 3)\n",
        "  "
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training on task:  0\n",
            "Train Epoch: 1 \tLoss: 0.664137\n",
            "Testing on task:  0\n",
            "Test set: Average loss: 0.0013, Accuracy: 9028/10000 (90%)\n",
            "\n",
            "Testing on task:  1\n",
            "Test set: Average loss: 0.0113, Accuracy: 618/10000 (6%)\n",
            "\n",
            "Testing on task:  2\n",
            "Test set: Average loss: 0.0101, Accuracy: 1266/10000 (13%)\n",
            "\n",
            "Avg acc:  36.373333333333335\n",
            "Training on task:  1\n",
            "Train Epoch: 1 \tLoss: 1.698266\n",
            "Testing on task:  0\n",
            "Test set: Average loss: 0.0140, Accuracy: 1905/10000 (19%)\n",
            "\n",
            "Testing on task:  1\n",
            "Test set: Average loss: 0.0050, Accuracy: 6432/10000 (64%)\n",
            "\n",
            "Testing on task:  2\n",
            "Test set: Average loss: 0.0101, Accuracy: 1257/10000 (13%)\n",
            "\n",
            "Avg acc:  31.98\n",
            "Training on task:  2\n",
            "Train Epoch: 1 \tLoss: 1.598729\n",
            "Testing on task:  0\n",
            "Test set: Average loss: 0.0230, Accuracy: 1716/10000 (17%)\n",
            "\n",
            "Testing on task:  1\n",
            "Test set: Average loss: 0.0075, Accuracy: 3393/10000 (34%)\n",
            "\n",
            "Testing on task:  2\n",
            "Test set: Average loss: 0.0038, Accuracy: 7142/10000 (71%)\n",
            "\n",
            "Avg acc:  40.836666666666666\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ANJfdFD3s0oT"
      },
      "source": [
        "**Questions to explore:**\n",
        "\n",
        "*   Does the order of the tasks effect the final results? \n",
        "\n",
        "Some tips here: http://proceedings.mlr.press/v78/lomonaco17a/lomonaco17a.pdf"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lCK0EYT-pJa8"
      },
      "source": [
        "### Rehearsal Strategy\n",
        "\n",
        "Another simple CL idea is to carry on *all* or *part* of the previously encountered examples (of the previous tasks), shuffling them with the data of the current task. Using *all* the past data is near to the optimal performance we can desire at the end of the task sequence but at the expense of much bigger memory usage.\n",
        "\n",
        "Let's start by defining a function to shuffle our data:\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FdWpT2jhfu3o"
      },
      "source": [
        "def shuffle_in_unison(dataset, seed, in_place=False):\n",
        "    \"\"\" Shuffle two (or more) list in unison. \"\"\"\n",
        "\n",
        "    np.random.seed(seed)\n",
        "    rng_state = np.random.get_state()\n",
        "    new_dataset = []\n",
        "    for x in dataset:\n",
        "        if in_place:\n",
        "            np.random.shuffle(x)\n",
        "        else:\n",
        "            new_dataset.append(np.random.permutation(x))\n",
        "        np.random.set_state(rng_state)\n",
        "\n",
        "    if not in_place:\n",
        "        return new_dataset"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "94hg1UrtqFmT"
      },
      "source": [
        "Now we can reset the model and optimizer and run our training over the tasks sequence:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "62TY0Ajgbsgk"
      },
      "source": [
        "model = Net().to(device)\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y_No-qvDbuZi",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "089494e6-1b3f-44cc-8c47-51b0494f4f14"
      },
      "source": [
        "rehe_accs = []\n",
        "for id, task in enumerate(tasks):\n",
        "  avg_acc = 0\n",
        "  print(\"Training on task: \", id)\n",
        "  \n",
        "  (x_train, t_train), _ = task\n",
        "  \n",
        "  # for previous task\n",
        "  for i in range(id):\n",
        "    (past_x_train, past_t_train), _ = tasks[i]\n",
        "    x_train = np.concatenate((x_train, past_x_train))\n",
        "    t_train = np.concatenate((t_train, past_t_train))\n",
        "  \n",
        "  x_train, t_train = shuffle_in_unison([x_train, t_train], 0)\n",
        "  \n",
        "  for epoch in range(1, 2):\n",
        "    train(model, device, x_train, t_train, optimizer, epoch)\n",
        "    \n",
        "  for id_test, task in enumerate(tasks):\n",
        "    print(\"Testing on task: \", id_test)\n",
        "    _, (x_test, t_test) = task\n",
        "    acc = test(model, device, x_test, t_test)\n",
        "    avg_acc = avg_acc + acc\n",
        "   \n",
        "  print(\"Avg acc: \", avg_acc / 3)\n",
        "  rehe_accs.append(avg_acc/3)"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training on task:  0\n",
            "Train Epoch: 1 \tLoss: 0.587807\n",
            "Testing on task:  0\n",
            "Test set: Average loss: 0.0014, Accuracy: 9053/10000 (91%)\n",
            "\n",
            "Testing on task:  1\n",
            "Test set: Average loss: 0.0113, Accuracy: 576/10000 (6%)\n",
            "\n",
            "Testing on task:  2\n",
            "Test set: Average loss: 0.0101, Accuracy: 1036/10000 (10%)\n",
            "\n",
            "Avg acc:  35.550000000000004\n",
            "Training on task:  1\n",
            "Train Epoch: 1 \tLoss: 0.840963\n",
            "Testing on task:  0\n",
            "Test set: Average loss: 0.0008, Accuracy: 9422/10000 (94%)\n",
            "\n",
            "Testing on task:  1\n",
            "Test set: Average loss: 0.0037, Accuracy: 7504/10000 (75%)\n",
            "\n",
            "Testing on task:  2\n",
            "Test set: Average loss: 0.0101, Accuracy: 1094/10000 (11%)\n",
            "\n",
            "Avg acc:  60.06666666666666\n",
            "Training on task:  2\n",
            "Train Epoch: 1 \tLoss: 0.526181\n",
            "Testing on task:  0\n",
            "Test set: Average loss: 0.0006, Accuracy: 9514/10000 (95%)\n",
            "\n",
            "Testing on task:  1\n",
            "Test set: Average loss: 0.0021, Accuracy: 8493/10000 (85%)\n",
            "\n",
            "Testing on task:  2\n",
            "Test set: Average loss: 0.0024, Accuracy: 8271/10000 (83%)\n",
            "\n",
            "Avg acc:  87.59333333333332\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jCna5k0DtN-X"
      },
      "source": [
        "**Questions to explore:**\n",
        "\n",
        "*   Can you find a way to reduce the number of examples of the previous tasks to maintain in memory? \n",
        "*   Can you find a good trade-off between memory overhead and final accuracy?\n",
        "*   Why is shuffling needed here?\n",
        "\n",
        "Some tips here: https://arxiv.org/abs/1809.05922"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AqofPHt01Zog"
      },
      "source": [
        "### Elastic Weights Consolidation (EWC) Strategy\n",
        "\n",
        "Elastic Weights Consolidation (EWC) is a common CL strategy firstly proposed in the paper: \"[Overcoming catastrophic forgetting in neural networks](https://arxiv.org/abs/1612.00796)\" for deep neural networks.\n",
        "\n",
        "It is based on the computation of the importance of each weight (fisher information) and a squared regularization loss, penalizing changes in the most important wheights for the previous tasks.\n",
        "\n",
        "It has the great advantage of **not using any** of the previous tasks data!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e9iry5W56xtO"
      },
      "source": [
        "fisher_dict = {}\n",
        "optpar_dict = {}\n",
        "ewc_lambda = 0.4"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0ZlyqcICEps_"
      },
      "source": [
        "model = Net().to(device)\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Clm_QFF12mO9"
      },
      "source": [
        "Now we need to define an additional function to compute the fisher information for each weight at the end of each task:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iEmBNkaO1Ykq"
      },
      "source": [
        "def on_task_update(task_id, x_mem, t_mem):\n",
        "\n",
        "  model.train()\n",
        "  optimizer.zero_grad()\n",
        "  \n",
        "  # accumulating gradients\n",
        "  for start in range(0, len(t_mem)-1, 256):\n",
        "      end = start + 256\n",
        "      x, y = torch.from_numpy(x_mem[start:end]), torch.from_numpy(t_mem[start:end]).long()\n",
        "      x, y = x.to(device), y.to(device)\n",
        "      output = model(x)\n",
        "      loss = F.cross_entropy(output, y)\n",
        "      loss.backward()\n",
        "\n",
        "  fisher_dict[task_id] = {}\n",
        "  optpar_dict[task_id] = {}\n",
        "\n",
        "  # gradients accumulated can be used to calculate fisher\n",
        "  for name, param in model.named_parameters():\n",
        "    \n",
        "    optpar_dict[task_id][name] = param.data.clone()\n",
        "    fisher_dict[task_id][name] = param.grad.data.clone().pow(2)"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2hGp0Qsf2wa-"
      },
      "source": [
        "We need also to modify our *train* function to add the new regularization loss:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IorTFus1Gs5H"
      },
      "source": [
        "def train_ewc(model, device, task_id, x_train, t_train, optimizer, epoch):\n",
        "    model.train()\n",
        "\n",
        "    for start in range(0, len(t_train)-1, 256):\n",
        "      end = start + 256\n",
        "      x, y = torch.from_numpy(x_train[start:end]), torch.from_numpy(t_train[start:end]).long()\n",
        "      x, y = x.to(device), y.to(device)\n",
        "      \n",
        "      optimizer.zero_grad()\n",
        "\n",
        "      output = model(x)\n",
        "      loss = F.cross_entropy(output, y)\n",
        "      \n",
        "      ### magic here! :-)\n",
        "      for task in range(task_id):\n",
        "        for name, param in model.named_parameters():\n",
        "          fisher = fisher_dict[task][name]\n",
        "          optpar = optpar_dict[task][name]\n",
        "          loss += (fisher * (optpar - param).pow(2)).sum() * ewc_lambda\n",
        "      \n",
        "      loss.backward()\n",
        "      optimizer.step()\n",
        "      #print(loss.item())\n",
        "    print('Train Epoch: {} \\tLoss: {:.6f}'.format(epoch, loss.item()))"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u754qi423EEc"
      },
      "source": [
        "Finally we can run the train over the three tasks sequence of th *Permuted MNIST*:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IkUCo4C6-QpT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d7769ad7-c06e-4407-fbb9-e1abb86d88d2"
      },
      "source": [
        "ewc_accs = []\n",
        "for id, task in enumerate(tasks):\n",
        "  avg_acc = 0\n",
        "  print(\"Training on task: \", id)\n",
        "  \n",
        "  (x_train, t_train), _ = task\n",
        "  \n",
        "  for epoch in range(1, 3):\n",
        "    train_ewc(model, device, id, x_train, t_train, optimizer, epoch)\n",
        "  on_task_update(id, x_train, t_train)\n",
        "    \n",
        "  for id_test, task in enumerate(tasks):\n",
        "    print(\"Testing on task: \", id_test)\n",
        "    _, (x_test, t_test) = task\n",
        "    acc = test(model, device, x_test, t_test)\n",
        "    avg_acc = avg_acc + acc\n",
        "   \n",
        "  print(\"Avg acc: \", avg_acc / 3)\n",
        "  ewc_accs.append(avg_acc / 3)"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training on task:  0\n",
            "Train Epoch: 1 \tLoss: 0.827138\n",
            "Train Epoch: 2 \tLoss: 0.669446\n",
            "Testing on task:  0\n",
            "Test set: Average loss: 0.0007, Accuracy: 9455/10000 (95%)\n",
            "\n",
            "Testing on task:  1\n",
            "Test set: Average loss: 0.0116, Accuracy: 643/10000 (6%)\n",
            "\n",
            "Testing on task:  2\n",
            "Test set: Average loss: 0.0097, Accuracy: 1179/10000 (12%)\n",
            "\n",
            "Avg acc:  37.589999999999996\n",
            "Training on task:  1\n",
            "Train Epoch: 1 \tLoss: 1.990289\n",
            "Train Epoch: 2 \tLoss: 1.587910\n",
            "Testing on task:  0\n",
            "Test set: Average loss: 0.0068, Accuracy: 5656/10000 (57%)\n",
            "\n",
            "Testing on task:  1\n",
            "Test set: Average loss: 0.0032, Accuracy: 7797/10000 (78%)\n",
            "\n",
            "Testing on task:  2\n",
            "Test set: Average loss: 0.0110, Accuracy: 1193/10000 (12%)\n",
            "\n",
            "Avg acc:  48.82\n",
            "Training on task:  2\n",
            "Train Epoch: 1 \tLoss: 1.963383\n",
            "Train Epoch: 2 \tLoss: 1.951831\n",
            "Testing on task:  0\n",
            "Test set: Average loss: 0.0066, Accuracy: 6235/10000 (62%)\n",
            "\n",
            "Testing on task:  1\n",
            "Test set: Average loss: 0.0077, Accuracy: 3587/10000 (36%)\n",
            "\n",
            "Testing on task:  2\n",
            "Test set: Average loss: 0.0038, Accuracy: 7401/10000 (74%)\n",
            "\n",
            "Avg acc:  57.410000000000004\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Nx4bR5K1uIwU"
      },
      "source": [
        "**Questions to explore:**\n",
        "\n",
        "*   How much the `ewc_lambda` parameter effect the final results? \n",
        "*   Can you find a better parametrization to improve stability?\n",
        "*   Can you find the memory overhead introduced by EWC with respect to the Naive approach?\n",
        "\n",
        "Some tips here: https://arxiv.org/pdf/1805.06370.pdf"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o3SM7U5fwTqV"
      },
      "source": [
        "### Plot Results\n",
        "\n",
        "To conclude, let's summerize our results in a nice plot! :-)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sIQEVVpDwPP5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "outputId": "47590431-f676-457d-cbea-6aaa6c419946"
      },
      "source": [
        "plt.plot([1, 2, 3], naive_accs, '-o', label=\"Naive\")\n",
        "plt.plot([1, 2, 3], rehe_accs, '-o', label=\"Rehearsal\")\n",
        "plt.plot([1, 2, 3], ewc_accs, '-o', label=\"EWC\")\n",
        "plt.xlabel('Tasks Encountered', fontsize=14)\n",
        "plt.ylabel('Average Accuracy', fontsize=14)\n",
        "plt.title('CL Strategies Comparison on MNIST', fontsize=14);\n",
        "plt.xticks([1, 2, 3])\n",
        "plt.legend(prop={'size': 16});"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEbCAYAAADXk4MCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOydZ3hU1daA3xUSCISeAAmEjggJRSAUC4iAgKKIiigK9nKvckVARfwsqChiQRD7xYpyFQVURFERFRQCBBSQXgRpoSaUkJ79/dgzw6RPkkkmZb3PM89k9jlnn3VOZvY6e68mxhgURVGUioufrwVQFEVRfIsqAkVRlAqOKgJFUZQKjioCRVGUCo4qAkVRlAqOKgJFUZQKjioCxeeIyG4RedDXcpRFRKS3iBgRCfG1LErZRRVBMSMiDURkuojsFJFkEdkvIt+JyOVu+xRoIBSReiLyhuO4ZBE5JCI/icilhe3Tg3MaERnqrf6y0BV4o5j6BkBEKovIQyLyh4icEZHjIhItIveISJXiPHcxsxwIA475WhBvIyLNHN+7dBFpkmVbHRFJdGyPcms3IpIiIi2y7P+BiHzj9nmiiPzl9rmSiIwXkc2O70eciMSIyP2O7bsdfef2+qXYbkQJ4O9rAcozItIM+B04BUwA1mGVb1/gLaBJbsfmw1ygGnAHsAOoD1wMBBdQPj9AjDHphZTDKxhjjhRn/yJSGfge6AQ8ASwD4rEKaCywFfilOGUoDkQkwBiTAsT6WpZiZj9wG/CUW9tNwGFy/g2lA88CwwtwjieBe4FRwCogCPt9aerY3hWo5Pi7I7AI6AbsdbSlFOBcpQ9jjL6K6QV8i/0SV89hW223v3cDD3rYZ23AAP3y2OcXxz6ul6P9VuA0cDnwF5AGtMN+yX8AjgIngd+A87PI597fbrdtVwJrgCTgb+wPsLLb9gbA10AisAf7g/4LmJjb9QO1gHewP/RTwK9AVJbtsxzbk4BdwAN53I+HgQz3Pty2+QE1HX9XAaYBhxz9RgMXue3b23H9lzmuORGrVMKxinid4/5+AwS7HfeBo+0xR9+ngfeBqm77DHT0FQccxyqutm7bmznOPRxY4jj3KDeZQjy5N9iBc77jvp4C5gHhbtsnOv4/NwA7Hft86ew/j3vcHljskOu445pr5XAPRmN/E3GOe1Atjz6d1/w09rslbtv+wCoGk+W7YYAXsMqgS9bzZ71Ot89/ApM8/A1GOc7TzNdjjLdeujRUTIhIXeyP+3VjzOms240x8YXs+rTjNVhEAnPZ5xpgH/YHFOZ4OQkEHgfuASKwg3MN7ODRE/uU8yfwrYg4ZxhdHe93OfrqCiAiA4BPgNeASOB2YCjwnNv5PsQ+VfUBrgJGcPYpKxsiIsBCoBFwBfapbCmwRESc1zEJO/BcAZzrOO/+3PrEPj0uNsbEZN1gjMkwxpx0fHwBuN7RXydgA7DI7bxOngIeALoDdYDPsDONu7EDcyR2oHHnYuyTZF/gWqA/MMVtexBWCXVz9HECWOCYzbgzGbuMFoEdoLOS671xzAC/wirnSxyvhsCXjvvupJnjPlztkLMTVsHniIgEYRXXaYf8VwMXAO9l2bUn9sGjn1v/o3Pr141vsd/bPo7zdQJaAnNy2X8Vdtb8ggd9O4kFeotIgwIcU37wtSYqry/sD8IAV3uw7248nBE49r8W+9SVBKwAXgK659cndkZgcHtSyqV/AQ4CI9zaDDA0y35LgceztA3BDgiCHYgM0MNte2Ps09rEnGTF/thP4/a07Gj/E3jY8ffXwHsFuF9ngOn57BOEnd7f7NZWCftUPMnxubfjega47TPK0dbZrW0imZ82P8AuRVV3axsBJANBeciTjmNGwtmn43FZ9nPK5JwR5HpvgEsdfTZza2uBnS31c5M9icxP8/8H7Mjj3t2FVVw1cpCrlds92AtUctvnv1gFnVu/zmuOwirNTxztrwEz3bdn/Z4CrRz/z4Fu589rRhABbHLci42O/q/BbRbitq/OCBSPkfx3KRzGmLnYJ7krge+wT1/RIvKoB4enYQdVFyJSX0TeFpFtInICuxxQn/xtGF2A/xOR084XMBs7iIUCbbA/LNeTuDFmL3Agnz6rAUey9NsO+xQI8CZwvYisE5GXROTifOT05H/REgjA2nScsqZjFW1Eln3Xu/19yPG+IUtb/azHmMwzwxVAZcd5EZGWIjLb4VRw0tGHH9n/B9lmNVnI6960BQ4YY3a7XeMu7P/D/Rr3GGNOuH0+kMP1uNPWcX2n3NqWY//37v1uMpntUfn16857wNUiEgrcCLyb187GmB1YRfO8YyaUJ8aYTdjvWHesEgjGzjgWenJ8WUeNxcXHduxTQ1vsmqxXMcYkAT86Xk+LyExgooi8ZKwBMTeSTXbj8IfY5YIx2KfzZOAn7ECVF37YZZLPc9hWWAOwH3YQ7JnDtpMAxpjvRKQpdq2+L/bH+rkx5rZc+tyG/T8UlqwpelOzbjPGZG0r6ODxDXY57x7sUk4a9gk16/8gIU9BC35v3GV2kprDtsIOhl7p1xizVUTWAv8DYo0xKxzOGHnxNNaZ4iYPz5EBrHa8XhGREdgl016UQWeCglDuNZ2vMMY4DX6jRKR61u0iUtvLp9yEVexOu0EKZ70c8uMiYIYxZqExZiN2RpB1XTw1h/7WAm2MMTtyeKUBW7DfsS7OA0QkHDubyY21WKWUkUOfh507GWOOGmNmGWNuxXpP3ZKHG+hsoJ+7m6GbPH4iUhO7BJQCXOi2rRJwPvbeFpX2jrV0Jz0c59vpsMW0AZ4zxiw2xmzG2m0K9aCWx73ZDDR0H0AdbpYNKdo1bsZeXw23tguw//vNReg3K+9il5yy2h5yxBhzCLts+gzWEaCgOO9Jtt9veUMVQfFyH3ZZIkZErhORc0WkjYj8m8zLC2B/oOdleWULEhKRYBFZIiIjRKSDiDQXkeuwnjE/mbOGz91ATxFplFM/WdgGjBCRCBHpCnxKdne43UBfEQkVkTqOtqeBG0XkaRFp57i2oSLyAtinOKwyfEtEeojIeVhPkTNkf8p2shi7PPOViFzmuL7zReQpEenpuAdPi8gQETlHRNpi13J3GWOSc+lzGtYT6kcRud9xb5uLyDWO9s7GmATsssoUEbnc0e+bWKXkjRgHf+A9EYkUG+/xPPBfx3njsB5bd4lIK8dyzlvYWUGByOfeLMZ+7z4RkSiHYvwEq3yXFOHaPsH+Tz8SkfYi0gt4G5jnWKLxFh8B9bD/T095GftwNCSvnUTkCxEZIyLdRaSpiPQGXsfOTpcXUt4ygyqCYsSx/toZu3wzBfsjXAIMxnqYuDMG6xLn/rohh25PY90aR2PdKjdivXRmYz0xnDyBNczuJP9lmtuxTz1rsErgPezA7844rJfJXodsGGO+BwY52lc5Xo8A/7gddyt2yeMXrCHzE866NmbDWGvc5dj79F+sj/8crOHZaVtIxnqxrMMqjRpYe0mOOAZB5+B7B3Z9fi1WeX7I2R/6eKwH0PtYO0oHrLHxYG59FwDn/+pn7FLhEsf5nUsS1zvO9xd2AHrccZ0FJdd747i3V2G/Dz87XrHAEMe2QmGMOQMMAGpivwNfYe/x7YXtM5fzpDtmOx4rSIdd5inOzpRzw/ld/hr7YDQL61HX1zG7L9dIEf7/ilJgHLOTA8Bwh9G73CMiH2C9eq7wtSyKkhNqLFaKFRHpg30q3YD1EHkWuwyyyJdyKYpyFlUESnETgA1yaoFdR44GejnWxhVFKQXo0pCiKEoFp0SNxSIyWkT+EpGNIvKAo62uiPwoItsd73Xy60dRFEXxHiU2IxCRdliPlG5Y18RFwL+w3jPHjTHPi8gjQB1jzPi8+goJCTHNmjUrZokVRVHKF2vWrDlqjKmXtb0kbQRtgZUOVzNE5Fesj/NV2CARsK58v2Dd+HKlWbNmxMTkF2mvKIqiuCMie3JqL8mlob+wAU7BIlIN6yveGGjg5qcdiw3gyYaI3C22UETMkSPFmr5eURSlQlFiisARNj8Fm/d+ETZgJz3LPs589zkd/44xJsoYE1WvXraZjaIoilJIStRYbIx51xjTxRjTCxtWvw04JI587473w3n1oSiKoniXkvYaqu94b4K1D8zGhnTf4tjlFmx4uqIoilJClHRA2VxHpsVU4D5jTLyIPA/MEZE7sLk9hpWwTIqiKBWaElUExphsOeaNMcewedMVRVGUnFg/B356Gk7sg1rh0PcJ6OC9Z+Zym2LixIkTHD16lJSUvGq0KBWNypUrExISQq1atXwtiqJ4xvo5sOB+SE20n0/stZ/Ba8qgXCqCpKQkDh06RHh4OFWrVkWk2KpGKmUIYwyJiYns27ePKlWqEBiYX2ZiRSkF/PT0WSXgJDXRtntJEZTLegRHjhyhXr16VKtWTZWA4kJEqFatGiEhIWgsilJmOLGvYO2FoFwqgqSkJKpXL/fV5ZRCUqNGDZKScqyLoyili8OboVJAzttqhXvtNOVSEaSlpeHvXy5XvRQv4O/vT1pagatAKkrJkZYMP0+Gt3qCXwBUqpx5e0BVazD2EuVSEQC6JKTkin43lFLN3lXwdi/49XmIHAIPrIerXodajQGx71e+ql5DiqIo5Y7k09YAvOodqNkIbvwcWve32zoM8+rAnxVVBIqiKL5m+2L45gFrAO52N/R9HKrUKLHTl9ulofLGBx98gIhQu3Zt4uLiMm1LS0tDRJg4cWKB+pw4caIukyiKL0k4BnPvgk+uhYBqcPv3cPkLJaoEQBVBmePEiRNMmTLFK33deeedrFixwit9KYpSAIyB9Z/D611h43y4+BH41zJo0t0n4ujSkAd8+cd+Xvx+KwfiE2lYuyoPDTiXIZ0a+USW/v37M2PGDMaMGUODBjmWbvCY8PBwwsO954KmKIoHxP8D34yFHT9CoygYPAMaRPhUJJ0R5MOXf+xnwrwN7I9PxAD74xOZMG8DX/6x3yfyPPbYYwBMmjQp132OHDnCPffcQ+vWralWrRqNGzfmxhtvZP/+zDJnXRqKjIzkmmuuydbfqlWrEBHmz5/valu3bh2DBw+mTp06VK1alQsvvJBly5YV9fIUpfySkQ4r34bXe8Ce5TBwCtzxg8+VAFSgGcFTCzay6cDJAh/3xz/xpKRnZGpLTE3n4S/W879V/xSor4iGNXnyysgCy+BOWFgYo0aNYtq0aTz44IM0bdo02z7Hjx8nMDCQyZMnU69ePQ4cOMDLL7/MhRdeyJYtW3JNrTBy5EgmTpxIXFwcderUcbXPmjWLunXrMmjQIADWrl1Lz5496dSpE//973+pVq0ab731Fv369WP58uV06dKlSNeoKOWOw1vg6//AvlXQqh9c8QrUbuJrqVzojCAfsiqB/NpLgvHjx1O1alWeeuqpHLefe+65TJ8+nWuvvZZevXpx3XXXMW/ePPbs2cN3332Xa7833XQTqampzJkzx9WWmprKp59+yvXXX0/lyjao5aGHHqJJkyYsWbKEoUOHcvnllzN//nxatGjBM888492LVZSyTFoK/PI8vHURHNsBV78DN31RqpQAVKAZQWGfxC98fgn74xOztTeqXZXP7jm/qGIVirp16zJu3Dieeuopxo8fT8uWLbPt8+abb/LWW2+xc+dOEhISXO1bt27Ntd/GjRvTu3dvZs2axT333APAokWLOHr0KCNHjgQgMTGRX3/9lUcffRQ/P79MEbr9+vXjk08+8dZlKkrZZu8q+Pp+OLIZ2l8HA5+HoBBfS5UjOiPIh4cGnEvVgEqZ2qoGVOKhAef6SCLLmDFjqFu3Lk88kT3MfMaMGdx7773069ePefPmsWrVKqKjowHyzbEzcuRIfv/9d/7++2/ALgu1atWK88+3Su/48eOkp6fzzDPPEBAQkOn12muvERcXR0aG72ZLiuJzkk/Dtw/Du/0h+RTcOAeunVlqlQBUoBlBYXF6B5UWryEn1atXZ8KECYwbN46HHnoo07ZPP/2Uvn378vLLL7vanAN7flx77bXcd999fPzxx9x///0sWLCACRMmuLbXrl0bPz8/7rvvPm6++eYc+/Dz0+cLpYKSKTDsLpsPqIRjAgqDKgIPGNKpkc8H/py49957mTp1qsuTyMmZM2eoWbNmprb333/foz5r1KjBkCFD+Pjjj2nYsCHJycmMGDHCtT0oKIiePXuybt06OnfurIO+ooANDFv0CGyYAyHn2sAwH8UEFAZVBGWYKlWq8MQTT3D33Xdnah84cCBTpkzhueeeo1u3bixZsoQvvvjC435HjhzJ7NmzefLJJ7nwwgtp0aJFpu1Tp06lV69eDBgwgDvuuIOwsDCOHj3K2rVrSU9P5/nnn/fK9SlKqccY2PAFLBoPSSdtYFjPseBfxdeSFQh9nCvj3HbbbZxzzjmZ2p544gnuueceXnnlFa6++mrWr1/P999/73Gfl156KaGhoezfv99lJHanc+fOrF69muDgYO6//3769+/P6NGj2bBhA7169SryNSlKmSB+L8weBvPuhDrN4Z6lcMmEMqcEAMQY42sZCkxUVJSJiYnJdfvmzZtp27ZtCUqklDX0O6IUmox0WP0u/PSUnRH0fcLaA/wq5X+sjxGRNcaYqKztujSkKIriKe6BYS372sCwOtmDOssaqggURVHyIy0FfpsKS1+yXkBXv2PrA5ST7L2qCBRFUfJi72o7CygDgWGFRRWBoihKTiSfhiXP2ERxNRvZwLDWA3wtVbGgikBRFCUrZTQwrLCUqCIQkTHAnYABNgC3AWHAp0AwsAYYaYxJKUm5FEVRABsY9v0EWP9ZmQwMKywlFkcgIo2A+4EoY0w7oBJwAzAFeMUY0wqIA+4oKZkURVGAzBXD/poHF4/3acWwkqakA8r8gaoi4g9UAw4CfQBn2OuHwJASlklRlIpMjoFhj5bJwLDCUmJLQ8aY/SLyEvAPkAj8gF0KijfGOHMZ7wNyTOojIncDdwM0aVK6cnkrilIGyciA1TPPBoYNnFJmAsO8TYkpAhGpA1wFNAfigc+BgZ4eb4x5B3gHbGRxccioKEoFoZwGhhWWklwa6gf8bYw5YoxJBeYBFwK1HUtFAOGAb4oBl3I++OADRMT1qly5Mi1btuTRRx/Nt8ZAVn755RdEhMWLFxeTtL7DeZ92797ta1GU0khaCvwyBd7uebZi2Ii5FVoJQMl6Df0D9BCRatilob5ADPAzMBTrOXQL8FUJylTm+PzzzwkPD+fUqVPMnz+fyZMnc+rUKWbMmOFr0RSldOMeGNZuqA0Mq17P11KVCkrSRrBSRL4A1gJpwB/YpZ6FwKciMsnR9m5JyeQx6+fAT09bn+Ja4danuMMwn4hy3nnn0apVK8BmCd2+fTvvvfce06dPL5O1AZKTk6lSpeIY5RQfUIECwwpLiY4cxpgnjTFtjDHtjDEjjTHJxphdxphuxphWxpjrjDHJJSlTvqyfAwvuhxN7AWPfF9xv20sBnTt35syZMxw9ehSwRWnGjx9P8+bNqVy5Ms2bN+fZZ5/NsXzkmTNnGDVqFCEhIYSEhDBixAji4+Mz7ZOWlsbkyZNp06YNVapUoWHDhowbNy7bctSTTz5J586dqVmzJiEhIfTp08dVHtOJc0lq3rx53HXXXdSrV48GDRoAsG3bNq6++mrq169PYGAgTZo04brrrnPVRE5KSmLMmDG0a9eO6tWrExoaypVXXsmWLVu8di+Vcsj2xfBGD6sEut0F90WrEsiBihNZ/N0jELuh4MftWw3pWXRTaiJ8NQrWfFiwvkLbw2XeLdqye/duatWqRXBwMGlpaQwYMIBNmzbx+OOP0759e6Kjo3nmmWc4fvx4ptKVAKNHj+aKK65g9uzZbN26lYcffphKlSrx4Ydnr2vEiBEsWLCA8ePHc8EFF7B582Yef/xxdu/ezdy5c1377d+/nzFjxhAeHk5CQgIff/wxvXr1Ys2aNbRv3z7Tef/zn/9w2WWXMWvWLJdCGTRoEHXq1OHNN98kJCSE/fv38+2337oUWHJyMqdOneKxxx4jLCyM48eP88Ybb3D++eezefNmQkNDvXpflTJOBQ0MKywVRxEUlqxKIL/2YiY9PZ20tDSXjWDu3LlMmzaNSpUqMWvWLH777Td+/fVXV4GYvn37AvDUU08xfvx46tev7+qrV69eLttC//792bp1KzNnznQZXJctW8Znn33Ghx9+6KpP3K9fP+rWrcuIESP4888/Oe+88wCYOXNmJhkHDhxIZGQkM2fOZPr06ZmuoVu3bpn2P3r0KDt27OCrr75i8ODBrvYbb7zR9XetWrWynWPAgAE0aNCA//3vf4wZM6ZoN1YpH2SrGDYeeo6rUDEBhaHiKILCPom/0s6xLJSFWo3htoVFk6kQtGnTJtPne++9l1GjRgGwaNEimjZtygUXXOBaUgE7yD/22GNER0dnGmgHDRqUqa/27duTnJzMoUOHCA0NZdGiRVSuXJmhQ4dm6w9g6dKlLkWwePFinn32WdavX8/x48dd+zZv3jzbNVx99dWZPgcHB9OiRQseeeQRDh06RO/evbNVXQOYM2cOL7/8Mlu3buXEiROu9q1bt+Zyt5QKRfxeWDgWtv8AjaJg8AxoEOFrqcoEZc+6WNL0fQICqmZuC6hq233A/PnzWb16Nd9++y39+vXjjTfe4KOPPgLg8OHD7Nmzh4CAgEyvbt26AXDs2LFMfdWtWzfTZ6fR1rlcc/jwYVJSUggKCsrUn3NW4exv7dq1XH755VSvXp13332X6OhoVq9eTceOHXN0bQ0LC8v0WUT48ccfiYqKYsKECbRu3ZoWLVrw5ptvuvZZsGAB119/PW3btmX27NmsXLmS1atXU69evQK7zyrljIwMWPmOtQXs/s16A93xgyqBAlBxZgSFxekdVEq8htq1a+fyGurTpw8dOnTgoYce4tprryU4OJjmzZszZ07OhuxmzZoV6FzBwcEEBgaybNmyHLc3bNgQgLlz5+Lv78+8efMICAhwbY+Li6N27drZjpMcinm0aNGCjz76CGMM69at47XXXuPee++lWbNmXHbZZXz66ae0atWKDz74wHVMampqptmHUgHRwDCv4JEiEJEhwAJjTHoxy1M66TDMZwN/XlSpUoUXX3yRq666ijfeeIOBAwcyd+5cqlevnm0JqTAMHDiQKVOmcOLECZetISfOnDlDpUqVMg3wS5Ys4Z9//slxaSgvRITzzjuPqVOn8u677/LXX39x2WWXcebMGfz9M39dZ82aRXp6xfxKVnjSUuC3V2DZS1C5ermrGFbSeDoj+AQ4JSIfAu8aY7YVo0xKARg8eDBdu3bl5ZdfZvv27bz//vv07duXcePG0bFjR1JSUti5cydff/01X375JdWqVfO47969ezN8+HCGDh3K2LFj6datG35+fuzevZtvv/2WKVOm0Lp1awYOHMi0adO49dZbue2229i2bRvPPPMMjRrlmDYqG+vXr2f06NFcf/31tGrVivT0dD744AP8/f3p06cPYJXSl19+yZgxY7jiiiuIiYlhxowZOc44lHKOBoZ5HU8VQShwI7Z+wIMisgIb+DXHGJNQXMIpnjFp0iQGDBjAzJkz+f7773n++ed55513+PvvvwkKCqJly5YMGjSIypUrF7jvjz/+mBkzZvDee+/x7LPPUqVKFZo1a+by2AEYMGAAr776KlOnTmXu3Lm0a9eOjz76iEmTJnl0jtDQUJo0acLUqVPZt28fgYGBtG/fnm+++YYuXboAcNddd7F3717ee+893n77bbp27cqCBQuyGZ6VckzyaVgyCVa+pYFhXkaMKVj+NhGJBG4HbsKmkv4MO0uIzvNALxIVFWViYmJy3b5582batm1bUuIoZRD9jpQxti+Gb8ZYD76ud0K/J8t1xbDiQkTWGGOisrYX2GvIGLMReAWbHqIycD2wTERWikiHIkuqKIriJOEYzLsbPrkWAgLh9kUw6CVVAl7GY0UgIgEiMkxEFgF/YwvK/AtoADQFNmNnB4qiKEUjx4phv0GTHr6WrFziqdfQDGA4ttbwLGCsMWaT2y6JIvIIcMD7IiqKUqHQwLASx1NjcQQwCpiXR2H5o8AlXpFKUZSKR6aKYRnWG6jb3RWyYlhJ45EiMMbk7kR+dp804NciS6QoSsXjyFbrErp3JbTsA1dM08CwEsTTpaFngb3GmLeytP8LaGSMebw4hFMUpZyTKTAsCK5+Gzpcr4FhJYynxuKR2KIxWVkD3Ow9cRRFqTDsXQ1v94JfnoO2g+G+1dDxBlUCPsBTG0F94EgO7cewXkOKoiieoYFhpQ5PFcE/QE9gV5b2XsA+r0qkKEr5ZcdiWKCBYaUNTxXB28ArIlIZWOJo6wtMBqYUh2CKopQjEo7B94/C+k8hpLUNDNOYgFKDRzYCY8zLWGXwKrDN8ZoO/NcY80Lxiac4cVYNy+lVu3ZtVqxYgYi4ahM4SU9Pp0aNGvj7+3Pq1KlM2xYuXIiI8M0332RqX7FiBcOGDaNhw4ZUrlyZ4OBgLr30Uj788EPN9qkUDGfFsNe7wV9fQK+HNTCsFOJxPQJjzAQRmYSNKQDYbIw5XTxiKbnx+eefEx4enqnN39+fjh07Uq1aNZYuXeoqKwm2aExCQgKBgYH8/vvvDBw40LVt6dKl+Pn5cdFFF7napk2bxtixY+nTpw9TpkyhadOmxMXF8cMPP/Dvf/+b2rVrc9VVVxX/hSpln0yBYV1g8NfQINLXUik5UKDCNI5Mo6uLSZZSy8JdC5m+djqxCbGEBoUyuvNoBrUYlP+BxcB5553nKkyTlfPPP5+lS5dmalu6dCmRkZE0aNCApUuXZlME7du3d6VyXrp0KWPHjmXUqFG8+uqrmfq56qqrGDt2LAkJmmxWyQcNDCtzeKwIROQSbJqJJthkcy6MMX28LFepYeGuhUxcPpGkdFsO8WDCQSYunwjgM2WQG7169eLJJ58kNjaW0NBQwA7uPXv2pH79+ixevNi175kzZ1izZg3//ve/XW1Tpkyhbt26vPBCzqt9LVu2LN4LUMo+GhhWJvE0oOxW4C1gPtAb+ApoDTQHPi4m2bzKlFVT2HJ8S4GPW39kPSkZmbNqJKUn8cTvT/DFti8K1Febum0Y34c0S28AACAASURBVG18gWVwJz09PVMheQA/Pz/8/Pzo1asXYAf/YcOGYYzht99+47XXXqN+/fpMnjyZpKQkAgMDWbFiBampqa5j0tPT+fnnnxkyZAiBgYFFklGpgGhgWJnG04CyB4FRxpjhQCowwRjTCasEyrWdIKsSyK+9uGnTpk224vSDBw8GoEePHlSpUsW1PPTXX39x/PhxevbsSY8ePcjIyCA62paNcO7jVARHjx4lMTGRpk316U0pIPti4J2LNTCsDOPp0lALwLmukAxUd/z9GvAL8Ih3xfI+hX0S7/9Ffw4mHMzWHhYUxvsD3y+qWAVm/vz52YzFzjX+wMBAunbt6hrkly5dSrNmzVz7d+rUiaVLl9K7d2+WLl1K27ZtqVdPS/wphSRTYFhDGP4ZnDsw/+OUUoeniuAY4Iz62A+0A9YDwUDVYpCr1DC68+hMNgKAwEqBjO482ifytGvXLldjMdgn/MmTJxMXF+eyDzjp2bMnS5cuJSUlhZUrV2byLgoODqZq1ars2bOnWOVXygmuwLB/oOtd0PcJCKzpa6mUQuLp0tAyoL/j7znAqyLyPvA/4EdPOhCRc0XkT7fXSRF5QETqisiPIrLd8V6n4JdRfAxqMYiJF0wkLCgMQQgLCmPiBRNLnaHYycUXX4wxhmXLlrFs2bJsiiA6Oprly5eTmJjoWhYC64Lau3dvfvzxR5KTk30hulIWSDgG8+6Bj50Vw763FcNUCZRtjDH5voC6QEPH337AeOBr4CWgtid9ZOmvEhCLrWz2AvCIo/0RYEp+x3fp0sXkxaZNm/LcXhZ5//33DWC2b9+e536nTp0y/v7+ZvDgwQYwmzdvdm07cuSIAVzb9u3bl+nYX3/91YiIuf/++3Pse9euXWbdunVFv5hSQHn8jhQrGRnGrP/cmCktjHmqrjE/TTImNcnXUikFBIgxOYyp+S4NiYg/cAPwpUNxZFD0tBJ9gZ3GmD0ichXWEwngQ6zNoWiuNeWYP//8k6NHj2Zrj4qKwt/fn+rVq9OpUycWLFhAvXr1aNOmjWufkJAQ2rRpw4IFC2jRogWNGjXK1EevXr2YOnUqY8eOZdOmTdx66600adKEuLg4fvrpJ2bOnMns2bPp0EFLU1coTuyDb8bC9u81MKyckq8iMMakiciLwEIvnvcG7LISQANjjNMaG0su2UxF5G7gboAmTZp4UZSyxXXXXZdj+5EjRwgJCQHs8tDq1aszRQw76dmzJ1u2bMm0LOTOAw88QLdu3XjllVd48MEHOXr0KDVq1CAqKoq3336bK6+80nsXo5RuMjIg5l1YPNEGhg2YDN3v0cCwcojY2UI+O4n8BLxujJlX5BPaxHUHgEhjzCERiTfG1HbbHmeMydNOEBUVZWJiYnLdvnnzZtq2bVtUUZVyjH5H8kEDw8olIrLGGBOVtd1Tr6H/Ai+JSBNsMZpMeQaMMWsLIMtlwFpjzCHH50MiEmaMOSgiYcDhAvSlKIo30cCwComnimC2431qDtsM1vjrKcM5uywE1uh8C/C84/2rAvSlKIq32BdjZwGHN0G7oTZHUHWNM6kIeKoImnvjZCISBFwK3OPW/DwwR0TuAPYAw7xxLkVRPEQDwyo8HikCY4xXooyMzV4anKXtGNaLSFGUkkYDwxQ8Tzp3TV7bvWFE9jbGGETXNZUc8MRBotxz5jgsmuBWMex7LRZTgfF0aSi3NJvOX1Sp8icLCAggMTGRatWq+VoUpRSSmJhIQECAr8XwDcbAX3Phu/GQFG8rhvUcZ6OElQqLp0tDmVJROILMOgEvAv9XDHIVifr167N//34aNWpE1apVdWagAHYmkJiYyP79+2nQIMdwlfKNBoYpuVCgCmVOjDFpwGoReRR4E+joVamKSM2ado3zwIEDpKam+lgapTQREBBAgwYNXN+RCoEGhin5UChF4EY8UCrLVtWsWbNi/dgVJSc0MEzxAE+NxZ2zNgFh2JxAf3hbKEVRikhaCvw+DZa+aAPDhrylxWKUXPF0RhCDNQxn/RZFA7d5VSJFUYpGpsCwa2HgFA0MU/KksAFlGcARY0xSTjsriuIDNDBMKSQlGlCmKEoxoYFhShHw1EbwLLDXGPNWlvZ/AY2MMY8Xh3CKouSDBoYpXsDTUpUjydkovAa4OYd2RVGKE2NgwxfwWlf46wsbGHbPMlUCSqHw1EZQHziSQ/sxcikkoyhKMXFiHywcB9sWaWCY4hU8VQT/AD2BXVnaewH7vCqRoig54woMewpMugaGKV7DU0XwNvCKo7rYEkdbX2AyRa9frChKfmQLDHsF6jTztVRKOcFTr6GXRSQEeBWo7GhOAaYbY14oLuEUpcKjgWFKCeBxigljzAQRmQREOJo2G2NOF49YiqJoYJhSUnjqPhoK+Btj9gGr3drDgVS3+sOKohSVlAQbGBb9pgaGKSWCp+6jH2OLzmdlADDLe+IoSgVnx0/wRg+IfgO63gH3RqsSUIodTxVBFLA0h/Zljm2KohSFM8dh/r/g42ugUhW4bREMelmjg5USwVMbgT9QJYf2wFzaFUXxhGwVwx6Cng9qxTClRPFUEawE/u14uXMfbjYDRVEKgAaGKaUETxXB/wFLRKQDZ+MI+mDLVfYrDsEUpdyigWFKKcPTOIJoETkfeAi4xtH8B3CvMWZdcQmnKOWOI9scgWHR0OISuHKaBoYpPqcgcQTrgBFZ20WknzFmsVelUpTyRloK/D4dlr6ggWFKqaNQNYtFpBG2MtntQFNA57SKkhv71jgCwzZC5DVw2RSoXt/XUimKC48VgYhUAq4C7gQuBdYDbwGfF49oilLGyRYY9imcm1M4jqL4lnwVgYicix38bwYSgNlYRTDSGLOpICcTkdrATKAdtgby7cBW4DOgGbAbGGaMiStIv4ric9bPgZ+etp5AtcJtSoiN8yD+H+h6J/R9UmMClFJLngFlIrIMW6C+DnaAbmGMeawI55sOLDLGtAE6ApuBR4CfjDHnAD85PitK2WH9HFhwP5zYCxj7/vs0SE3WwDClTJBfZPH5wEfAK8aYX4tyIhGpha1f8C6AMSbFGBOPXW760LHbh8CQopxHUUqcn56G1MTs7ZUCoOn5JS+PohSQ/BRBV+zy0W8i8oeIjHEkoCsMzbFVzt539DVTRIKABsaYg459Ysml4pmI3C0iMSISc+RITsXSFMVHnMilNtPJ/SUrh6IUkjwVgTHmD2PMfUAYMBUYDOx1HDdIROoU4Fz+QGfgTWNMJ6y9IdMykDHGYG0HOcnyjjEmyhgTVa+epuJVSgHxe+GLO8jlK2ttBYpSBvAo6ZwxJskYM8sYcwnQFngRGAPEish3Hp5rH7DPGLPS8fkLrGI4JCJhAI73wwW5AEUpcVIS4OfnbOH4Ld/AuVeAf9XM+wRUhb5P+EY+RSkgnmYfdWGM2WGMeQRoDAzDVirz5LhYYK/DCwlsqctNwNfALY62W4CvCiqTopQIGRmw7jOYEQW/ToE2l8OoGBj+CQx+FWo1BsS+X/kqdBjma4kVxSPErsaU0MlEzsO6j1YGdmGD0vyAOUATYA/WO+l4Xv1ERUWZmJiYYpZWUdzYuxoWPQL7Y6BhJxj4PDTp4WupFKVAiMgaY0y20gGFiiwuLMaYP8m5fkHfkpRDUTzmxH5YPBE2zIHqoTDkTehwA/gVeDKtKKWWElUEilJmSDkDy1+F36aBybA1Ai4aA1Wq+1oyRfE6qggUxR1jYMMXsPhJ6/4ZeTX0ewrqNPW1ZEoFZuGuhUxfO53YhFhCg0IZ3Xk0g1oM8lr/qggUxcm+NdYOsG8VhHWEa2dC0wt8LZVSwVm4ayETl08kKT0JgIMJB5m4fCKA15RBQZLOtQfuAVoCtxtjDorIEGCPMeYPr0ijKL7g5AFbJGb9p1C9AVz1OnS8Ue0Ais84mXKSzcc2s+nYJt748w2XEnCSlJ7E9LXTS1YRiEh/rJvnd9jKZE6n6ZbArWhaCKUskpoIy2fAb69ARjpcNBZ6joUqNXwtmVKBOJVyyjXobzy2kU3HNvHPqX/yPS42IdZrMng6I3gGGGuMeUNETrm1/wKM85o0ilISGGMzg/74pE0QF3EVXPq0VgpTip3TKafZfNwx6B/dyKbjm9hzco9re1hQGBHBEQxpNYSI4AgigiO4/pvrOZhwMFtfoUGFzfaTHU8VQTvg2xzajwN1vSaNohQ3+9fCogm2VGRoe7j6LWh2ka+lUsohmQb9YxvZfGwzu0/udm0PDQolom4Eg1sOdg36dQOzD6ejO4/OZCMACKwUyOjOo70mq6eK4DjQCFsvwJ3O2NQRilK6OXkQljwDf34CQfVg8Aw47yYtGK94hYTUBDYf2+xa2tl0bFOmQb9BtQZEBkdyRYsrXIN+cNVgj/p22gFKg9fQbOBFERmGzbDlLyIXAy8B73tNGkXxNqmJsOI1WPYKZKTChQ9Az3FaH0ApNM5Bf9OxTWw6bpd49pzcg3EkH6xfrT6RwZEMajGIyODIAg36uTGoxSCvDvxZ8VQRPAZ8gE0BIdgcQYJVEM8Wi2SKUhSMgY3zHXaAf6DtldYOULeFryVTyhBnUs9kWt7ZdGwTu0/szjToRwRHcHmLy12DfkjVEB9LXXA8UgTGmFTgJhF5AuiEzQ/0hzFme3EKpyiF4sCf1g7wz3Jo0B6GLIDmvXwtlVLKOZN6hi3Ht2Qa9P8+8ffZQb+qHfQva35ZmR70c6JAAWXGmJ3AzmKSRVGKxqlDtlrYn59AtWC4cjp0Gql2ACUbZ1LPsDVuq/Xccazp7zqxyzXo16taj4jgCAY2G+ha069XrfzWQfE0juC9XDYZIAnYAXxmjDngLcEUxWNSkyD6dVg2FdKS4YL/QK8HIbCWryVTSgGJaYlsPb41kyF314ldZJgMAEKqhhAZHEn/Zv1dg379avV9LHXJ4umMoB7QE8gA/nK0tcPaCdYA1wBPi0hPR4ZRRSl+jIHNX8MPj0P8Hjh3EPR/BoJb+loyxUfkN+gHBwYTGRJJv6b9XMs7FW3QzwlPFcHvwGngDmPMGQARqQb8F1gHXI4tcv8ymlJaKQkOroNFj8Ke36B+JNz8FbTo7WuplBLEOei7r+m7D/p1A+sSGRxJ3yZ9Mw36IuJjyUsfniqC0UAfpxIAMMacEZFngZ+MMS+IyBRgcXEIqSguTh+28QBrZ0G1unDFK9DpZqik+RPLM0lpSWyN25opIndX/C7STTpgB/2I4Aj6NOnjGvQbVGugg76HePrrqY4tYL85S3uoYxvAyQL0pygFIy0Zot+EpS9BWiKcfx/0egiq1va1ZIqXSUpLYlvctkzLOzvjd2Ya9NsGt+WSxpcQERxBZHCkDvpFxNOBez7wrog8DKx2tHUFXgDmOT53A7Z5VzylwmOMLRD/w2MQtxtaXwb9J0FIK19LpniB5PRkth3PPOjviN/hGvTrVKlDRHAEF4df7HrSDw0K1UHfy3iqCP4FTAU+djsmDXgPeNDxeTNwl1elUyo2sRtsPMDuZVCvLYycDy37+FoqpZA4B333iNyd8TtJM2kA1K5Sm8jgSHqF93I96eugXzJ4GlB2BviXiIzDpp4G2GmMSXDbR72FFO9w+gj8PAnWfgSBteHyl6DLbWoHKEOkpKewLW5bJkPujrgdrkG/VpVaRAZH0jO8p+tJPywoTAd9H1HQgLIEYH0xyaJUdNKSYeXbsPRFSD0D3f8FFz8MVev4WjIlD1LSU9getz3T8s72+O2kZZwd9CPqRnBL5C1EhthBv2FQQx30SxEFqVB2CTAcaAJUdt9mjNH5ulJ4jIGt38L3/wdxf8M5A2DAsxByjq8lU7KQkp7C9vjtmSJy3Qf9mpVrEhEcwS0Rt7iCsxpVb6SDfinH08jiW4G3sEbj3sBXQGugOdZuoCiF49BGawf4+1eo1wZGzIVW/XwtlQKkpqeyLX6ba8DfeHRjpkG/RuUaRARHcHPEza5BP7x6uA76ZRBPZwQPAqOMMTMdFcomGGN2ichr2EAzRSkYCUfh52dhzQc2FYTaAXxKanoq2+O3nx30j21ke9x2UjNSgbOD/siIkS5Drg765QdPf3UtOBsslszZ2IHXsOUqH/GuWEq5JS0FVr0Dv74AKaeh291w8XgbHKaUCKnpqeyI35HJkLstbtvZQT/ADvoj2o4gIiSCyLqRhNfQQb8846kiOAY4K3rvx+YZWg8Ec7aQvaLkjjGwbZG1AxzfCa0utXaAeuf6WrJyTWpGKjvjd2Za098atzXToN82uC03tb3J5b0TXiMcP/HzseRKSeKpIlgG9Ac2AHOAV0XkUmxeoR+LSTalvHBoE3z/KOz6GUJaw01fwDmX+lqqckdqRiq74ndl8t7ZenwrKRkpAFQPqO4a9J1r+o1rNNZBX/FYEYwCAh1/T8YGk12IVQqTPD2ZiOwGTgHpQJoxJkpE6gKfAc2wNZGHGWPiPO1TKcUkHINfnoOY96BKTRg4BbreAZUCfC1Zmcc56Lsv77gP+kEBQbSt25bhbYa7Bv0mNZvooK/kSL6KQET8gRuALwGMMRnAlCKc8xJjzFG3z49gE9c9LyKPOD6PL0L/iq9JT4VV/4Vfn4fk09D1Tug9Qe0AhSQtI42d8TtdT/nO5Z3k9GQAqvlXIyI4ghva3OAy5OqgrxSEfBWBMSZNRF4EFhaTDFdhXVIBPsQan1URlEWMge0/WDvAse02HcSA56B+W19LVmZIy0hj14ldmbJsbju+jaT0JMAO+m2D2zLs3GGuQb9pzaY66CtFwtOloWigC7Z4fVEwwA8iYoC3jTHvAA2MMQcd22OBBjkdKCJ3A3cDNGnSpIhiKF7n8BZrB9j5EwS3ghvnwDn9QT1NciUtI42/T/ydbU3ffdBvU7cNQ1sPdUXkNqvZTAd9xet4qgj+C7wkIk2wFckS3DcaY9Z62M9Fxpj9IlIf+FFEtmTpxziURDYcSuMdgKioqBz3UXzAmePwy2RY/S5UqQ4DJtulIP/K+R9bjli4ayHT104nNiGW0KBQRncezaAWg1zb0zPSsw36W45vcQ36Vf2r0rZuW4a2HprpSb+S1ltWSgAxJv8xVUQy8thsjDEF/raKyERsMNpdQG9jzEERCQN+Mcbk6VMYFRVlYmJiCnpKxZukp9rB/5fJkHwSom6H3o9CULCvJStxFu5ayMTlE12DOkCVSlUY3HIwlStVZuPRjWyN20piWiJgB/02ddu43DWdT/o66CvFjYisMcZEZW33dEbQ3AsCBAF+xphTjr/7A08DXwO3AM873r8q6rmUYmb7j3YZ6Og2Wx5ywGRoEOFrqXzGtDXTMikBsCmXP9/2uWvQv+aca1xP+jroK6UNT9NQF9U2AHbtf74jOtEfmG2MWSQiq4E5InIH1gYxzAvnUoqDI1utIXjHj1C3JQz/FFoPrHB2gLSMNDYe28jKgytZeXAlsWdic9xPEFYMX6GDvlLqKUj20cuA+7DpJgYYY/aKyJ3A38aYn/I73hizC+iYQ/sxtOB96ebMcfh1inUJrVwd+j9rU0NUEDuAMYad8TuJPhjNyoMriTkUw+lUm2Lr3DrnEuQfREJaQrbjQoNCVQkoZQJPs4/ehM0+OhM7aDsjgioBDwP5KgKlDJKeZoPBfnkOkk5Al1vhkv+DoBBfS1bsHDh9gJUHVxJ9MJpVsas4mmhDX8KrhzOg2QB6NOxBt9Bu1A2sm6ONILBSIKM7j/aV+IpSIDydETwM3GWM+dQxC3ASjV3nV8obOxbbZaAjW6B5L2sHCG3na6mKjbikOFbFrnI99e89tRewhdK7h3WnR1gPuod1p1H1RtmOdXoH5eU1pCilGU8VwTnAihzaTwM1vSeO4nOObrcKYPv3UKc53DAbzr283NkBzqSeYc2hNXadP3YlW45bT+aggCCiGkQxvM1wuod155za53iUdXNQi0E68CtlFk8VwQFsIZqsRuNewE6vSqT4hsQ4mxp61TsQUA0ufQa63wP+VXwtmVdIzUhlw5ENruWe9UfXk5aRRoBfAOfVP49R542ie1h3IkMiCfDTXEhKxcJTRfAONuOoc1mosYj0BF4AJhaHYEoJkZ4Ga96Hn5+zyqDLLXDJY1C9nq8lKxIZJoNtcdtcA/+aQ2tITEtEENoGt2VkxEh6hPagU4NOVPXXTOpKxcZT99EXRKQWNuV0IPAztkDNS8aY14tRPqU42fmzjQc4vAma9YSBkyG0va+lKhTGGPad2kd0rF3jX3VwFXHJNolts5rNGNxyMD3CetA1tCu1qtTysbSKUrrw2H3UGPN/IvIsEAH4AZuMMVqmsixybKe1A2z7Duo0g+s/hjZXlDk7wNHEoy5f/pUHV3Ig4QAA9avW56JGF9E9rDvdw7oTGhTqY0kVpXTjqfvoA9gAsMOA5nYoqyTGw9IXYeXb4B8I/Z6CHv8uM3aA0ymniTkU41ru2RG/A7D1dLs26Mqt7W6le1h3mtdsrmUVFaUAeDojGAu8ICJLgFnAfGPMmeITS/EqGemw9kNYMskGh3UaAX0ehxo5JnotNaSkp7DuyDpWHFjBytiVbDy6kXSTTpVKVehUvxODWgyiR1gP2tZtq4FbilIEPFUETbE1A24EZgBvichXwMfAD45iNUppZNevsGgCHN4ITS+0doCwbAHepYL0jHS2HN/i8uX/4/AfJKUn4Sd+tAtux+3tbqdHWA861u9IlUplYxajKGUBT43FBmsg/llE7gOuwCqFeUA80LDYJFQKx7Gd8MPjsHUh1G4Cwz6CtoNLlR3AGMPuk7tdSz2rY1dzMuUkAK1qt+La1tfSPbQ7UaFR1Khcw8fSKkr5xWNjsRNjTIqIrMBmJI0E8kwZrZQwSSesHSD6Lbv23/cJ6HEfBATmf2wJcCjhECtjV7oG/8NnDgMQFhRGnyZ9XBG8IVXLfxoLRSktFCTpXA1gKHATcDGwA5iNXR5SfE1GOqz9yGEHOAbn3QR9H4cavvWYOZF8gpjYGLvcE7uSv0/8DUDtKrXpFtrNlb6hcY3GauBVFB/hqdfQF8DlwEngM2CCMWZ1cQqmFIC/l1k7wKEN0OR8GPgFNOzkE1GS0pJYe3ity6Vz8/HNZJgMqvpXpXODzlzT6hp6NOxB6zqtteSiopQSPJ0RJAPXYg3D6e4bRKSfMWax1yVT8uf43/DDY7DlG6jVBK77ACKGlKgdIGtu/j8P/0lKRgr+4k+Heh24p8M9dA/rToeQDgRU0tQNilIa8dRYfJP7ZxFpBNwG3I71KFLfvZIk6SQsewmi3wS/AOsKev59EFD8qRLyy81/Q5sb6B7WnagGUVQLqFbs8iiKUnQKYiOoBFwF3AlcCqzH1ij4vHhEU7KRkQ5/fgI/PQMJh6HjjdYYXDOsWE+bV27+gc0H0j2suys3v6IoZY98FYGInIsd/G8GErAG4kuBkcaYTcUrnuJi9++w6BGIXQ+Ne8CNn0GjzsVyKmdufudyzz+n/gE8y82vKErZI09FICLLgHbAXGCYMeZXR/v4EpBNAYjbDT8+AZu+glqNYeh7EHmNV+0Anubmb1W7lXr2KEo5JL8ZwfnA68A7xpiNJSCP4iT5FCybCiteB79KtkTkBf/xih3A09z87ULa4e9X4FATRVHKGPn9yrtil4V+E5HdwEfA/4pbqApNRgasmw0/PQ2nD0GHG6Dfk1Cz8MHbGSaD7XHbiT4YnS03f0RwBDdH3Ez3sO50qq+5+RWlIpKnIjDG/AHcJyLjgOuwXkIvYNNQDxKRg8aYuOIXs4KwZ7m1AxxcB+Hd4Ib/QXiXAnejufkVRSkInrqPJmGzjs4SkVbYWcIYYJKILDHGXFaMMpZ/4vbA4idh43yo2QiumQnthxbIDnA08SirDq5iZexKog9EZ8vN36NhD7qFdtPc/IqiZKMwuYZ2AI+IyP9hk8/d7nWpKgrJp+G3qbD8NRA/6D0BLrgfKufvf59Xbv5uod00N7+ilCO+/GM/L36/lQPxiTSsXZWHBpzLkE7e89ortCXQEWH8leOlFISMDFj/KSx+Ck7HQvth1g5QKzzXQ5y5+Z2BXH8d/Stbbv7zw86nTd02mptfUcoRX/6xnwnzNpCYapM67I9PZMK8DQBeUwbqElLS/BNt7QAH/oBGXWyZyMZds+2WW27+SlKJyJBIzc2vKBWEF7/f6lICThJT03nx+61lVxE4IpRjgP3GmCtEpDnwKRAMrMEGqqWUtFzFTvxeawf4ay7UaAhXvwPtrwM/m3jNPTf/yoMrWRW7SnPzK0oFxBhD7Mkk1u09wfp98eyPT8xxvwO5tBcGX8wIRgObgZqOz1OAV4wxn4rIW8AdwJs+kKt4SEmA36bB8lft54vHw4WjoXJQnrn5+zbp6yq+rrn5FaX8cjwhhXX74lnvGPjX7z/BkVPJAPj7CQGVhNR0k+24hrW95+pdoopARMKBQcCzwFixVsw+2GpnAB8CEykPiiAjAzbMgcUT4dRBaDeUE73GEZN4kOi10zQ3v6JUQE4np7Fhn2PA33eCdfvi2Rdnn+xFoEVIED1bhdAhvBYdGtcmIqwmi/6KzWQjAKgaUImHBnivJlhJzwimAQ8DzrWNYCDeGJPm+LwPyHHRS0TuBu4GaNKkSTGLWUT2roJFj5B0YC1/NIxkZefBRCf8w+ZFw125+bs06MK151xL97DumptfUcohSanpbD540jXgr993gp1HTmMcD/fhdarSMbw2I3s0pUN4bdo1qkmNwOyp2p12gOL0GhJjsk85igMRuQK43Bhzr4j0Bh4EbgWijTGtHPs0Br4zxrTLq6+oqCgTExNTzBIXnLS4PWz88WFW7l/Oyuo1+bNKACkm3ZWb37nUo7n5FaV8kZaewfbDp1m/L551jif+rbGnXEs6IdWr0DG8Fh3Ca9OhcS06NKpFcPWSd/IQkTXGmKis7SU5I7gQGCwilwOBWBvBdKC2iPg7ZgXhwP4SlKlIOHPzr9y3jOjNnxGTsJfTfn5QBVQx2wAAESBJREFUtzZtap/D8IYX0D2sO10adNHc/IpSTjDGsPvYGTvoO9b1/zpwgqTUDABqBPrTIbwWd/Zs4Rr8w2oFlurl3hJTBMaYCcAEAOeMwBhzk4h8jq2F/ClwC6U8LiG33PyNU1MZWDWM7ufdSbeWl2lufkUpBxhjOHgiyfWk71zfP5lkV7MDA/yIbFiL4d2a0DG8Nh3Ca9EsOAg/v9I76OdEaYgjGA98KiKTgD+Ad30sTyZyy80fXLkm3ZNS6HHsGN1rtKDhwBeh6fk+llZRlKKQ1YNn3b4THD191oOnTVgNBnVo6HrSb92gOv6Vyr59zyeKwBjzC/CL4+9dQDdfyJETeeXm79qgK8ObXU73nStotXEBUr0B9H0ROg53xQMoilI2yM+Dp2W96vRqHeJ60m8bVpPAgPIZtV8aZgQlwsJdC5m+djqxCbGEBoUyuvNoBrUYlG9u/v90+g/dw7oTWaM5/tFvwneTbMnInuPgojFQRYO7FKW04y0PnvJKiXkNeZOCeg0t3LWQicsnkpSe5Grz9/OnRc0W7D29N1Nufqdnjys3vzE2GvjHJ+HkPogYApc+BXWaFcOVKYpSVNLSM9h26DQb9p/14Nly8BRpGaXLg8cXlAavIZ8xfe30TEoAIC0jjZ0ndjK09dDcc/PvXwPfPQL7VkFoB7jmHWh2YQlKrihKXmRkGHYfS8j0pL8xBw+eu3qVHQ8eX1AhFEFsQmyO7Rkmg8d6PJZ9w8kDtkLYuv9BUH0Y/Bqcd6MtGakoik/I6sHjXNs/5ebB065hLW7s1tRG5pZRDx5fUCEUQWhQKAcTDubYnonURFsb4LepkJEGFz5gbQGBNbMdqyhK8eKJB8+VHc968JxTv3x48PiCCqEILqw7ks9PvYL4/X97dx4dV3necfz70zrahRwt3lhNHYeC7SQQQoJ7CkkgLVCX5nQLW8op/aclnBT4I8khiWlLOSSB9lCghFCWJJAUaMtSEihlSVIwOAYUwEACYbORZGNLI1vWWJae/vHeka5GI8kyoxlZ9/mcc4/vXN3llWZ8n3nf57n3Do0us5FKPtFydvTC4MV7Qh6g721YcTp8+nJoOaxELXYuWbyCp7QSEQgeenoxgyNnUt36E1TZiw01k9l6Cg++u4iv/NZGKh76Mrz1JLQfDWuvh8NOLHWTnZu3BoeGeendNL/0Cp45IxGBYEvvbk4v28Wlg1tYpG1ssQFu2NvDyrJrKPvOT3lPjfyg/kI6a06j7bla2l//FR2NKdoaq2lvTNHRmKK5ttITTM7NULaCJ/cePLkVPKcfsyhxFTxzSSICwbn1T3Pp0E3UKjzvZom2cXnlrQxL/GLRWdzb9Oe8tauC7t5BNrzVx46BoQn7qCovGxcYsvPto/+Gqb46EX9S5yaYroKnMVXBMUuauWDN4aF0c0mTV/DMEYk4a11a+UNq945/6JkEQ6lWjr3gWnIfFJnZO0xPOkNP/yBdfRm604N09w/Sk87Q1TfIpq40j7+aYWdmL7nqqytCkGiIgkRTKpofCxptjdVUV/j4pjtwzaSCZ+XSkMw9pKXWK3jmqEQEgtrd+ctHawa35l1eXVHO0pZalrZMfcfQnZm9IUikoyARm+9OD7LhzR30pDPsGR6ZsO1BtZWxnsT4XkV7YzUdjSkW1FdT7v9x3BzgFTzzWyICAU1LQjVQvuXvQ311BfWt9RzRWj/pOmZG78DQhCARXodex8tdabb2ZxjJuci7TNDaUD0+YGR7F01jrz1/4Qqpf3CIX27uozO626ZX8Mx/yQgEJ18G910YrhPIqqwJy2eZJA6qq+KguipWLJz8eoS9wyO8t2tPCBJ9g3T3Z+iJgkdXOsPb2wfY8Mb2/PmLirLxQSJnGKojWlbn+QuXI1vB0/n2WNnm69t2jVbwLG3xCp4kSMaZ4Zg/Dv8+sg763gk9gZMvG1s+B1SUl42exI+ZoqMyODTM1v4obxENR2UDRnc6w6Z30zz6Sg8De4YnbFtfXTFuGCoeJNobq2lr8PzFfDZdBU9rQ6jg+YNVi6Mrc5tpqasqcatdMSTipnNJNJq/6AuJ7u4o0d0TzWeHqfLlL1rqqmhrqKZjNNFdTVtULZUNGp6/mNv2tYIne8JfubSJjkav4JnvEn3TuSTa1/zFjoGh0ZxFT3osSGR7GC9tSbNt58T8RXmZaK2vzgkSYb499rqpxvMXs21/KngOXVDr74sb5YEgwSTRUldFyz7kL7bt3BMLELGA0Z/hrfcGeOaN7fROkb8I116EHkZHU5S/iHobHU0paqv8o7iv3tuZofOdvmjyCh73/vn/PjetivIyOppSdDSlplwvm7/oyg0W0bRpS5pH0/nzFw3Z6y9GL9hLjQ8gUQ6jqiJZJ7R4BU/2Yembe72CxxWWBwJXMKnK6a+/MLMofxGqorpiASNcwDfI+t9sp6d/kKHhifmrBXVVo4GhvWF8GW1HU0h2L6g7MPMX+1LBs+rgZs49IVvB0+RXsruC8E+RKypJNKQqaUhVsqxt8vzFyIixY2BPCBL9IX/R1RebTw/yYpS/yK13GM1fNKVoj67D6GhK0Ra7JqOjMUVjTUXJxsm9gsfNJR4I3JxUViYW1IfqpA8xff5i7IK9sbLa7vQgb743wPrfbKdv98T8RXVFGROv7J54lfe+5C/+89nNXPWTV9jSu5tFzTVccspy1q5eDMz8HjxeweOKzctHXSIMDoX7R3VHw09hKCozegFfT38or909lCd/kaoYf2V3rKfR3pTi+bd7ufLHL4+e2AGqysWJR7YyuHc4bwVP9oTvFTyumLx81CVaqrKcgxfUcvCCqfMX/Zm948pou2K3BelOT52/iNszbDzycg9HL27ijJWLWLmkmaOXNHkFj5uTPBA4F5FEY6qSxlQly9oaJl0vm7/IBokv3PJM/v0B9/3NJ2eptc4VjgcC52Yonr84ahEsbq4ZLemMW9RcU4LWOTdz3kd17n265JTl1OTU7tdUlnPJKctL1CLnZsZ7BM69T9nqoMmqhpyb64oWCCSlgCeA6ui4d5nZ1yQdBtwJLAB+AZxtZnsm35Nzc8/a1Yv9xO8OWMUcGsoAJ5nZSmAVcKqk44ErgavNbBmwAzi/iG1yzrnEK1ogsGBn9LIymgw4CbgrWn4rsLZYbXLOOVfkZLGkcknPAT3Aw8BrQK+ZZZ8C/w6Qt38t6QJJGyRt2Lo1/7OGnXPOzVxRA4GZDZvZKmAJcBzwwRlse6OZfdTMPtra2jprbXTOuaQpSfmomfUCjwIfB5olZZPWS4DNpWiTc84lVdHuNSSpFRgys15JNcBDhETxucDdZnanpBuATjO7bpp9bQXe3M+mfADYtp/bOjcd/3y52fR+P1+HmNmEIZViBoJjCMngckJP5Edmtk7S4YTy0RbgWeAsM8vMYjs25LvpknOF4J8vN5tm6/NVtOsIzKwTWJ1n+euEfIFzzrkS8FtMOOdcwiUxENxY6ga4ec0/X242zcrn64B8MI1zzrnCSWKPwDnnXIwHAuecS7jEBAJJN0vqkfRCqdvi5h9JSyU9KuklSS9K+mKp2+TmD0kpSU9Lej76fH2joPtPSo5A0hpgJ3Cbmf12qdvj5hdJC4GFZrZRUgPhluprzeylEjfNzQOSBNSZ2U5JlcDPgC+a2VOF2H9iegRm9gSwvdTtcPOTmb1rZhuj+X5gE5PcQNG5mZri7s0FkZhA4FyxSDqUcPHk+tK2xM0nuXdvNrOCfb48EDhXQJLqgbuBi8wsXer2uPkj9+7Nkgo2xO2BwLkCicZu7wa+b2b3lLo9bn6K3b351ELt0wOBcwUQJfO+C2wys2+Xuj1ufpHUKqk5mq8BPg28XKj9JyYQSLoDeBJYLukdSf5sZFdInwDOBk6S9Fw0/V6pG+XmjYXAo5I6gWcIOYL7C7XzxJSPOuecyy8xPQLnnHP5eSBwzrmE80DgnHMJ54HAOecSzgOBc84lnAcCd0CQ9HW/c+zcJulaSY+Vuh1u5jwQuIKRZNNMt5S6jVmSzpuinalSt2+mJL0h6eJSt8MdmCpK3QA3ryyMzZ8GfCdn2e7iNmdaA8ARuQvNbLAEbZkTJFWZ2Z5St8MVl/cIXMGYWVd2Anrjy4A64DZJXZJ2Sdoo6bT49pLOlNQpabek7ZIel9Se71iSDpb0sqRbJVVIapJ0e/TwoUFJr0u6aPomj7U51tbsMR6TdJ2kf5C0Ldr3NyWVxdapin7+pqRMdNwLYz9fI2l91KZuSVdLqso5xrU5v9stku7PWWfSdkTDMYcAV2V7NbFtT4j+jgOSNku6XlJjzr6vj/a3Ffh5tPxDkh6Q1B8d7w5JHbHtyqNtdkTTNUD5NH9vN0d5IHDFUg88SLhHykrCzdnukfRBgOgkcydwK7ACWAPcnm9HklYQTlj/DZxnZnuBvwOOJvRElgN/AWwuQLs/D+wFTgD+GrgI+JPYz28FzgG+FLX7fKIgKGlx9Ds/S7gt9fnAnwFXFLgdZwLvAOsIPbCF0fGPBh4C7iX8zc8EVgE35+z7LEDAicA5Cg/ZeQJ4ATgO+BTh/fuvWBD8W+Avgb8CPk4IAp/fj9/LzQVm5pNPBZ+Az4WP15TrPAV8NZr/MOFBG4dMsu7XCSemjwHbgK/k/Pxe4OYZtO+86Hg7c6b/i63zGPBkznYPAzdF80dG+zh1kmP8PfAroCznuBmgNnaMa3O2uwW4f1/bEb1+A7g4Z53bgO/mLFsVtbkttu/OnHXWAY/kLDso2u646PWW+HtA+FL5KvBYqT97Ps188hyBKwpJdcDXCN/YFxKesJQCOqNVngf+B3hB0kPR/F1mtjW2m8XR8nVmdlXOIa4H7pL0EcJJ8j4ze3yaZg0QToxxmZzXnTmvtwBt0fxqYIRwS+B8VgBPmdlIbNnPgCpgWZ59T2WqdkzmI8AySfEejKJ/jyA84ATCYzVzt1sjaScTHSHpFcJ7+GR2oZmNSFoPLJ2mTW4O8kDgiuWbhPunX0z4ljxA+MZaBeGhG5I+AxwPfIYwjHKFpN8xs+ejfWwjfPP9U0k3mdmO7M7N7EFJhwCfBU4GHpD072b2hSnaZGb262naPZS7DYUZUs2O448wdnLOqixQO8qAm4Cr8/wsPmy2K892DxDeq1zd+3Bcd4DxN9QVyyeB28zsbjPrJIxpj6vYseBJM/sGcCzhW2/822wGOAPYATys6P7sse23mdntZnYeIZCcK6l61n4jeI7wf+h3J/n5JuD4eHKZ8HfYA7wWvd7K+MoqCOP5M7WHicnajcBRZvbrPNNUFVwbgaOAN/Ns129mfcC7hKANjD6P4bj9aLebAzwQuGJ5FfhDSR+OkpjfIwwNASDpeElflXSspIMJJ/ylwEvxnUQnsNOBPmLBQNI6SWslHRklk88EXjez3KGeOEnqyDPtU/WLmb0K/Ai4SdIfSTpM0omSzo5WuQ5YBFwnaYWk3wf+kZATGIjW+V/gs5LOkLRc0rfZv+GVN4ATJS2W9IFo2ZWERxreIGm1pGWSTpP0r9Ps61+AJuCHkj4m6XBJn5J0o6SGaJ1/Ai6V9DlJy4FrmBjQ3AHCA4Erli8RxqR/SqikeSqaz+ojPNzlfsLQ0beAy83se7k7ioLBaUCasWCQISRnnydUFDUQAsZUagnfbHOnw2bwe50D/AD4Z8ITo24hnEQxs82EoarVhN7DzcAdwJdj298cm34O9AP/MYPjZ11GCCCvEXoZRD2vNcChwOOEv80VhOGdSZnZFsJ7MQL8GHiREBwyjOVQvgX8G2HoaT3hXPL9/Wi3mwP8wTTOOZdw3iNwzrmE80DgnHMJ54HAOecSzgOBc84lnAcC55xLOA8EzjmXcB4InHMu4TwQOOdcwv0/E33jplglCFUAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9x1D3O7iunxS"
      },
      "source": [
        "**Questions to explore:**\n",
        "\n",
        "*   What's the difference in terms of memory utilization among the three methods? \n",
        "*   Can you plot a similar graph highlighting the memory increase over time?\n",
        "\n",
        "Some tips here: https://stackoverflow.com/questions/449560/how-do-i-determine-the-size-of-an-object-in-python/30316760"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OS9gHzZM7HQr"
      },
      "source": [
        "**Copyright (c) 2018. Continual AI. All rights reserved. **\n",
        "\n",
        "See the accompanying LICENSE file in the GitHub repository for terms. \n",
        "\n",
        "*Date: 29-09-2018                                                             \n",
        "Author: Vincenzo Lomonaco                                                    \n",
        "E-mail: contact@continualai.org                                           \n",
        "Website: continualai.org*                                               "
      ]
    }
  ]
}