Storage backup/restore for hosted TypeAgents (#488)

TypeAgent using the API endpoint will now backup/restore storage writes
to cloud storage. Restore happens at startup and blocks API until sync
is complete. Backup happens lazily as files change on disk.

android/samples/mobile/app/src/main/AndroidManifest.xml

@@ -39,7 +39,7 @@
         android:fullBackupContent="@xml/backup_rules"
         android:icon="@mipmap/ic_launcher"
         android:label="@string/app_name"
-        android:roundIcon="@mipmap/ic_launcher_round"
+        android:roundIcon="@mipmap/ic_launcher"
         android:supportsRtl="true"
         android:theme="@style/Theme.TypeAgentAndroidSample"
         android:usesCleartextTraffic="true">

android/samples/mobile/app/src/main/java/com/microsoft/typeagent/sample/MainActivity.kt

@@ -57,6 +57,7 @@ class MainActivity : ComponentActivity() {
          */
         private const val SPEECH_TO_TEXT_COMPLETE : Int = 1
 
+        @SuppressLint("StaticFieldLeak")
         public var currentActivity : MainActivity? = null
     }
 
@@ -98,7 +99,7 @@ class MainActivity : ComponentActivity() {
             TypeAgentAndroidSampleTheme {
                 Scaffold(
                     modifier = Modifier.fillMaxSize(),
-                    topBar = { TopAppBar(title = { Text("WebView - Test", color = Color.White) }) },
+                    topBar = { TopAppBar(title = { Text("TypeAgent Sample") }) },
                 ) { innerPadding ->
                     Browser()
                     Greeting(
@@ -213,7 +214,7 @@ class MainActivity : ComponentActivity() {
     @Composable
     fun Greeting(name: String, modifier: Modifier = Modifier) {
         Text(
-            text = "Hello $name!",
+            text = "Connected to $name!",
             modifier = modifier
         )
     }

android/samples/mobile/app/src/main/res/mipmap-anydpi/ic_launcher.xml

@@ -1,6 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
-    <background android:drawable="@drawable/ic_launcher_background" />
-    <foreground android:drawable="@drawable/ic_launcher_foreground" />
-    <monochrome android:drawable="@drawable/ic_launcher_foreground" />
+<!--    <background android:drawable="@drawable/ic_launcher_background" />-->
+<!--    <foreground android:drawable="@drawable/ic_launcher_foreground" />-->
+<!--    <monochrome android:drawable="@drawable/ic_launcher_foreground" />-->
+    <foreground android:drawable="@drawable/typeagent_logo" />
 </adaptive-icon>

android/samples/mobile/app/src/main/res/mipmap-anydpi/ic_launcher_round.xml

@@ -1,6 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
-    <background android:drawable="@drawable/ic_launcher_background" />
-    <foreground android:drawable="@drawable/ic_launcher_foreground" />
-    <monochrome android:drawable="@drawable/ic_launcher_foreground" />
+<!--    <background android:drawable="@drawable/ic_launcher_background" />-->
+<!--    <foreground android:drawable="@drawable/ic_launcher_foreground" />-->
+<!--    <monochrome android:drawable="@drawable/ic_launcher_foreground" />-->
+    <foreground android:drawable="@drawable/typeagent_logo" />
 </adaptive-icon>

ts/Dockerfile

@@ -28,12 +28,33 @@ RUN apt-get update
 RUN apt install -y gnome-keyring
 RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash
 
+######################################### BLOB STORAGE MOUNT ######################################### 
+## blobfuse - https://github.com/Azure/azure-storage-fuse/tree/main
+## Uncomment the section below to use blobfuse to mount blob storage
+#RUN apt-get update \
+#    && apt-get install -y wget apt-utils \
+#    && wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb \
+#    && dpkg -i packages-microsoft-prod.deb \
+#    && apt-get update \
+#    && apt-get install -y libfuse3-dev fuse3 libcurl3-gnutls libgnutls30
+#
+#RUN apt-get install -y blobfuse2
+#COPY --from=build /usr/src/app/mount-blobfuse.yaml ./mount-blobfuse.yaml
+#RUN mkdir /mnt/blob
+#RUN mkdir /blob-cache
+######################################################################################################
+
 EXPOSE 80:3000
 
-# setup start actions
+# start actions
+# az login
 RUN echo az login --identity >> start.sh
+## mount blob storage locally
+#RUN echo blobfuse2 mount /mnt/blob --config-file=mount-blobfuse.yaml >> start.sh
+# run getKeys
 RUN echo node ../tools/scripts/getKeys.mjs >> start.sh
+# start API server
 RUN echo pnpm start >> start.sh
 
-# azure login, then start node application
+# run start script
 CMD bash start.sh

ts/mount-blobfuse.yaml

@@ -0,0 +1,36 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+logging:
+  level: log_debug
+  file-path: "blog.log"
+  type: base
+
+components:
+  - libfuse
+  - file_cache
+  - attr_cache
+  - azstorage
+
+libfuse:
+  attribute-expiration-sec: 0
+  entry-expiration-sec: 0
+  negative-entry-expiration-sec: 0
+  ignore-open-flags: true
+
+file_cache:
+  path: "/blob-cache"
+  timeout-sec: 30
+  max-size-mb: 2048
+  allow-non-empty-temp: true
+  cleanup-on-start: true
+
+attr_cache:
+  timeout-sec: 3600
+
+azstorage:
+  type: block
+  use-http: false
+  mode: azcli
+  container: sessions
+  tier: hot

ts/packages/aiclient/src/openai.ts

@@ -102,6 +102,9 @@ export enum EnvVars {
     AZURE_MAPS_CLIENTID = "AZURE_MAPS_CLIENTID",
 
     ENABLE_MODEL_REQUEST_LOGGING = "ENABLE_MODEL_REQUEST_LOGGING",
+
+    AZURE_STORAGE_ACCOUNT = "AZURE_STORAGE_ACCOUNT",
+    AZURE_STORAGE_CONTAINER = "AZURE_STORAGE_CONTAINER",
 }
 
 export const MAX_PROMPT_LENGTH_DEFAULT = 1000 * 60;

ts/packages/api/README.md

@@ -6,7 +6,7 @@ TypeAgent API is a HTTP+WS API server for **TypeAgent sample code**. It explores
 
 ## Running
 
-After setting up and building at the workspace root, there are several additional ways to start the server in this directory.
+After setting up and building at the workspace root, there are several additional ways to start the server in this directory. It is also possible to use the [docker image](../../Dockerfile) to host TypeAgent either locally or in a cloud hosted environment such as [Azure App Service](https://learn.microsoft.com/en-us/azure/app-service/quickstart-custom-container?tabs=dotnet&pivots=container-linux-vscode).
 
 ### Globally Link the package
 

ts/packages/api/data/config.json

@@ -1,5 +1,6 @@
 {
   "wwwroot": "../shell/out/renderer",
   "port": 3000,
-  "broadcast": true
+  "broadcast": true,
+  "blobBackupEnabled": true
 }

ts/packages/api/package.json

@@ -24,6 +24,8 @@
     "tsc": "tsc -b"
   },
   "dependencies": {
+    "@azure/identity": "^4.2.1",
+    "@azure/storage-blob": "^12.26.0",
     "@typeagent/agent-sdk": "workspace:*",
     "agent-cache": "workspace:*",
     "agent-dispatcher": "workspace:*",
@@ -35,7 +37,9 @@
     "find-config": "^1.0.0",
     "node-http": "0.0.5",
     "rimraf": "^5.0.5",
+    "telemetry": "workspace:*",
     "ts-node": "^10.9.1",
+    "typeagent": "workspace:*",
     "typechat": "^0.1.1",
     "typescript": "^5.4.3",
     "ws": "^8.17.1"