feat: experimental local ai model (#254)

Signed-off-by: Jan <hey@janrietveld.com>

apps/easypid/README.md

@@ -388,6 +388,9 @@ The following standards and specifications were implemented.
 - Fixed an issue where the PIN screen would get stuck in a loading state when an incorrect PIN was entered  [commit](https://github.com/animo/paradym-wallet/commit/0f65ef98f5f26c3afc0968e4f848bf538a86cfd7)
 - Fixed an issue with redirect based auth flow if the authorization flow left the in-app browser (e.g. when requiring authentication using the native AusweisApp with the eID card) [commit](https://github.com/animo/paradym-wallet/commit/eb333b81fe5662cc2f010e1ee9bbdc83a7e19aa3)
 - Fixed an issue where the PID setup would get stuck if you skipped it during onboarding [commit](https://github.com/animo/openid4vc-playground-funke/commit/65178e776bc421b9ca413542ea0e86db4ad1ead4)
+- Added support for on-device local AI model for oversharing detection on higher-end devices (can be enabled in the settings) [commit](https://github.com/animo/paradym-wallet/commit)
+
+
 
 #### 28-11-2024
 

apps/easypid/app.config.js

@@ -117,6 +117,9 @@ const config = {
       ],
     },
     associatedDomains: associatedDomains.map((host) => `applinks:${host}`),
+    entitlements: {
+      'com.apple.developer.kernel.increased-memory-limit': true,
+    },
   },
   android: {
     adaptiveIcon: {
@@ -145,6 +148,9 @@ const config = {
         }))
       ),
     ],
+    config: {
+      largeHeap: true,
+    },
   },
   experiments: {
     tsconfigPaths: true,

apps/easypid/package.json

@@ -38,6 +38,7 @@
 		"expo-blur": "^13.0.2",
 		"expo-constants": "~16.0.2",
 		"expo-dev-client": "~4.0.16",
+		"expo-device": "~6.0.2",
 		"expo-font": "~12.0.7",
 		"expo-haptics": "~13.0.1",
 		"expo-image": "~1.13.0",
@@ -55,6 +56,7 @@
 		"react": "catalog:",
 		"react-native": "catalog:",
 		"react-native-argon2": "^2.0.1",
+		"react-native-executorch": "^0.1.2",
 		"react-native-fs": "^2.20.0",
 		"react-native-gesture-handler": "~2.16.2",
 		"react-native-get-random-values": "~1.11.0",

apps/easypid/src/app/_layout.tsx

@@ -6,6 +6,7 @@ import { DefaultTheme, ThemeProvider } from '@react-navigation/native'
 import { Slot } from 'expo-router'
 import * as SplashScreen from 'expo-splash-screen'
 
+import { useCheckIncompleteDownload } from '@easypid/llm'
 import tamaguiConfig from '../../tamagui.config'
 
 void SplashScreen.preventAutoHideAsync()
@@ -17,6 +18,7 @@ export const unstable_settings = {
 
 export default function RootLayout() {
   useTransparentNavigationBar()
+  useCheckIncompleteDownload()
 
   return (
     <Provider config={tamaguiConfig}>

apps/easypid/src/features/menu/FunkeSettingsScreen.tsx

@@ -1,15 +1,15 @@
-import { Button, FlexPage, Heading, HeroIcons, Paragraph, ScrollView, Stack, XStack, YStack } from '@package/ui'
+import { FlexPage, Heading, HeroIcons, ScrollView, Stack, Switch, YStack } from '@package/ui'
 import React from 'react'
-import { useRouter } from 'solito/router'
-import { Label, Switch } from 'tamagui'
+
+import { TextBackButton } from 'packages/app/src'
+import { LocalAiContainer } from './components/LocalAiContainer'
 
 import { useScrollViewPosition } from '@package/app/src/hooks'
 import { useDevelopmentMode } from '../../hooks/useDevelopmentMode'
 
 export function FunkeSettingsScreen() {
   const { handleScroll, isScrolledByOffset, scrollEventThrottle } = useScrollViewPosition()
-  const router = useRouter()
-  const [isDevelpomentModeEnabled, setIsDevelopmentModeEnabled] = useDevelopmentMode()
+  const [isDevelopmentModeEnabled, setIsDevelopmentModeEnabled] = useDevelopmentMode()
 
   return (
     <FlexPage gap="$0" paddingHorizontal="$0">
@@ -27,26 +27,19 @@ export function FunkeSettingsScreen() {
         contentContainerStyle={{ minHeight: '85%' }}
       >
         <YStack fg={1} px="$4" jc="space-between">
-          <YStack>
-            <Paragraph color="$grey-700" py="$4">
-              This page is under construction. More options will be added.
-            </Paragraph>
-            <XStack jc="space-between" ai="center">
-              <Label>Development Mode</Label>
-              <Switch
-                size="$5"
-                checked={isDevelpomentModeEnabled}
-                onCheckedChange={setIsDevelopmentModeEnabled}
-                animation="quick"
-                backgroundColor={isDevelpomentModeEnabled ? '$primary-500' : '$primary-300'}
-              >
-                <Switch.Thumb animation="quick" backgroundColor="$grey-200" />
-              </Switch>
-            </XStack>
+          <YStack gap="$4" py="$2">
+            <Switch
+              id="development-mode"
+              label="Development Mode"
+              icon={<HeroIcons.CommandLineFilled />}
+              value={isDevelopmentModeEnabled ?? false}
+              onChange={setIsDevelopmentModeEnabled}
+            />
+            <LocalAiContainer />
+          </YStack>
+          <YStack btw="$0.5" borderColor="$grey-200" pt="$4" mx="$-4" px="$4" bg="$background">
+            <TextBackButton />
           </YStack>
-          <Button.Text color="$primary-500" fontWeight="$semiBold" fontSize="$4" onPress={() => router.back()}>
-            <HeroIcons.ArrowLeft mr={-4} color="$primary-500" size={20} /> Back
-          </Button.Text>
         </YStack>
       </ScrollView>
     </FlexPage>

apps/easypid/src/features/menu/components/LocalAiContainer.tsx

@@ -0,0 +1,99 @@
+import { HeroIcons } from '@package/ui/src/content/Icon'
+
+import { Switch } from '@package/ui/src/base/Switch'
+
+import { useIsDeviceCapable, useLLM } from '@easypid/llm'
+import { ConfirmationSheet } from '@package/app/src/components/ConfirmationSheet'
+import { useHasInternetConnection, useIsConnectedToWifi } from 'packages/app/src/hooks'
+import { useToastController } from 'packages/ui/src'
+import React, { useState } from 'react'
+
+export function LocalAiContainer() {
+  const toast = useToastController()
+  const isConnectedToWifi = useIsConnectedToWifi()
+  const hasInternetConnection = useHasInternetConnection()
+  const isDeviceCapable = useIsDeviceCapable()
+
+  const [isAiModelConfirmationOpen, setIsAiModelConfirmationOpen] = useState(false)
+  const { loadModel, isModelReady, downloadProgress, removeModel, isModelActivated, isModelDownloading } = useLLM()
+
+  const onActivateModel = () => {
+    if (!isDeviceCapable) {
+      toast.show('Device not supported', {
+        message: 'This device is not powerful enough to run local AI models',
+        customData: {
+          preset: 'warning',
+        },
+      })
+      setIsAiModelConfirmationOpen(false)
+      return
+    }
+    if (!isConnectedToWifi && !hasInternetConnection) {
+      toast.show('WiFi connection required', {
+        message: 'Please connect to WiFi to activate and download the model',
+        customData: {
+          preset: 'warning',
+        },
+      })
+      setIsAiModelConfirmationOpen(false)
+      return
+    }
+
+    setIsAiModelConfirmationOpen(false)
+    loadModel()
+  }
+
+  const handleAiModelChange = (value: boolean) => {
+    if (isModelDownloading) {
+      toast.show('Model download in progress', {
+        message: 'Force close the app to cancel the download',
+        customData: {
+          preset: 'warning',
+        },
+      })
+      return
+    }
+
+    if (value) {
+      setIsAiModelConfirmationOpen(true)
+    } else {
+      removeModel()
+    }
+  }
+
+  return (
+    <>
+      <Switch
+        id="local-ai-model"
+        label="Use local AI model"
+        icon={<HeroIcons.CpuChipFilled />}
+        value={isModelActivated}
+        description={
+          isModelActivated
+            ? isModelReady
+              ? 'Model active and ready to use'
+              : downloadProgress
+                ? `Downloading model ${(downloadProgress * 100).toFixed(2)}%`
+                : 'Getting ready...'
+            : ''
+        }
+        onChange={handleAiModelChange}
+        beta
+      />
+      <ConfirmationSheet
+        type="floating"
+        variant="regular"
+        isOpen={isAiModelConfirmationOpen}
+        setIsOpen={setIsAiModelConfirmationOpen}
+        title="Enable local AI model"
+        confirmText="Enable"
+        cancelText="Cancel"
+        description={[
+          'This will download a local AI model to your device which will take up to around 1.3GB of space.',
+          'This is an experimental feature. Only supported on high-end devices.',
+        ]}
+        onConfirm={onActivateModel}
+      />
+    </>
+  )
+}

apps/easypid/src/features/receive/FunkeCredentialNotificationScreen.tsx

@@ -448,6 +448,8 @@ export function FunkeCredentialNotificationScreen() {
                   logo={credentialsForRequest.verifier.logo}
                   submission={credentialsForRequest.formattedSubmission}
                   isAccepting={isSharingPresentation}
+                  // Not supported for this flow atm
+                  overAskingResponse={{ validRequest: 'could_not_determine', reason: '' }}
                 />
               ),
             }

apps/easypid/src/features/receive/slides/CredentialErrorSlide.tsx

@@ -1,15 +1,18 @@
-import { Button, Heading, HeroIcons, Paragraph, Stack, XStack, YStack } from '@package/ui'
+import { Button, Heading, HeroIcons, Paragraph, ScrollView, Stack, XStack, YStack } from '@package/ui'
+import { useState } from 'react'
 
 interface CredentialErrorSlideProps {
   reason?: string
   onCancel: () => void
 }
 
 export const CredentialErrorSlide = ({ reason, onCancel }: CredentialErrorSlideProps) => {
+  const [scrollViewHeight, setScrollViewHeight] = useState(0)
+
   return (
     <YStack fg={1} jc="space-between">
-      <YStack gap="$6">
-        <YStack gap="$2">
+      <YStack gap="$6" fg={1} onLayout={(event) => setScrollViewHeight(event.nativeEvent.layout.height)}>
+        <ScrollView fg={1} maxHeight={scrollViewHeight} contentContainerStyle={{ gap: '$4' }}>
           <YStack gap="$4">
             <Heading>Something went wrong</Heading>
             <Stack alignSelf="flex-start">
@@ -22,13 +25,15 @@ export const CredentialErrorSlide = ({ reason, onCancel }: CredentialErrorSlideP
               again later.
             </Paragraph>
           </YStack>
-          {reason && (
-            <Paragraph variant="sub">
-              <Paragraph variant="caption">Reason: </Paragraph>
-              {reason}
-            </Paragraph>
+          {reason && scrollViewHeight !== 0 && (
+            <YStack>
+              <Paragraph variant="sub">
+                <Paragraph variant="caption">Reason: </Paragraph>
+                {reason}
+              </Paragraph>
+            </YStack>
           )}
-        </YStack>
+        </ScrollView>
       </YStack>
       <Stack borderTopWidth="$0.5" borderColor="$grey-200" py="$4" mx="$-4" px="$4">
         <Button.Solid scaleOnPress onPress={onCancel}>

apps/easypid/src/features/share/FunkeOpenIdPresentationNotificationScreen.tsx

@@ -12,9 +12,8 @@ import React, { useEffect, useState, useCallback } from 'react'
 
 import { useAppAgent } from '@easypid/agent'
 import { InvalidPinError } from '@easypid/crypto/error'
+import { useOverAskingAi } from '@easypid/hooks'
 import { useDevelopmentMode } from '@easypid/hooks'
-import { analyzeVerification } from '@easypid/use-cases/ValidateVerification'
-import type { VerificationAnalysisResponse } from '@easypid/use-cases/ValidateVerification'
 import { usePushToWallet } from '@package/app/src/hooks/usePushToWallet'
 import { setWalletServiceProviderPin } from '../../crypto/WalletServiceProviderClient'
 import { useShouldUsePinForSubmission } from '../../hooks/useShouldUsePinForPresentation'
@@ -62,26 +61,24 @@ export function FunkeOpenIdPresentationNotificationScreen() {
       })
   }, [credentialsForRequest, params.data, params.uri, toast.show, agent, pushToWallet, toast, isDevelopmentModeEnabled])
 
-  const [verificationAnalysis, setVerificationAnalysis] = useState<{
-    isLoading: boolean
-    result: VerificationAnalysisResponse | undefined
-  }>({
-    isLoading: false,
-    result: undefined,
-  })
+  const { checkForOverAsking, isProcessingOverAsking, overAskingResponse, stopOverAsking } = useOverAskingAi()
 
   useEffect(() => {
     if (!credentialsForRequest?.formattedSubmission || !credentialsForRequest?.formattedSubmission.areAllSatisfied) {
       return
     }
-    setVerificationAnalysis((prev) => ({ ...prev, isLoading: true }))
+
+    if (isProcessingOverAsking || overAskingResponse) {
+      // Already generating or already has result
+      return
+    }
 
     const submission = credentialsForRequest.formattedSubmission
     const requestedCards = submission.entries
       .filter((entry): entry is FormattedSubmissionEntrySatisfied => entry.isSatisfied)
       .flatMap((entry) => entry.credentials)
 
-    analyzeVerification({
+    void checkForOverAsking({
       verifier: {
         name: credentialsForRequest.verifier.name ?? 'No name provided',
         domain: credentialsForRequest.verifier.hostName ?? 'No domain provided',
@@ -93,8 +90,8 @@ export function FunkeOpenIdPresentationNotificationScreen() {
         subtitle: credential.credential.display.description ?? 'Card description',
         requestedAttributes: getDisclosedAttributeNamesForDisplay(credential),
       })),
-    }).then((result) => setVerificationAnalysis((prev) => ({ ...prev, isLoading: false, result })))
-  }, [credentialsForRequest])
+    })
+  }, [credentialsForRequest, checkForOverAsking, isProcessingOverAsking, overAskingResponse])
 
   const onProofAccept = useCallback(
     async (pin?: string): Promise<PresentationRequestResult> => {
@@ -106,6 +103,7 @@ export function FunkeOpenIdPresentationNotificationScreen() {
           },
         }
 
+      stopOverAsking()
       setIsSharing(true)
 
       if (shouldUsePin) {
@@ -190,10 +188,11 @@ export function FunkeOpenIdPresentationNotificationScreen() {
         }
       }
     },
-    [credentialsForRequest, agent, shouldUsePin, isDevelopmentModeEnabled]
+    [credentialsForRequest, agent, shouldUsePin, stopOverAsking, isDevelopmentModeEnabled]
   )
 
   const onProofDecline = async () => {
+    stopOverAsking()
     if (credentialsForRequest) {
       await addSharedActivityForCredentialsForRequest(
         agent,
@@ -219,7 +218,7 @@ export function FunkeOpenIdPresentationNotificationScreen() {
       trustedEntities={credentialsForRequest?.verifier.trustedEntities}
       lastInteractionDate={lastInteractionDate}
       onComplete={() => pushToWallet('replace')}
-      verificationAnalysis={verificationAnalysis}
+      overAskingResponse={overAskingResponse}
     />
   )
 }

apps/easypid/src/features/share/FunkePresentationNotificationScreen.tsx

@@ -1,6 +1,6 @@
 import type { DisplayImage, FormattedSubmission, TrustedEntity } from '@package/agent'
 
-import type { VerificationAnalysisResult } from '@easypid/use-cases/ValidateVerification'
+import type { OverAskingResponse } from '@easypid/use-cases/OverAskingApi'
 import { type SlideStep, SlideWizard } from '@package/app'
 import { LoadingRequestSlide } from '../receive/slides/LoadingRequestSlide'
 import { VerifyPartySlide } from '../receive/slides/VerifyPartySlide'
@@ -14,7 +14,7 @@ interface FunkePresentationNotificationScreenProps {
   verifierName?: string
   logo?: DisplayImage
   lastInteractionDate?: string
-  verificationAnalysis: VerificationAnalysisResult
+  overAskingResponse?: OverAskingResponse
   trustedEntities?: Array<TrustedEntity>
   submission?: FormattedSubmission
   usePin: boolean
@@ -35,7 +35,7 @@ export function FunkePresentationNotificationScreen({
   isAccepting,
   submission,
   onComplete,
-  verificationAnalysis,
+  overAskingResponse,
   trustedEntities,
 }: FunkePresentationNotificationScreenProps) {
   return (
@@ -74,7 +74,7 @@ export function FunkePresentationNotificationScreen({
                 logo={logo}
                 submission={submission}
                 isAccepting={isAccepting}
-                verificationAnalysis={verificationAnalysis}
+                overAskingResponse={overAskingResponse}
               />
             ),
           },