Merge branch 'development' into refactor/handover-clarifications

functions/index.js

@@ -21,10 +21,10 @@ exports.getEventDataFromCode = v1_oncall( getEventDataFromCode )
 exports.check_code_status = v1_oncall( check_code_status )
 
 // Refresh all codes ( trigger from frontend on page mount of EventView )
-exports.requestManualCodeRefresh = v1_oncall( [ 'memory_512MiB', 'long_timeout'  ], refresh_unknown_and_unscanned_codes )
+exports.requestManualCodeRefresh = v1_oncall( [ 'memory_512MB', 'long_timeout'  ], refresh_unknown_and_unscanned_codes )
 
 // Allow frontend to trigger updates for scanned codes, ( triggers on a periodic interval from EventView ), is lighter than requestManualCodeRefresh as it checks only scanned and claimed == true codes
-exports.refreshScannedCodesStatuses = v1_oncall( [ 'memory_512MiB', 'long_timeout' ], refreshScannedCodesStatuses )
+exports.refreshScannedCodesStatuses = v1_oncall( [ 'memory_512MB', 'long_timeout' ], refreshScannedCodesStatuses )
 
 // Directly mint a code to an address
 const { mint_code_to_address } = require( './modules/minting' )
@@ -34,12 +34,16 @@ exports.mint_code_to_address = v2_oncall( [ 'memory_512MiB', 'long_timeout' ], m
 const { recalculate_available_codes_admin } = require( './modules/codes' )
 exports.recalculate_available_codes = v2_oncall( recalculate_available_codes_admin )
 
+// On event registration, recalculate available codes
+const { recalculate_available_codes } = require( './modules/codes' )
+exports.recalculate_available_codes_on_event_registration = functions.runWith( generousRuntime ).firestore.document( `events/{eventId}` ).onCreate( ( { params } ) => recalculate_available_codes( params.eventId ) )
+
 // ///////////////////////////////
 // Event data
 // ///////////////////////////////
 
 const { registerEvent, deleteEvent, getUniqueOrganiserEmails } = require( './modules/events' )
-exports.registerEvent = v1_oncall( [ 'memory_1GiB', 'long_timeout' ], registerEvent )
+exports.registerEvent = v2_oncall( [ 'memory_1GiB', 'long_timeout' ], registerEvent )
 exports.deleteEvent = v1_oncall( deleteEvent )
 
 // Email export to update event organisers
@@ -73,7 +77,6 @@ exports.delete_data_of_deleted_event = functions.firestore.document( `events/{ev
 exports.updatePublicEventData = functions.firestore.document( `events/{eventId}` ).onWrite( updatePublicEventData )
 exports.updateEventAvailableCodes = functions.firestore.document( `codes/{codeId}` ).onUpdate( updateEventAvailableCodes )
 
-
 /* ///////////////////////////////
 // Security
 // /////////////////////////////*/

functions/modules/events.js

@@ -63,6 +63,7 @@ async function validate_and_write_event_codes( event_id, expiration_date, codes,
 
     // Parse out codes that are expected to be new, so keep only codes that are not found in the existing_code array
     const new_codes = saneCodes.filter( ( { qr_hash } ) => !existing_codes?.find( existing_code => existing_code.qr_hash == qr_hash ) )
+
     // First check if all codes are unused by another event
     const code_clash_queue = new_codes.map( code => async () => {
 
@@ -75,30 +76,78 @@ async function validate_and_write_event_codes( event_id, expiration_date, codes,
 
     } )
 
-    /* ///////////////////////////////
-	// Step 2: Throttled code writing, see https://cloud.google.com/firestore/docs/best-practices and https://cloud.google.com/firestore/quotas#writes_and_transactions */
-
     // Check for code clashes in a throttled manner
     await Throttle.all( code_clash_queue, { maxInProgress } )
 
-    // Load the codes into firestore
-    const code_writing_queue = saneCodes.map( code => async () => {
+    /* ///////////////////////////////
+	// Step 2: Throttled code writing using firestore patches */
+
+    // Batch config
+    const batch_size = 499
+
+    // Split into chunks of batch_size
+    const code_chunks = []
+    for( let index = 0; index < saneCodes.length; index += batch_size ) {
+        const chunk = saneCodes.slice( index, index + batch_size )
+        code_chunks.push( chunk )
+    }
 
-        return db.collection( 'codes' ).doc( code.qr_hash ).set( {
-            claimed: !!code.claimed,
-            scanned: false,
-            amountOfRemoteStatusChecks: 0,
-            created: Date.now(),
-            updated: Date.now(),
-            updated_human: new Date().toString(),
-            event: event_id,
-            expires: new Date( expiration_date ).getTime() + weekInMs
-        }, { merge: true } )
+    // Create batches for each chunk
+    const code_batches = code_chunks.map( chunk => {
+
+        // Make a batch for this chunk
+        const batch = db.batch()
+
+        // For each entry in the chunk, add a batch set
+        chunk.forEach( code => {
+
+            if( !code ) return
+
+            const ref = db.collection( `codes` ).doc( code.qr_hash )
+            batch.set( ref, {
+                claimed: !!code.claimed,
+                scanned: false,
+                amountOfRemoteStatusChecks: 0,
+                created: Date.now(),
+                updated: Date.now(),
+                updated_human: new Date().toString(),
+                event: event_id,
+                expires: new Date( expiration_date ).getTime() + weekInMs
+            }, { merge: true } )
 
+        } )
+
+        // Return batch
+        return batch
+
     } )
 
-    // Write codes to firestore with a throttle
-    await Throttle.all( code_writing_queue, { maxInProgress } )
+    // Create writing queue
+    const writing_queue = code_batches.map( batch => () => batch.commit() )
+
+    // Write the watches with retry
+    const { throttle_and_retry } = require( './helpers' )
+    await throttle_and_retry( writing_queue, maxInProgress, `validate_and_write_event_codes`, 2, 5 )
+
+    // Old non-batchified way
+    // // Load the codes into firestore
+    // const code_writing_queue = saneCodes.map( code => async () => {
+
+    //     return db.collection( 'codes' ).doc( code.qr_hash ).set( {
+    //         claimed: !!code.claimed,
+    //         scanned: false,
+    //         amountOfRemoteStatusChecks: 0,
+    //         created: Date.now(),
+    //         updated: Date.now(),
+    //         updated_human: new Date().toString(),
+    //         event: event_id,
+    //         expires: new Date( expiration_date ).getTime() + weekInMs
+    //     }, { merge: true } )
+
+    // } )
+
+    // // Write codes to firestore with a throttle
+    // await Throttle.all( code_writing_queue, { maxInProgress } )
 
     // Return the sanitised codes
     return saneCodes
@@ -217,9 +266,10 @@ const update_event_data_of_kiosk = async ( kiosk_id, public_kiosk_data ) => {
 
 exports.update_event_data_of_kiosk = update_event_data_of_kiosk
 
-exports.registerEvent = async function( data, context ) {
+exports.registerEvent = async request => {
 
     let new_event_id = undefined
+    const { data } = request
 
     try {
 
@@ -229,9 +279,6 @@ exports.registerEvent = async function( data, context ) {
         // Add a week grace period in case we need to debug anything
         const weekInMs = 1000 * 60 * 60 * 24 * 7
 
-        // Appcheck validation
-        throw_on_failed_app_check( context )
-
         // Validations
         const { name='', email='', date='', dropId, codes=[], challenges=[], game_config={ duration: 30, target_score: 5 }, css, collect_emails=false, claim_base_url } = data
         if( !codes.length ) throw new Error( 'Csv has 0 entries' )
@@ -241,7 +288,7 @@ exports.registerEvent = async function( data, context ) {
 
         // Get the ip this request came from
         const { get_ip_from_request } = require( './firebase' )
-        const created_from_ip = get_ip_from_request( context ) || 'unknown'
+        const created_from_ip = get_ip_from_request( request ) || 'unknown'
 
         // Create event document
         const authToken = uuidv4()
@@ -280,9 +327,11 @@ exports.registerEvent = async function( data, context ) {
         // Check code validity and write to firestore
         await validate_and_write_event_codes( id, date, formatted_codes )
 
+        // NOTE: this was moved to an onCreate trigger in index.js, leaving this here as a debug breadcrumb in case it did not solve out issue with creating large events
+        // NOTE: safe to delete after jan 2024
         // Calculate publicly available codes for this new event
-        const { recalculate_available_codes } = require( './codes' )
-        await recalculate_available_codes( id )
+        // const { recalculate_available_codes } = require( './codes' )
+        // await recalculate_available_codes( id )
 
         // Grab the latest drop data form api, adding the event_data is important because the publicEventData does not exist yet
         await update_event_data_of_kiosk( id, event_data )

functions/package.json

@@ -2,7 +2,8 @@
   "name": "functions",
   "description": "Cloud Functions for Firebase",
   "scripts": {
-    "serve": "firebase emulators:start --only functions",
+    "serve": "firebase use development && NODE_ENV=development ENVIRONMENT_LOCAL=true firebase emulators:start --only functions,firestore,pubsub | tee functions-emulator.log",
+    "kill": "sudo lsof -i :8085,8080 | sed -n '2p' | awk '{print $2}' | xargs kill -9",
     "shell": "firebase functions:shell",
     "start": "firebase use development && npm run runtime && npm run shell",
     "start:production": "development=true && firebase use production && npm run runtime:production && npm run shell",
@@ -39,4 +40,4 @@
   "devDependencies": {
     "firebase-tools": "^12.9.1"
   }
-}
+}

functions/runtime/on_call_runtimes.js

@@ -4,7 +4,7 @@ const debug = false
 
 /**
  * Return a V1 oncall with runtimes. NOTE: v1 appcheck is enforced through code and not config
- * @param {Array.<"high_memory"|"long_timeout"|"keep_warm">} [runtimes] - Array of runtime keys to use
+ * @param {Array.<"X MiB/GiB"|"long_timeout"|"keep_warm">} [runtimes] - Array of runtime keys to use
  * @param {Function} handler - Function to run
  * @returns {Function} - Firebase function 
 */
@@ -13,14 +13,17 @@ exports.v1_oncall = ( runtimes=[], handler ) => {
     if( debug ) log( `Creating handler with: `, typeof runtimes, runtimes, typeof handler, handler )
 
     const functions = require( "firebase-functions" )
-    const { v1_runtimes } = require( './runtimes_settings' )
+    const { v1_runtimes, validate_runtime_settings } = require( './runtimes_settings' )
 
     // If the first parameter was a function, return the undecorated handler
     if( typeof runtimes === 'function' ) {
         if( debug ) log( 'v1_oncall: no runtimes specified, returning undecorated handler' )
         return functions.https.onCall( runtimes )
     }
 
+    // Validate runtime settings
+    validate_runtime_settings( runtimes, v1_runtimes )
+
     // Config the runtimes for this function
     const runtime = runtimes.reduce( ( acc, runtime_key ) => ( { ...acc, ...v1_runtimes[ runtime_key ] } ), {} )
     if( debug ) log( 'v1_oncall: returning decorated handler with runtime: ', runtime )
@@ -29,7 +32,7 @@ exports.v1_oncall = ( runtimes=[], handler ) => {
 
 /**
  * Return a V2 oncall with runtimes
- * @param {Array.<"long_timeout"|"high_memory"|"keep_warm"|"max_concurrency">} [runtimes] - Array of runtime keys to use, the protected runtime is ALWAYS ADDED
+ * @param {Array.<"long_timeout"|"X MiB/GiB"|"keep_warm"|"max_concurrency">} [runtimes] - Array of runtime keys to use, the protected runtime is ALWAYS ADDED
  * @param {Function} handler - Firebase function handler
  * @returns {Function} - Firebase function
  */
@@ -38,14 +41,17 @@ exports.v2_oncall = ( runtimes=[], handler ) => {
     if( debug ) log( `Creating handler with: `, typeof runtimes, runtimes, typeof handler, handler )
 
     const { onCall } = require( "firebase-functions/v2/https" )
-    const { v2_runtimes } = require( './runtimes_settings' )
+    const { v2_runtimes, validate_runtime_settings } = require( './runtimes_settings' )
 
     // If the first parameter was a function, return the handler as 'protected' firebase oncall
     if( typeof runtimes === 'function' ) {
         if( debug ) log( 'v2_oncall: no runtimes specified, returning undecorated handler' )
         return onCall( { ...v2_runtimes.protected }, runtimes )
     }
 
+    // Validate runtime settings
+    validate_runtime_settings( runtimes, v2_runtimes )
+
     const runtime = runtimes.reduce( ( acc, runtime_key ) => ( { ...acc, ...v2_runtimes[ runtime_key ] } ), { ...v2_runtimes.protected } )
     if( debug ) log( 'v2_oncall: returning decorated handler with runtime: ', runtime )
     return onCall( runtime, handler )

functions/runtime/on_request_runtimes.js

@@ -7,15 +7,13 @@
 exports.v1_onrequest = ( runtimes=[], handler ) => {
 
     const functions = require( "firebase-functions" )
-    const { v1_runtimes } = require( './runtimes_settings' )
+    const { v1_runtimes, validate_runtime_settings } = require( './runtimes_settings' )
 
     // If the first parameter was a function, return the undecorated handler
     if( typeof runtimes === 'function' ) return functions.https.onRequest( runtimes )
 
-    // Check that all runtime keys exist in the v2_runtimes object
-    const runtime_keys = Object.keys( v1_runtimes )
-    const invalid_runtime_keys = runtimes.some( runtime_key => !runtime_keys.includes( runtime_key ) )
-    if( invalid_runtime_keys.length ) throw new Error( `Invalid runtime keys: ${ invalid_runtime_keys }` )
+    // Validate runtime settings
+    validate_runtime_settings( runtimes, v1_runtimes )
 
     // Config the runtimes for this function
     const runtime = runtimes.reduce( ( acc, runtime_key ) => ( { ...acc, ...v1_runtimes[ runtime_key ] } ), {} )
@@ -31,16 +29,14 @@ exports.v1_onrequest = ( runtimes=[], handler ) => {
 exports.v2_onrequest = ( runtimes=[], handler ) => {
 
     const { onRequest } = require( "firebase-functions/v2/https" )
-    const { v2_runtimes } = require( './runtimes_settings' )
+    const { v2_runtimes, validate_runtime_settings } = require( './runtimes_settings' )
     const runtime_basis = { cors: true }
 
     // If the first parameter was a function, return the handler as 'protected' firebase oncall
     if( typeof runtimes === 'function' ) return onRequest( runtime_basis, runtimes )
 
-    // Check that all runtime keys exist in the v2_runtimes object
-    const runtime_keys = Object.keys( v2_runtimes )
-    const invalid_runtime_keys = runtimes.some( runtime_key => !runtime_keys.includes( runtime_key ) )
-    if( invalid_runtime_keys.length ) throw new Error( `Invalid runtime keys: ${ invalid_runtime_keys }` )
+    // Validate runtime settings
+    validate_runtime_settings( runtimes, v2_runtimes )
 
     const runtime = runtimes.reduce( ( acc, runtime_key ) => ( { ...acc, ...v2_runtimes[ runtime_key ] } ), runtime_basis )
 

functions/runtime/runtimes_settings.js

@@ -31,4 +31,29 @@ exports.v2_runtimes = {
     ...[ "128MiB", "256MiB", "512MiB", "1GiB", "2GiB", "4GiB", "8GiB", "16GiB", "32GiB" ].reduce( reduce_memory_declarations, {} ),
     keep_warm: { minInstances: 1 },
     max_concurrency: { concurrency: 1000 }
+}
+
+/**
+ * 
+ * @param {Array} runtimes - runtime object as applied to a function
+ * @param {Object} allowed_runtimes - allowed runtime object as defined in this file
+ * @param {boolean} throw_on_fail - throw an error if validation fails
+ * @returns {boolean} - true if validation passes, false if validation fails and throw_on_fail is false
+ */
+exports.validate_runtime_settings = ( runtimes, allowed_runtimes, throw_on_fail=true ) => {
+
+    // const { log } = require( '../modules/helpers' )
+    // log( `Validating runtime settings against allowed rintimes: `, runtimes, allowed_runtimes )
+
+    // Check that all runtime keys exist in the v2_runtimes object
+    const runtime_keys = Object.keys( allowed_runtimes )
+    const invalid_runtime_keys = runtimes.filter( runtime_key => !runtime_keys.includes( runtime_key ) )
+    if( invalid_runtime_keys.length ) {
+        console.error( `Invalid runtime keys: `, invalid_runtime_keys )
+        if( throw_on_fail ) throw new Error( `Invalid runtime keys: ${ invalid_runtime_keys.length }` )
+        else return false
+    }
+
+    return true
+
 }

src/components/pages/Claim.jsx

@@ -59,6 +59,8 @@ export default function ClaimPOAP() {
     // If the game is done, and the user is valid, redirect for claiming
     useEffect( (  ) => {
 
+        log( `Determining whether to forward. User is ${ user_valid ? 'valid' : 'invalid' }, claim link is `, claim_link )
+
         // Game version do their claiming through a subcomponent of <Stroop />
         // for that reason we never forward game-based challenges
         if( has_game_challenge ) return

src/hooks/claim_codes.js

@@ -75,7 +75,7 @@ export const useClaimcodeForChallenge = ( captchaResponse, fetch_code=false ) =>
             try {
 
                 // Check for presence of challenge data
-                if( !user_valid ) return log( 'User not (yet) validated' )
+                if( user_valid !== true ) return log( 'User not (yet) validated' )
 
                 // Validate for expired challenge, note that the claim_link check here exists because challenges are expired once links are retreived, so once a claim_code is loaded the challenge is expired while the page is still open
                 if( !claim_link && user_valid && challenge?.deleted ) {
@@ -90,7 +90,7 @@ export const useClaimcodeForChallenge = ( captchaResponse, fetch_code=false ) =>
                 // If we already have a link in the state, do not get a new one
                 if( claim_link ) return
 
-                // If we are not ready to fetch the code ret, return undefined
+                // If we are not ready to fetch the code yet, return undefined
                 if( !fetch_code ) return
 
                 // If no game challenge, get a code

src/hooks/user_validation.js

@@ -13,7 +13,7 @@ export const useValidateUser = ( captchaResponse ) => {
     // States
     const { challenge_code, error_code } = useParams(  )
     const [ message, set_message ] = useState( `${ t( 'claim.setLoading' ) }` )
-    const [ user_valid, set_user_valid ] = useState( true )
+    const [ user_valid, set_user_valid ] = useState( undefined )
     const challenge = useChallenge( challenge_code )
 
     // Stakking function 
@@ -45,6 +45,10 @@ export const useValidateUser = ( captchaResponse ) => {
     const slow_users_down = false
     useEffect( f => {
 
+        // If no challenge code is present, do nothing
+        if( !challenge_code ) return
+
+        // Effect cancellation flag
         let cancelled = false;
 
         // Validate client