Stop file from being fetched or processed multiple times.

src/scripts/datasource.ts

@@ -1,12 +1,13 @@
-import { openDB, DBSchema } from 'idb';
+import { openDB, DBSchema } from 'idb'
+import { MutexGroup } from './mutex'
 
-const repos = 'Dimbreath/StarRailData/';
-const apiBase = 'https://api.github.com/repos/' + repos;
-const jsonBase = 'https://raw.githubusercontent.com/' + repos;
+const repos = 'Dimbreath/StarRailData/'
+const apiBase = 'https://api.github.com/repos/' + repos
+const jsonBase = 'https://raw.githubusercontent.com/' + repos
 
-console.log(`apiBase: ${apiBase}\njsonBase: ${jsonBase}`);
+console.log(`apiBase: ${apiBase}\njsonBase: ${jsonBase}`)
 
-const configDBVersion = 1;
+const configDBVersion = 1
 interface ConfigDBv1 extends DBSchema 
 {
     files: {
@@ -23,7 +24,7 @@ const configDB = openDB<ConfigDBv1>('configDB', configDBVersion,
 {
     upgrade(db, oldVersion, newVersion, _transaction, _event) 
     {
-        console.log(`fileDB upgrading from ${oldVersion} to ${newVersion}`);
+        console.log(`fileDB upgrading from ${oldVersion} to ${newVersion}`)
         db.createObjectStore('files', { keyPath: ['hash', 'path'], })
     },
     blocked(currentVersion, blockedVersion, _event) 
@@ -40,31 +41,32 @@ const configDB = openDB<ConfigDBv1>('configDB', configDBVersion,
     },
 })
 
+const retrieveJsonMutex = new MutexGroup()
 export async function retrieveJson(request:string, commit:string, useApi:boolean) : Promise<any>
 {
     let url = useApi ? apiBase : jsonBase
     if (!useApi && commit != null)
         url += `${commit}/`
     url += request
 
+    return await retrieveJsonMutex.runExclusive(url, async () =>
     {
         const existing = await (await configDB).get('files', IDBKeyRange.only([commit, request]));
         if (existing?.hash)
-            return JSON.parse(existing.content);
-    }
-
-    const cache = useApi ? 'force-cache' : 'default'
-    const result = await fetch(url, { headers: { 'Accept': 'application/json', }, cache })
-        .then(response => response.json())
-        .catch(error => console.log(`fileDB error: ${error}`))
+            return JSON.parse(existing.content)
 
-    if (result)
-    {
-        await (await configDB).put('files', { hash:commit, path:request, content:JSON.stringify(result) })
-        console.log(`fileDB stored ${request} @${commit}`)
-    }
-
-    return result
+        const cache = useApi ? 'force-cache' : 'default'
+        const result = await fetch(url, { headers: { 'Accept': 'application/json', }, cache })
+            .then(response => response.json())
+            .catch(error => console.log(`fileDB error: ${error}`))
+
+        if (result)
+        {
+            await (await configDB).put('files', { hash:commit, path:request, content:JSON.stringify(result) })
+            console.log(`fileDB stored ${request} @${commit}`)
+        }
+        return result
+    })
 }
 
 export interface DataSourceCommit
@@ -80,8 +82,8 @@ export interface DataSourceCommit
     }
 }
 
-export function retrieveCommits() : Promise<DataSourceCommit[]>
+export async function retrieveCommits() : Promise<DataSourceCommit[]>
 {
     return fetch(apiBase + 'commits', { headers: { 'Accept': 'application/json', } })
-        .then(response => response.json());
+        .then(response => response.json())
 }

src/scripts/mutex.ts

@@ -0,0 +1,65 @@
+type ReleaseFunc = () => void
+
+export class MutexGroup
+{
+    private mutexes:{[key:string]:Mutex} = {}
+
+    public async runExclusive<T>(key:string, action:() => Promise<T>) : Promise<T>
+    {
+        let mutex = this.mutexes[key]
+        if (!mutex)
+            this.mutexes[key] = mutex = new Mutex()
+
+        return mutex.runExclusive(action)
+    }
+}
+
+export class Mutex 
+{
+    private locked = false
+    private queue:{resolve:(release:ReleaseFunc) => void}[] = []
+
+    public acquire() : Promise<ReleaseFunc>
+    {
+        return new Promise<ReleaseFunc>((resolve) => 
+        {
+            this.queue.push({resolve})
+            this.dispatch()
+        })
+    }
+
+    public async runExclusive<T>(action:() => Promise<T>) : Promise<T>
+    {
+        const release = await this.acquire()
+        try 
+        {
+            return await action()
+        } 
+        finally 
+        {
+            release()
+        }
+    }
+
+    private dispatch() : void
+    {
+        if (this.locked)
+            return
+
+        const next = this.queue.shift()
+        if (!next) 
+            return
+
+        this.locked = true
+        next.resolve(this.createReleaseFunc())
+    }
+
+    private createReleaseFunc() : ReleaseFunc 
+    {
+        return () => 
+        {
+            this.locked = false
+            this.dispatch()
+        }
+    }
+}

src/scripts/sources/ability.ts

@@ -1,8 +1,7 @@
 import { retrieveJson } from '../datasource';
+import { MutexGroup } from '../mutex';
 import { GamecoreContext, GamecoreNode, GamecoreTargetType, DynamicExpression, DynamicValues, } from './gamecore';
 
-// TODO look at StatusConfig
-
 export interface ModifierEventHandler
 {
   CallbackConfig: GamecoreNode[]
@@ -304,46 +303,50 @@ const contextTypeToPaths =
 }
 
 const abilityContextCache:{[commitId: string]: {[type: string]: AbilityContext}} = {}
+const abilityContextMutex = new MutexGroup()
 export async function getAbilityContext(commitId:string, type:AbilityContextType) : Promise<AbilityContext>
 {
-  let result 
-  let container = abilityContextCache[commitId]
-  if (container === undefined)
-    container = abilityContextCache[commitId] = {}
-  else result = container[type]
-
-  if (result == undefined)
+  return abilityContextMutex.runExclusive(type, async () => 
   {
-    const context:AbilityContext = 
+    let result 
+    let container = abilityContextCache[commitId]
+    if (container === undefined)
+      container = abilityContextCache[commitId] = {}
+    else result = container[type]
+
+    if (result == undefined)
     {
-      Abilities: {},
-      Modifiers: {},
-      TaskListTemplates: {},
-    }
-    const paths = contextTypeToPaths[type]
-    for (const path of paths.Abilities)
-      if (path.endsWith('.json'))
-        mergeAbilityConfig(context, await getAbilities(commitId, path) as AbilityConfig)
-      else
+      const context:AbilityContext = 
       {
-        const response = await retrieveJson(`git/trees/${commitId}:${path}`, commitId, true)
-        const tree = response.tree
-        if (tree !== undefined)
-          for (const treePath of tree.map((t:any) => t.path))
-            if (treePath.endsWith('.json'))
-              mergeAbilityConfig(context, await getAbilities(commitId, `${path}/${treePath}`) as AbilityConfig)
+        Abilities: {},
+        Modifiers: {},
+        TaskListTemplates: {},
       }
-
-    for (const path of paths.Modifiers)
-      mergeModifierConfig(context, await getModifiers(commitId, path) as ModifierConfig)
-
-    for (const path of paths.TaskListTemplates)
-      mergeTaskListTemplateConfig(context, await getTaskListTemplates(commitId, path) as TaskListTemplateConfig)
-
-    result = container[type] = context
-    console.log(`cached ${type} ability context for ${commitId}`)
-  }
-  return result
+      const paths = contextTypeToPaths[type]
+      for (const path of paths.Abilities)
+        if (path.endsWith('.json'))
+          mergeAbilityConfig(context, await getAbilities(commitId, path) as AbilityConfig)
+        else
+        {
+          const response = await retrieveJson(`git/trees/${commitId}:${path}`, commitId, true)
+          const tree = response.tree
+          if (tree !== undefined)
+            for (const treePath of tree.map((t:any) => t.path))
+              if (treePath.endsWith('.json'))
+                mergeAbilityConfig(context, await getAbilities(commitId, `${path}/${treePath}`) as AbilityConfig)
+        }
+
+      for (const path of paths.Modifiers)
+        mergeModifierConfig(context, await getModifiers(commitId, path) as ModifierConfig)
+
+      for (const path of paths.TaskListTemplates)
+        mergeTaskListTemplateConfig(context, await getTaskListTemplates(commitId, path) as TaskListTemplateConfig)
+
+      result = container[type] = context
+      console.log(`cached ${type} ability context for ${commitId}`)
+    }
+    return result
+  })
 }
 
 export function findTaskTemplate(templateName:string, gamecoreContext:GamecoreContext, abilityContext:AbilityContext) : TaskListTemplate | undefined

src/scripts/sources/avatar.ts

@@ -3,6 +3,7 @@ import translate, { Translatable, } from '../translate';
 import { GamecoreParam } from './gamecore';
 import { Creature } from './creature';
 import { cleanupFilename } from '../common';
+import { Mutex } from '../mutex';
 
 // Eidolons
 
@@ -98,54 +99,58 @@ export interface AvatarConfig
 }
 
 const avatarConfigCache:{[commitId: string]: AvatarConfig} = {}
+const avatarConfigMutex = new Mutex()
 export async function getAvatars(commitId:string) : Promise<AvatarConfig>
 {
-    let config = avatarConfigCache[commitId]
-    if (config == undefined)
+    return avatarConfigMutex.runExclusive(async () => 
     {
-        const eidolons = await retrieveJson('ExcelOutput/AvatarRankConfig.json', commitId, false) as AvatarRankConfig
-        // No point translating any of these, it doesn't work
-        //await translate(commitId, eidolon.Trigger)
-        //eidolon.Name = await translateHash(commitId, getHash(eidolon.Name))
-        //eidolon.Desc = await translateHash(commitId, getHash(eidolon.Desc))
-
-        const avatars = await retrieveJson('ExcelOutput/AvatarConfig.json', commitId, false) as AvatarConfig
-        for (const key in avatars)
+        let config = avatarConfigCache[commitId]
+        if (config == undefined)
         {
-            const avatar = avatars[key]
-            await translate(commitId, avatar.AvatarName, (name) => 
+            const eidolons = await retrieveJson('ExcelOutput/AvatarRankConfig.json', commitId, false) as AvatarRankConfig
+            // No point translating any of these, it doesn't work
+            //await translate(commitId, eidolon.Trigger)
+            //eidolon.Name = await translateHash(commitId, getHash(eidolon.Name))
+            //eidolon.Desc = await translateHash(commitId, getHash(eidolon.Desc))
+
+            const avatars = await retrieveJson('ExcelOutput/AvatarConfig.json', commitId, false) as AvatarConfig
+            for (const key in avatars)
             {
-                if (!name)
-                    return avatar.AvatarID.toString()
-
-                if (avatar.AvatarID > 8000)
-                    name = name.replace('{NICKNAME}', `Trailblazer (${avatar.DamageType})`)
-                return name
-            })
-            avatar.Eidolons = avatar.RankIDList.map(v => eidolons[v])
-            avatar.Traces = []
-
-            avatar.SearchKeywords = []
-            avatar.SearchKeywords.push(avatar.AvatarName.Text.toLowerCase())
-            avatar.SearchKeywords.push(avatar.DamageType.toLowerCase())
-            avatar.SearchKeywords.push(avatar.AvatarBaseType.toLowerCase())
-            avatar.SearchKeywords.push(cleanupFilename(avatar.JsonPath).toLowerCase())
-        }
-
-        const traceRanks = await retrieveJson('ExcelOutput/AvatarSkillTreeConfig.json', commitId, false) as AvatarSkillTreeConfig
-        // Fields that look translatable are just not
-        for (const key in traceRanks)
-        {
-            const ranks = traceRanks[key]
-            const trace = ranks[Object.keys(ranks).length]
-            const avatar = avatars[trace.AvatarID]
-            avatar.Traces.push(trace)
+                const avatar = avatars[key]
+                await translate(commitId, avatar.AvatarName, (name) => 
+                {
+                    if (!name)
+                        return avatar.AvatarID.toString()
+
+                    if (avatar.AvatarID > 8000)
+                        name = name.replace('{NICKNAME}', `Trailblazer (${avatar.DamageType})`)
+                    return name
+                })
+                avatar.Eidolons = avatar.RankIDList.map(v => eidolons[v])
+                avatar.Traces = []
+
+                avatar.SearchKeywords = []
+                avatar.SearchKeywords.push(avatar.AvatarName.Text.toLowerCase())
+                avatar.SearchKeywords.push(avatar.DamageType.toLowerCase())
+                avatar.SearchKeywords.push(avatar.AvatarBaseType.toLowerCase())
+                avatar.SearchKeywords.push(cleanupFilename(avatar.JsonPath).toLowerCase())
+            }
+
+            const traceRanks = await retrieveJson('ExcelOutput/AvatarSkillTreeConfig.json', commitId, false) as AvatarSkillTreeConfig
+            // Fields that look translatable are just not
+            for (const key in traceRanks)
+            {
+                const ranks = traceRanks[key]
+                const trace = ranks[Object.keys(ranks).length]
+                const avatar = avatars[trace.AvatarID]
+                avatar.Traces.push(trace)
+            }
+
+            config = avatarConfigCache[commitId] = avatars
+            console.log('cached avatar config for ' + commitId)
         }
-
-        config = avatarConfigCache[commitId] = avatars
-        console.log('cached avatar config for ' + commitId)
-    }
-    return config
+        return config
+    })
 }
 
 export async function getAvatar(commitId:string, avatarId:number) : Promise<Avatar>