Skip to content

Commit

Permalink
Improve member sync function
Browse files Browse the repository at this point in the history
  • Loading branch information
@AndrewCraswell
AndrewCraswell committed Dec 29, 2023
1 parent ca9ef32 commit eacc429
Show file tree
Hide file tree
Showing 2 changed files with 79 additions and 64 deletions.
89 changes: 52 additions & 37 deletions api/MembersSync/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,44 +66,59 @@ const scheduledIngest: AzureFunction = async function (
context: Context,
timer: any
): Promise<void> {
// Request the members page
const { data: response } = await axios.get<string>(
process.env.USA_FENCING_MEMBERS_CSV_URL
)

// Parse the URL to the latest document
const match = response.match(
/<a href="(.*)" class="card p-2 text-white bg-blued tight d-inline-block no-link">/
context.log(
`Fetching the USA Fencing members page... [${process.env.USA_FENCING_MEMBERS_CSV_URL}]`
)
let membersCsvUrl = ""
if (match.length === 2) {
membersCsvUrl = he.decode(match[1])
} else {
// TODO: Warn that the member file wasn't parsed
}

context.log(membersCsvUrl)

// Fetch the CSV file
const members: Array<IAssociationMember> = []
await csv()
.fromStream(request.get(membersCsvUrl))
.subscribe((row: CsvMemberRow) => {
return new Promise((resolve, reject) => {
const member = parseMember(row)

if (member.AssociationMemberId) {
members.push(member)
}

resolve()
})
})

context.log("Found", members.length, "members to ingest.")
// Request the members page
try {
const { data: response } = await axios.get<string>(
process.env.USA_FENCING_MEMBERS_CSV_URL
)

context.log("Fetched the USA Fencing members page.")

// Parse the URL to the latest document
const match = response.match(
/<a href="(.*)" class="card p-2 text-white bg-blued tight d-inline-block no-link">/
)
let membersCsvUrl = ""
if (match.length === 2) {
membersCsvUrl = he.decode(match[1])
context.log(membersCsvUrl)
} else {
context.log("Could not locate download link for the members CSV.")
// TODO: Send an email alert that the file was not parsed correctly
}

// Upsert the results into the database
await updateDatabase(context, members)
// Fetch the CSV file
try {
const members: Array<IAssociationMember> = []
await csv()
.fromStream(request.get(membersCsvUrl))
.subscribe((row: CsvMemberRow) => {
return new Promise((resolve, reject) => {
const member = parseMember(row)

if (member.AssociationMemberId) {
members.push(member)
}

resolve()
})
})

context.log("Found", members.length, "members to ingest.")

// Upsert the results into the database
await updateDatabase(context, members)
} catch (e) {
// TODO: Send an email alert that the file was not parsed correctly
context.log("Could not parse the members CSV file.")
}
} catch (e) {
context.log("Could not fetch the USA Fencing members page.")
}
}

async function updateDatabase(
Expand Down Expand Up @@ -179,13 +194,13 @@ function getUpsertQuery(members: IAssociationMember[]) {
query = `
${query}
UPDATE dbo.AssociationMembers WITH (UPDLOCK, SERIALIZABLE) SET
UPDATE dbo.AssociationMembers WITH (UPDLOCK, SERIALIZABLE) SET
${columns
.map((col) => `[${col}] = ${getSqlValue(member[col])},`)
.join("")
.slice(0, -1)}
WHERE AssociationMemberId = '${member.AssociationMemberId}';
IF @@ROWCOUNT = 0
BEGIN
INSERT dbo.AssociationMembers(
Expand Down
54 changes: 27 additions & 27 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,16 +22,16 @@
},
"dependencies": {
"@apollo/client": "^3.6.9",
"@auth0/auth0-react": "^1.11.0",
"@auth0/auth0-react": "^2.2.4",
"@azure/app-configuration": "^1.3.1",
"@emotion/react": "^11.10.4",
"@emotion/styled": "^11.10.4",
"@fluentui/react": "^8.96.1",
"@fluentui/react-components": "^9.7.1",
"@fluentui/react-icons": "^2.0.190",
"@fluentui/react-icons": "^2.0.224",
"@fluentui/theme-samples": "^8.7.3",
"@loadable/component": "^5.15.2",
"@microsoft/applicationinsights-react-js": "^3.3.5",
"@loadable/component": "^5.16.3",
"@microsoft/applicationinsights-react-js": "^17.0.3",
"@microsoft/applicationinsights-web": "^2.8.7",
"@nylas/components-availability": "^1.1.4",
"@nylas/components-schedule-editor": "^1.1.4-canary.0",
Expand All @@ -49,35 +49,35 @@
"react-markdown": "^8.0.3",
"react-redux": "^8.0.4",
"react-router-dom": "^6.4.1",
"stripe": "^11.6.0"
"stripe": "^14.10.0"
},
"devDependencies": {
"@azure/static-web-apps-cli": "^1.0.2",
"@graphql-codegen/cli": "^2.12.1",
"@graphql-codegen/introspection": "^2.2.1",
"@graphql-codegen/typescript": "^2.7.3",
"@graphql-codegen/typescript-operations": "^2.5.3",
"@graphql-codegen/typescript-react-apollo": "^3.3.3",
"@types/loadable__component": "^5.13.4",
"@types/node": "17.0.40",
"@types/react": "^17.0.44",
"@azure/static-web-apps-cli": "^1.1.6",
"@graphql-codegen/cli": "^5.0.0",
"@graphql-codegen/introspection": "^4.0.0",
"@graphql-codegen/typescript": "^4.0.1",
"@graphql-codegen/typescript-operations": "^4.0.1",
"@graphql-codegen/typescript-react-apollo": "^4.1.0",
"@types/loadable__component": "^5.13.8",
"@types/node": "20.10.5",
"@types/react": "^18.2.46",
"@types/react-dom": "^17.0.16",
"@vitejs/plugin-react-swc": "^3.0.0",
"azure-functions-core-tools": "^4.0.4785",
"@vitejs/plugin-react-swc": "^3.5.0",
"azure-functions-core-tools": "^4.0.5455",
"cross-env": "^7.0.3",
"eslint": "^8.29.0",
"eslint-config-prettier": "^8.5.0",
"eslint": "^8.56.0",
"eslint-config-prettier": "^9.1.0",
"eslint-config-react-app": "^7.0.1",
"eslint-plugin-react-refresh": "^0.3.1",
"eslint-plugin-react-refresh": "^0.4.5",
"gql": "^1.1.2",
"graphql": "^16.6.0",
"husky": "^8.0.1",
"lint-staged": "^13.0.3",
"prettier": "^2.7.1",
"graphql": "^16.8.1",
"husky": "^8.0.3",
"lint-staged": "^15.2.0",
"prettier": "^3.1.1",
"rollup-plugin-visualizer": "^5.8.3",
"sass": "^1.56.1",
"typescript": "^4.8.2",
"vite": "^4.0.0",
"vite-plugin-static-copy": "^0.12.0"
"sass": "^1.69.6",
"typescript": "^5.3.3",
"vite": "^5.0.10",
"vite-plugin-static-copy": "^1.0.0"
}
}

0 comments on commit eacc429

Please sign in to comment.