Update youtube data (#139)

.powershell/dev/Update-YoutubeChannelData.ps1

@@ -0,0 +1,106 @@
+# Define variables
+$apiKey = $env:google_apiKey
+$channelId = "UCkYqhFNmhCzkefHsHS652hw"
+$outputDir = "site\content\resources\videos\youtube"
+$dataDirectory = ".\site\data"
+$refreshData = $false
+
+$maxResults = 800
+
+# Create the output directory if it doesn't exist
+if (-not (Test-Path $outputDir)) {
+    New-Item -Path $outputDir -ItemType Directory
+}
+
+# Function to fetch video list from YouTube API and save a single youtube.json file
+function Fetch-YoutubeVideoList {
+    param ()
+
+    $nextPageToken = $null
+    $page = 1;
+    $allVideosData = @()
+
+    do {
+        # YouTube API endpoint to get videos from a channel, including nextPageToken
+        $searchApiUrl = "https://www.googleapis.com/youtube/v3/search?key=$apiKey&part=snippet&channelId=$channelId&type=video&maxResults=$maxResults&pageToken=$nextPageToken"
+
+        # Fetch video list
+        $searchResponse = Invoke-RestMethod -Uri $searchApiUrl -Method Get
+
+        $allVideosData += $searchResponse.items
+
+        # Get the nextPageToken to continue fetching more videos
+        $nextPageToken = $searchResponse.nextPageToken
+        $page++
+    } while ($nextPageToken)
+
+    # Save all video data to a single youtube.json file
+    $dataFilePath = Join-Path $dataDirectory "youtube.json"
+    $allVideosData | ConvertTo-Json -Depth 10 | Set-Content -Path $dataFilePath
+
+    Write-Host "All video data saved to youtube.json."
+}
+
+# Function to update data.json for a single video
+function Update-YoutubeDataFile {
+    param (
+        [string]$videoId
+    )
+
+    # Create the directory named after the video ID
+    $videoDir = Join-Path $outputDir $videoId
+    if (-not (Test-Path $videoDir)) {
+        New-Item -Path $videoDir -ItemType Directory
+    }
+
+    # File path for data.json
+    $jsonFilePath = Join-Path $videoDir "data.json"
+    if ($videoId -eq "xo4jMxupIM0") {
+        Write-Host "Updating data.json for video: $videoId"
+    }
+    # Only update if $refreshData is true or data.json doesn't exist
+    if ($refreshData -or -not (Test-Path $jsonFilePath)) {
+        # Fetch full video details from YouTube API
+        $videoDetailsUrl = "https://www.googleapis.com/youtube/v3/videos?key=$apiKey&id=$videoId&part=snippet,contentDetails"
+        $videoDetails = Invoke-RestMethod -Uri $videoDetailsUrl -Method Get
+        $videoData = $videoDetails.items[0]
+
+        if ($videoData) {
+            # Save updated video data to data.json
+            $videoData | ConvertTo-Json -Depth 10 | Set-Content -Path $jsonFilePath
+            Write-Host "Updated data.json for video: $videoId"
+        }
+        else {
+            Write-Host "No data found for video: $videoId"
+        }
+    }
+    else {
+        Write-Host "Data for video $videoId is already up to date."
+    }
+}
+
+# Function to iterate through youtube.json and update data.json for each video
+function Update-YoutubeDataFilesFromJson {
+    param ()
+
+    $dataFilePath = Join-Path $dataDirectory "youtube.json"
+    if (-not (Test-Path $dataFilePath)) {
+        Write-Host "youtube.json file not found. Please run Fetch-YoutubeVideoList first."
+        return
+    }
+
+    # Load video list from youtube.json
+    $allVideosData = Get-Content -Path $dataFilePath | ConvertFrom-Json
+
+    foreach ($video in $allVideosData) {
+        $videoId = $video.id.videoId
+
+        # Call the function to update the data for a single video
+        Update-YoutubeDataFile -videoId $videoId
+    }
+
+    Write-Host "All video data files updated from youtube.json."
+}
+
+#Fetch-YoutubeVideoList          # Call this to fetch video list and save to youtube.json
+Update-YoutubeDataFilesFromJson # Call this to update data.json files from youtube.json

.powershell/dev/Update-YoutubeMarkdownFiles.ps1

@@ -0,0 +1,122 @@
+# Define variables
+$outputDir = "site\content\resources\videos\youtube"
+$excludedTags = @("martin hinshelwood", "nkd agility")  # List of tags to exclude
+
+# Function to generate markdown content for a video
+function Get-NewMarkdownContents {
+    param (
+        [pscustomobject]$videoData,
+        [string]$videoId,
+        [string[]]$excludedTags
+    )
+
+    $videoSnippet = $videoData.snippet
+    $fullDescription = $videoSnippet.description
+    $durationISO = $videoData.contentDetails.duration
+
+    # Convert duration to seconds
+    $timeSpan = [System.Xml.XmlConvert]::ToTimeSpan($durationISO)
+    $durationInSeconds = $timeSpan.TotalSeconds
+
+    # Check if the video is a short (60 seconds or less)
+    $isShort = $durationInSeconds -le 60
+
+    # Get the highest resolution thumbnail
+    $thumbnails = $videoSnippet.thumbnails
+    $thumbnailUrl = $thumbnails.maxres.url
+    if (-not $thumbnailUrl) {
+        $thumbnailUrl = $thumbnails.high.url  # Fallback to high resolution
+    }
+
+    # Format the title to be URL-safe and remove invalid characters
+    $title = $videoSnippet.title -replace '[#"]', ' ' -replace ':', ' - ' -replace '\s+', ' '  # Ensure only one space in a row
+    $publishedAt = Get-Date $videoSnippet.publishedAt -Format "yyyy-MM-ddTHH:mm:ssZ"
+    $urlSafeTitle = ($title -replace '[:\/\\*?"<>|#%.]', '-' -replace '\s+', '-').ToLower()
+
+    # Remove consecutive dashes
+    $urlSafeTitle = $urlSafeTitle -replace '-+', '-'
+
+    # Create the external URL for the original video
+    $externalUrl = "https://www.youtube.com/watch?v=$videoId"
+
+    # Get the tags from the snippet and filter out excluded tags
+    $tags = @()
+    if ($videoSnippet.tags) {
+        $tags = $videoSnippet.tags | Where-Object { -not ($excludedTags -contains $_.ToLower()) }
+    }
+    $tagsString = $tags -join ", "
+
+    # Return the markdown content without etag and with properly formatted title
+    return @"
+---
+title: "$title"
+date: $publishedAt
+videoId: $videoId
+url: /resources/videos/:slug
+slug: $urlSafeTitle
+canonicalUrl: $externalUrl
+aliases:
+ - /resources/videos/$videoId
+# - /resources/videos/$urlSafeTitle
+preview: $thumbnailUrl
+duration: $durationInSeconds
+isShort: $isShort
+tags: [$tagsString]
+sitemap:
+  filename: sitemap.xml
+  priority: 0.4
+---
+
+{{< youtube $videoId >}}
+
+# $title
+
+$fullDescription
+
+[Watch on YouTube]($externalUrl)
+"@
+}
+
+# Function to generate markdown files from existing data.json files
+function Update-YoutubeMarkdownFiles {
+    param ()
+
+    # Iterate through each video folder
+    Get-ChildItem -Path $outputDir -Directory | ForEach-Object {
+        $videoDir = $_.FullName
+        $markdownFile = Join-Path $videoDir "index.md"
+        $jsonFilePath = Join-Path $videoDir "data.json"
+
+        # Check if index.md exists and if it contains 'canonicalUrl'
+        $shouldUpdate = $false
+        if (-not (Test-Path $markdownFile)) {
+            # If index.md does not exist, we should create it
+            $shouldUpdate = $true
+        }
+        elseif ((Test-Path $markdownFile) -and (Get-Content -Path $markdownFile -Raw) -match 'canonicalUrl:') {
+            # If index.md exists and contains 'canonicalUrl', we should update it
+            $shouldUpdate = $true
+        }
+        else {
+            Write-Host "Markdown file for video $($videoDir) has been customised. Skipping."
+        }
+
+        # Proceed to update or create the markdown file if necessary
+        if ($shouldUpdate -and (Test-Path $jsonFilePath)) {
+            # Load the video data from data.json
+            $videoData = Get-Content -Path $jsonFilePath | ConvertFrom-Json
+            $videoId = $videoData.id
+
+            # Generate markdown content
+            $markdownContent = Get-NewMarkdownContents -videoData $videoData -videoId $videoId -excludedTags $excludedTags
+
+            # Write the markdown content to index.md
+            Set-Content -Path $markdownFile -Value $markdownContent
+            Write-Host "Markdown created or updated for video: $($videoData.snippet.title)"
+        }
+    }
+
+    Write-Host "All markdown files processed."
+}
+
+Update-YoutubeMarkdownFiles  # Call this to update markdown files from existing data.json files

.powershell/single-use/videos/AddResourceType.ps1

@@ -0,0 +1,62 @@
+# Define variables
+$outputDir = "site\content\resources\videos\youtube"
+
+# Function to update the front matter with resourceTypes
+function Update-ResourceTypes {
+    param (
+        [string]$markdownPath
+    )
+
+    # Load the existing content
+    $content = Get-Content -Path $markdownPath -Raw
+
+    # Extract front matter and body content
+    if ($content -match '(?ms)^---\s*(.*?)---\s*(.*)$') {
+        $frontMatterContent = $matches[1]
+        $bodyContent = $matches[2]
+
+        # Convert front matter content to ordered hash table
+        $frontMatter = ConvertFrom-Yaml $frontMatterContent -Ordered
+
+        # Check and update resourceTypes
+        if ($frontMatter.Contains("resourceTypes")) {
+            # Ensure resourceTypes is a list and add "video" only if not present
+            if ($frontMatter["resourceTypes"] -notcontains "video") {
+                $frontMatter["resourceTypes"] += "video"
+            }
+        }
+        else {
+            # Add resourceTypes as a list with "video"
+            $frontMatter.Add("resourceTypes", @("video"))
+        }
+
+        # Convert the updated front matter back to YAML
+        $updatedFrontMatter = ConvertTo-Yaml $frontMatter
+
+        # Combine updated front matter and body content
+        $updatedContent = "---`n$updatedFrontMatter`n---`n$bodyContent"
+
+        # Write updated content back to file
+        Set-Content -Path $markdownPath -Value $updatedContent
+        Write-Host "resourceTypes updated for file: $markdownPath"
+    }
+    else {
+        Write-Host "Failed to parse front matter for file: $markdownPath"
+    }
+}
+
+# Iterate through each video folder and update markdown files
+Get-ChildItem -Path $outputDir -Directory | ForEach-Object {
+    $videoDir = $_.FullName
+    $markdownFile = Join-Path $videoDir "index.md"
+
+    if (Test-Path $markdownFile) {
+        # Update the resourceTypes in the existing markdown file
+        Update-ResourceTypes -markdownPath $markdownFile
+    }
+    else {
+        Write-Host "Skipping folder: $videoDir (missing index.md)"
+    }
+}
+
+Write-Host "All markdown files processed."

.powershell/single-use/videos/updateYoutubeTagsToFrontMatter.ps1

@@ -0,0 +1,73 @@
+# Define variables
+$outputDir = "site\content\resources\videos\youtube"
+$excludedTags = @("martin hinshelwood", "nkd agility")  # List of tags to exclude
+
+# Function to add tags to existing markdown front matter using an ordered hash table
+function Add-TagsToMarkdown {
+    param (
+        [string]$markdownPath,
+        [pscustomobject]$videoData,
+        [string[]]$excludedTags
+    )
+
+    # Load the existing content
+    $content = Get-Content -Path $markdownPath -Raw
+
+    # Extract front matter and body content
+    if ($content -match '(?ms)^---\s*(.*?)---\s*(.*)$') {
+        $frontMatterContent = $matches[1]
+        $bodyContent = $matches[2]
+
+        # Convert front matter content to ordered hash table
+        $frontMatter = ConvertFrom-Yaml $frontMatterContent -Ordered
+
+        # Parse the tags from video data
+        $videoTags = @()
+        if ($videoData.snippet.tags) {
+            $videoTags = $videoData.snippet.tags | Where-Object { -not ($excludedTags -contains $_.ToLower()) }
+        }
+
+        # Add or merge tags into the front matter
+        if ($frontMatter.Contains("tags")) {
+            $existingTags = $frontMatter["tags"]
+            $mergedTags = ($existingTags + $videoTags) | Sort-Object -Unique
+            $frontMatter["tags"] = $mergedTags
+        }
+        else {
+            $frontMatter.Add("tags", $videoTags)
+        }
+
+        # Convert the updated front matter back to YAML
+        $updatedFrontMatter = ConvertTo-Yaml $frontMatter
+
+        # Combine updated front matter and body content
+        $updatedContent = "---`n$updatedFrontMatter`n---`n$bodyContent"
+
+        # Write updated content back to file
+        Set-Content -Path $markdownPath -Value $updatedContent
+        Write-Host "Tags updated for file: $markdownPath"
+    }
+    else {
+        Write-Host "Failed to parse front matter for file: $markdownPath"
+    }
+}
+
+# Iterate through each video folder and update tags in markdown files
+Get-ChildItem -Path $outputDir -Directory | ForEach-Object {
+    $videoDir = $_.FullName
+    $markdownFile = Join-Path $videoDir "index.md"
+    $jsonFilePath = Join-Path $videoDir "data.json"
+
+    if ((Test-Path $markdownFile) -and (Test-Path $jsonFilePath)) {
+        # Load the video data from data.json
+        $videoData = Get-Content -Path $jsonFilePath | ConvertFrom-Json
+
+        # Update the tags in the existing markdown file
+        Add-TagsToMarkdown -markdownPath $markdownFile -videoData $videoData -excludedTags $excludedTags
+    }
+    else {
+        Write-Host "Skipping folder: $videoDir (missing index.md or data.json)"
+    }
+}
+
+Write-Host "All markdown files processed."