No commit message

.gitignore

@@ -15,4 +15,5 @@ list.scroll
 *.key
 *.txt
 trash/
-now.txt
+now.txt
+ipToGeo/

ScrollHub.js

@@ -21,6 +21,9 @@ const httpBackend = require("git-http-backend")
 // PPS
 const { Particle } = require("scrollsdk/products/Particle.js")
 
+// This
+const { Dashboard } = require("./dashboard.js")
+
 express.static.mime.define({ "text/plain": ["scroll", "parsers"] })
 
 const parseUserAgent = userAgent => {
@@ -57,7 +60,11 @@ class ScrollHub {
     this.templatesFolder = path.join(__dirname, "templates")
     this.trashFolder = path.join(__dirname, "trash")
     this.folderCache = {}
+    this.sseClients = new Set()
     this.storyCache = ""
+    this.globalLogFile = path.join(__dirname, "log.txt")
+    this.storyLogFile = path.join(__dirname, "now.txt")
+    this.dashboard = new Dashboard(this.globalLogFile)
   }
 
   startAll() {
@@ -78,6 +85,7 @@ class ScrollHub {
     this.initHistoryRoutes()
     this.initZipRoutes()
     this.initCommandRoutes()
+    this.initSSERoute()
 
     this.enableStaticFileServing()
 
@@ -86,6 +94,39 @@ class ScrollHub {
     return this.startServers()
   }
 
+  initSSERoute() {
+    const { app, globalLogFile } = this
+    app.get("/requests.htm", (req, res) => {
+      req.headers["accept-encoding"] = "identity"
+      res.writeHead(200, {
+        "Content-Type": "text/event-stream",
+        "Cache-Control": "no-cache",
+        Connection: "keep-alive"
+      })
+
+      // Send initial ping
+      res.write(": ping\n\n")
+
+      const id = Date.now()
+
+      const client = {
+        id,
+        res
+      }
+
+      this.sseClients.add(client)
+
+      req.on("close", () => this.sseClients.delete(client))
+    })
+  }
+
+  async broadCastMessage(log, ip) {
+    if (!this.sseClients.size) return
+    const geo = await this.dashboard.ipToGeo(ip)
+    log = [log.trim(), geo.lat, geo.lon].join(" ")
+    this.sseClients.forEach(client => client.res.write(`data: ${JSON.stringify({ log })}\n\n`))
+  }
+
   enableCompression() {
     this.app.use(compression())
   }
@@ -143,8 +184,6 @@ class ScrollHub {
   }
 
   initAnalytics() {
-    this.globalLogFile = path.join(__dirname, "log.txt")
-    this.storyLogFile = path.join(__dirname, "now.txt")
     if (!fs.existsSync(this.storyLogFile)) fs.writeFileSync(this.storyLogFile, "", "utf8")
     const { app, folderCache, storyCache } = this
     app.use(this.logRequest.bind(this))
@@ -158,9 +197,8 @@ class ScrollHub {
       res.send(storyCache)
     })
 
-    const { Dashboard } = require("./dashboard.js")
     app.get("/dashboard.csv", async (req, res) => {
-      const dashboard = new Dashboard(this.globalLogFile)
+      const { dashboard } = this
       await dashboard.processLogFile()
       const { csv } = dashboard
       res.setHeader("Content-Type", "text/plain")
@@ -183,6 +221,8 @@ class ScrollHub {
       if (err) console.error("Failed to log request:", err)
     })
 
+    this.broadCastMessage(logEntry, ip)
+
     if (folderName && folderCache[folderName]) {
       const folderPath = path.join(rootFolder, folderName)
       const folderLogFile = path.join(folderPath, "log.txt")

dashboard.js

@@ -1,12 +1,64 @@
 const fs = require("fs")
 const readline = require("readline")
+const fsp = require("fs").promises
+const path = require("path")
 
 class Dashboard {
   constructor(inputFile) {
     this.inputFile = inputFile
     this.stats = {} // Combined daily and folder stats
     this.totalLines = 0
     this.parsedLines = 0
+    this.ipGeoMap = new Map()
+  }
+
+  async ipToGeo(ip4) {
+    const { ipGeoMap } = this
+    // Check if the IP exists in the in-memory map
+    if (ipGeoMap.has(ip4)) return ipGeoMap.get(ip4)
+
+    // Prepare the cache file path
+    const firstPart = ip4.split(".")[0]
+    const cacheDir = path.join(__dirname, "ipToGeo", firstPart)
+    const cacheFile = path.join(cacheDir, `${ip4}.json`)
+
+    try {
+      // Check if the IP data exists in the cache folder
+      const cachedData = await fsp.readFile(cacheFile, "utf-8")
+      const geoData = JSON.parse(cachedData)
+
+      // Store in the in-memory map and return
+      ipGeoMap.set(ip4, geoData)
+      return geoData
+    } catch (error) {
+      // If file doesn't exist or there's an error reading it, proceed to download
+      if (error.code !== "ENOENT") {
+        console.error(`Error reading cache file: ${error.message}`)
+      }
+    }
+
+    try {
+      // Download data from ip-api.com using fetch
+      const response = await fetch(`http://ip-api.com/json/${ip4}`)
+      if (!response.ok) {
+        throw new Error(`HTTP error! status: ${response.status}`)
+      }
+      const geoData = await response.json()
+
+      // Store the data in the in-memory map
+      ipGeoMap.set(ip4, geoData)
+
+      // Ensure the cache directory exists
+      await fsp.mkdir(cacheDir, { recursive: true })
+
+      // Write the data to the cache file
+      await fsp.writeFile(cacheFile, JSON.stringify(geoData, null, 2))
+
+      return geoData
+    } catch (error) {
+      console.error(`Error fetching or caching geo data: ${error.message}`)
+      throw error
+    }
   }
 
   parseLogEntry(entry) {

globe.css

@@ -0,0 +1,22 @@
+
+      body {
+        margin: 0;
+        font-family: Arial, sans-serif;
+      }
+      canvas {
+        display: block;
+      }
+ #log-container {
+     padding: 10px;
+     height: 100px;
+     overflow-y: auto;
+     font-family: monospace;
+     white-space: pre-wrap;
+     word-wrap: break-word;
+     position: fixed;
+     bottom: 0px;
+     position: absolute;
+     left: 0;
+     right: 0;
+     color: white;
+ }

globe.js

@@ -0,0 +1,109 @@
+class Globe {
+  constructor() {}
+  main() {
+    let scene, camera, renderer, earth
+    const spikes = []
+
+    function init() {
+      scene = new THREE.Scene()
+
+      camera = new THREE.PerspectiveCamera(75, window.innerWidth / window.innerHeight, 0.1, 1000)
+      camera.position.z = 2
+
+      renderer = new THREE.WebGLRenderer()
+      renderer.setSize(window.innerWidth, window.innerHeight)
+      document.body.prepend(renderer.domElement)
+
+      const geometry = new THREE.SphereGeometry(0.5, 32, 32)
+      const texture = new THREE.TextureLoader().load("earth_atmos_2048.jpg")
+      const material = new THREE.MeshBasicMaterial({ map: texture })
+      earth = new THREE.Mesh(geometry, material)
+      scene.add(earth)
+
+      animate()
+    }
+
+    function animate() {
+      requestAnimationFrame(animate)
+
+      earth.rotation.y += 0.001
+
+      spikes.forEach((spike, index) => {
+        spike.scale.y *= 0.95
+        if (spike.scale.y < 0.01) {
+          earth.remove(spike)
+          spikes.splice(index, 1)
+        }
+      })
+
+      renderer.render(scene, camera)
+    }
+
+    function latLongToVector3(lat, lon) {
+      const phi = (90 - lat) * (Math.PI / 180)
+      const theta = (lon + 180) * (Math.PI / 180)
+      const x = -0.5 * Math.sin(phi) * Math.cos(theta)
+      const y = 0.5 * Math.cos(phi)
+      const z = 0.5 * Math.sin(phi) * Math.sin(theta)
+      return new THREE.Vector3(x, y, z)
+    }
+
+    function visualizeHit(request) {
+      console.log(request)
+      const { lat, long, type, page } = request
+      const position = latLongToVector3(lat, long)
+
+      const spikeGeometry = new THREE.ConeGeometry(0.005, 0.2, 8)
+      spikeGeometry.translate(0, 0.05, 0)
+      spikeGeometry.rotateX(Math.PI / 2)
+
+      const color = type === "read" ? 0xffffff : 0x00ff00
+      const spikeMaterial = new THREE.MeshBasicMaterial({ color })
+
+      const spike = new THREE.Mesh(spikeGeometry, spikeMaterial)
+      spike.position.set(position.x, position.y, position.z)
+      spike.lookAt(new THREE.Vector3(0, 0, 0))
+
+      earth.add(spike) // Add spike to earth instead of scene
+      spikes.push(spike)
+    }
+
+    init()
+
+    const logContainer = document.getElementById("log-container")
+    const eventSource = new EventSource("/requests.htm")
+    eventSource.onmessage = function (event) {
+      const data = JSON.parse(event.data)
+      const logEntry = document.createElement("div")
+      logEntry.textContent = data.log
+      logContainer.appendChild(logEntry)
+      logContainer.scrollTop = logContainer.scrollHeight
+
+      const parts = data.log.split(" ")
+      const long = parseFloat(parts.pop())
+      const lat = parseFloat(parts.pop())
+      const type = parts.shift()
+      const page = parts.shift()
+
+      const request = {
+        lat,
+        long,
+        type,
+        page
+      }
+      visualizeHit(request)
+    }
+    eventSource.onerror = function (error) {
+      console.error("EventSource failed:", error)
+      eventSource.close()
+    }
+    // Log when the connection is opened
+    eventSource.onopen = function (event) {
+      console.log(event)
+      console.log("SSE connection opened")
+    }
+  }
+}
+
+window.globe = new Globe()
+globe.main()

globe.scroll

@@ -0,0 +1,8 @@
+buildHtml
+title Globe: ScrollHub Real-time Request Viewer
+
+div#log-container
+
+globe.css
+external/three.min.js
+globe.js