diff --git a/infra/testApi.mjs b/infra/testApi.mjs index 1a81be1e0..5ec1d78eb 100644 --- a/infra/testApi.mjs +++ b/infra/testApi.mjs @@ -3,9 +3,78 @@ import { strict as assert } from 'node:assert'; // use strict equality everywhe // Future TODO: before this API becomes set in stone/offered publicly, we should change the results of these methods to be just the output data rather than duplicating input values. // Reusable testing data +const SAMPLE_SIZE = 8000 const FPCoreFormula = '(FPCore (x) (- (sqrt (+ x 1)) (sqrt x)))' const FPCoreFormula2 = '(FPCore (x) (- (sqrt (+ x 1))))' const eval_sample = [[[1], -1.4142135623730951]] +const analyzeBody = JSON.stringify({ + formula: FPCoreFormula, sample: [[[ + 14.97651307489794 + ], 0.12711304680349078]] +}) +const alternativesBody = JSON.stringify({ + formula: FPCoreFormula, sample: [[[ + 14.97651307489794 + ], 0.12711304680349078]] +}) +const exactsBody = JSON.stringify({ + formula: FPCoreFormula2, sample: eval_sample +}) +const calculateBody = JSON.stringify({ + formula: FPCoreFormula2, sample: eval_sample +}) +const costBody = JSON.stringify({ + formula: FPCoreFormula2, sample: eval_sample +}) +// -------------------------------------- +// TEST ASYNC APIs +// -------------------------------------- + +// Sample +const sampleStartData = await testAsyncAPI("sample-start", JSON.stringify({ formula: FPCoreFormula2, seed: 5 })) +assert.ok(sampleStartData.points) +assert.equal(sampleStartData.points.length, SAMPLE_SIZE, `sample size should be ${SAMPLE_SIZE}`) + +// Analyze +const analyzeStartData = await testAsyncAPI("analyze-start", analyzeBody) +assertIdAndPath(analyzeStartData) +assert.deepEqual(analyzeStartData.points, [[[14.97651307489794], "2.3"]]) + +// Localerror +const localErrorBody = JSON.stringify({ + formula: FPCoreFormula, sample: sampleStartData.points +}) +const localerrorStartData = await testAsyncAPI("localerror-start", analyzeBody) +assertIdAndPath(localerrorStartData) +assert.equal(localerrorStartData.tree['avg-error'] > 0, true) + +// Alternatives +const alternativesStartData = await testAsyncAPI("alternatives-start", alternativesBody) +assertIdAndPath(alternativesStartData) +assert.equal(Array.isArray(alternativesStartData.alternatives), true) + +// Exacts endpoint +const exactsStartData = await testAsyncAPI("exacts-start", exactsBody) +assertIdAndPath(exactsStartData) +assert.deepEqual(exactsStartData.points, [[[1], -1.4142135623730951]]) + +// Calculate endpoint +const calculateStartData = await testAsyncAPI("calculate-start", calculateBody) +assertIdAndPath(calculateStartData) +assert.deepEqual(calculateStartData.points, [[[1], -1.4142135623730951]]) + +// Cost endpoint +const costStartData = await testAsyncAPI("cost-start", costBody) +assertIdAndPath(costStartData) +assert.equal(costStartData.cost > 0, true) + +const explainStartData = await testAsyncAPI("explanations-start",localErrorBody) +assertIdAndPath(explainStartData) +assert.equal(explainStartData.explanation.length > 0, true, 'explanation should not be empty'); + +// -------------------------------------- +// END ASYNC APIS +// -------------------------------------- // improve endpoint const improveResponse = await callHerbie(`/improve?formula=${encodeURIComponent(FPCoreFormula2)}`, { method: 'GET' }) @@ -53,7 +122,6 @@ assert.notEqual(jid, null) const sample = await sampleRSP.json() assertIdAndPath(sample) -const SAMPLE_SIZE = 8000 assert.ok(sample.points) const points = sample.points assert.equal(points.length, SAMPLE_SIZE, `sample size should be ${SAMPLE_SIZE}`) @@ -68,20 +136,14 @@ assert.deepEqual(points[1], points2[1]) // Analyze endpoint const errors = await callHerbie("/api/analyze", { - method: 'POST', body: JSON.stringify({ - formula: FPCoreFormula, sample: [[[ - 14.97651307489794 - ], 0.12711304680349078]] - }) + method: 'POST', body: analyzeBody }) assertIdAndPath(errors) assert.deepEqual(errors.points, [[[14.97651307489794], "2.3"]]) // Local error endpoint const localError = await callHerbie("/api/localerror", { - method: 'POST', body: JSON.stringify({ - formula: FPCoreFormula, sample: sample2.points - }) + method: 'POST', body: localErrorBody }) assertIdAndPath(localError) assert.equal(localError.tree['avg-error'] > 0, true) @@ -217,5 +279,31 @@ async function callHerbie(endPoint, body) { function assertIdAndPath(json) { assert.equal(json.job.length > 0, true) + // TODO regex for valid hashes? assert.equal(json.path.includes("."), true) +} + +async function testAsyncAPI(endpointName, fetchBodyString) { + const serverURL = `http://127.0.0.1:8000` + const sampleStartURL = new URL(`${serverURL}${`/api/${endpointName}`}`) + + const rsp = await fetch(sampleStartURL, { method: 'POST', body: fetchBodyString }) + assert.equal(rsp.status, 200) + const rspJSON = await rsp.json() + assertIdAndPath(rspJSON) + + const checkStatusURL = new URL(`${serverURL}${"/check-status/"}${rspJSON.job}`) + let counter = 0 + let cap = 100 + let checkStatus = await fetch(checkStatusURL, { method: 'GET' }) + // Loop to wait for for job to finnish + while (checkStatus.status != 201 && counter < cap) { + counter += 1 + checkStatus = await fetch(checkStatusURL, { method: 'GET' }) + await new Promise(r => setTimeout(r, 100)); // ms + } + assert.equal(checkStatus.statusText, 'Job complete') + const resultsURL = new URL(`${serverURL}${"/api/results/"}${rspJSON.job}`) + const results = await fetch(resultsURL, { method: 'GET' }) + return await results.json() } \ No newline at end of file diff --git a/src/api/demo.rkt b/src/api/demo.rkt index 4b39079e3..74b2f3f93 100644 --- a/src/api/demo.rkt +++ b/src/api/demo.rkt @@ -54,16 +54,25 @@ [("check-status" (string-arg)) check-status] [("timeline" (string-arg)) get-timeline] [("up") check-up] + [("api" "results" (string-arg)) get-results] [("api" "sample") #:method "post" sample-endpoint] + [("api" "sample-start") #:method "post" sample-start-endpoint] [("api" "analyze") #:method "post" analyze-endpoint] + [("api" "analyze-start") #:method "post" analyze-start-endpoint] [("api" "localerror") #:method "post" local-error-endpoint] + [("api" "localerror-start") #:method "post" local-error-start-endpoint] [("api" "alternatives") #:method "post" alternatives-endpoint] + [("api" "alternatives-start") #:method "post" alternatives-start-endpoint] [("api" "exacts") #:method "post" exacts-endpoint] + [("api" "exacts-start") #:method "post" exacts-start-endpoint] [("api" "calculate") #:method "post" calculate-endpoint] + [("api" "calculate-start") #:method "post" calculate-start-endpoint] [("api" "cost") #:method "post" cost-endpoint] + [("api" "cost-start") #:method "post" cost-start-endpoint] [("api" "mathjs") #:method "post" ->mathjs-endpoint] [("api" "translate") #:method "post" translate-endpoint] [("api" "explanations") #:method "post" explanations-endpoint] + [("api" "explanations-start") #:method "post" explanations-start-endpoint] [((hash-arg) (string-arg)) generate-page] [("results.json") generate-report])) @@ -360,6 +369,27 @@ (header #"Access-Control-Allow-Origin" (string->bytes/utf-8 "*"))) (λ (out) (write-json (hash-ref job-result 'timeline) out)))])) +(define (get-results req job-id) + (match (get-results-for job-id) + [#f + (response 404 + #"Job Not Found" + (current-seconds) + #"text/plain" + (list (header #"X-Job-Count" (string->bytes/utf-8 (~a (job-count)))) + (header #"X-Herbie-Job-ID" (string->bytes/utf-8 job-id)) + (header #"Access-Control-Allow-Origin" (string->bytes/utf-8 "*"))) + (λ (out) `()))] + [job-result + (response 201 + #"Job complete" + (current-seconds) + #"text/plain" + (list (header #"X-Job-Count" (string->bytes/utf-8 (~a (job-count)))) + (header #"X-Herbie-Job-ID" (string->bytes/utf-8 job-id)) + (header #"Access-Control-Allow-Origin" (string->bytes/utf-8 "*"))) + (λ (out) (write-json job-result out)))])) + ; /api/sample endpoint: test in console on demo page: ;; (await fetch('/api/sample', {method: 'POST', body: JSON.stringify({formula: "(FPCore (x) (- (sqrt (+ x 1))))", seed: 5})})).json() (define sample-endpoint @@ -374,6 +404,20 @@ (define id (start-job command)) (wait-for-job id)))) +; /api/sample endpoint: test in console on demo page: +;; (await fetch('/api/sample', {method: 'POST', body: JSON.stringify({formula: "(FPCore (x) (- (sqrt (+ x 1))))", seed: 5})})).json() +(define sample-start-endpoint + (post-with-json-response + (lambda (post-data) + (define formula-str (hash-ref post-data 'formula)) + (define formula (read-syntax 'web (open-input-string formula-str))) + (define seed* (hash-ref post-data 'seed)) + (define test (parse-test formula)) + (define command + (create-job 'sample test #:seed seed* #:pcontext #f #:profile? #f #:timeline-disabled? #t)) + (define job-id (start-job command)) + (hasheq 'job job-id 'path (make-path job-id))))) + (define explanations-endpoint (post-with-json-response (lambda (post-data) (define formula-str (hash-ref post-data 'formula)) @@ -392,6 +436,24 @@ (define id (start-job command)) (wait-for-job id)))) +(define explanations-start-endpoint + (post-with-json-response (lambda (post-data) + (define formula-str (hash-ref post-data 'formula)) + (define formula (read-syntax 'web (open-input-string formula-str))) + (define sample (hash-ref post-data 'sample)) + (define seed (hash-ref post-data 'seed #f)) + (define test (parse-test formula)) + (define pcontext (json->pcontext sample (test-context test))) + (define command + (create-job 'explanations + test + #:seed seed + #:pcontext pcontext + #:profile? #f + #:timeline-disabled? #t)) + (define job-id (start-job command)) + (hasheq 'job job-id 'path (make-path job-id))))) + (define analyze-endpoint (post-with-json-response (lambda (post-data) (define formula-str (hash-ref post-data 'formula)) @@ -410,6 +472,24 @@ (define id (start-job command)) (wait-for-job id)))) +(define analyze-start-endpoint + (post-with-json-response (lambda (post-data) + (define formula-str (hash-ref post-data 'formula)) + (define formula (read-syntax 'web (open-input-string formula-str))) + (define sample (hash-ref post-data 'sample)) + (define seed (hash-ref post-data 'seed #f)) + (define test (parse-test formula)) + (define pcontext (json->pcontext sample (test-context test))) + (define command + (create-job 'errors + test + #:seed seed + #:pcontext pcontext + #:profile? #f + #:timeline-disabled? #t)) + (define job-id (start-job command)) + (hasheq 'job job-id 'path (make-path job-id))))) + ;; (await fetch('/api/exacts', {method: 'POST', body: JSON.stringify({formula: "(FPCore (x) (- (sqrt (+ x 1))))", points: [[1, 1]]})})).json() (define exacts-endpoint (post-with-json-response (lambda (post-data) @@ -429,6 +509,24 @@ (define id (start-job command)) (wait-for-job id)))) +(define exacts-start-endpoint + (post-with-json-response (lambda (post-data) + (define formula + (read-syntax 'web (open-input-string (hash-ref post-data 'formula)))) + (define sample (hash-ref post-data 'sample)) + (define seed (hash-ref post-data 'seed #f)) + (define test (parse-test formula)) + (define pcontext (json->pcontext sample (test-context test))) + (define command + (create-job 'exacts + test + #:seed seed + #:pcontext pcontext + #:profile? #f + #:timeline-disabled? #t)) + (define job-id (start-job command)) + (hasheq 'job job-id 'path (make-path job-id))))) + (define calculate-endpoint (post-with-json-response (lambda (post-data) (define formula @@ -447,6 +545,24 @@ (define id (start-job command)) (wait-for-job id)))) +(define calculate-start-endpoint + (post-with-json-response (lambda (post-data) + (define formula + (read-syntax 'web (open-input-string (hash-ref post-data 'formula)))) + (define sample (hash-ref post-data 'sample)) + (define seed (hash-ref post-data 'seed #f)) + (define test (parse-test formula)) + (define pcontext (json->pcontext sample (test-context test))) + (define command + (create-job 'evaluate + test + #:seed seed + #:pcontext pcontext + #:profile? #f + #:timeline-disabled? #t)) + (define job-id (start-job command)) + (hasheq 'job job-id 'path (make-path job-id))))) + (define local-error-endpoint (post-with-json-response (lambda (post-data) (define formula @@ -466,6 +582,25 @@ (define id (start-job command)) (wait-for-job id)))) +(define local-error-start-endpoint + (post-with-json-response (lambda (post-data) + (define formula + (read-syntax 'web (open-input-string (hash-ref post-data 'formula)))) + (define sample (hash-ref post-data 'sample)) + (define seed (hash-ref post-data 'seed #f)) + (define test (parse-test formula)) + (define expr (prog->fpcore (test-input test))) + (define pcontext (json->pcontext sample (test-context test))) + (define command + (create-job 'local-error + test + #:seed seed + #:pcontext pcontext + #:profile? #f + #:timeline-disabled? #t)) + (define job-id (start-job command)) + (hasheq 'job job-id 'path (make-path job-id))))) + (define alternatives-endpoint (post-with-json-response (lambda (post-data) (define formula @@ -484,6 +619,24 @@ (define id (start-job command)) (wait-for-job id)))) +(define alternatives-start-endpoint + (post-with-json-response (lambda (post-data) + (define formula + (read-syntax 'web (open-input-string (hash-ref post-data 'formula)))) + (define sample (hash-ref post-data 'sample)) + (define seed (hash-ref post-data 'seed #f)) + (define test (parse-test formula)) + (define pcontext (json->pcontext sample (test-context test))) + (define command + (create-job 'alternatives + test + #:seed seed + #:pcontext pcontext + #:profile? #f + #:timeline-disabled? #t)) + (define job-id (start-job command)) + (hasheq 'job job-id 'path (make-path job-id))))) + (define ->mathjs-endpoint (post-with-json-response (lambda (post-data) (define formula @@ -503,6 +656,16 @@ (define id (start-job command)) (wait-for-job id)))) +(define cost-start-endpoint + (post-with-json-response + (lambda (post-data) + (define formula (read-syntax 'web (open-input-string (hash-ref post-data 'formula)))) + (define test (parse-test formula)) + (define command + (create-job 'cost test #:seed #f #:pcontext #f #:profile? #f #:timeline-disabled? #f)) + (define job-id (start-job command)) + (hasheq 'job job-id 'path (make-path job-id))))) + (define translate-endpoint (post-with-json-response (lambda (post-data) ; FPCore formula and target language