Skip to content

Commit

Permalink
more test unification
Browse files Browse the repository at this point in the history
  • Loading branch information
@khrome
khrome committed Jan 3, 2022
1 parent f74eec6 commit c770dd8
Show file tree
Hide file tree
Showing 3 changed files with 4 additions and 128 deletions.
2 changes: 1 addition & 1 deletion package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "@open-automaton/automaton-engine",
"homepage": "https://github.com/open-automaton/automaton-engine",
"version": "0.0.4",
"version": "0.0.5",
"main": "src/automaton-engine.js",
"description": "A base class for scraping engines",
"keywords": [],
Expand Down
128 changes: 2 additions & 126 deletions test/test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,135 +4,11 @@ const Automaton = require('@open-automaton/automaton');
const fs = require('fs');
const express = require('express');
const bodyParser = require('body-parser');
const AutomatonPuppeteerEngine = require('../src/puppeteer.js');

describe('strip-mine', function(){
describe('automaton', function(){
it('loads a canonical definition', function(done){
fs.readFile(
path.join(
__dirname,
'./data/bots/whitepages-peoplesearch.xml'
), function(err, body){
should.not.exist(err);
let scraper = new Automaton(body.toString());
scraper.root.children.length.should.equal(5);
done();
}
);
});

it('scrapes a static page', function(done){
console.log('B-1')
let app = express();
app.get('/', (req, res)=>{
res.setHeader('content-type', 'text/html')
res.send(fs.readFileSync(path.join(
__dirname, './data/table-scrape.html'
)));
});
let server = app.listen(8080, (err)=>{
should.not.exist(err);
fs.readFile(path.join(__dirname, './data/table-scrape-bot.xml'), function(err, body){
should.not.exist(err);
let scraper = new Automaton(
body.toString(),
new AutomatonPuppeteerEngine()
);
scraper.run((err, data)=>{
console.log('B-2')
should.not.exist(err);
should.exist(data);
should.exist(data.matches);
data.matches.length.should.equal(3);

should.exist(data.matches[0]);
should.exist(data.matches[0].name);
data.matches[0].name.should.equal('1');
should.exist(data.matches[0].age);
data.matches[0].age.should.equal('something');
should.exist(data.matches[0].location);
data.matches[0].location.should.equal('23.4');

should.exist(data.matches[1]);
should.exist(data.matches[1].name);
data.matches[1].name.should.equal('7');
should.exist(data.matches[1].age);
data.matches[1].age.should.equal('another thing');
should.exist(data.matches[1].location);
data.matches[1].location.should.equal('23.45');

should.exist(data.matches[2]);
should.exist(data.matches[2].name);
data.matches[2].name.should.equal('9');
should.exist(data.matches[2].age);
data.matches[2].age.should.equal('nothing');
should.exist(data.matches[2].location);
data.matches[2].location.should.equal('7.38');

server.close(()=>{
done();
});
});
});
});
});

it('scrapes a form', function(done){
console.log('A-1')
let app = express();
app.use(bodyParser.urlencoded({extended:false}));
app.get('/', (req, res)=>{
res.setHeader('content-type', 'text/html')
res.send(fs.readFileSync(path.join(
__dirname, './data/form-search.html'
)));
});
app.post('/submit', (req, res)=>{
res.setHeader('content-type', 'text/html')
res.send(fs.readFileSync(path.join(
__dirname, './data/form-search-response.html'
)).toString()
.replace( /\[\[name\]\]/g, req.body.name)
.replace( /\[\[title\]\]/g, req.body.title)
);
});
let server = app.listen(8080, (err)=>{
should.not.exist(err);
fs.readFile(
path.join(__dirname, './data/form-search-bot.xml'),
(err, body)=>{
should.not.exist(err);
let scraper = new Automaton({
body: body.toString(),
environment : {
incomingName : 'foo',
incomingTitle: 'bar'
}
}, new AutomatonPuppeteerEngine());
scraper.run((err, data)=>{
console.log('A-2')
data.matches.length.should.equal(2);

should.exist(data.matches[0]);
should.exist(data.matches[0].name);
data.matches[0].name.should.equal('foo-augmented');
should.exist(data.matches[0].title);
data.matches[0].title.should.equal('bar-augmented');

should.exist(data.matches[1]);
should.exist(data.matches[1].name);
data.matches[1].name.should.equal('foo-segmented');
should.exist(data.matches[1].title);
data.matches[1].title.should.equal('bar-segmented');

server.close(()=>{
done();
});
});
}
);
});
it.skip('generic test', function(done){

});
});
});

0 comments on commit c770dd8

Please sign in to comment.