diff --git a/README.md b/README.md index e4b6dfe..2e7d014 100644 --- a/README.md +++ b/README.md @@ -123,4 +123,4 @@ from the same record on the same website. * [MyHeritage](https://www.myheritage.com) historical records and tree profiles * [Open Archives](https://www.openarch.nl) * [WeRelate](http://www.werelate.org/) -* [WikiTree](http://www.wikitree.com/) person profiles +* [WikiTree](https://www.wikitree.com/) person profiles diff --git a/src/scrapers/wikitree.js b/src/scrapers/wikitree.js index c5f6172..12231ff 100644 --- a/src/scrapers/wikitree.js +++ b/src/scrapers/wikitree.js @@ -4,7 +4,7 @@ var debug = require('debug')('genscrape:scrapers:wikitree'), schema = require('../schema'); var urls = [ - utils.urlPatternToRegex('http://www.wikitree.com/wiki/*-*') + utils.urlPatternToRegex('https://www.wikitree.com/wiki/*-*') ]; module.exports = function(register){ diff --git a/test/data/wikitree/output/Alice Roosevelt.json b/test/data/wikitree/output/Alice Roosevelt.json index 9ddf106..399e204 100644 --- a/test/data/wikitree/output/Alice Roosevelt.json +++ b/test/data/wikitree/output/Alice Roosevelt.json @@ -499,10 +499,10 @@ "id": "1", "citations": [ { - "value": "Alice Hathaway (Lee) Roosevelt | WikiTree: The FREE Family Tree (http://www.wikitree.com/wiki/Lee-16 : accessed 17 April 2013)" + "value": "Alice Hathaway (Lee) Roosevelt | WikiTree: The FREE Family Tree (https://www.wikitree.com/wiki/Lee-16 : accessed 17 April 2013)" } ], - "about": "http://www.wikitree.com/wiki/Lee-16", + "about": "https://www.wikitree.com/wiki/Lee-16", "titles": [ { "value": "Alice Hathaway (Lee) Roosevelt | WikiTree: The FREE Family Tree" diff --git a/test/data/wikitree/output/Edward M Salty Smith.json b/test/data/wikitree/output/Edward M Salty Smith.json index 4d42ffa..6c812d2 100644 --- a/test/data/wikitree/output/Edward M Salty Smith.json +++ b/test/data/wikitree/output/Edward M Salty Smith.json @@ -325,10 +325,10 @@ "id": "1", "citations": [ { - "value": "Edward Smith | Family Tree Unknown (http://www.wikitree.com/wiki/Smith-9130 : accessed 17 April 2013)" + "value": "Edward Smith | Family Tree Unknown (https://www.wikitree.com/wiki/Smith-9130 : accessed 17 April 2013)" } ], - "about": "http://www.wikitree.com/wiki/Smith-9130", + "about": "https://www.wikitree.com/wiki/Smith-9130", "titles": [ { "value": "Edward Smith | Family Tree Unknown" diff --git a/test/data/wikitree/output/Edward Smith.json b/test/data/wikitree/output/Edward Smith.json index e6bc978..121ae9a 100644 --- a/test/data/wikitree/output/Edward Smith.json +++ b/test/data/wikitree/output/Edward Smith.json @@ -322,10 +322,10 @@ "id": "1", "citations": [ { - "value": "Edward Smith | Family Tree Unknown (http://www.wikitree.com/wiki/Smith-19124 : accessed 17 April 2013)" + "value": "Edward Smith | Family Tree Unknown (https://www.wikitree.com/wiki/Smith-19124 : accessed 17 April 2013)" } ], - "about": "http://www.wikitree.com/wiki/Smith-19124", + "about": "https://www.wikitree.com/wiki/Smith-19124", "titles": [ { "value": "Edward Smith | Family Tree Unknown" diff --git a/test/data/wikitree/output/Theodore Roosevelt Jr.json b/test/data/wikitree/output/Theodore Roosevelt Jr.json index 2a963ac..874be5c 100644 --- a/test/data/wikitree/output/Theodore Roosevelt Jr.json +++ b/test/data/wikitree/output/Theodore Roosevelt Jr.json @@ -719,10 +719,10 @@ "id": "1", "citations": [ { - "value": "Theodore Roosevelt Jr. | WikiTree: The FREE Family Tree (http://www.wikitree.com/wiki/Roosevelt-18 : accessed 17 April 2013)" + "value": "Theodore Roosevelt Jr. | WikiTree: The FREE Family Tree (https://www.wikitree.com/wiki/Roosevelt-18 : accessed 17 April 2013)" } ], - "about": "http://www.wikitree.com/wiki/Roosevelt-18", + "about": "https://www.wikitree.com/wiki/Roosevelt-18", "titles": [ { "value": "Theodore Roosevelt Jr. | WikiTree: The FREE Family Tree" diff --git a/test/data/wikitree/pages/Alice Roosevelt.html b/test/data/wikitree/pages/Alice Roosevelt.html index e8562c0..df995be 100644 --- a/test/data/wikitree/pages/Alice Roosevelt.html +++ b/test/data/wikitree/pages/Alice Roosevelt.html @@ -4,7 +4,7 @@ - + Alice Hathaway (Lee) Roosevelt | WikiTree: The FREE Family Tree @@ -38,7 +38,7 @@ var wgArticlePath = "/wiki/$1"; var wgScriptPath = ""; var wgScript = "/index.php"; -var wgServer = "http://www.wikitree.com"; +var wgServer = "https://www.wikitree.com"; var wgCanonicalNamespace = ""; var wgCanonicalSpecialPageName = false; var wgNamespaceNumber = 0; @@ -160,7 +160,7 @@

Alice Hathaway (Lee) Roosevelt (1861 - 1884)
This page has been accessed 14,001 times.
-
+

Categories: Wives of US Presidents | Spouses of US Presidents.


First wife of Theodore Roosevelt and mother for two days to Alice Lee Roosevelt. @@ -185,7 +185,7 @@

Alice Hathaway (Lee) Roosevelt (1861 - 1884)
- +

More: Family Tree & Genealogy Tools



@@ -196,7 +196,7 @@

Alice Hathaway (Lee) Roosevelt (1861 - 1884)


- +
@@ -225,7 +225,7 @@

Alice Hathaway (Lee) Roosevelt (1861 - 1884) carriers of Alice's mitochondrial DNA have taken an mtDNA test and no close relatives have taken a 23andMe, AncestryDNA, or Family Tree DNA "Family Finder" test.

Have you taken a DNA test for genealogy? If so, login to add it. If not, see our friends at Family Tree DNA.



Images: 2 - +
@@ -242,7 +242,7 @@

Alice Hathaway (Lee) Roosevelt (1861 - 1884)
- + Collaboration
    @@ -291,9 +291,9 @@

    Alice Hathaway (Lee) Roosevelt (1861 - 1884) © 2008 - 2016 INTERESTING.COM, INC. CONTENT MAY BE COPYRIGHTED BY WIKITREE COMMUNITY MEMBERS.

diff --git a/test/data/wikitree/pages/Edward M Salty Smith.html b/test/data/wikitree/pages/Edward M Salty Smith.html index 876aebd..73f2974 100644 --- a/test/data/wikitree/pages/Edward M Salty Smith.html +++ b/test/data/wikitree/pages/Edward M Salty Smith.html @@ -4,7 +4,7 @@ - + Edward Smith | Family Tree Unknown @@ -38,7 +38,7 @@ var wgArticlePath = "/wiki/$1"; var wgScriptPath = ""; var wgScript = "/index.php"; -var wgServer = "http://www.wikitree.com"; +var wgServer = "https://www.wikitree.com"; var wgCanonicalNamespace = ""; var wgCanonicalSpecialPageName = false; var wgNamespaceNumber = 0; @@ -88,7 +88,7 @@
- no image + no image
@@ -138,7 +138,7 @@

Edward Smith (abt. 1847)

[mother unknown]
[sibling(s) unknown]
- +
Husband of — married @@ -156,7 +156,7 @@

Edward Smith (abt. 1847)

This page has been accessed 121 times.
-
+

Contents

    @@ -179,22 +179,22 @@

    Edward Smith (abt. 1847)

    Name: Edward /Smith/[1]

    Birth

    -
    Birth: +
    Birth:
    Date: abt 1847
    Place: Calstone, Wiltshire, England[2]

    Residence

    -
    Residence: +
    Residence:
    Date: 1891
    Place: Calne Without, Wiltshire, England[3]

    Sources

    -
    • WikiTree profile Smith-19124 created through the import of Freegard Family Tree.ged on Aug 20, 2011 by Maxi Freegard. See the Changes page for the details of edits by Maxi and others. +
      • WikiTree profile Smith-19124 created through the import of Freegard Family Tree.ged on Aug 20, 2011 by Maxi Freegard. See the Changes page for the details of edits by Maxi and others.
      -
      • Source: S-2082312640 Repository: #R-2141871862 Title: 1891 England Census Author: Ancestry.com Publication: Online publication - Provo, UT, USA: Ancestry.com Operations Inc, 2005.Original data - Census Returns of England and Wales, 1891. Kew, Surrey, England: The National Archives of the UK (TNA): Public Record Office (PRO), 1891. Data imaged from The National A Note: APID: 1,6598::0 -
      • Repository: R-2141871862 Name: Ancestry.co.uk Address: http://www.Ancestry.co.uk Note: +
        • Source: S-2082312640 Repository: #R-2141871862 Title: 1891 England Census Author: Ancestry.com Publication: Online publication - Provo, UT, USA: Ancestry.com Operations Inc, 2005.Original data - Census Returns of England and Wales, 1891. Kew, Surrey, England: The National Archives of the UK (TNA): Public Record Office (PRO), 1891. Data imaged from The National A Note: APID: 1,6598::0 +
        • Repository: R-2141871862 Name: Ancestry.co.uk Address: http://www.Ancestry.co.uk Note:
        1. Source: #S-2082312640 Page: Class: RG12; Piece: 1597; Folio 26; Page 4; GSU roll: 6096707. Note: http://search.ancestry.co.uk/cgi-bin/sse.dll?db=uki1891&h=23683651&ti=5538&indiv=try&gss=pt Note: Data: Text: Birth date: abt 1847Birth place: Calstone, Wiltshire, EnglandResidence date: 1891Residence place: Calne Without, Wiltshire, England APID: 1,6598::23683651
        2. Source: #S-2082312640 Page: Class: RG12; Piece: 1597; Folio 26; Page 4; GSU roll: 6096707. Note: http://search.ancestry.co.uk/cgi-bin/sse.dll?db=uki1891&h=23683651&ti=5538&indiv=try&gss=pt Note: Data: Text: Birth date: abt 1847Birth place: Calstone, Wiltshire, EnglandResidence date: 1891Residence place: Calne Without, Wiltshire, England APID: 1,6598::23683651
        3. @@ -203,7 +203,7 @@

          Edward Smith (abt. 1847)



          - +

          More: Family Tree & Genealogy Tools



          @@ -214,7 +214,7 @@

          Edward Smith (abt. 1847)





          - +
          @@ -285,9 +285,9 @@

          Edward Smith (abt. 1847)

          © 2008 - 2016 INTERESTING.COM, INC. CONTENT MAY BE COPYRIGHTED BY WIKITREE COMMUNITY MEMBERS.

          diff --git a/test/data/wikitree/pages/Edward Smith.html b/test/data/wikitree/pages/Edward Smith.html index 8503c67..f4dc59a 100644 --- a/test/data/wikitree/pages/Edward Smith.html +++ b/test/data/wikitree/pages/Edward Smith.html @@ -3,7 +3,7 @@ - + Edward Smith | Family Tree Unknown @@ -37,7 +37,7 @@ var wgArticlePath = "/wiki/$1"; var wgScriptPath = ""; var wgScript = "/index.php"; - var wgServer = "http://www.wikitree.com"; + var wgServer = "https://www.wikitree.com"; var wgCanonicalNamespace = ""; var wgCanonicalSpecialPageName = false; var wgNamespaceNumber = 0; @@ -88,7 +88,7 @@
          - no image + no image
          @@ -164,7 +164,7 @@

          Edward Smith (abt. 1847)

          This page has been accessed 96 times.
          -
          +

          Contents

            @@ -199,7 +199,7 @@

            Edward Smith (abt. 1847)

            Sources

            -
            • WikiTree profile Smith-19124 created through the import of Freegard Family Tree.ged on Aug 20, 2011 by Maxi Freegard. See the Changes page for the details of edits by Maxi and others. +
              • WikiTree profile Smith-19124 created through the import of Freegard Family Tree.ged on Aug 20, 2011 by Maxi Freegard. See the Changes page for the details of edits by Maxi and others.
              • Source: S-2082312640 Repository: #R-2141871862 Title: 1891 England Census Author: Ancestry.com Publication: Online publication - Provo, UT, USA: Ancestry.com Operations Inc, 2005.Original data - Census Returns of England and Wales, 1891. Kew, Surrey, England: The National Archives of the UK (TNA): Public Record Office (PRO), 1891. Data imaged from The National A Note: APID: 1,6598::0
              • Repository: R-2141871862 Name: Ancestry.co.uk Address: http://www.Ancestry.co.uk Note: diff --git a/test/data/wikitree/pages/Theodore Roosevelt Jr.html b/test/data/wikitree/pages/Theodore Roosevelt Jr.html index e6ce971..74ddecf 100644 --- a/test/data/wikitree/pages/Theodore Roosevelt Jr.html +++ b/test/data/wikitree/pages/Theodore Roosevelt Jr.html @@ -4,7 +4,7 @@ - + Theodore Roosevelt Jr. | WikiTree: The FREE Family Tree @@ -38,7 +38,7 @@ var wgArticlePath = "/wiki/$1"; var wgScriptPath = ""; var wgScript = "/index.php"; -var wgServer = "http://www.wikitree.com"; +var wgServer = "https://www.wikitree.com"; var wgCanonicalNamespace = ""; var wgCanonicalSpecialPageName = false; var wgNamespaceNumber = 0; @@ -67,9 +67,9 @@ @@ -175,7 +175,7 @@

                Theodore Roosevelt Jr. (1858 - 1919)

                This page has been accessed 15,428 times.
                -
                +

                Categories: US Presidents | US Vice Presidents | New York Governors | New York Political History | Nobel Laureates of the 20th Century | Nominated Profiles.

                @@ -215,7 +215,7 @@

                Theodore Roosevelt Jr. (1858 - 1919)


                -
                +

                Biography

                Theodore (Teddy) Roosevelt was characterized as being a depressed, neurotic, imperalistic and opportunistic individual. He served as Governor of NY, was married twice, became Vice President under McKinley and was the youngest President as he was 42 years old when he took the office. @@ -224,7 +224,7 @@

                Theodore Roosevelt Jr. (1858 - 1919)

                September 14, 1901, is when Theodore Roosevelt became president. The way he became president was because president McKinley was assassinated. President Roosevelt wasn’t only a vice president or a president, he was also a deputy sheriff in Dakota Territory. He was also police commissioner of New York City, Assistant Secretary of the Navy, and Colonel of the Rough Riders. All this happened by the age of 42.

                President Theodore Roosevelt’s interventionist foreign policy subverted the Constitution and helped transform America into the most powerful nation on Earth.

                Describing his foreign policy, Roosevelt quoted a West African proverb: "Speak softly and carry a big stick." As president, Roosevelt used "big stick diplomacy" to seize unprecedented executive power in handling international affairs. -

                He was the first world leader to make an argument to the Court of Arbitration at The Hague. He also was first to call for convening of what became the Second Hague Peace Conference at which he got the Latin American Nations to equal the rest of the world. +

                He was the first world leader to make an argument to the Court of Arbitration at The Hague. He also was first to call for convening of what became the Second Hague Peace Conference at which he got the Latin American Nations to equal the rest of the world.

                It was January 6, 1919, when Theodore passed away from coronary embolism. He was buried in Young's Memorial Cemetery at Oyster Bay, New York.

                Theodore Roosevelt had a tremendous amount of education. He had attended Harvard, where he had developed an interest in naval affairs. Not only was Roosevelt intelligent, but he was also quiet athletic boxing in college. He stayed at Harvard for a year, left in 1881 to run for a seat in the state legislature.

                Theodore Roosevelt later became the 26th president of the United States. He was part of the leadership of Progressive Movements, as well as a leader of the Republican Party. He was a naturalist, explorer, hunter, author, as well as a soldier. Roosevelt had accomplished many things in his career, and he was recognized as one of the greatest presidents ever. @@ -234,20 +234,20 @@

                Theodore Roosevelt Jr. (1858 - 1919)

                He was a sickly child, but even tho he was weak he lived a very active life which included boxing.

                He published his first historical book in 1882 called The Naval War of 1812.

                Roosevelt married twice. He married Alice Hathaway Lee on his 22nd birthday October 26, 1880. Then on Febuary 14th, 1884 Alice passed away from blights disease and on the same day Roosevelt's mother passed away from typhoid fever. Then in 1886 Roosevelt married Edith Kermit Carow, who he had 5 children with. -

                Teddy was a New York state legislator. He was appointed to the United States Civil Service Commission by Benjamin Harrison in 1888 and served until 1895. He worked as New York City Police Commissioner, for 2 years. He became Assistant Secretary of the Navy in 1898. Later in 1898 he became governor of New York. He became Vice President of the U.S. and after 6 months when president William Mckinley was assassinated he became the 26th president of the United States until 1909. -

                When Roosevelt was president he tried to make the Republican Party more progressive, which included trust busting, and increasing the regulation of businesses. +

                Teddy was a New York state legislator. He was appointed to the United States Civil Service Commission by Benjamin Harrison in 1888 and served until 1895. He worked as New York City Police Commissioner, for 2 years. He became Assistant Secretary of the Navy in 1898. Later in 1898 he became governor of New York. He became Vice President of the U.S. and after 6 months when president William Mckinley was assassinated he became the 26th president of the United States until 1909. +

                When Roosevelt was president he tried to make the Republican Party more progressive, which included trust busting, and increasing the regulation of businesses.

                Roosevelt helped create the Panama Canal and negotiated an end to the Russo-Japanese war, which earned him the Nobel Peace Prize.

                -
                +

                In 1912 Roosevelt when on a expedition to South America, where he got malaria. The malaria damaged his health.

                It was January 6, 1919 when Teddy passed away from a heart attack. He was buried in New York, at Youngs Memorial Cemetery.

                He was a successful president, naturalist, historian, lawyer and explorer.

                -
                +

                Sources

                • Gary Boyd Roberts, Notable Kin, Volume One Publisher Carl Boyer, 3rd; Location: Santa Clarita, California; Date: 1998; Published in cooperation with the New England Historic genealogical Society, Boston, Massachusetts
                -
                • Wikipedia - Theodore "T.R." Roosevelt, Jr. + @@ -264,7 +264,7 @@

                  Theodore Roosevelt Jr. (1858 - 1919)


                  Acknowledgements

                  -
                  @@ -337,7 +337,7 @@

                  Theodore Roosevelt Jr. (1858 - 1919)

                  No known carriers of Theodore's Y-chromosome or his mother's mitochondrial DNA have taken yDNA or mtDNA tests and no close relatives have taken a 23andMe, AncestryDNA, or Family Tree DNA "Family Finder" test.

                  Have you taken a DNA test for genealogy? If so, login to add it. If not, see our friends at Family Tree DNA.



                  Images: 12 - +
                  @@ -377,7 +377,7 @@

                  Theodore Roosevelt Jr. (1858 - 1919)



                  - + Collaboration
                    @@ -453,12 +453,12 @@

                    Theodore Roosevelt Jr. (1858 - 1919)

                    Quotes by Teddy Roosevelt -

                    "Believe you can and you're halfway there." What he is trying to say is that if u believe in yourself then you can do anything basically. +

                    "Believe you can and you're halfway there." What he is trying to say is that if u believe in yourself then you can do anything basically.

                    - +

                  @@ -484,9 +484,9 @@

                  Theodore Roosevelt Jr. (1858 - 1919)

                  © 2008 - 2016 INTERESTING.COM, INC. CONTENT MAY BE COPYRIGHTED BY WIKITREE COMMUNITY MEMBERS.

                diff --git a/test/src/scrapers/wikitree.js b/test/src/scrapers/wikitree.js index 95de7a5..79931a2 100644 --- a/test/src/scrapers/wikitree.js +++ b/test/src/scrapers/wikitree.js @@ -4,22 +4,22 @@ describe('wikitree', function(){ it('basic male', setupTest( 'Theodore Roosevelt Jr', - 'http://www.wikitree.com/wiki/Roosevelt-18' + 'https://www.wikitree.com/wiki/Roosevelt-18' )); it('basic female', setupTest( 'Alice Roosevelt', - 'http://www.wikitree.com/wiki/Lee-16' + 'https://www.wikitree.com/wiki/Lee-16' )); - + it('basic nickname', setupTest( 'Edward M Salty Smith', - 'http://www.wikitree.com/wiki/Smith-9130' + 'https://www.wikitree.com/wiki/Smith-9130' )); - + it('Much missing information and year of birth', setupTest( 'Edward Smith', - 'http://www.wikitree.com/wiki/Smith-19124' + 'https://www.wikitree.com/wiki/Smith-19124' )); });