Correcting the cdep and senat parser scripts

python/src/ro/vivi/cdep_crawler/1_get_votes_pages.py

@@ -114,7 +114,7 @@ def process_vote(vote_id, outdir):
 # Go through all the ids starting with 4829 (the first vote on February 4th 2009
 # from this session. Stop when you don't get anything anymore.
 vote_id = 4828
-last_vote_id = 8828 #int(getMostRecentVoteId())
+last_vote_id = int(getMostRecentVoteId())
 print "Max vote id:", last_vote_id
 
 outdir = sys.argv[1]

tools/cdep.sh

@@ -14,6 +14,19 @@ fi
 echo 'System: ' $unamestr
 echo 'Temporary dir: '$TEMP
 
+set +x
+
+read -s -p "Enter Password: " PASS
+
+mysqldump --host=mysql5.s701.sureserver.com --port=3307 -u vivi -p$PASS \
+    --add-drop-table -v hartapoliticii_pistruiatul \
+    people \
+    people_history > $TEMP/hartapoliticii_pistruiatul.sql
+
+set -x
+mysql -u root -proot hartapoliticii_pistruiatul < $TEMP/hartapoliticii_pistruiatul.sql
+
+
 # Get the pages with the votes.
 /usr/bin/python ./python/src/ro/vivi/cdep_crawler/1_get_votes_pages.py $TEMP
 
@@ -54,13 +67,16 @@ mysqldump -u root -proot -v --port=3306 \
     cdep_2008_votes_agg \
     cdep_2008_votes_details \
     people \
-    people_history \
-  > $TEMP/cdep_data.sql
+    people_history > $TEMP/cdep_data.sql
 
 
 set +x
 
-read -s -p "Enter Password: " PASS
+read -p "Do you want to push this to prod now? [y/n] " -n 1
+if [[ ! $REPLY =~ ^[Yy]$ ]]
+then
+    exit 1
+fi
 
 # And push them online.
 mysql -C \

tools/senat.sh

@@ -9,13 +9,13 @@ if [ ! -d $TEMP ]; then
 fi
 
 # Get the pages with the days.
-#/usr/bin/python ./python/src/ro/vivi/sawler/s01_days_get.py $TEMP
+/usr/bin/python ./python/src/ro/vivi/senat_crawler/s01_days_get.py $TEMP
 
 # From the files with the days, get the votes and the law pages.
-/usr/bin/python ./python/src/ro/vivi/sawler/s02_votes_get.py $TEMP
+/usr/bin/python ./python/src/ro/vivi/senat_crawler/s02_votes_get.py $TEMP
 
 # From all these files now generate the aggregate monster file.
-/usr/bin/python ./python/src/ro/vivi/sawler/s03_votes_parse.py \
+/usr/bin/python ./python/src/ro/vivi/senat_crawler/s03_votes_parse.py \
   $TEMP $TEMP/senat_2008_agg.txt
 
 # Transform the big file with all votes into a database, plus aggregate stuff.
@@ -52,6 +52,12 @@ mysqldump -u root -proot -v --port=3306 \
 
 set +x
 
+read -p "Do you want to push this to prod now? [y/n] " -n 1
+if [[ ! $REPLY =~ ^[Yy]$ ]]
+then
+    exit 1
+fi
+
 read -s -p "Enter Password: " PASS
 
 # And push them online.