From a020864da74520f6f3e04902d8af269fcd43542c Mon Sep 17 00:00:00 2001 From: Srayman Date: Sat, 29 Jun 2019 23:05:21 -0400 Subject: [PATCH] adding readme and scripts --- README.md | 34 ++++++++++- node_stats.py | 121 ++++++++++++++++++++++++++++++++++++++ record_confirmations.py | 126 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 279 insertions(+), 2 deletions(-) create mode 100644 node_stats.py create mode 100644 record_confirmations.py diff --git a/README.md b/README.md index dc01c72..3b47da2 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,32 @@ -# nano-stats-recorder -Recorder Scripts for Nanocurrency Node Stats +# Nano Stats Recorder +Python scripts to record Nanocurrency node stats - used by https://nano-faucet.org/beta/chart/ +- Confirmation History +- Block Count +- Unchecked Count +- Cemented Count +- Active Confirmation Count +- Confirmation Height Processor Count +- Active Difficulty Multiplier + +# Install +These scripts use the requests library as well as a few others. Install any missing libraries + +`pip install requests` + +# Usage +Confirmation History is pulled from the confirmation_history RPC. The script by default runs every 10 seconds and saves to a file every 60 seconds. The file save amount should be in multiples of the RPC delay. The process will save to a file with the date in the filename (eg. confirmation_history_2019-06-29.json). + +It will read the contents of any file that matches the filename and combine the results for that day. + +After quiting the execution (eg. Ctrl-C) it will save the current run to confirmation_history.json. When the script is started again it will automatically rename confirmation_history.json to include the timestamp at the end to preserve prior attempts. + +`confirmation_history.py` + +Node Stats will collect other statistical measures to help with plotting the node performance. The script by default checks every 15 seconds and saves to a file every 60 seconds. The file save amount should be in multiples of the RPC delay. The process will save to a file with the date in the filename (eg. stats_2019-06-29.json). + +It will read the contents of any file that matches the filename and combine the results for that day. + +After quiting the execution (eg. Ctrl-C) it will save the current run to stats.json. When the script is started again it will automatically rename confirmation_history.json to include the timestamp at the end to preserve prior attempts. + +`node_stats.py` + diff --git a/node_stats.py b/node_stats.py new file mode 100644 index 0000000..8427a56 --- /dev/null +++ b/node_stats.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 + +import asyncio +import argparse +import json +import csv +import os +# importing the requests library +# pip install requests +import requests +import json +import time +import datetime +from collections import defaultdict +from sys import exit +from time import sleep +from datetime import datetime + +parser = argparse.ArgumentParser() +parser.add_argument('-host', '--node_url', type=str, help='Nano node url', default='localhost') +parser.add_argument('-port', '--node_port', type=str, help='Nano node port', default='55000') +parser.add_argument('-save', '--save', type=int, help='Save blocks to disk how often (in seconds) should be multiple of --delay', default=60) +parser.add_argument('-delay', '--delay', type=int, help='recorder delay (in seconds)', default=15) +args = parser.parse_args() + +json_data = [] +timeString = datetime.utcnow().strftime("%Y-%m-%d") +filename = 'stats_'+timeString+'.json' +#Rename existing file +try: + os.rename('stats.json','stats.json.'+datetime.utcnow().strftime("%Y%m%d%H%M%S")) + print('Renaming stats.json ...') +except: + print('stats.json does not exist, create new file ...') + +def writeBkup(): + global json_data + global filename + print('Writing to '+filename+' ...') + try: + with open(filename, 'w') as jsonfile: + jsonfile.write(json.dumps(json_data)) + except: + print('unable to write to file ...') + +async def main(): + global json_data + global filename + global timeString + try: + with open(filename) as jsonfile: + json_data = json.load(jsonfile) + except: + print(filename+' does not exist, create new file ...') + loop_count = 0 + # api-endpoint + URL = "http://"+args.node_url+":"+args.node_port + print("Connecting to: "+URL) + + # defining a params dict for the parameters to be sent to the API + data1 = {'action':'active_difficulty'} + data2 = {'action':'confirmation_active'} + data3 = {'action':'stats','type':'objects'} + data4 = {'action':'block_count','include_cemented':'true'} + + while 1: + filename2 = 'stats_'+datetime.utcnow().strftime("%Y-%m-%d")+'.json' + if filename2 != filename: + writeBkup() + timeString = datetime.utcnow().strftime("%Y-%m-%d") + json_data = [] + filename = filename2 + loop_count += 1 + currentTime = time.time() + # sending get request and saving the response as response object + try: + r = requests.post(url = URL, json = data1) + r2 = requests.post(url = URL, json = data2) + r3 = requests.post(url = URL, json = data3) + r4 = requests.post(url = URL, json = data4) + # extracting data in json format + response = r.json() + response2 = r2.json() + response3 = r3.json() + response4 = r4.json() + except: + print("Error connecting to RPC server. Make sure you have enabled it in ~/Nano/config.json and check " + "./sample_client.py --help") +# print(response2) + try: + data = {} + data['timestamp'] = str(time.time()) + data['confirmation_active'] = str(len(response2['confirmations'])) + data['network_minimum'] = response['network_minimum'] + data['network_current'] = response['network_current'] + data['multiplier'] = response['multiplier'] + data['confirmation_height_count'] = response3['node']['pending_confirmation_height']['pending']['count'] + data['block_count'] = response4['count'] + data['unchecked_count'] = response4['unchecked'] + data['cemented_count'] = response4['cemented'] + json_data.append(data) + except Exception as e: print(e) +# print('\nAn error occurred getting data') + if loop_count%(round(args.save/args.delay)) == 0: + writeBkup() + endTime = time.time() + print('{} records - '.format(len(json_data))+'Time to Process: '+str(endTime-currentTime)+' - Active Confirmations: '+str(len(response2['confirmations']))) + if (args.delay-(endTime - currentTime)) < 0: + sleep(0) + else: + sleep(args.delay-(endTime - currentTime)) + +try: + asyncio.get_event_loop().run_until_complete(main()) +except KeyboardInterrupt: + pass + +print('\nWriting to stats.json .....') +with open('stats.json', 'w') as jsonfile: + jsonfile.write(json.dumps(json_data)) +print('Done') \ No newline at end of file diff --git a/record_confirmations.py b/record_confirmations.py new file mode 100644 index 0000000..4926c54 --- /dev/null +++ b/record_confirmations.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 + +import asyncio +import argparse +import json +import csv +import os +# importing the requests library +# pip install requests +import requests +import json +import time +import datetime +from collections import defaultdict +from sys import exit +from time import sleep +from datetime import datetime + +parser = argparse.ArgumentParser() +parser.add_argument('-host', '--node_url', type=str, help='Nano node url', default='localhost') +parser.add_argument('-port', '--node_port', type=str, help='Nano node port', default='55000') +parser.add_argument('-save', '--save', type=int, help='Save blocks to disk how often (in seconds) should be multiple of --delay', default=60) +parser.add_argument('-delay', '--delay', type=int, help='recorder delay (in seconds)', default=10) +args = parser.parse_args() + +json_data = [] +hashes = [] +distinct = [] +timeString = datetime.utcnow().strftime("%Y-%m-%d") +filename = 'confirmation_history_'+timeString+'.json' +#Rename existing file +try: + os.rename('confirmation_history.json','confirmation_history.json.'+datetime.utcnow().strftime("%Y%m%d%H%M%S")) + print('Renaming confirmation_history.json ...') +except: + print('confirmation_history.json does not exist, create new file ...') + +def writeBkup(): + global json_data + global filename + print('Writing to '+filename+' ...') + try: + with open(filename, 'w') as jsonfile: + jsonfile.write(json.dumps(json_data)) + except: + print('unable to write to file ...') + +async def main(): + global json_data + global hashes + global distinct + global filename + global timeString + try: + with open(filename) as jsonfile: + json_data = json.load(jsonfile) + except: + print(filename+' does not exist, create new file ...') + loop_count = 0 + # api-endpoint + URL = "http://"+args.node_url+":"+args.node_port + print("Connecting to: "+URL) + + # defining a params dict for the parameters to be sent to the API + data = {'action':'confirmation_history'} + + while 1: + filename2 = 'confirmation_history_'+datetime.utcnow().strftime("%Y-%m-%d")+'.json' + if filename2 != filename: + writeBkup() + writeString = timeString+'|'+str(len(json_data))+'\n' + with open('files.txt', 'a') as files: + files.write(writeString) + timeString = datetime.utcnow().strftime("%Y-%m-%d") + json_data = [] + filename = filename2 + loop_count += 1 + currentTime = time.time() + # sending get request and saving the response as response object + try: + r = requests.post(url = URL, json = data) + + # extracting data in json format + response = r.json() + except: + print("Error connecting to RPC server. Make sure you have enabled it in ~/Nano/config.json and check " + "./sample_client.py --help") +# print(response) + try: + json_data += response['confirmations'] + # To sort the list in place... + json_data.sort(key=lambda x: x.get('time', 0), reverse=True) + json_data = list({ each['hash'] : each for each in json_data }.values()) + except: + print('\nAn error occurred getting data') + print('{} blocks confirmed - '.format(len(json_data))+"execution: %s seconds" % (time.time() - currentTime)) + if loop_count%(round(args.save/args.delay)) == 0: + writeBkup() + endTime = time.time() + if (args.delay-(endTime - currentTime)) < 0: + sleep(0) + else: + sleep(args.delay-(endTime - currentTime)) + +try: + asyncio.get_event_loop().run_until_complete(main()) +#except ConnectionRefusedError: +# print("Error connecting to RPC server. Make sure you have enabled it in ~/Nano/config.json and check " +# "./sample_client.py --help") +# exit(1) +except KeyboardInterrupt: + pass + +json_data.sort(key=lambda x: x.get('time', 0), reverse=True) +json_data = list({ each['hash'] : each for each in json_data }.values()) +# reformat data +confirmations = {'hashes':{}} +for item in json_data: + hash = item['hash'] + confirmations['hashes'][hash] = item +#confirmations = json_data +print('{} distinct confirmations'.format(len(confirmations['hashes']))) +print('\nWriting to confirmation_history.json .....') +with open('confirmation_history.json', 'w') as jsonfile: + jsonfile.write(json.dumps(json_data)) +print('Done') \ No newline at end of file