Skip to content

Commit

Permalink
try to reduce batch size de 50
Browse files Browse the repository at this point in the history
  • Loading branch information
bailletced committed Jan 21, 2025
1 parent 8ea56bb commit d46e280
Showing 1 changed file with 4 additions and 3 deletions.
7 changes: 4 additions & 3 deletions scripts/synchro.py
Original file line number Diff line number Diff line change
Expand Up @@ -416,17 +416,18 @@ def populate_fields(self, values, wikidata_id):
return fields

class Processor(object):
batch_size = 100
client = OpenChurchClient()
redis_url = os.getenv('REDIS_URL')
redis_client = redis.from_url(redis_url)

def __init__(self, verbosity_level, type):
def __init__(self, verbosity_level, type, batch_size):
self.q = Query(verbosity_level=verbosity_level)
self.verbosity_level = verbosity_level
self.type = type
self.batch_size = batch_size

def process_batch(self, data, method, run_id):
print(self.batch_size)
batches = Query.split_into_batches(data, self.batch_size)
self.redis_client.hset(self.type, "batchCount", len(batches))
iteration = 1
Expand Down Expand Up @@ -508,5 +509,5 @@ def get_redis_key(type, origin, to):
parser.add_argument("-v", "--verbose", action="count", default=0, help="Augmente le niveau de verbosité (utilisez -vvv pour plus de détails).")
args = parser.parse_args()

processor = Processor(verbosity_level=args.verbose, type=args.entity_only)
processor = Processor(verbosity_level=args.verbose, type=args.entity_only, batch_size=50)
processor.process_entity()

0 comments on commit d46e280

Please sign in to comment.