Merge pull request #30 from WIPACrepo/condor_status_update

Condor status to es 8.14

Dockerfile

@@ -15,7 +15,7 @@ RUN dnf -y install epel-release && \
     dnf clean all && yum clean all && \
     ln -s /usr/bin/python3.11 /usr/bin/python && \
     wget https://github.com/WIPACrepo/rest-tools/archive/refs/tags/v1.8.2.tar.gz && \
-    pip3.11 install --no-cache-dir elasticsearch elasticsearch htcondor requests prometheus_client setuptools  ./v1.8.2.tar.gz
+    pip3.11 install --no-cache-dir elasticsearch elasticsearch_dsl htcondor requests prometheus_client setuptools  ./v1.8.2.tar.gz
 
 COPY . /monitoring
 

condor_history_to_es.py

@@ -25,6 +25,7 @@
 parser.add_argument('--client_id',help='oauth2 client id',default=None)
 parser.add_argument('--client_secret',help='oauth2 client secret',default=None)
 parser.add_argument('--token_url',help='oauth2 realm token url',default=None)
+parser.add_argument('--token',help='oauth2 token',default=None)
 parser.add_argument("positionals", nargs='+')
 
 options = parser.parse_args()
@@ -57,8 +58,10 @@ def es_generator(entries):
 
 token = None
 
-if None not in (options.token_url, options.client_secret, options.client_id):
-    api = ClientCredentialsAuth(address='https://elasticsearch.icecube.aq',
+if options.token is not None:
+    token = options.token
+elif None not in (options.token_url, options.client_secret, options.client_id):
+    api = ClientCredentialsAuth(address='https://elastic.icecube.aq',
                                 token_url=options.token_url,
                                 client_secret=options.client_secret,
                                 client_id=options.client_id)
@@ -68,7 +71,8 @@ def es_generator(entries):
 logging.info('connecting to ES at %s',url)
 es = Elasticsearch(hosts=[url],
                    request_timeout=5000,
-                   bearer_auth=token)
+                   bearer_auth=token,
+                   sniff_on_connection_fail=True)
 
 def es_import(document_generator):
     if options.dry_run:
@@ -78,7 +82,7 @@ def es_import(document_generator):
             json.dump(hit, sys.stdout)
         success = True
     else:
-        success, _ = bulk(es, document_generator, max_retries=20, initial_backoff=2, max_backoff=360)
+        success, _ = bulk(es, document_generator, max_retries=20, initial_backoff=10, max_backoff=360)
     return success
 
 failed = False

condor_queue_to_es.py

@@ -81,7 +81,7 @@ def es_generator(entries):
 es = Elasticsearch(hosts=[url],
                    timeout=5000,
                    bearer_auth=token)
-es_import = partial(bulk, es, max_retries=20, initial_backoff=2, max_backoff=3600)
+es_import = partial(bulk, es, max_retries=20, initial_backoff=10, max_backoff=3600)
 
 failed = False
 if options.collectors: