-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
ba17b02
commit 590f407
Showing
11 changed files
with
657 additions
and
26 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| __pycache__ | ||
| app/__pycache__ | ||
| app/routers/__pycache__ | ||
| schema/__pycache__ | ||
| schema/model.py | ||
| schema/__init__.py | ||
| tests/ | ||
| Dockerfile | ||
| http_ca.crt | ||
| Dockerfile |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| from ipaddress import ip_address, IPv6Address, IPv4Address | ||
| from datetime import datetime, timezone, timedelta | ||
| from elasticsearch import Elasticsearch | ||
| from opensearchpy import OpenSearch | ||
| import json | ||
| from fastapi import HTTPException | ||
| import os | ||
|
|
||
| def map_dbspec(record): | ||
|
|
||
| ip_version = set() | ||
|
|
||
| for ip in record['addresses']: | ||
| if type(ip_address(ip)) is IPv6Address: | ||
| ip_version.add(6) | ||
| elif type(ip_address(ip)) is IPv4Address: | ||
| ip_version.add(4) | ||
|
|
||
| record['ip_versions'] = list(ip_version) | ||
| record['@timestamp'] = datetime.now(timezone.utc) | ||
| record['expires'] = record['@timestamp'] + timedelta(hours=24) | ||
|
|
||
| return record | ||
|
|
||
| def post_to_elastic(record): | ||
| # Create the client instance | ||
| esclient = Elasticsearch( | ||
| os.environ['ELASTIC_HOST'], | ||
| ca_certs=os.environ.get('ELASTIC_CA_CERT'), | ||
| basic_auth=(os.environ['ELASTIC_USER'], os.environ['ELASTIC_PASS']) | ||
| ) | ||
| record_id = record['host']['client_uuid'] | ||
| for ip in record['addresses']: | ||
| record_id += '-' + str(ip) | ||
| try: | ||
| if not esclient.indices.exists(index=os.environ['ELASTIC_INDEX']): | ||
| with open('app/mapping/es_mapping.json') as file: | ||
| es_mapping = json.load(file) | ||
|
|
||
| with open('app/mapping/es_settings.json') as file: | ||
| es_settings = json.load(file) | ||
|
|
||
| settings = { | ||
| "settings": es_settings, | ||
| "mappings": es_mapping | ||
| } | ||
|
|
||
| esclient.indices.create(index=os.environ['ELASTIC_INDEX'], ignore=400, body=settings) | ||
| resp = esclient.index(index=os.environ['ELASTIC_INDEX'], document=record, id=record_id) | ||
|
|
||
| except Exception as e: | ||
| raise HTTPException(status_code=500, detail="Error handling elasticsearch connection - " + str(e)) | ||
|
|
||
|
|
||
| return resp |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| from fastapi import FastAPI | ||
|
|
||
| from slowapi import Limiter, _rate_limit_exceeded_handler | ||
| from slowapi.util import get_ipaddr | ||
| from slowapi.errors import RateLimitExceeded | ||
| from opentelemetry import trace | ||
| from opentelemetry.sdk.trace import TracerProvider | ||
| from opentelemetry.sdk.trace.export import ConsoleSpanExporter | ||
| from opentelemetry.sdk.trace.export import BatchSpanProcessor | ||
| from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor | ||
| from opentelemetry.instrumentation.elasticsearch import ElasticsearchInstrumentor | ||
| from .routers import records | ||
|
|
||
| app = FastAPI() | ||
|
|
||
| limiter = Limiter(key_func=get_ipaddr) | ||
| app.state.limiter = limiter | ||
| app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) | ||
|
|
||
| # Acquire a tracer | ||
| trace.set_tracer_provider(TracerProvider()) | ||
| tracer = trace.get_tracer(__name__) | ||
| trace.get_tracer_provider().add_span_processor( | ||
| BatchSpanProcessor(ConsoleSpanExporter())) | ||
| FastAPIInstrumentor().instrument_app(app) | ||
| ElasticsearchInstrumentor().instrument() | ||
|
|
||
| app.include_router(records.router) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,4 @@ | ||
| { | ||
| "number_of_shards": 1, | ||
| "number_of_replicas": 0 | ||
| } |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| from fastapi import APIRouter, Request, Response, HTTPException | ||
| from ..validate_record import RecordValidation | ||
| from ..database import map_dbspec, post_to_elastic | ||
| try: | ||
| from schema.model import PerfsonarLookupServiceSchema | ||
| except Exception: | ||
| PerfsonarLookupServiceSchema = dict | ||
|
|
||
| router = APIRouter() | ||
| validation = RecordValidation() | ||
|
|
||
| @router.post("/record/") | ||
| def register_record(request: Request, response: Response, registration_record: dict): | ||
|
|
||
| registration_record = validation.validate_record(registration_record) | ||
|
|
||
| if not registration_record['validated']: | ||
| raise HTTPException(status_code=422, | ||
| detail="Registration record Validation failed. {}".format(registration_record['error'].message)) | ||
|
|
||
| registration_record = map_dbspec(registration_record['record']) | ||
| registration_record = map_dbspec(registration_record) | ||
| response = post_to_elastic(registration_record) | ||
|
|
||
| return response |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| from jsonschema import validate | ||
| import json | ||
|
|
||
| class RecordValidation(object): | ||
|
|
||
| def __init__(self): | ||
| with open('schema/schema.json') as file: | ||
| self.schema = json.load(file) | ||
|
|
||
| def validate_record(self, record={}): | ||
| #validate | ||
| try: | ||
| validate(instance=record, schema=self.schema) | ||
| validated = True | ||
| error = None | ||
|
|
||
| except Exception as e: | ||
| validated = False | ||
| error = e | ||
|
|
||
| return {'validated': validated, 'error': error, 'record': record} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| opensearch-py |
Oops, something went wrong.