diff --git a/charts/pagila/scripts/generate_efg_data.py b/charts/pagila/scripts/generate_efg_data.py new file mode 100644 index 0000000..3bc9ff3 --- /dev/null +++ b/charts/pagila/scripts/generate_efg_data.py @@ -0,0 +1,146 @@ +import pandas as pd +import random +import string + +# Function to generate random strings +def random_string(length=10): + return ''.join(random.choices(string.ascii_uppercase, k=length)) + +# Function to generate XML row +def generate_xml_row(row_id, c1, c2, c3, c4, c5): + xml_template = f""" + + {c1} + {c2} + {c3} + {c4} + {c5} + CH + CHF + S + 101 + CHF + 0 + 12.02 + 20241114 + 1 + 84 + CH0022268228 + CRIMS13871745 + 0022268220 + QUOTED + NO + NO + NO + N + NO + NO + NO + NO + NO + NO + NO + 20050926 + 84 + CH + CH + 20050926 + 3753677191 + 1 + 9000 + SHA.GUI + EFGN + BSI.166027 + CH0010002 + 20150131 + EFGN SW Equity + EQT1 + 74 + NA + N + 30911 + N + 20100216 + EFGIS + BLOCKED/UNBLOCKED CLASSEUR FMI - BSE 07.02.12 + 00 + CHF + TELEKURS + Y + 1M + 20241115 + 11.86 + 20241113 + CH0010001.GLC + ID-PRC + CHKLG + MASTER + ESVTFR + 4.5757 + 8.9 + 9 + 1 + 20240326 + 20240328 + IDC + 4.5757 + 27.57 + 22.78 + 78269 + BS1P + CH1P + CYB + GB1P + HK1P + LI1P + LU1P + MC1P + SG1P + US1P + OUT + 20140701 + EQUITIES-EQ_CH + 222682-200 + 0.25 + 222682-200 + 171922 + EFGN + NO + YES + 66.12 + 2 + MAPPED_OFS-EXPL + 43_CONV.SECURITY.MASTER.G15.2.00 + 45_CONV.SECURITY.MASTER.R07 + 45_CONV.SECURITY.MASTER.R08 + 45_CONV.SECURITY.MASTER.R09 + 45_CONV.SECURITY.MASTER.201804 + 2411150824 + MAPPED_SHDN.SM.NACE + CH0010001 + 8100 + + """ + return xml_template + +# Generate 10,000 rows +data = [] +for i in range(100000): + row_id = f"222682-{200 + i}" + c1 = f"Company_{i} {random_string(5)}" + c2 = f"Alias_{i} {random_string(4)}" + c3 = f"Description_{i}" + c4 = f"SYM_{random_string(3)}" + c5 = f"Country_{random.choice(['US', 'CH', 'DE', 'FR', 'UK'])}" + xml_row = generate_xml_row(row_id, c1, c2, c3, c4, c5) + data.append((row_id, xml_row)) + +# Convert to DataFrame +df = pd.DataFrame(data, columns=['key', 'value']) + +# Generate SQL INSERT statements +with open("insert_security_master.sql", "w") as f: + for index, row in df.iterrows(): + f.write(f"INSERT INTO trading_schema.SECURITY_MASTER (key, value) VALUES (\'{row['key']}\', '{row['value']}');\n") + +print("SQL File Generated: insert_security_master.sql") diff --git a/charts/pagila/templates/configmap.yaml b/charts/pagila/templates/configmap.yaml index 756a483..23e5be6 100644 --- a/charts/pagila/templates/configmap.yaml +++ b/charts/pagila/templates/configmap.yaml @@ -1,12 +1,13 @@ +{{- range .Values.databases }} apiVersion: v1 kind: ConfigMap metadata: - name: {{ .Release.Name }}-custom-scripts + name: {{ .name }}-custom-scripts labels: - app: {{ .Chart.Name }} + app: {{ $.Chart.Name }} data: -{{- if .Values.configMap.customScripts }} -{{- range $filename, $content := .Values.configMap.customScripts }} +{{- if and .configMap .configMap.customScripts }} +{{- range $filename, $content := .configMap.customScripts }} {{ $filename }}: | {{ $content | indent 4 }} {{- end }} @@ -14,11 +15,5 @@ data: dummy.sql: | -- No custom scripts provided {{- end }} - - {{- if .Values.configMap.env }} - {{- range $key, $value := .Values.configMap.env }} - {{ $key }}: "{{ $value }}" - {{- end }} - {{- else }} - debug-env: "No environment variables defined" - {{- end }} +--- +{{- end }} diff --git a/charts/pagila/templates/secret.yaml b/charts/pagila/templates/secret.yaml index b174cef..a7c01a7 100644 --- a/charts/pagila/templates/secret.yaml +++ b/charts/pagila/templates/secret.yaml @@ -1,10 +1,13 @@ +{{- range .Values.databases }} apiVersion: v1 kind: Secret metadata: - name: {{ .Release.Name }}-postgres-secret + name: {{ .name }}-postgres-secret labels: - app: {{ .Chart.Name }} + app: {{ $.Chart.Name }} type: Opaque data: - POSTGRES_USER: {{ .Values.secret.postgresUser | b64enc | quote }} - POSTGRES_PASSWORD: {{ randAlphaNum (int .Values.secret.postgresPasswordLength) | b64enc | quote }} + POSTGRES_USER: {{ .secret.postgresUser | b64enc | quote }} + POSTGRES_PASSWORD: {{ .secret.postgresPassword | default (randAlphaNum (int .secret.postgresPasswordLength)) | b64enc | quote }} +--- +{{- end }} diff --git a/charts/pagila/templates/service.yaml b/charts/pagila/templates/service.yaml index cb0da63..03e1560 100644 --- a/charts/pagila/templates/service.yaml +++ b/charts/pagila/templates/service.yaml @@ -1,14 +1,16 @@ +{{- range .Values.databases }} apiVersion: v1 kind: Service metadata: - name: {{ .Release.Name }}-pagila + name: {{ .name }} labels: - app: {{ .Chart.Name }} + app: {{ $.Chart.Name }} spec: - clusterIP: None - type: {{ .Values.service.type }} + type: {{ .service.type }} ports: - - port: {{ .Values.service.port }} + - port: {{ .service.port }} targetPort: 5432 selector: - app: {{ .Chart.Name }} + app: {{ $.Chart.Name }}-{{ .name }} +--- +{{- end }} diff --git a/charts/pagila/templates/statefulset.yaml b/charts/pagila/templates/statefulset.yaml index 78b7fbc..953bb09 100644 --- a/charts/pagila/templates/statefulset.yaml +++ b/charts/pagila/templates/statefulset.yaml @@ -1,64 +1,61 @@ +{{- range .Values.databases }} apiVersion: apps/v1 kind: StatefulSet metadata: - name: {{ .Release.Name }}-pagila + name: {{ .name }} labels: - app: {{ .Chart.Name }} + app: {{ $.Chart.Name }} spec: - replicas: {{ .Values.replicaCount }} + replicas: 1 selector: matchLabels: - app: {{ .Chart.Name }} - serviceName: {{ .Release.Name }}-pagila + app: {{ $.Chart.Name }}-{{ .name }} + serviceName: {{ .name }} template: metadata: labels: - app: {{ .Chart.Name }} + app: {{ $.Chart.Name }}-{{ .name }} spec: containers: - - name: pagila - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} + - name: {{ .name }} + image: "{{ .image.repository }}:{{ .image.tag }}" + imagePullPolicy: {{ .image.pullPolicy }} ports: - containerPort: 5432 env: - name: POSTGRES_USER valueFrom: secretKeyRef: - name: {{ .Release.Name }}-postgres-secret + name: {{ .name }}-postgres-secret key: POSTGRES_USER - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: - name: {{ .Release.Name }}-postgres-secret + name: {{ .name }}-postgres-secret key: POSTGRES_PASSWORD + - name: PGDATA + value: /var/lib/postgresql/data/pgdata volumeMounts: - {{- $index := 0 }} - {{- range $filename, $content := .Values.configMap.customScripts }} - - name: custom-script-{{ $index }} - mountPath: /docker-entrypoint-initdb.d/{{ $filename }} - subPath: {{ $filename }} - {{- $index = (add $index 1) }} + {{- if and .configMap .configMap.customScripts }} + - name: custom-scripts + mountPath: /docker-entrypoint-initdb.d {{- end }} + - name: data + mountPath: /var/lib/postgresql/data volumes: - {{- $index := 0 }} - {{- range $filename, $content := .Values.configMap.customScripts }} - - name: custom-script-{{ $index }} + {{- if and .configMap .configMap.customScripts }} + - name: custom-scripts configMap: - name: {{ $.Release.Name }}-custom-scripts - items: - - key: {{ $filename }} - path: {{ $filename }} - {{- $index = (add $index 1) }} + name: {{ .name }}-custom-scripts {{- end }} volumeClaimTemplates: - metadata: name: data - labels: - app: {{ .Chart.Name }} spec: accessModes: - - {{ .Values.persistence.accessMode }} + - {{ .persistence.accessMode }} resources: requests: - storage: {{ .Values.persistence.size }} + storage: {{ .persistence.size }} +--- +{{- end }} diff --git a/charts/pagila/values.yaml b/charts/pagila/values.yaml index a779c1d..3846360 100644 --- a/charts/pagila/values.yaml +++ b/charts/pagila/values.yaml @@ -1,44 +1,62 @@ -replicaCount: 1 - -image: - repository: synthesizedio/pagila - tag: "1.8" - pullPolicy: IfNotPresent - -service: - type: ClusterIP - port: 5432 - -persistence: - enabled: true - accessMode: ReadWriteOnce - size: 10Gi - resources: {} nodeSelector: {} tolerations: [] affinity: {} -configMap: - customScripts: - 3_create_apiuser.sql: | - CREATE ROLE apiuser WITH LOGIN SUPERUSER PASSWORD 'apipassword'; - 4_create_input_db.sql: | - CREATE USER input_source; - CREATE DATABASE input_source WITH TEMPLATE postgres OWNER input_source; - GRANT ALL PRIVILEGES ON DATABASE input_source TO input_source; - GRANT ALL PRIVILEGES ON DATABASE input_source TO apiuser; - GRANT SELECT ON ALL TABLES IN SCHEMA public TO input_source; - 5_create_output_db.sql: | - CREATE USER output_source; - CREATE DATABASE output_source; - GRANT ALL PRIVILEGES ON DATABASE output_source TO output_source; - GRANT ALL PRIVILEGES ON DATABASE output_source TO apiuser; - GRANT ALL ON ALL TABLES IN SCHEMA public TO output_source; - ALTER SCHEMA public OWNER TO output_source; - env: - PGDATA: /mnt/data/pgdata - -secret: - postgresUser: "postgres" - postgresPasswordLength: 16 +databases: + - name: api-backend + image: + repository: postgres + tag: "15" + pullPolicy: IfNotPresent + service: + type: ClusterIP + port: 5432 + secret: + postgresUser: backend_user + postgresPassword: "backend_password" + persistence: + accessMode: ReadWriteOnce + size: 1Gi + - name: input-source + image: + repository: synthesizedio/pagila + tag: "1.8" + pullPolicy: IfNotPresent + service: + type: ClusterIP + port: 5432 + secret: + postgresUser: postgres + postgresPassword: "input_source" + persistence: + accessMode: ReadWriteOnce + size: 10Gi + - name: custom-source + image: + repository: postgres + tag: "15" + pullPolicy: IfNotPresent + service: + type: ClusterIP + port: 5432 + secret: + postgresUser: custom-source + postgresPassword: "custom-source" + persistence: + accessMode: ReadWriteOnce + size: 10Gi + - name: output-source + image: + repository: postgres + tag: "15" + pullPolicy: IfNotPresent + service: + type: ClusterIP + port: 5432 + secret: + postgresUser: output_source + postgresPassword: "output_source" + persistence: + accessMode: ReadWriteOnce + size: 10Gi diff --git a/filebrowser/filebrowser-values.yaml b/filebrowser/filebrowser-values.yaml new file mode 100644 index 0000000..9b7bbfa --- /dev/null +++ b/filebrowser/filebrowser-values.yaml @@ -0,0 +1,19 @@ +rootDir: + pvc: + existingClaim: api-pvc + +ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: azure/application-gateway + cert-manager.io/cluster-issuer: "letsencrypt-prod" + kubernetes.io/tls-acme: "true" + hosts: + - host: filebrowser-investcloud.synthesized.io + paths: + - path: / + pathType: ImplementationSpecific + tls: + - secretName: filebrowser-tls-secret + hosts: + - filebrowser-investcloud.synthesized.io \ No newline at end of file