Skip to content

Commit

Permalink
setup separate pagila backend database
Browse files Browse the repository at this point in the history
  • Loading branch information
@dmeatriy
dmeatriy committed Jan 6, 2025
1 parent f7901be commit 291076e
Show file tree
Hide file tree
Showing 7 changed files with 270 additions and 90 deletions.
146 changes: 146 additions & 0 deletions charts/pagila/scripts/generate_efg_data.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
import pandas as pd
import random
import string

# Function to generate random strings
def random_string(length=10):
return ''.join(random.choices(string.ascii_uppercase, k=length))

# Function to generate XML row
def generate_xml_row(row_id, c1, c2, c3, c4, c5):
xml_template = f"""
<row id=\"{row_id}\" xml:space=\"preserve\">
<c1>{c1}</c1>
<c2>{c2}</c2>
<c3>{c3}</c3>
<c4>{c4}</c4>
<c5>{c5}</c5>
<c6>CH</c6>
<c7>CHF</c7>
<c8>S</c8>
<c9>101</c9>
<c10>CHF</c10>
<c11>0</c11>
<c15>12.02</c15>
<c16>20241114</c16>
<c17>1</c17>
<c18>84</c18>
<c47>CH0022268228</c47>
<c48>CRIMS13871745</c48>
<c51>0022268220</c51>
<c53>QUOTED</c53>
<c54>NO</c54>
<c55>NO</c55>
<c56>NO</c56>
<c57>N</c57>
<c58>NO</c58>
<c59>NO</c59>
<c60>NO</c60>
<c61>NO</c61>
<c62>NO</c62>
<c63>NO</c63>
<c64>NO</c64>
<c68>20050926</c68>
<c73>84</c73>
<c74>CH</c74>
<c76>CH</c76>
<c115>20050926</c115>
<c125>3753677191</c125>
<c126>1</c126>
<c359>9000</c359>
<c359 m="9">SHA.GUI</c359>
<c359 m="17">EFGN</c359>
<c359 m="28">BSI.166027</c359>
<c359 m="32">CH0010002</c359>
<c359 m="33">20150131</c359>
<c359 m="35">EFGN SW Equity</c359>
<c359 m="49">EQT1</c359>
<c359 m="50">74</c359>
<c359 m="51">NA</c359>
<c359 m="52">N</c359>
<c359 m="53">30911</c359>
<c359 m="54">N</c359>
<c359 m="55">20100216</c359>
<c359 m="56">EFGIS</c359>
<c359 m="58">BLOCKED/UNBLOCKED CLASSEUR FMI - BSE 07.02.12</c359>
<c359 m="69">00</c359>
<c359 m="87">CHF</c359>
<c359 m="90">TELEKURS</c359>
<c359 m="126">Y</c359>
<c359 m="133">1M</c359>
<c359 m="144">20241115</c359>
<c359 m="147">11.86</c359>
<c359 m="148">20241113</c359>
<c359 m="202">CH0010001.GLC</c359>
<c359 m="203">ID-PRC</c359>
<c359 m="212">CHKLG</c359>
<c359 m="213">MASTER</c359>
<c359 m="226">ESVTFR</c359>
<c359 m="230">4.5757</c359>
<c359 m="235">8.9</c359>
<c359 m="259">9</c359>
<c359 m="260">1</c359>
<c359 m="264">20240326</c359>
<c359 m="265">20240328</c359>
<c359 m="266">IDC</c359>
<c359 m="272">4.5757</c359>
<c359 m="280">27.57</c359>
<c359 m="281">22.78</c359>
<c359 m="282">78269</c359>
<c359 m="290">BS1P</c359>
<c359 m="290" s="2">CH1P</c359>
<c359 m="290" s="3">CYB</c359>
<c359 m="290" s="4">GB1P</c359>
<c359 m="290" s="5">HK1P</c359>
<c359 m="290" s="6">LI1P</c359>
<c359 m="290" s="7">LU1P</c359>
<c359 m="290" s="8">MC1P</c359>
<c359 m="290" s="9">SG1P</c359>
<c359 m="290" s="10">US1P</c359>
<c359 m="310">OUT</c359>
<c359 m="311">20140701</c359>
<c359 m="324">EQUITIES-EQ_CH</c359>
<c359 m="325">222682-200</c359>
<c359 m="326">0.25</c359>
<c359 m="332">222682-200</c359>
<c359 m="333">171922</c359>
<c359 m="346">EFGN</c359>
<c359 m="350">NO</c359>
<c359 m="370">YES</c359>
<c359 m="386">66.12</c359>
<c362>2</c362>
<c363>MAPPED_OFS-EXPL</c363>
<c363 m="2">43_CONV.SECURITY.MASTER.G15.2.00</c363>
<c363 m="3">45_CONV.SECURITY.MASTER.R07</c363>
<c363 m="4">45_CONV.SECURITY.MASTER.R08</c363>
<c363 m="5">45_CONV.SECURITY.MASTER.R09</c363>
<c363 m="6">45_CONV.SECURITY.MASTER.201804</c363>
<c364>2411150824</c364>
<c365>MAPPED_SHDN.SM.NACE</c365>
<c366>CH0010001</c366>
<c367>8100</c367>
<c369/>
</row>"""
return xml_template

# Generate 10,000 rows
data = []
for i in range(100000):
row_id = f"222682-{200 + i}"
c1 = f"Company_{i} {random_string(5)}"
c2 = f"Alias_{i} {random_string(4)}"
c3 = f"Description_{i}"
c4 = f"SYM_{random_string(3)}"
c5 = f"Country_{random.choice(['US', 'CH', 'DE', 'FR', 'UK'])}"
xml_row = generate_xml_row(row_id, c1, c2, c3, c4, c5)
data.append((row_id, xml_row))

# Convert to DataFrame
df = pd.DataFrame(data, columns=['key', 'value'])

# Generate SQL INSERT statements
with open("insert_security_master.sql", "w") as f:
for index, row in df.iterrows():
f.write(f"INSERT INTO trading_schema.SECURITY_MASTER (key, value) VALUES (\'{row['key']}\', '{row['value']}');\n")

print("SQL File Generated: insert_security_master.sql")
19 changes: 7 additions & 12 deletions charts/pagila/templates/configmap.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,24 +1,19 @@
{{- range .Values.databases }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ .Release.Name }}-custom-scripts
name: {{ .name }}-custom-scripts
labels:
app: {{ .Chart.Name }}
app: {{ $.Chart.Name }}
data:
{{- if .Values.configMap.customScripts }}
{{- range $filename, $content := .Values.configMap.customScripts }}
{{- if and .configMap .configMap.customScripts }}
{{- range $filename, $content := .configMap.customScripts }}
{{ $filename }}: |
{{ $content | indent 4 }}
{{- end }}
{{- else }}
dummy.sql: |
-- No custom scripts provided
{{- end }}

{{- if .Values.configMap.env }}
{{- range $key, $value := .Values.configMap.env }}
{{ $key }}: "{{ $value }}"
{{- end }}
{{- else }}
debug-env: "No environment variables defined"
{{- end }}
---
{{- end }}
11 changes: 7 additions & 4 deletions charts/pagila/templates/secret.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
{{- range .Values.databases }}
apiVersion: v1
kind: Secret
metadata:
name: {{ .Release.Name }}-postgres-secret
name: {{ .name }}-postgres-secret
labels:
app: {{ .Chart.Name }}
app: {{ $.Chart.Name }}
type: Opaque
data:
POSTGRES_USER: {{ .Values.secret.postgresUser | b64enc | quote }}
POSTGRES_PASSWORD: {{ randAlphaNum (int .Values.secret.postgresPasswordLength) | b64enc | quote }}
POSTGRES_USER: {{ .secret.postgresUser | b64enc | quote }}
POSTGRES_PASSWORD: {{ .secret.postgresPassword | default (randAlphaNum (int .secret.postgresPasswordLength)) | b64enc | quote }}
---
{{- end }}
14 changes: 8 additions & 6 deletions charts/pagila/templates/service.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
{{- range .Values.databases }}
apiVersion: v1
kind: Service
metadata:
name: {{ .Release.Name }}-pagila
name: {{ .name }}
labels:
app: {{ .Chart.Name }}
app: {{ $.Chart.Name }}
spec:
clusterIP: None
type: {{ .Values.service.type }}
type: {{ .service.type }}
ports:
- port: {{ .Values.service.port }}
- port: {{ .service.port }}
targetPort: 5432
selector:
app: {{ .Chart.Name }}
app: {{ $.Chart.Name }}-{{ .name }}
---
{{- end }}
55 changes: 26 additions & 29 deletions charts/pagila/templates/statefulset.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,64 +1,61 @@
{{- range .Values.databases }}
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: {{ .Release.Name }}-pagila
name: {{ .name }}
labels:
app: {{ .Chart.Name }}
app: {{ $.Chart.Name }}
spec:
replicas: {{ .Values.replicaCount }}
replicas: 1
selector:
matchLabels:
app: {{ .Chart.Name }}
serviceName: {{ .Release.Name }}-pagila
app: {{ $.Chart.Name }}-{{ .name }}
serviceName: {{ .name }}
template:
metadata:
labels:
app: {{ .Chart.Name }}
app: {{ $.Chart.Name }}-{{ .name }}
spec:
containers:
- name: pagila
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
- name: {{ .name }}
image: "{{ .image.repository }}:{{ .image.tag }}"
imagePullPolicy: {{ .image.pullPolicy }}
ports:
- containerPort: 5432
env:
- name: POSTGRES_USER
valueFrom:
secretKeyRef:
name: {{ .Release.Name }}-postgres-secret
name: {{ .name }}-postgres-secret
key: POSTGRES_USER
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Release.Name }}-postgres-secret
name: {{ .name }}-postgres-secret
key: POSTGRES_PASSWORD
- name: PGDATA
value: /var/lib/postgresql/data/pgdata
volumeMounts:
{{- $index := 0 }}
{{- range $filename, $content := .Values.configMap.customScripts }}
- name: custom-script-{{ $index }}
mountPath: /docker-entrypoint-initdb.d/{{ $filename }}
subPath: {{ $filename }}
{{- $index = (add $index 1) }}
{{- if and .configMap .configMap.customScripts }}
- name: custom-scripts
mountPath: /docker-entrypoint-initdb.d
{{- end }}
- name: data
mountPath: /var/lib/postgresql/data
volumes:
{{- $index := 0 }}
{{- range $filename, $content := .Values.configMap.customScripts }}
- name: custom-script-{{ $index }}
{{- if and .configMap .configMap.customScripts }}
- name: custom-scripts
configMap:
name: {{ $.Release.Name }}-custom-scripts
items:
- key: {{ $filename }}
path: {{ $filename }}
{{- $index = (add $index 1) }}
name: {{ .name }}-custom-scripts
{{- end }}
volumeClaimTemplates:
- metadata:
name: data
labels:
app: {{ .Chart.Name }}
spec:
accessModes:
- {{ .Values.persistence.accessMode }}
- {{ .persistence.accessMode }}
resources:
requests:
storage: {{ .Values.persistence.size }}
storage: {{ .persistence.size }}
---
{{- end }}
Loading

0 comments on commit 291076e

Please sign in to comment.