This repository has been archived by the owner on Jan 5, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 6
/
Copy pathconfig_sync.py
executable file
·197 lines (175 loc) · 8.93 KB
/
config_sync.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
#!/usr/bin/env python
from __future__ import print_function
import sys
import yaml
import os
from string import Template
import pwd
import re
class Config:
config = {'volumes': {}}
supervisor_conf_folder = '/etc/supervisor.conf.d/'
unison_template_path = '/etc/supervisor.unison.tpl.conf'
unison_defaults = ' -copyonconflict -auto -numericids -batch -maxerrors 10 -repeat watch'
def read_yaml(self, config_file):
"""
Read YAML file and returns the configuration
"""
with open(config_file, 'r') as stream:
try:
return yaml.load(stream, Loader=yaml.FullLoader)
except yaml.YAMLError as exc:
print(exc)
def write_supervisor_conf(self):
"""
Generates supervisor configuration for unison
"""
if 'volumes' in self.config:
for i, (volume, conf) in enumerate(self.config['volumes'].items(), 1):
conf.update({'port': 5000 + int(i)})
template = open(self.unison_template_path)
with open(self.supervisor_conf_folder + 'unison' + conf['name'] + '.conf', 'w') as f:
f.write(Template(template.read()).substitute(conf))
def create_user(self, user, uid):
"""
Create the user on the system. If the user and the uid doesn't exist, simply create it.
If the uid is not used, but user exists, modify existing user to set the appropriate uid.
If the uid is used, but user doesn't exists, rename existing user and create home directory.
"""
if uid:
uid_str = " -u " + str(uid) + " "
# if uid doesn't exist on the system
if int(uid) not in [x[2] for x in pwd.getpwall()]:
# if user doesn't exist on the system
if user not in [y[0] for y in pwd.getpwall()]:
cmd="useradd " + user + uid_str + " -m"
os.system(cmd)
else:
cmd="usermod " + uid_str + user
os.system(cmd)
else:
# get username with uid
for existing_user in pwd.getpwall():
if existing_user[2] == int(uid):
user_name = existing_user[0]
cmd="mkdir -p /home/" + user + " && usermod --home /home/" + user + " --login " + user + " " + str(user_name) + " && chown -R " + user + " /home/" + user
os.system(cmd)
else:
if user not in [x[0] for x in pwd.getpwall()]:
cmd="useradd " + user + " -m"
os.system(cmd)
else:
print("user already exists")
self.debug("CMD:" + cmd)
def set_permissions(self, user, folder, recursive=False):
"""
Set permissions to the folder
"""
args = ' -R ' if recursive else ''
if user != 'root':
os.system("chown " + args + user + " " + folder)
def generate_ignore_string(self, ignores, sync_method='unison', ignore_type='name'):
"""
Generates an ignore string depending on the type of sync command, currently supports:
- unison
- tar
"""
if type(ignores) is str:
ignores = ignores.split(':')
if sync_method == 'unison':
if ignore_type == 'path':
separator = "' -ignore 'Path "
else:
separator = "' -ignore 'Name "
return separator[1:] + separator.join(ignores) + "' "
elif sync_method == 'tar':
separator = " --exclude "
return separator + separator.join(ignores) + ' '
def set_defaults(self):
"""
Set values for configured volumes to sync:
- volume: the volume path
- name: autogenerated from path, replacing / by -
- user: user to be mapped to the folder, set in config by volumes, or globally by ENV variable SYNC_USER
- homedir: autogenerated, path to the home directory of the user
- uid: the uid of the user, set in config by volumes, or globally by ENV variable SYNC_UID
:return:
"""
if 'volumes' in self.config:
for i, (volume, conf) in enumerate(self.config['volumes'].items(), 1):
self.config['volumes'][volume]['volume'] = volume
self.config['volumes'][volume]['name'] = re.sub(r'\/', '-', volume)
if 'user' in conf:
user = conf['user']
self.config['volumes'][volume]['homedir'] = '/home/' + conf['user']
elif 'SYNC_USER' in os.environ:
user = os.environ['SYNC_USER']
self.config['volumes'][volume]['user'] = os.environ['SYNC_USER']
self.config['volumes'][volume]['homedir'] = '/home/' + os.environ['SYNC_USER']
else:
user = self.config['volumes'][volume]['user'] = self.config['volumes'][volume]['name'][1:8]
self.config['volumes'][volume]['homedir'] = '/home/' + self.config['volumes'][volume]['name'][1:8]
if 'uid' in conf:
self.config['volumes'][volume]['uid'] = uid = conf['uid']
elif 'SYNC_UID' in os.environ:
self.config['volumes'][volume]['uid'] = uid = os.environ['SYNC_UID']
else:
raise Exception('Unable to grab uid from config file or env variable. Ensure you have set SYNC_UID ! ')
self.config['volumes'][volume]['unison_ignore'] = ''
if 'ignore_name' in conf:
self.config['volumes'][volume]['unison_ignore'] = self.generate_ignore_string(conf['ignore_name'], 'unison')
elif 'SYNC_IGNORE_NAMES' in os.environ:
self.config['volumes'][volume]['ignore_name'] = os.environ['SYNC_IGNORE_NAMES']
self.config['volumes'][volume]['unison_ignore'] = self.generate_ignore_string(os.environ['SYNC_IGNORE_NAMES'], 'unison')
else:
self.config['volumes'][volume]['ignore_name'] = ''
if 'ignore_path' in conf:
self.config['volumes'][volume]['unison_ignore'] += self.generate_ignore_string(conf['ignore_path'], 'unison', 'path')
elif 'SYNC_IGNORE_PATHS' in os.environ:
self.config['volumes'][volume]['ignore_path'] = os.environ['SYNC_IGNORE_PATHS']
self.config['volumes'][volume]['unison_ignore'] += self.generate_ignore_string(os.environ['SYNC_IGNORE_PATHS'], 'unison', 'path')
else:
self.config['volumes'][volume]['ignore_path'] = ''
if 'unison_defaults' not in conf and 'SYNC_UNISON_DEFAULTS' in os.environ:
self.config['volumes'][volume]['unison_defaults'] = os.environ['SYNC_UNISON_DEFAULTS']
elif 'unison_defaults' not in conf:
self.config['volumes'][volume]['unison_defaults'] = '-prefer ' + volume + '.magic ' + self.unison_defaults
self.create_user(user, uid)
self.set_permissions(user, volume)
# In case of multiple users with the same uid, they may be renamed
# Ensure that we use the adequate name in configuration
for i, (volume, conf) in enumerate(self.config['volumes'].items(), 1):
for existing_user in pwd.getpwall():
if existing_user[2] == int(conf['uid']):
conf['user'] = existing_user[0]
def merge_discovered_volumes(self):
"""
Read config file auto generated on container start by docker-gen.
Merges the `magic` folders with the ones configured in the provided config file, if any.
"""
volumes = self.read_yaml('/volumes.yml')
for volume in volumes['volumes']:
if '.magic' in volume:
if not self.config or volume not in self.config['volumes']:
self.config['volumes'][volume.replace('.magic', '')] = {}
def initial_sync(self):
"""
When starting the container, copies the files from host to container
"""
if 'volumes' in self.config:
for volume, conf in self.config['volumes'].items():
command = 'unison ' + volume + '.magic ' + volume + ' -prefer ' + volume + '.magic -copyonconflict -numericids -auto -batch ' + self.generate_ignore_string(conf['ignore_name'], 'unison') + self.generate_ignore_string(conf['ignore_path'], 'unison', 'path')
self.debug(command)
os.system(command)
self.set_permissions(conf['user'], volume, True)
def debug(self, message):
print(message)
def set(self, config_file):
if config_file:
self.config = self.read_yaml(config_file)
self.merge_discovered_volumes()
self.set_defaults()
self.write_supervisor_conf()
#self.initial_sync()
c = Config()
c.set(sys.argv[1])