Proxychains setup

Joystream · Nov 20, 2024 · c7eba65 · c7eba65
1 parent 427396e
commit c7eba65
Show file tree

Hide file tree

Showing 13 changed files with 148 additions and 81 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -4,6 +4,8 @@ FROM node:20
 # Set the working directory to /youtube-synch
 WORKDIR /youtube-synch
 
+ENV DEBIAN_FRONTEND=noninteractive
+
 # Install AWS CLI
 RUN apt-get update && \
     apt-get install -y awscli && \
@@ -17,6 +19,9 @@ RUN curl -fsSL https://get.docker.com -o get-docker.sh && \
 # Install node-gyp
 RUN npm install -g node-gyp
 
+# Install proxychains
+RUN apt-get update && apt-get install -y proxychains4
+
 # Copy the package.json and yarn.lock (or package-lock.json for npm) files
 COPY package.json package-lock.json ./
 

diff --git a/config.yml b/config.yml
@@ -14,7 +14,6 @@ sync:
     dailyApiQuota:
       sync: 9500
       signup: 500
-    bandwidthPerDownload: 1000
     maxConcurrentDownloads: 2
     preDownloadSleep:
       min: 0
@@ -42,9 +41,9 @@ youtube:
   clientSecret: google-client-secret
   # adcKeyFilePath: path/to/adc-key-file.json
   # maxAllowedQuotaUsageInPercentage: 95
-proxy:
-  urls:
-    ["socks5://chisel-client:1080"]
+# proxy:
+#   urls:
+#     ["socks5://proxy1:1080", "socks5://proxy2:1080"]
 aws:
   endpoint: http://localhost:4566
   region: us-east-1

diff --git a/docker-compose.elasticsearch.yml b/docker-compose.elasticsearch.yml
@@ -18,7 +18,10 @@ services:
         hard: -1
     volumes:
       - es-data:/usr/share/elasticsearch/data
-    network_mode: "container:gluetun"
+    ports:
+      - 127.0.0.1:9200:9200
+    networks:
+      - youtube-synch
 
   # Ref: https://www.elastic.co/guide/en/kibana/8.7/docker.html
   kibana:
@@ -27,11 +30,14 @@ services:
     depends_on:
       - elasticsearch
     environment:
-      ELASTICSEARCH_HOSTS: http://localhost:9200
+      ELASTICSEARCH_HOSTS: http://elasticsearch:9200
       ELASTICSEARCH_SERVICEACCOUNTTOKEN: ${ELASTICSEARCH_SERVICEACCOUNTTOKEN}
       # Ref: https://www.elastic.co/guide/en/kibana/current/xpack-security-secure-saved-objects.html#xpack-security-secure-saved-objects
       XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: ${XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY}
-    network_mode: "container:gluetun"
+    ports:
+      - 127.0.0.1:5601:5601
+    networks:
+      - youtube-synch
 
   # Ref: https://www.elastic.co/guide/en/apm/guide/8.7/running-on-docker.html
   apm-server:
@@ -40,10 +46,13 @@ services:
       - elasticsearch
     command: |
       --strict.perms=false -e 
-      -E output.elasticsearch.hosts=["localhost:9200"] 
+      -E output.elasticsearch.hosts=["elasticsearch:9200"] 
       -E output.elasticsearch.username=${ELASTIC_USERNAME:-elastic} 
       -E output.elasticsearch.password=${ELASTIC_PASSWORD:-password}
-    network_mode: "container:gluetun"
+    ports:
+      - 8200:8200
+    networks:
+      - youtube-synch
 
 volumes:
   es-data:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,32 +1,5 @@
 version: '3.4'
 services:
-  gluetun:
-    image: qmcgaw/gluetun:latest
-    container_name: gluetun
-    restart: on-failure
-    cap_add:
-      - NET_ADMIN
-    devices:
-      - /dev/net/tun:/dev/net/tun
-    env_file:
-      - .env
-    environment:
-      - VPN_SERVICE_PROVIDER=ivpn
-      - VPN_TYPE=openvpn
-      - OPENVPN_USER=${VPN_USER}
-      - OPENVPN_PASSWORD=password
-    ports:
-      - 127.0.0.1:${YT_SYNCH__HTTP_API__PORT}:${YT_SYNCH__HTTP_API__PORT}
-      - 127.0.0.1:${YT_SYNCH__ENDPOINTS__REDIS__PORT}:${YT_SYNCH__ENDPOINTS__REDIS__PORT}
-      # Elasticsearch
-      - 127.0.0.1:9200:9200
-      # Kibana
-      - 127.0.0.1:5601:5601
-      # Apm server
-      - 127.0.0.1:8200:8200
-    profiles:
-      - gluetun
-
   youtube-synch_httpApi:
     image: youtube-synch:${IMAGE_TAG}
     container_name: youtube-synch_httpApi
@@ -51,7 +24,7 @@ services:
       # YT_SYNCH__LOGS__ELASTIC: "{\"level\":\"debug\",\"endpoint\":\"http://elasticsearch:9200\"}"
       # YT_SYNCH__ENDPOINTS__QUERY_NODE: ${YT_SYNCH__ENDPOINTS__QUERY_NODE}
       # YT_SYNCH__ENDPOINTS__JOYSTREAM_NODE_WS: ${YT_SYNCH__ENDPOINTS__JOYSTREAM_NODE_WS}
-      YT_SYNCH__ENDPOINTS__REDIS__HOST: localhost
+      YT_SYNCH__ENDPOINTS__REDIS__HOST: redis
       YT_SYNCH__ENDPOINTS__REDIS__PORT: ${YT_SYNCH__ENDPOINTS__REDIS__PORT}
       # YT_SYNCH__HTTP_API__PORT: ${YT_SYNCH__HTTP_API__PORT}
       # YT_SYNCH__HTTP_API__OWNER_KEY: ${YT_SYNCH__HTTP_API__OWNER_KEY}
@@ -67,7 +40,11 @@ services:
       # YT_SYNCH__JOYSTREAM__FAUCET__ENDPOINT: ${YT_SYNCH__JOYSTREAM__FAUCET__ENDPOINT}
       # YT_SYNCH__JOYSTREAM__FAUCET__CAPTCHA_BYPASS_KEY: ${YT_SYNCH__JOYSTREAM__FAUCET__CAPTCHA_BYPASS_KEY}
       OTEL_EXPORTER_OTLP_ENDPOINT: ${TELEMETRY_ENDPOINT}
-    network_mode: "container:gluetun"
+    ports:
+      - 127.0.0.1:${YT_SYNCH__HTTP_API__PORT}:${YT_SYNCH__HTTP_API__PORT}
+    networks:
+      - youtube-synch
+      - ip6net
     command: ['./scripts/start-youtube-synch.sh', '--service', 'httpApi']
 
   youtube-synch_sync:
@@ -84,6 +61,7 @@ services:
       # - /var/run/docker.sock:/var/run/docker.sock
       - ./local/logs:/youtube-synch/local/logs
       - ./local/data:/youtube-synch/local/data
+      - ./proxychains4.conf:/etc/proxychains4.conf
       # mount Google Cloud's Application Default Credentials file. A default bind mount is created
       # as workaround for scenario when `YT_SYNCH__YOUTUBE__ADC_KEY_FILE_PATH` will be undefined,
       # since docker-compose does not support creating bind-mount volume with empty path.
@@ -98,7 +76,7 @@ services:
       # YT_SYNCH__LOGS__ELASTIC: "{\"level\":\"debug\",\"endpoint\":\"http://elasticsearch:9200\"}"
       # YT_SYNCH__ENDPOINTS__QUERY_NODE: ${YT_SYNCH__ENDPOINTS__QUERY_NODE}
       # YT_SYNCH__ENDPOINTS__JOYSTREAM_NODE_WS: ${YT_SYNCH__ENDPOINTS__JOYSTREAM_NODE_WS}
-      YT_SYNCH__ENDPOINTS__REDIS__HOST: localhost
+      YT_SYNCH__ENDPOINTS__REDIS__HOST: redis
       YT_SYNCH__ENDPOINTS__REDIS__PORT: ${YT_SYNCH__ENDPOINTS__REDIS__PORT}
       # YT_SYNCH__HTTP_API__PORT: ${YT_SYNCH__HTTP_API__PORT}
       # YT_SYNCH__HTTP_API__OWNER_KEY: ${YT_SYNCH__HTTP_API__OWNER_KEY}
@@ -113,21 +91,48 @@ services:
       # YT_SYNCH__JOYSTREAM__APP__ACCOUNT_SEED: ${YT_SYNCH__JOYSTREAM__APP__ACCOUNT_SEED}
       # YT_SYNCH__JOYSTREAM__FAUCET__ENDPOINT: ${YT_SYNCH__JOYSTREAM__FAUCET__ENDPOINT}
       # YT_SYNCH__JOYSTREAM__FAUCET__CAPTCHA_BYPASS_KEY: ${YT_SYNCH__JOYSTREAM__FAUCET__CAPTCHA_BYPASS_KEY}
-    network_mode: "container:gluetun"
+    networks:
+      - youtube-synch
+      - ip6net
     command: ['./bin/run', 'start', '--service', 'sync']
 
   redis:
     image: redis:7.2.1
     container_name: redis
     command: ['redis-server', '--maxmemory-policy', 'noeviction']
-    network_mode: "container:gluetun"
+    ports:
+      - 127.0.0.1:${YT_SYNCH__ENDPOINTS__REDIS__PORT}:${YT_SYNCH__ENDPOINTS__REDIS__PORT}
+    networks:
+      - youtube-synch
     volumes:
       - redis-data:/data
 
+  # ipv6-assigner:
+  #   image: node:18
+  #   container_name: ipv6-assigner
+  #   working_dir: /ipv6-assigner
+  #   env_file:
+  #     # relative to working directory where docker-compose was run from
+  #     - .env
+  #   volumes:
+  #     - .:/ipv6-assigner
+  #     - ~/.aws/:/root/.aws:ro
+  #   restart: on-failure
+  #   command: sh -c "npx ts-node src/utils/assignIpv6Adress.ts"
+
 volumes:
   logs:
     driver: local
   data:
     driver: local
   redis-data:
     driver: local
+
+networks:
+  youtube-synch:
+    name: youtube-synch
+  ip6net:
+    enable_ipv6: true
+    ipam:
+      config:
+        - subnet: 2001:db8::/64
diff --git a/...elated-settings-properties-limits-properties-predownloadsleep-properties-max.md b/...elated-settings-properties-limits-properties-predownloadsleep-properties-max.md
@@ -0,0 +1,3 @@
+## max Type
+
+`integer`
diff --git a/...elated-settings-properties-limits-properties-predownloadsleep-properties-min.md b/...elated-settings-properties-limits-properties-predownloadsleep-properties-min.md
@@ -0,0 +1,3 @@
+## min Type
+
+`integer`
diff --git a/...yncronization-related-settings-properties-limits-properties-predownloadsleep.md b/...yncronization-related-settings-properties-limits-properties-predownloadsleep.md
@@ -0,0 +1,46 @@
+## preDownloadSleep Type
+
+`object` ([Details](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-predownloadsleep.md))
+
+# preDownloadSleep Properties
+
+| Property    | Type      | Required | Nullable       | Defined by                                                                                                                                                                                                                                                                                           |
+| :---------- | :-------- | :------- | :------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [min](#min) | `integer` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-predownloadsleep-properties-min.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/preDownloadSleep/properties/min") |
+| [max](#max) | `integer` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-predownloadsleep-properties-max.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/preDownloadSleep/properties/max") |
+
+## min
+
+Minimum value to sleep (in miliseconds)
+
+`min`
+
+*   is required
+
+*   Type: `integer`
+
+*   cannot be null
+
+*   defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-predownloadsleep-properties-min.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/preDownloadSleep/properties/min")
+
+### min Type
+
+`integer`
+
+## max
+
+Maximum value to sleep (in miliseconds)
+
+`max`
+
+*   is required
+
+*   Type: `integer`
+
+*   cannot be null
+
+*   defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-predownloadsleep-properties-max.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/preDownloadSleep/properties/max")
+
+### max Type
+
+`integer`
diff --git a/...nition-properties-yt-synch-syncronization-related-settings-properties-limits.md b/...nition-properties-yt-synch-syncronization-related-settings-properties-limits.md
@@ -8,11 +8,11 @@
 | :------------------------------------------------------ | :-------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | [dailyApiQuota](#dailyapiquota)                         | `object`  | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/dailyApiQuota") |
 | [maxConcurrentDownloads](#maxconcurrentdownloads)       | `number`  | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-maxconcurrentdownloads.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/maxConcurrentDownloads")                                                |
-| [bandwidthPerDownload](#bandwidthperdownload)           | `string`  | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-bandwidthperdownload.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/bandwidthPerDownload")                                                    |
 | [createVideoTxBatchSize](#createvideotxbatchsize)       | `number`  | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-createvideotxbatchsize.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/createVideoTxBatchSize")                                                |
 | [maxConcurrentUploads](#maxconcurrentuploads)           | `number`  | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-maxconcurrentuploads.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/maxConcurrentUploads")                                                    |
 | [pendingDownloadTimeoutSec](#pendingdownloadtimeoutsec) | `integer` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-pendingdownloadtimeoutsec.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/pendingDownloadTimeoutSec")                                          |
 | [storage](#storage)                                     | `string`  | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-storage.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/storage")                                                                              |
+| [preDownloadSleep](#predownloadsleep)                   | `object`  | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-predownloadsleep.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/preDownloadSleep")                                                            |
 
 ## dailyApiQuota
 
@@ -58,34 +58,6 @@ The default value is:
 50
 ```
 
-## bandwidthPerDownload
-
-Bandwidth limit per each individual video download (in bytes per second, e.g. 500K, 1M etc.)
-
-`bandwidthPerDownload`
-
-*   is optional
-
-*   Type: `string`
-
-*   cannot be null
-
-*   defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-bandwidthperdownload.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/bandwidthPerDownload")
-
-### bandwidthPerDownload Type
-
-`string`
-
-### bandwidthPerDownload Constraints
-
-**pattern**: the string must match the following regular expression: 
-
-```regexp
-[1-9][0-9]*(K|M)
-```
-
-[try pattern](https://regexr.com/?expression=%5B1-9%5D%5B0-9%5D\*\(K%7CM\) "try regular expression with regexr.com")
-
 ## createVideoTxBatchSize
 
 No. of videos that should be created in a batched 'create_video' tx
@@ -187,3 +159,21 @@ Maximum total size of all downloaded assets stored in `downloadsDir`
 ```
 
 [try pattern](https://regexr.com/?expression=%5E%5B0-9%5D%2B\(B%7CK%7CM%7CG%7CT\)%24 "try regular expression with regexr.com")
+
+## preDownloadSleep
+
+Specifies the time to sleep before each download is started
+
+`preDownloadSleep`
+
+*   is optional
+
+*   Type: `object` ([Details](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-predownloadsleep.md))
+
+*   cannot be null
+
+*   defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-predownloadsleep.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/preDownloadSleep")
+
+### preDownloadSleep Type
+
+`object` ([Details](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-predownloadsleep.md))
diff --git a/proxychains4.conf b/proxychains4.conf
@@ -0,0 +1,8 @@
+strict_chain
+
+tcp_read_time_out 15000
+tcp_connect_time_out 8000
+
+[ProxyList]
+
+socks5 	172.20.0.2 1080
diff --git a/scripts/start-elasticsearch.sh b/scripts/start-elasticsearch.sh
@@ -31,7 +31,7 @@ export XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=$(openssl rand -base64 24)
 curl -X DELETE -u "${ELASTIC_USERNAME}":"${ELASTIC_PASSWORD}" "http://localhost:9200/_security/service/elastic/kibana/credential/token/my_kibana_token" -H 'Content-Type: application/json'
 # Generate the service token
 # Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/service-accounts.html#service-accounts-tokens
-response=$(curl -s -w "\n%{http_code}\n" -X GET -u "${ELASTIC_USERNAME}":"${ELASTIC_PASSWORD}" "http://localhost:9200/_security/service/elastic/kibana/credential/token/my_kibana_token" -H 'Content-Type: application/json')
+response=$(curl -s -w "\n%{http_code}\n" -X POST -u "${ELASTIC_USERNAME}":"${ELASTIC_PASSWORD}" "http://localhost:9200/_security/service/elastic/kibana/credential/token/my_kibana_token" -H 'Content-Type: application/json')
 response_body=$(echo "$response" | head -n1)
 status_code=$(echo "$response" | tail -n1)
 

diff --git a/src/schemas/config.ts b/src/schemas/config.ts
@@ -336,10 +336,6 @@ export const configSchema: JSONSchema7 = objectSchema({
               type: 'number',
               default: 50,
             },
-            bandwidthPerDownload: {
-              description: 'Bandwidth limit per each individual video download (in KB/s)',
-              type: 'integer',
-            },
             createVideoTxBatchSize: {
               description: `No. of videos that should be created in a batched 'create_video' tx`,
               type: 'number',