From 76223f65a9fd8060364821ca7f0387bbc3464cf9 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Wed, 31 Jul 2024 10:57:39 +0800 Subject: [PATCH] arbiter: remove all codes Signed-off-by: lance6716 --- Makefile | 7 +- arbiter/README.md | 67 ----- arbiter/README_CN.md | 150 ---------- arbiter/arbiter.json | 567 ------------------------------------- arbiter/arbiter.png | Bin 47544 -> 0 bytes arbiter/arbiter.rules.yml | 25 -- arbiter/checkpoint.go | 107 ------- arbiter/checkpoint_test.go | 87 ------ arbiter/config.go | 200 ------------- arbiter/config_test.go | 147 ---------- arbiter/metrics.go | 93 ------ arbiter/metrics_test.go | 49 ---- arbiter/server.go | 317 --------------------- arbiter/server_test.go | 468 ------------------------------ cmd/arbiter/arbiter.toml | 24 -- cmd/arbiter/main.go | 82 ------ 16 files changed, 2 insertions(+), 2388 deletions(-) delete mode 100644 arbiter/README.md delete mode 100644 arbiter/README_CN.md delete mode 100644 arbiter/arbiter.json delete mode 100644 arbiter/arbiter.png delete mode 100644 arbiter/arbiter.rules.yml delete mode 100644 arbiter/checkpoint.go delete mode 100644 arbiter/checkpoint_test.go delete mode 100644 arbiter/config.go delete mode 100644 arbiter/config_test.go delete mode 100644 arbiter/metrics.go delete mode 100644 arbiter/metrics_test.go delete mode 100644 arbiter/server.go delete mode 100644 arbiter/server_test.go delete mode 100644 cmd/arbiter/arbiter.toml delete mode 100644 cmd/arbiter/main.go diff --git a/Makefile b/Makefile index e8fd7dc81..5c6afd083 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ ### Makefile for tidb-binlog -.PHONY: build test check update clean pump drainer fmt reparo integration_test arbiter binlogctl +.PHONY: build test check update clean pump drainer fmt reparo integration_test binlogctl PROJECT=tidb-binlog @@ -37,7 +37,7 @@ all: dev install dev: check test -build: pump drainer reparo arbiter binlogctl +build: pump drainer reparo binlogctl pump: $(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/pump cmd/pump/main.go @@ -45,9 +45,6 @@ pump: drainer: $(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/drainer cmd/drainer/main.go -arbiter: - $(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/arbiter cmd/arbiter/main.go - reparo: $(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/reparo cmd/reparo/main.go diff --git a/arbiter/README.md b/arbiter/README.md deleted file mode 100644 index 156894cfb..000000000 --- a/arbiter/README.md +++ /dev/null @@ -1,67 +0,0 @@ -Arbiter -========== - -**Arbiter** is a tool used for syncing data from Kafka to TiDB incrementally. - -![](./arbiter.png) - -The complete import process is as follows: - -1. Read Binlog from Kafka in the format of [Protobuf](https://github.com/pingcap/tidb-tools/blob/master/tidb-binlog/proto/proto/binlog.proto). -2. While reaching a limit data size, construct the SQL according the Binlog and write to downstream concurrently(notice: Arbiter will split the upstream transaction). -3. Save the checkpoint. - - -## Checkpoint -`arbiter` will write a record to the table `tidb_binlog.arbiter_checkpoint` at downstream TiDB. -``` -mysql> select * from tidb_binlog.arbiter_checkpoint; -+-------------+--------------------+--------+ -| topic_name | ts | status | -+-------------+--------------------+--------+ -| test_kafka4 | 405809779094585347 | 1 | -+-------------+--------------------+--------+ -``` -- topic_name: the topic name of Kafka to consume. -- ts: the timestamp checkpoint -- status: - * 0 - All Binlog data <= ts has synced to downstream. - * 1 - means `Arbiter` is running or quit unexpectedly, Binlog with timestamp bigger than ts may partially synced to downstream. - - - -## Monitor - -Arbiter supports metrics collection via [Prometheus](https://prometheus.io/). - -###Metrics - -* **`binlog_arbiter_checkpoint_tso`** (Gauge) - - Corresponding to ts in table `tidb_binlog.arbiter_checkpoint` - -* **`binlog_arbiter_query_duration_time`** (Histogram) - - Bucketed histogram of the time needed to wirte to downstream. Labels: - - * **type**: `exec` `commit` time takes to execute and commit SQL. - -* **`binlog_arbiter_event`** (Counter) - - Event times counter. Labels: - - * **type**: e.g. `DDL` `Insert` `Update` `Delete` `Txn` - -* **`binlog_arbiter_queue_size`** (Gauge) - - Queue size. Labels: - - * **name**: e.g. `kafka_reader` `loader_input` - -* **`binlog_arbiter_txn_latency_seconds`** (Histogram) - - Bucketed histogram of the time duration between the time write to downstream and commit time of upstream transaction(phsical part of commitTS). - - diff --git a/arbiter/README_CN.md b/arbiter/README_CN.md deleted file mode 100644 index 7003e8b45..000000000 --- a/arbiter/README_CN.md +++ /dev/null @@ -1,150 +0,0 @@ -Arbiter -========== - -**Arbiter** 是一个从 Kafka 获取 Binlog 增量同步数据到 TiDB 的工具. - -![](./arbiter.png) - -整体工作原理如下： - -1. 读取 Kafka 的 [Protobuf](https://github.com/pingcap/tidb-tools/blob/master/tidb-binlog/proto/proto/binlog.proto) 格式 Binlog 。 -2. 达到一定数据量后 根据 Binlog 构造对应 SQL 并发写入下游（注意 Arbiter 会拆分上游事务)。 -3. 保存 checkpoint 。 - - -## Checkpoint -`arbiter` 会在下游 TiDB `tidb_binlog.arbiter_checkpoint` 表里保存一条 checkpoint 记录。 -``` -mysql> select * from tidb_binlog.arbiter_checkpoint; -+-------------+--------------------+--------+ -| topic_name | ts | status | -+-------------+--------------------+--------+ -| test_kafka4 | 405809779094585347 | 1 | -+-------------+--------------------+--------+ -``` -- topic_name: 消费的 Kafka 主题名。 -- ts: 当前同步到了哪个 ts -- status: - * 0 - 表示 <= ts 的数据都同步到下游了。 - * 1 - 运行中或者异常退出，> ts 后的部分 Binlog 可能同步到下游。 - - - -## 监控告警 - -Arbiter 支持给 [Prometheus](https://prometheus.io/) 采集度量 (metrics)。本节介绍 Arbiter 的监控配置与监控指标。 - -### 监控配置 - -只要 Prometheus 能发现 **Arbiter** 的监控地址，就能收集监控指标。 - -监控的端口可在 arbiter.toml 中配置： - -```toml -# addr (i.e. 'host:port') to listen on for Arbiter connections -addr = "0.0.0.0:8251" -``` - -要让 Prometheus 发现 Arbiter，可以将地址直接写入其配置文件，例如： -```yml -scrape_configs: - - job_name: 'arbiter' - honor_labels: true # don't overwrite job & instance labels - static_configs: - - targets: ['192.168.20.10:8251'] -``` - -#### 导入 Grafana 面板 - -执行以下步骤，为 Arbiter 导入 Grafana 面板： - -1. 点击侧边栏的 Grafana 图标。 - -2. 在侧边栏菜单中，依次点击 **Dashboards** > **Import** 打开 **Import Dashboard** 窗口。 - -3. 点击 **Upload .json File** 上传对应的 JSON 文件（下载 [Arbiter 配置文件](./arbiter.json))。 - -4. 点击 **Load**。 - -5. 选择一个 Prometheus 数据源。 - -6. 点击 **Import**，Prometheus 面板即导入成功。 - -### 监控指标 - -本节将详细介绍 **arbiter** 的监控指标. - -* **`binlog_arbiter_checkpoint_tso`** (测量仪) - - 对应 `tidb_binlog.arbiter_checkpoint` 表里的 ts - -* **`binlog_arbiter_query_duration_time`** (直方图) - - 写下游需时的直方图。标签: - - * **type**: `exec` `commit` 执行 SQL 跟提交时的耗时。 - -* **`binlog_arbiter_event`** (计数器) - - 计算事件次数 - - * **type**: `DDL` `Insert` `Update` `Delete` `Txn` - -* **`binlog_arbiter_queue_size`** (测量仪) - - 内部队列数据囤积大小。标签： - - * **name**: `kafka_reader` `loader_input` - -* **`binlog_arbiter_txn_latency_seconds`** (直方图) - - 上游事务提交(commitTS物理时间) 到对应事务写入下游的花时。 - -### 报警配置 - -执行以下步骤，为 Arbiter 导入 Prometheus 告警规则： - -1. 下载 Arbiter 报警规则文件 [Arbiter 配置文件](./arbiter.rules.yml) 放到 Prometheus 的配置文件目录下。 - -2. 修改 Prometheus 配置文件 `prometheus.yml`，在 rule_files 添加对应文件: - -```yml -# Load and evaluate rules in this file every 'evaluation_interval' seconds. -rule_files: - - 'arbiter.rules.yml' -``` - -### 报警规则 - -本节介绍 Arbiter 的告警规则。 - -#### `binlog_arbiter_checkpoint_high_delay` - -* 报警规则: - - `(time() - binlog_arbiter_checkpoint_tso / 1000) > 3600` - -* 规则描述: - - Arbiter 同步落后超过一个小时则报警。 - -* 处理方法: - - 同步慢或者同步历史数据造成的，需要查看监控定位问题。 - -#### `binlog_arbiter_checkpoint_tso_no_change_for_1m` - -* 报警规则: - - `changes(binlog_arbiter_checkpoint_tso[1m]) < 1` - -* 规则描述: - - Arbiter checkpoint 一分钟没更新则报警。 - -* 处理方法: - - 没有数据源或者同步异常阻塞造成的, 需要查看监控定位问题。 - diff --git a/arbiter/arbiter.json b/arbiter/arbiter.json deleted file mode 100644 index 619147667..000000000 --- a/arbiter/arbiter.json +++ /dev/null @@ -1,567 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_TIDB-CLUSTER", - "label": "tidb-cluster", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "4.6.3" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "id": null, - "links": [], - "refresh": "5s", - "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_TIDB-CLUSTER}", - "decimals": null, - "format": "dateTimeAsIso", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 1, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 4, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "binlog_arbiter_checkpoint_tso{instance = \"$arbiter_instance\"}", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": "", - "title": "Checkpoint TS", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TIDB-CLUSTER}", - "description": "Bucketed histogram of seconds of a txn between loaded to downstream and committed at upstream", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 8, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, rate(binlog_arbiter_txn_latency_seconds_bucket{instance = \"$arbiter_instance\"}[1m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "99% Txn Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6" - }, - { - "collapse": false, - "height": 250, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TIDB-CLUSTER}", - "fill": 1, - "id": 2, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "irate(binlog_arbiter_event{instance = \"$arbiter_instance\"}[1m])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Events", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6" - }, - { - "collapse": false, - "height": 250, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TIDB-CLUSTER}", - "fill": 1, - "id": 3, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, rate(binlog_arbiter_query_duration_time_bucket{instance = \"$arbiter_instance\"}[1m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%: {{type}}", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.95, rate(binlog_arbiter_query_duration_time_bucket{instance = \"$arbiter_instance\"}[1m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%: {{type}} ", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Query Time", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "decimals": null, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6" - }, - { - "collapse": false, - "height": 250, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TIDB-CLUSTER}", - "fill": 1, - "id": 4, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "binlog_arbiter_queue_size{instance = \"$arbiter_instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Queue Size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "transparent": false, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "allValue": null, - "current": {}, - "datasource": "${DS_TIDB-CLUSTER}", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "arbiter_instance", - "options": [], - "query": "label_values(binlog_arbiter_checkpoint_tso,instance)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "arbiter", - "version": 10 -} \ No newline at end of file diff --git a/arbiter/arbiter.png b/arbiter/arbiter.png deleted file mode 100644 index 94ea9171839498398f4afcf37f41012e66d26773..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 47544 zcmeFZbC6`)^Do+(=Cn0!+qP|+)pqx^?VdKLZDZQDZQHi(+vj_}_ugN`ixcm^7x5xa zMb)ldd*?#tS}QZxCv#V*ysQ{3G$u3<5D=_{xUeD+5ZJ@l`3EHU*ReS{oD2x)JDItV zki3MD5Wc*43ax?fm%t92to|y5~KuvPGB;E z@YztVKPoDWs-ueVbUjebQWaIWZ#yrFg2IA`j#Y7+1jy0JboXF3$lJ0KseNZatN~j=5=Ss_;E+`_ zfaj1^*?|m_(f$szg94;nj2%Uc1H{5XXM)Ry^P->yT^$fYh9a;ASxu+bwJq?5v7@lO z%z;p&*YXLBQKm0B*#wA4oxFMjP z#LO1lC%I*05O#i$8DP96c#B`6#FXyBR~RtRU0Dd%evYo%*||yZH`yPfRjSW*5awYU z-2EZaS13DH2yiu9J}CzAX6%|Et{^NuS-B1bi_}YdUVO*NCqkSC^b8?CM86ZPY`rwm zT||m?74ccgHbGo)^qr>hY*XY!UM1TFObIkG$*7Y*`*9#gK;Xm}4(U6WY?=mZ3079P z;$MKi3vIH4EFaK$V^N3YD zdDSF$D}YlAXIZ;7#4Wo#84S3*!lS9~Ujm$gIAgX3vKTn|hGz7gRGXYNVZ%>Gl7%%h zpwNhRlOew#exO7%{ZP_nDG{m{YV1Ai=8YMhoJ^oYQ;l*h6i%icG6wHjPn|RhZ^{#6 z$iT+t0gkzRA1U9b5vndCiWuE=3>pQx`ib%5huK?UxIDxbWcqFVge|8fT~qoka8pw{ z)$ctDr=~XnAe3x4wJ=J`m+)W^w-LQ7>6A83eNO%4&*?0m7NPiEtf z$UA;dgx%s`NCCtI6yltOf<{ADMg)ps@dQ}0=i;b0SoYsJBDDoqVz`Eg^0C|kJOvrz z!z}WC6_ap+-}8e`*)hXq_!s8QPJ`4z>O@tDQB8Z9(YRx_`)}}bO(mJ(fzzu*_@(GS zBts1JatSgW3<&3Cs(0use4|+&~N!2A>W)f?L!@ zVhCUlZ;Ee9arJc#TM|7Zeuxz(=83}@4n`5#iLsaH&xg(T%Kx43tcZRb5cVgmI_$S( zh-6Kec!7L@wsGYMRDyGYDdoEnY|XdjaBPu-SkpMQ7?(J#A+=$hJ?}8m{;(aP&^#k~ zH2DQ33-UwqM+#dN6%`p39u=};XqCg#3N@Z$cZK`d6R|&N^NAz5vV|kXqb02+Ma3?~ zw&u*KB;#3$(xU}5MJm%1Q@As{#nbbmrPcGU7PoVha~AU}^SUL!=V|6Z=ZgNI7blhC zsdoO^$P-won$tJ8V?MA@Hp8B6Dwr;kSah6=m>FBp{?n{>8+YJ96PH+^T7@Q8DsCCP zfKe{XDaw{td!+p$_#*ql;hw~*uqv5Wo{^*Tr(LgAq*YB!YEXnsfK0DQ@UTf10@P5Q zaVdg&0D6F6KxUu_@fQ(@=#>~Xh5}QrSD(jvY|BTB=Q`3lXbYPo-!a3z|2^Zq>hb(B z&(z?(^*!0q{=`d5qxBur7z|ZX5Wu7#wEa%)s(53(GLr9)-`?C zfvUj}z@BY9pl;wOP@$1weQ8nMtZm>a{~D$cj>$KrUwe{!^o)gp9*QY8#fr_G;h3$S zF_MjjnMePnE5hIakZ95{P&Qn)xVGfk_H1=)aodV!1OpG_*UuRagLX+%PK`{1uCbuO z+c;BipoOZgqP5;wsPU>H*H}^4VIx#`T{T>{Y&B~UzS6P+ZAE2Oxsu~-<1E-Z?rfb0 zna3>_DykCIVk}n=Uo}}<+2F&R%D`#hxP8bfXWAyc8Zwzf`e!(y|3 zad1I#k+x>*X67O6L4DtJBysiqF8y%%=y`f{oNDnjW>IhBVIyV3qdhCS9&;40)xoML zx&(gEv{f#cZkDazCCxTCD->_Aa^|wSGlg%`C+V{mD9ty**9*831P6q~PYiUkwoTnZ zy~w{Rdn!9FJ5j%U>uD>p8(v6SXh!Idkh_qMkZDjRG*Mtsa5;2qKdX_$vdY=U&x_rI z2revH*l5_zA0O;Jtb$0>^td>DHg6}_k3=+tBC$>i_+dEpoF@WxDtC8Bn`8)N!g9sQ zSIh$pCeC9)>h5QCOYPN{J0QENee2#FILqGr&&OO@$zV~5+^#fNx}dm(^m5uw z3UYtR-d#bB5?jH8y~di;S`xR<2Bt3CX0YCXPvXIE)x-zxbuup}_r@+cdD^7nj+lgGCwwguE|S+|ajAF;&NRdL-&wPy4an%>T!kJodR*;lNVLl& zBKb=B_suI;G3(T|iSL9S@rP+@v5x@Jo`~M()aTr{gOmoQq%lub1jVaUYELv5)mf|G z%P=NMyEXTt;E6>vwi??F z2BVTYlG7t;rEjX|6tmP+jKciI?~nL#x-0 zracodo$ha%^-?}+#}#y6Due5x?~FrhUst7mQU5Z`qE1tFz1{YvW8q!k_0;mrvH?#Y zo16y{ZUnC5iDthsEw~4s1?PcX+o5k~s4(v?kECAwd~9WdQ$R<|OVMk!`sZLOW*b)2 z2k*<{dL(Hi>Y!6pCA$EJf}_EfQTK3%(vA{O(X5;ctBhrfi^)#PsVP~yH3z!)#m;lg zttl;-HoM#Op}{UmrPLs;hjvBlj&ni7a{coWUWE^~^-s%^_7m^=j``a99BZr9Q0LBv z*+u_~jLwM;?|sp|$cD6QHrvfq&(dAi%ZFPo4HwOJTyKgZ)GRVv$q2mF&7KyzjYnkQ992q zDL#~LCuJ(j^l+X>KPwM5LfdsJNh%>WsJ%)bogX)8ec0|r9(g>I-YY)6h9TPFJ$T(c z+pZ|CXKOz&vLvz=eX5^lu2|kq-~G<~fEmPk6tKV1FHO}k18tmv`zC?`wWDU@ky2cm zKnuh_N_+$Pgrd5%h5iYe4c@UgXtzKC1lMB%g=NhPbVKBt&c~mT5xkl$KAWBq-%)%u zAoxU2f3<)BqzDS6M+^%3o(2KlaTaKF31nx{awAKQ{*2?87LHxdbaNPhx3}p@ZC9~i zKJfXqbAYlF*Khy=LL>R>11zCPd<6sqoN2DC?x-#!&1qIB>f15dMdP^XvSt%XEbJ{~>X-qG!AYy zjsRC08waAlEBW8`2pc;X+MC-sn%mmo|5Y!*z}Cr;hmi2EhW^j%?|vG)n*ZOHY#jc} z))#|xf8C*Dprxn#KXrc<<^Jm`r@XnVv6Z^8xwWy4!&e);42*2_-2W-?|G4#koBSU| zHU4i=Mi%D(S@M6}`EN;Xy1zL19~}KHuK!&9k{2&DH{Jh{Juh_8y}<0(aA2DY%PD_- zgZwo%UoW4p5BYz;zs{xTpB|G&fq?jdB!mT&U4hRsf_-%66E2fRR4&`mwfbwUM%$;L zX!s%70||&@_ytYkl7_Et4&I^Yj2V9eMqWK{3iE^6rH_os{SfFcL=ugiMlBMJ?l)Go zI%)8lSR1OnqAz$BUvPNQbw(f0YA0Q?TH79ME9;%R{Be5%z{Gw3;oVgU2`0aETY7C^ zD)Emr2-#JG5PttqgD==y3{VgcA2zdd%0H5jpIu|%10(&zn~#_uFbNM4sYCjYB+wq7 z;Q!17Jk%Ety#j_w|Hy-H4{p{!R*rNf1cLJLW;eBD{7+p(`VxWZKMCkx5qu5Jzq8<9 z5&Sy~{#^)v*?@m_!T&Er&_x&29vTXDe0iC4kcd{otw3^re?K!fN35fx!^p_p_JoNM zdwu;gB_$=XxtSx18Hzp8cXU7Q>IxDH7Up*hLXxnUn3RD*oRO)i1OSkOf{s3V5{EQA zG7^`Z`~wOW*6FOh%J2a(IW;v7P;Xi!mz8*uidj-x`uo=}aC6(snC@-?!^44rfryN> z)YSOpWsO(biA3=&XsC3{LBR+dDM`tAt9q?O9I70gR>40XpFc+(585Pe*OaF@%{Y#kPr~Ijtks60-&@<=&&&{nR$1Y$lnDa z;0am}KtT>)ifBV4wCjo_BqZW8GijSSI25xV>b5Yeio6NV#wRCB`Pqn)`yL+Q#r5>E z?jG(XU0k9a(~sjaH+aAB$N1@Lg_O)xv>&`}c5uQx>5~Cq>5};5HrXJ=;_OP%3zH?Ysw5`YhjtVyn_abQ@TshOGg zh`18-e@__I8Ey83QA4+ z!Gpz&?d>H==S>4}wC)y7jw?Bg@t5JVkRZQn-@ya~pq-nGL-w`Phek%lsVU=F3JPifz3#(9L-hXms^)G_ ze8AL#`u?jM8?^iTgZY0X^J1}1oVpFP4FJ1y3;icljFNe*)p3t zpOe0Po88UM-y5tX(O1yUm##FpSj!N-)8G&Up_U$#JZAU z$qjG32j`0t-L*`ZIc_wCIaj!P^@lbY+@n$e`%&_O1%-gM+_g*jfJ^vZZPvo*?^h;b z_yq;IX(TTm&DT~2J)3JJr^Wj=|Ab1%O2}?T^n#VDl9sgFYRfMw5MOSoAE>C%td}G! z9@c(p2hU7#cVA)AM!Hg#;XhF5kd5-WL+sdUCVRXI$=i>rEtaM+-@*qY)-yj_RNh-L1T3_4Z5rY$Xc;JeXr!c&x<^kwtN6MMQP{0jwQ$nY<6B!>&lDr? zSkybvlv`t-&lKhAgcuneii?^=<(n8u2pZIcc_^3^m@91c3WB3C_+#6PoekI%+c9vl zZrKdZC3Z)VkdY)SGsgc=Z?+FInu&UTCt@m4AVj!}kpT87Z)s3~!zoQrqA@ixb5uM4 zB_bi@mFh8MBBmyl5hUCpVJGfs^VbGgbPC>m^NV&KyTk+E@EruBZ%$9kQ1sxSxpheO zVrI%IIWc79VjyNherIG_Aezn7VdPn$nnezX`+-ba(BE9s;#v?Nk0Rx*TOlf5bRsAj zKMcOjJ+w=3Z2_8f1tXU+BN8XliJ)O|2h+}p-Ikn%T=h1Kkzr`q!BUcb z++G+lIiAK${1y;};68`tKg3qS5~A+a2cFd|J^e{m#Q~8?v>t z?E@CW%fQTRq-wC~d1@^{Lz%Qv8bZs)~y7_$t8=kRl!1V%V`Cd6Qia9X-=Kt$Fuz1 ziBWA~gTTmk<@qLEc?KvWPC6m*_AwG>8Vy10FcS73LLu>{6L3g)Np#b8`io&l@I_-KWU|_KiGF!~yUuh4N_y zCCwU9D%gv;*Y_<7qNSWcfSNy!87OF#iEsXdk=hcB3Y5|U4j)NjJZ^e zT|{zFDl}Iw-jhYF+p&G?Rp zyGcXe5v?l#vwbk6tu%0K7F(!mJu0#*s=gGbv_!Mc5R@nNA%=*{g%{s)p$rfhYAAX> z%QaKzYIE!MreC5=m@T@GSIE~X0=fq&!z zMA6A~&{#I_@j#Tz$h%k;gmi_()ka)0>}K!d^|DB(9i*`1?aN`&n>nmc;sX8+5*??$ zYe?!|S=&DURHF0ZU}Yq)x_8LAk;y5!!~`zm;#pQwp1{{}Tfv-e6@Q$*aEVDkY*DvgTh?lHJc#4EY?5KMUL$`_%?!uEHSgsPYzr*v)x%QsDgX0Fbi?%&hr@X%-w}3fL@;oH zTh}c>n50r!fNo}PT&u^X%KQtlVFS7S?ZwEogm0*$GP9{lt3iQi|ESYAyg-u{p%Xc~ zj%Tj6?#ppR^X1FIg0~8SNCMSgU|E(H6iTAgj>1dTT}XQm;xvvP$Y7Ds!F&I?JuArz zY|qTpEvYDp&1)gr<1L?NY(yYWYuqINRw`LPq1+-o*ypLt6)lGiMXd%x{=vb@Y9Sa% zMHMA3X8Bt>0uIXJ!V7RHyN+soZ*Wq$ptHNp4Pv)(1fW+`nVw!E|>fEg1IsGNi zP+Oc2!V|guW#9tKOz6*ug|*i=ug4Km9f=`vIkupq_g&dUH97^MIz1O1&Ofp4lRTff zckq$o%Z1j8+AXR#bq%=Aj|T!Eqd`E<1eV36i7%;=C9{~h6fK4qLh!ugSgd79KXk@m z=GV{N*n@rdD+%&;*7@4$f}`ifg-)}jn`cP8=B+VH)UKv~{g^mr@el+V4e#nv?kwD6 z9dgu*jBw6~=~)wg-U-ml)^R#A8e2mEYGwneU3U#xEKA&0Pl9~0z z^5hEj?$C4|=d_nDc<`W6@4~5!q^7D(hlIrL2v4ULu^lZi)lR{&QO4Qod*%h>GEs^- zh31+={hH&fjETpE!{k<0I)C=<7zPDhC4HJPb|p6fK!L%2*4ia)oRQG!Q0wesJOC^a z2Dp}xlacyflkM)^Wg{l&Sfcr(<-Eq*E2Sa~O~Sxn$6jC?{n4B>`~4lW#6>ytz4ume zk&$@&Sc`Pw%>c7?537IP5b6=#LXgl`oIE;*#~I0aTMuFvKR4NpyBO*>-t!w4e^OOTQL^wLJO4{8I-iyCr45i;rAN%gM=-m0* zATcZx5uO;QB4U49-YDsb;Q<#39gpflB~7B1O|!_U&~(L%jWTaX%|7}9iD7^FwNCjUo4lHT+| z3q8LY0KCiR^WboNa-1~IWivOl$2hz&wD*I?;;%uc^84t}v(B0GR_}a z?088Qy(l-8A&1~WVX1ez=?SMB%?T6bU_zq2MdXN!Zxtcjsa02+6xi*nhyj8elY1CV zFWj5>K4)>Kn@~nE7Bh(J74JMx5FNaaj>~b1o|&U}6cYDsiv)Ha zbwKX@#4uVyLnCFnl-KRoqtC7bz}D4OoiYk9Y4hy>=qRacaMO0I0l_xAJ!u0Ji73=55R04v!cJ|S4%5{XJ^F_V*rD8j`n^V0c z>(vH}J@2Mvgd@q7UDIspEKNXfBzX&74RC8_lC zSYKGM15#prcYe`H?4*g4#R76F#g1#(`qC4unTnOJ#a>eC&MV(EJy*PULHLpzM}o5BHQ8)AP$d+JuYBQPs@i!N6--i~Ga)B)S;D&>Rg48XTqBrj9Dy z1?-Uy_q#K`d?pOMrywUMSkw0&o)&(=&M6ARUhkN$XRx*Rxy8&)65uPP%T7I2bIA*E z)W>wYC+#JV{RFlAf_fR=wN|+v!reUwJy-QG$K26z#`G)KD)xT*%a|c+)P^ZMNNqZNg!Id5bfEW@l9GKPYjuXTqRG@N^TaDESy6;c?_Tujz;3X6_ z3$=Z2&q-m?d;T(q{8p{~ny=6U&rg_1@_d;4qyax1dg z9ozYB4=-3VOLMuUHJlU^!$~zVtHd-JB{L1ZCIQ}3+2sYpXE*Dc>lDLK)h$dmeiytA zLiRPa?+=+3K~ar$bwd!XE~7byPN~1R@C|_};9otwL5eLl0mMaxH0L_CPDo9^pqLjHhgK`w z@sWE$Ys?K(%CtOKcSty%F*_^|H&?DlkZ;B=z3GZrG%K^4lvPp6)&Oyi6R z81)R87A4gkPl9+SHe-RH*h9K{gd4|T(_>;vW|2-Y6IuF~?H|`&qyz-~0^FP5S`6NL zbWYHni}ji$-+KgKf<`YXws2mt{BZA;{P>R8yF|7U!ZuQUCH||)oAEk*)N;-^g zbQr&Tp4uk5nf-jUompoiXY)RgrO;^ARaBgn;hDBPAax@m-Xiep_VYHkv5HZGt+)Sk z5olCcFhms1O>L<`H78V)nc=;6r0^X5`PQsF=OiV*;tVCFY3&ui;IX9yjGFt4-|DkvbFU0%X z>`?a8dRnURAhrv1P}E})BX7b<(vR~)uCKTnwTu$DK5stnm@mcgK0 zQ@X^wE$)AEx{@%12 z6OnY!Q>{(Lv{Ie$$5XmT;m#>N(Xe-7zi6nY*OJRGyX&ZE36EF#^aRT5dFTLuM|3w% zYZy`MeAdZ0vOzyv`$CD2goxXip`O_MN`@scFtKA={y;z9YE?=aXvcFu@BOYZpC$e5 zVBouahrl~ay!82eW#r_blJh?F86L1+{P}3hho|Jc=?(H1`dg^rK}H!hWI1^r*K`Ew zc4FH>wc21^|7l=#T?3^L^3sd^jAMuJbTZvErEe)QK6e4KAIB=wlg z=-XIwgtMx++&qk8Djv<|`-O|~sp^Uf9&Bhc_21h}iLDbzb#IG=my$}}PN|bps(mA& z_jwkRTa74$hXvaWS~<^C3ZgRroAyPBDKRaj_Qy-* zXDVEJbe8{m>({`&T?I21_H3|q-0~n-i)kJ13T*G=i0MC^U2G)m2k!^zbjBwS(LZk} z`4Fu(+PY}#B`F1?1-0XqRIyQfWAn6BSW@zdy(Yqeq@7nk1D zhDx<21fVdO++WeK2@-lfK03Qp%2D{?+RhL7P&oM5ORr0x5jbd%L;stbEZ#S}=r{3m zHK)_~vbGO865imFX!h%WxtMbU*KAL^|&w1;W8>EXd}3#`^2mELDV*?JeMJ{@;-TOT8VognOZ zh2EPop-{ka7l;1XG!B zAF`x6r@RZTV!_%(x^9#x&YBU6st&;eY$duxzw$_|0?mb#9Jj1|he#7)fky$kRfiD- zUjz|9_36aBkiRInb4)**?_kbNN6;j?w7{tO5_&SRenNZL&_5-DS^bp^Bbv<0ECMkB z9%5y%jPT!X*#Fy&f2Hy7-1v8;{Hr(qKi8ra-*pwV;TU~oGM*;RxADtKq733;+d+9))4!sD zS`zo|$&948HVcDWOAtuUNQn&%{h6ZXH@OcTtDhw09}qwy|BV-Sk@vsmqq^|_-F(&S?Smd)u!= z7XuHA36X9aaXT(Pof2`y1$Q*~E1A@c4(k<-gZRH6Hz}CiD z(u8!L_1MzHU0NUVE55n|9T^uB;(Gh>*0Fkh*nA9?f_eIT*_@hnOu|mQDNuy$baJTF z!rR+f)1*-JKF9+j!52R%FpsA_+Lyn67PG(#&M@ilp7qek+h*Y@(@)?e;KbB5rYqqq zwbP}bFY)14>Kt8GMt0z{DLESE>lKs!E#oKo{=Zsz3 ziPHC)w-&z--PvF%-A^l3QG|*->!4r1PS&`3#yV;E;8U>BxoAByeHwr*GvWD`TV#ce z@-?J8mRO!VkgEQk4&fmmcZF4pOY&4cZpFQ=wVOmP1Q>-lS!Z_hX0@EGoIl=TG?oZP#d-eb*aV5^Ebt}GcYRi4_s zM1*B_dIX~m4(-Xj90cL93#($|_>=2n8u{IY zZAjeqcbF9u65?-C28y7#iP*KDRH zrgQM;vy*cp+wUIIc(n6=zFk02Ga@ixv-$)a3=|@a4))`3SPoaoOHEP}opdglw)X_W zK|*&N3L+!a)_hcKa!#SBEoRuZUV(fPTe<)B?FhdX=U^0ejNbcd=g9SqP|FU26`{h_ zEpmwt*e?Fe9n8>>BsEl{vXiKJ4De^%LnBx zBbc|LAzW07@PwL=DY7LT3IrY-K4R$Lm@;5`nxctTYc0J?(FG_EFYX@sIfar_m+g3M zROi;?Cim#2y;ak0RN;GxTo()qRJNTTabrg#x3GBj+K}Ly}oV!-^$*ZWqo>mjIM<@ptvE_L?QkrHNZ;EKE4&k3&ix4=Md8yUw zMWgKY8R6yre6-;(nrLy)gOyTq*@0lrg3A36xBCX%SbEe`RVy#yOT*t`p3|aBTI$J3 z$DslkSZ2RdLJ16`g?bECq{kMI`kG2xXyr6F{^P>GC9TkLudh%;Jhss#3`U914!dI< zU2;JHy>+_PGYv%~Pm3tOJuBWC8p+|m8jion*lz|zi(7fNDIcx1>|6=lufrwt|8Vhf zaJP_rpK^X56L3>HEcnVn(yj3tE84yUd_X^wj%k?l=f+d+4bhh>89VU-F|_6xZ#I&bS=Y;D4+0dqpn0g>}tMpQYBQ z2UTR_h|AAH>qT=g`!fBYnfEHfd;C2C`2bMonhMd`CsPtKpMub^RqtLJ%yPB5{E{X+ z&d}j{nK=dr2@B1__%aIQGJd|s)1<<=E*h7$@<*q34ZH3U z+GxwK_}rIDcNj>1(OuG_Mx`(ASt;qAl(ZcMq~PK)Pg@Ce=fowq3#eLo%t3lS)vlNP z2|===@DX^CQyaS5tLiIA5butvln{KFr%8^2Z%Bje+9(V-Ae(WY*0uj&B|7^C27YSf zA^B@!PQqi}XLL_I8rr>?QoCZt(q?L?kJ*&_$61gbVs_PG*|w%dXH{OiOq3b8S~=}f zL%KzLclU<_(3m9@3UypmbY$uc8F{FE$uq+Vj*xNy-NW*0M>a1YzM)es>VyUEbh2*` zg=w?yZM>vJ;+WZMsq}Vcv(#42XZXRXl9?x_qm}*h7v70|&>#KeVgZ)tmaaB|t!R_d zDVLno(FN|$rR{gp475iitR#IxpvmUzTA$?SBTh|S0g(vp=~!jqS*^NtsE2 zkPd@k2DxBz@s?ePw`QhjLFf7uw07Thu^umC-Z{`g#*F0jSay51ALMyVQ|7eQ&RVns z2ma57vJ_qFyh~%Tq_iSF>GO(9#^XsoZ$pz0bz!_?*1A;aMZag-|{R6&~YWsFk{wC+*ZColAwbe`}G=E56S^9cT|_ zj>?ekk+0q~pc(U+^N2Pl>jZyjSVf4A8oZYw4RaJiHD76W-n6ccCIj5t6N!9cIvVUF zbCm#eUiB!tpRh#4x-z{8C*WMAMpb#5KB{SuZDlNv$<6^n=%)`knR>Rj1RTwmNsJt8 z#(krGZ8ULh2EoDuraI%sZWD}u*-s7lF2&cN_o2YAAhc=*Iw28gH|0feiBD1I>B$+r zgQ)P5j)nL4YZwJ9gay{qT*`w!CzS+{&RSJHaX`N(GB1X3{ctGMy-+=O{7W9)=Qf4_ z*ml9nVvGPXOUMj$fTk42?d8S=EfKGvuyFQ>=|!8Gs-^F^!TFPeWxD3AC)ky+1y0vo z-S_b$b*E`0#-q(G{;WI4tk{WJYtgmQ#p3q+w8qhLmkq;QFglgAl?;+iI=;VWj`f)r z!CAeT5p?cF*~r+z3635%$vU`;zZhu`{dhMk?31q zm1*Ql{o|~C@)z7`%SaZ})aya!z!|F!I4LoMmFmkLc}GM>Nj91YC|lxm@O@sxV$25V zTG~Wp9Zp8e{3%Nm3!QSKKba(2M}?_gfI;+H+WTJr7y}MNFs-xHJ7AV%frscpd7k1v zne5Y8&paF(;8CD(xXD9#1?a7V<$N5-W%4exEcx-+i}YY2i#^*>lxDeZ|GKU@QZV&y zh}SMwx07!}>5s&E*?C(i%EForL#lb%E!Odc$y+rWS)W8>?xYz@{0%X|cG3m%Ksq7p zHCT0a3wz*Keh5OGq~S|9pGpUZng|dNcel4jwq&KpnGkARbbJ!%q^Q>>$CZM^$sVko z?Oa5;M&tO)e|H4n{<0VdMBmk<_ZFvPDrvK6{ygvu6g|C&p=U=xuLi|}E7kQ?@cHQ3#`A6>gS}b?e$wU2PK_xP{)?b zRtPVl4rZx{ekgCQH7*`$)lFect7Q7 z@a4M#rbQ_;s^Qe!pVl~W9xDT2Ef9~Oc|G>13+=1*z<0lL2fgP$Q#)MV=u1cAXGFJ$ zM!w&_SE9+Cpts-F^B%sQleMf`;ap;(*2FL0YfK_IOW`fS>2ECIF*J>{A~74I_&?_%fE{zma@ zC7Zka$7Qd-m&;iREHjABg|#fU$4Ra41s?(LHb0f1%P9&TEPtYFE(HMX+K!LQTHKys zeX}S*z2B~aIhR*k|3>tHRI_k0fTnapH-1DB-hsK&dw}kjiuC;x<~-A(l@VTY*KZS&6;@(oJ@>J3dCkzsGwRthyYTEp;*hn{kgM`s zkejA`c34(t$U-}lh4deRKNQ_bgs8f)!zXD9rbThbAVaX{a=Ivh>G(>e#(K;Vszjo+DOuRt+vD)Ygxf?`$a$hTOeR6@F1sQ?I=s3%q?(_IzM*{R-Qh%M<{611I*D0uf!T$xgS+4vhd5znDLZCk`J$WQK z6tY~Pm^SC14MhqKeyCCMbAb;s+0GC>XPbZ~wd_#8CIS>hRoyj^;wUBG={v0|H;TEV z(2AawHJk#yGAz;wvc8x!_5w7VV8%3Eh~bk53VQ#@M2t-&Z+0oA8H6l_V&qSrvaW2J zKm)y#s_!=;Ha<~Pm$BOdM19-XYeJ6F=@O^m3-so7J{>}48MZ$IcOydxJe>uXqs;KC zy|_qEph783zCn1k$(Q#@-QQ|{azT4YarK15jqeD<`Og&uDsN3`z8tX> z5#IHHYjw!oDQ^3i!OPcBlg?w_&$Ux2Su3L|e+rFy zG-92Hgj3SWyFa{PFomt><D7)Zy5uUDQ49EoaJtv8fiw+p)Cbqe2uFcP$L%WZ&cxV_vX#;2 zX7gFov!l|iIQjK1B4X}|HL4TL8#~@FV=2WYs%D|qzK^bZR>6b%fM5iS2ZQMM6#5MC zw-E;BgZbDjo&c>BU8+6UOdcd0f&b{jqHLQ#?VT^vVF-&8;gUdO3mvaIYc-3RLQ1D;e^KkY1iyDU}2U;`30vJ{N|bndb7klBL{scW~3XMRb=oHRJeCcv$x^t!V;NUMFk=da`iR7qT zcC;3#`oYpa+N$gzJ!xk%@A)iqqkPC;+*?)VxD1{51LZ@8`QX-stgPD1$>#6)<{^Jw zAfNY(r$uw@;8vMK^&wY@tZs$+@OJ_GBCqR?#!jBz6LO z>VoDB;v^!^{xoZG(v49QSS!-HV7+p7dCpRTjVFd-2;e7Ml^FIXH)R(W-}LIABliYXHY ziH2h5)7*j~?!R!{Yh%gDuV(EMo9Qot6%vp|dgT2EcD0`HchE;o@cf1k=5>B<&gdf} zdj6Z}5^dYndTp-dy4Gz5e8+-)q!AW<>n57+V^f$J;N?|K`nWi zxN;wKX^U!T7lcdFUlZanVSd+?qAkkgX(@AcyE$;kBq8IRF;QHloxKMb2zOUKUOTtj z)yJHEx@$la`Hz2MAOvjf^Q8t@oZQ?!hlDRM0CF0EW`F3M5#gdGbWon|+(nnTM~B0V zgge1G{}{>kkMvtMeaEUAY!?*$9w`|!6qrsT;nmRTUfy^qxFPw1kE^k0*ic{kU2^S^=VSfNgXPENjXhBPCWyX+f6CHd9ZyB1O$Ec z*M9-9u&^fYq=yUy7@}jK=B)^?V%Im_OVM_eZh8^hzQmMGO(q{>@hRpp~T8*}9^)}LdZxmT(!EVeIOoo>PNBht*NOWJEM;^+rLETCf4I(B@Y4v+37ai=`~nK* zgn}{UGfNkUd=Ub}ZJ^!!a}Hoal73TYMF?Il7>XoX=~_5}yslWt$e@Qi!Xu2{L5}-A z*jk6C=C=T{d(H5r4e@H#!(7N@(_>7tyj0!Y-Hp~;oNCf^#6<*;T%f_F+mT}x;9ta}4#ijkhuI>jn_fg2qE<+Iwyg_mh!-PUSd%ABt? zH8XbB`5VOzQM+F`sQ!x1mFwC1q{GI?abTr|{GB-4lUllY90RhOX%x8pVm@COWfE;F za&oSsuC4UJuaBo`U%K*z0@DE#{vGCj+}7QE!NAX->hq1x>E7Pns#aF(Dm96TI1ek1 z4VfYK%lHn zbW`3luFCE2qLo`#2UU(^$KYA2&6p&o^ea806w|p*nrYd}s|-$fTHet6U(45DZt3Kw z^l!>>a%9}12G){QnX2VM)KqWsz|Vzjsn)_=y~>iwXmdI1T8EXL)?tLXmA>dH;r7oP zFqd&q9+V&`t<&z%6`iA7^aj597x@L7$!?{SK7SRNjxv9UH&!&WGM=9Z%6L`X1hp$4 zGjMvCKZU4I`}Cmi{_I=EUHB@`Bobo#zR?|u;T>HLt$0u~uQO9>(Nk`&7cw-{c{x{g1S-6*|ROtoYZfItY-}Mvw0ngzx@tJ=J62f_4g+$_I ze(mYcKy|p18)_xe1ckfc!Y|Ii2*oa|E(vcGJa~k%C2qeo_urL(V-$e)E9)dDP4qOa zs*SZw?`Yj+UZ|Pi{zPon{_vM5wD@q9U1LDl;id~4RC})rEYdyh)ewDn(iz>o&F!Z@ z&+J&NQ;)D3l%l2K%yPO^iOj^IMe&V|BAK}k1j{k(Qs8F#pEyKjP>+ntqo8X6qTz`9 zM|D_jZwu5?{wG%y7yOeVoVxb|+4GtK`m@KD zjsObcJ*DtVqB*S6E=YX`zK~CgT8rx}FuU&yKRaH&sf`N}qS<)D`3Fhv=nv-WRYLAJ z>3)^ze6`&iglQWab3T9m{6l7DSs6xN;@V=@Q`r6Q-LG$5-i;k-I|tJp9u;3a_^py$ zaos*x6keh=3Z6cwXi6if!Heq<;E$qu`%vOqVtH~i@L~Lz&IP|S&yZZMZhc{IlQa2( zN-_*LP@#dAjWxIyyEHeJ#H2`j?f67mrDt>By;IX#cgfvZ{4FU_(oO_gJei_5*to{1 z`^^w;Z}2Pg&iJJLQDn4&9IHsaw0NotY%W7F#~F5Mu@E+)4)O$?+Lq#dk-Z03X)3c^ zo7>o(U9Us8uXP-?!BFlPe(YF6f@xKO@JOtX0OKeMJeGx*<0B$mZQFtydD3i_;u?VP z!ag9Ml>1K4#Kaja4cw*QzI|gB7dP5ytOhY&CzI0UGv2NDslY=X5_Fr&kLwOG<3>$G z^X*)pXr@e3ZsRlsDe>8B^f5LQy@>lQ}R^&PHUoGlbV8W=8y*AHZ-4h^oxAhK)pHuhHpE*p zM#?BD4DV$a1QnUNfAA(i6IWjz51G2)iT>bEW!v~U-<#)({JpG=$uC0hin7GKCL3oXDgZz7wo__zELJ zjkjr-Q3Lm*_!ZAu(V7PO)&ZU!IM7DEGmata14j!wfK_c@z&)BirbWQF9H76D_@IH2 zT{~s9v7LPu&ZT8j`v;%=R{dv2h-!3X2?-!?_Y#VC(Ne^?t?()6PsZQyQ8s4PVP@YU zt>W>aj?=$E4fzv$+-2i?9ZeT+na&q-gDL3l%g3;}g0`CaKi(Wd3LNst?wLXYM}Euk zr=xUMWc&?koL{3&!TM7|oc(6Wy-{Q_JPZQ!Q7m6~fMy`10tRQQ@lAO%xWF7G4zg=H zzX+q^l|3t!?AVPTDmk3{75`Dm?r(wv8C<;QcS~`L78`sg8dMEzwHS4RFV7Mg-+MLfe^wRpH zTJOBK3`QyFITKUi%sd`^r^iM3z?lE8(n>UU<0&J)t$(~5oi2!8_PPhZ2rv$QtFIpg zlRGv;{W-X}?!c(4;V9_Aj0f`2+n+?MBDzi#+s?rDm`B{0cP5X8! z-MSPili28aT|wNlOP7}JkzwatQ#9TNna8Z8OtqP4Ke2^!D+SAN*&`B(Wx=bxR-Bw1 z!pKj_yp)y#C>-D0^gIr9RR|^n#VL)@7vpCTsIZ8{DR%pu_XCmGz@_;6Rg+Nm^AGqd z!7pp+!_q9ke^)&AX}+l!ezj6uE9%LQJd_+daw_K|<^6jh^Ckg5fk2gmR`-;MqXV~5 z*lx%i8nQE2QZ+ZMZ1y|s>_=b7|F-QR-yJ5i!f0iJ4D+tYAFhP7yQnBNYbFB$LHUR5 zY%UfS7UhG;;EJ-ck6)i|p42rL8w{eQp{9LiS*Krmmv?_YeF`A?^+Me(J?;LGH4AUj z8R`)!H|y6}m%=z;0!PNlN3tyc6) zO5}C0q3s87KSQIKoHPu}X(e6O2EDv^JrwDSwYCDC9(vh^p`HEsYS`+TUqdO~{k?CEL85asdz z-t7k)h0OCWznbDWicSy zW!WdXQo};H;tUTTU+uL)I`vAILCDh%U-5cS)$d9pE)zd<4$b){Vg_tb+W;cst;+*(dQNis{i zSG6PwepUbT$3B#gEs5qs=V#hs3j_Amkn7lQo@XLwK|X(s0-mboekYp`Wu_Ltkda|9 z5)b%hHBCD2IWY4fyX%$p=H^?cciIcaCPnAhRsq`=>3MSZkr$nkLJ&7M zJ~jxWdfOL+8x!#ARZ>xL@$l~SpTB%hccH%~cHqzRh5}yOE zl1wQQLz}VqKUxI87M0b>-{0Re2kVs(yDzecu7;#nYTXf&lb3c~<%w)W%YGLUCSQ!G zOQKz3L%RLEY+%Nh4ewm5r^riY4{;(JR30kosxkOEL)&pR$hCF0vSe6uStUn%-(xYn zb=by2PyeRmyi4nkzVAI$>gso>`k%~q58Nv1@>VH9yY&Ijhl@gHV`mqPVWi=riO<$#DlaWrSw+W5&64ti8~g2a4Htan)ZU#|jzx!^T(^c5Z!r z_ZAbvdS6us@A7N(s2}}^WEnp&=hcILpz*l6s(*Wb%>sAkfS=m=PoKcx$1Tc3`<+rr zRuZ}=7IYuexWXAyUdTwWhxZT=W+wAQN^5FEejDWug;4A` zfu6cSahhE4GnbU29BIPLjN&frnHHR1mVSlapJiEbzpi|GymUI%kt9q1uBGq(5Yu1c zu6sxBU=D06UoVtjW(z1X60*3D_XV0Qve2_@91msST=}U&F67H3&WjRvHm^r`K785k zyE%J!?#*1_=2MsoDNsCRm z&wdh{tA8!Uc~~|2&3vxl`pV7P z+H|hyVQOa5%{#o@%btZ>O2PM_MSZTYO)nL}I$Xwe=)P$~*c|q3miqPm5TEyZjH zB0z>Og>!#|O`3IX}N9_{@wNkQQ`9)j#1uS{W@s z-O}F==HTS~@Xxy%2)X;6-u2j+sCjz_&{(4!h}z*GG-qA-v4i{Ez*+{@VF+R6c-5*Q zx5dM46Kpj6OY|r-fd+J`o&zHp|-U7mg<9ga*`G%u2Df4 zHY}X8-CDO}7WanVOBG2=LQ_Fk-MCNuGBGmN29r_J0>xFJUK;rd9BQC3Tq1=knipv7i-EQ z5sYPy6dwu;m=9h%W$|jmx_9jlj!FX_!r}|={_L)nFISaMIIHSFT6EkV_M|^F&D!|! z4Fb_3xz8SR@5Wyr8oYb=0Xz{hO0%J~g4703p+&xBa~Od{oYsBg-CY%+#=hR(kzOHI zR^^TJ)!$kV_cOWf$s}4W9~$@hbPwAt6X)8b1jy7o71Z1n`i_ zQTwj-1~|2qs08pfH-eFR)~6N&k94e#!*IMsRSh+=Y)O~P&Jeuob*&7Trz0auWQm+Y z8yBPYvi!(&{T(OG-oq;)d7UBI0f!zI+41&*7?63%#8NR~VUEH`Y2|atrelGgeFi($ zy&J-F$-%yx7>G%6eJ6NG;>-43YUK%j^7)0ui*2NeH7F_*^Tvj!H0Th`=g~EA*=NW? zs5c-YKF;o-GFQ2w9viaE^GEqUVVwPwUUp|e?{?r@A{UXcldFxd+mT-|>psNZVS4Au zGS1wEaM`0ggaWJ6FE8+4OJA6bt9ufKDE!k0R_e+b$1q|(`&_i=!?v^}=43AAC!Q+V( zY3U_O!b*O=JV}dFv2LM8j8GPj64^(`-ArYWA}VDRKl$fVfgpp;!Z9DCVS<3i;h{u& zjCGGxEQsII84por?CCZ@Hl^HN+|h|ldYp2CEskpnAbl%!3@g2KFX98tP=@cR@MgQ@T8WcjDC*3P&tA z`;)lLbk1*I1!&)o2L1c0_#J#9d{G@+TXsDP#2do>K0(T$;e=zv7dhq^1y{9d;XfX4 zV^~iU#?InuW0mH9_h(?)3>EKn1IWdH3aSbP|5|zEA-b_^(g}9`$f|%x;2}Nwl|_hi zrkz(S`o-(0`REWB%K^hX-0k=|hmLmXRpFEn=!p&nncRV!#M>xoFfLx%57Lhf%71-j zOOYv22gk8ZpbPsGbKm!ZjK>azt<7dG5&9b|Jx>lp-Iy?4zWi+0C<+uB&NLBUO~kJekj@=058!PC%7XMVX+B!o zC>GqwbWbpn^re4DcK92Z||fs ziGZuk@(h+&?(A|6IWc2n`>H|uYmBQ}SSzHW-iw8oVc0gsD4cmk?C;gDRA9oRIJ(m& zy>zM=cg(0MiOqs`)MTfbw9!XLGZm=SHiN;~Q)Y+gQ>s=cb+uq(s>GMjlI@tiS1l^0 z*)IqJQ63&xi>sQNOad=_R4pD(-n(aF$VN#sO^tPS=Ty|jmi8S#p0L?{hOULHTFFxG zIsNG#qARU!GAKN_%w+GmqkBvV2)4CtAUI1_4?36#(bU<0qxIsY+YDT`=oym)0rvB| z@O@eXn?Pk9h{YUy8SPILR0`a^B%=76i7_g<1a}Nl-7*=azc{qHnTg3y;%CSE>S0!W zfM+fZ8zbO4mac5iKkeh-EO$>9d=kbm7_4D*{W-Y8_^KT6B))ajCP!v`+=pL(K&Jr|w;iI~4P<-ds>trB@)Q*$Gy58AUfzcht%z zxGQFHF6QsnfQmpIO$tnhQ?tvqQMDOxALWy8skKoZ)q~nFJG>d8CF|i8-x{@=T;9F) zV4#2PG<#|;!(MSDAJAMm{@hn45*J-IPr~tbfSX5Wj^9kpfq2;*_q$x8q>Jj4oPSmm z_0lp@I=}BI&B4n$Hz!gnMb3$wi2tcgtNdM@x6nd;}zyLnH)p5EB0Cx>KbfcuNvrkUxm!vNcHN`Eb z_e#+neCh)OC1=UEWhd`bdL~z9t$$j}%YnObA~Zn`5kB1aZ*8Mb>BDyc#!{25K$efi z3Lks6x*9Jv{_eL;*9#5{Ll6`3b;;tz@+$2*(OPB}5hdJ9ZHuxMpR< zP!06g|@d)cQJ9GutJLL_7g)Y=3mEek$>8cPPv(q?JJ zYf_s3Wiga26kx8-#M4oD=6JjJY~2g14V3}^5_^mcRS;48 z_-L-PNrXovf}RB@aMnV?(;D+z^2&?ze)a3xtGv%NTKcuON9(-Q4*cBOCAswCgAY~0 z?;{8uKh<+Bb8UVPvy}thE!i) zB|Dre?#o%vr3*e!0o>}3g$Ju$7KH(d8f9qUZ7Un(OQ%0SKPn@Dr+`GMNIrhl`&-jB zp?fokNpea1VW**hLfPlY0O~-sQz_VMcv!dHm}FW3X4}rD^sh6n;T7V%iAaUvZC-4{ zp2Y?R>2M+xD;1+moF0V<3A%10=1h8Nqw3a93}J|AT^(>ynWgBo6(+=tvIU(AqmJMC zm$b56_U=U~E@oYwUg6(zd|94NrmU2@Ehj;`9Bcx{R~Qg8W0V?gCQ60_7@jY-(*?_) z0}=;FtLe1i zor9ZG9>2AWx;Udk){&QFnr@?4czwz+ASXQw##xEASM9Ub%9!kFDf0cF%j&*P`iNCR zJEq?bjoy;igv&Ka-1T;6d8U28g}X<0#ovyzqv#YON%|-H9QaTs*BgnpYe`bd2;dWD zyaF^;MmFQhb{v=x!<|MU;c}pZoBQuIm52~AD!*wz%c4qA$nwmmFOEU?*#XtXZc(hD za$xu`bCNTDdycJA)LC#u)ZZ>(mZo^#mn7k2-Gej~L5YSJE7-e2d9(idsiGxmw}5u+ zRH>IX%ZXlyvaH!MYU9rH6tSLRMBbetMvv_6>rIYM=7U3zwsN_7O@$U;?9+%dOzL;>_dQTbPpHrzR z!cPZph;;Qk&f&cYX0AA}x61<*OVDW+pL#$Q4K^by@*BzO}I4T`bA$Zjq5W-VENqG75^=E$Pcc_V90!&vHe8}i)Ux+3rIR~lG z(D~d=z)#=1KlCKN$04hHd05s-hSN*;N%}txhW(w6p?WWEZ&)2v>GZP6JXju0$Qp0g z831$#Eek@q*E#gj?1I6x1#oZqpX8+6$$HPga4Kz=U#~wo5nQ7YRmE;IV9-mWVW0AA zk|e|@38ZlBa3>6~X^$qq6X}9VV?dtr)O;F&$Hr(wQfgzzT1#Qkw?<40nS=vZlVK>! z(RN;6uz9O-F9_c>5cl?(ij8ei|2h@^2luM!WJ648AqN7E4i5Ygv{4}YrJ?lWV-rC012_TG= zc!&`%z3&IMP^*cAPkhY+a_w?{ost`H-*A%# zGiARhxIO=@+(Irn*f=@5S`D(=5 zu-E+c8O8hBubRyIuC0pE9ru~$ZhOEa?n_~X-%HiHz#CN?d%hX}zVT&|@`0uUnZ*ev zOkDt1)-DIpU{SEEB=@E{E=pt7ggOb}Fm^t&b&_4r7v@OyX- z=@b!)(}F^+2(OEay*YEoApHtR70ZFuG({)tz4_fOf(l)y&uD*?R{SK+?#qMvM1z3& z{yu~Cv2zyEw`vb7CV0naLYJTD|0=HS^-DE`qCYQIU}3Q+X>TcAEXV@G?|MSTk#ofq zl>@xNFoP|L?cR0yA+3TzK^GfbtyfX6p#>=#QL4y5m*-QUafQC!0pDzKjb~$YIUI^5 zs8YABXF0<|N@8ihR_7kOx2*abaiHy1=PfarwH%=pC_npk$ zGJ9*M?~L=FTk0-z!XSQy=&EbM7^sJ=bkb3bR}Zo8CQ+4*9zxim&2$;xRgwY`{CcDS zrJfZ6Cc_6-J<_rk?3DNfXonXkbpaJWR<8=PZT3RM3hK{sIU{^;Ce%ro0|H3~bbCMm z0kmIl(ue%8AlMMP^ho1e_ z^Akdr*!)B-(eUujSjBn(R_(aAtn)B~V?&UsP-`z$5K{myR_hs7QxEgCfMPBASeE8i*1` z*fsSAdWj}oK@%Z&`=P?4Y}WJ#j_5xIh%Y&RzRskCu)U@ho_t}bx_WjPYa1D&V*Ep6 zx5F}?hOlTXzmNr09(2(eE9?7s6|q z5W`BpYweqawcI)nd2RA0=`k3Yl6HNYm z?P&<(%g&bM@=;dLjqK3>)XGE2=KK?;hv_v@(y!GIzmz@2s5Y&9swbpQvDCID0`?Y$ ziC?J>o|D#|>hvhZ*!1QzI zYc>9}wtMTO&Ar_^OXzb=)rXIc6~8N1ufLy4*HK{S;|~>MK@3xSyQD+TzeqhEKN&no zbv!*UxOW6wxZ&??wWtEJcGPG)0~P|$M(!ht zlXt!et8s=+-FxRAWa?dr*SZ5$7QQPmsm)t*cPnmsl&-zEDZ%L2{d)cp#a`*4!1fuK z%f0urI%Lp}0d~2RInCJ`%zfN>Q!di%z;9MQdAL8DM^fEQjD`4B!Gf4<1}xVf76 z5N;SVwmRpi*|v0l!D>vv(G0c2!+^7c8cj0#hUnVuv+zDTq|rsmu|-M4Svocrs5Hx; z`|4P+G3+pW+*lU15P(U}x=mF7Hg+B-Hh2_@@NZonC>cGfs7{3qEHax&A~WqxxcQ~1 zONp2H8a(e)c{;Svm`8Q#)^Il#dI&k&z9IN8#N}*r=WyAp(V`Q8XY(*Rcc&F1*`<=+!5s*J65&e?8;q)PgPLUr2J=iHuAPauaV;;h%&i1}RG!prtZG z9X}S&YRh_Jcf_gd2@n96(fRu=Qxu*<5o7DZ*5OSxv%{4ouva63*`sA&cIB^-!s4a! zgIa|L#I+R~DAa6&K3$_nkL|Tj$85fT8TB8+)ADMp==_V_Vb-`9EzQ-3+<0v6f9ok; z)%dZ8+~}Rf)1@Em#Ey@`tE`t+8?wG?ZF5~`;mBhojXw+c*N)TNWlp5$YvTwYD~~as z&W?HhW#JswSZYN{n>iHJO6+YigwksAmC%9m-sloNhierP`J8y5q`X)}c&z9z9+`7Y zL|5g-8x&?4OOnZBm34r2ClXQ0I(iuLDXrdpH7ecOK^7Ew9OqyI9+u42S;xViv;XX` z`^FyM=eY+Dur~>|3y19kD)x*W+V)c2r*GpT{qx7vtcb94BcYb-4dJx{o?;dv&(jdpKVxEGeG;_F168oeSe#8p1miZPdQ1SXN5S;|u-c zY?s4*2ugC0-Va+=YFbVDF4E{uH(c-JH>pYfWH<8NH$j-RYaI5Uhc}z5DLSwb=$cGo zj>MGUnv%a)=kN4?{EVPVTzmVz`OG)>a|wus^Ak-PC(Ksy+*}p>0n5xpQ{63bcF-gr zw^L#~-NChWRrQ$K`Ji!*R==)p8NTt4={+QG)0;Ld^bzcv9zh{+*U6flS=C~y+Lj@0 z@2psliX$FbxYSfL?N|oW1FOvubA;lRF_`<6yY=L%78eh&RZ(CE53QYu4c`l0h;iE$ zqdJ-?fK5z>ij?t^VCRm^w$UOi^IS}n>*TcAc8rWR)1qo`3~{2Rv1i|2=k|VXbNj#6 z>SgtM4pMIt>MRGd^SSW$$>ePWT_zh%t<1Wpm(UmX3k|TpfmC0P{*^xLrM(x*+5g@q z^|-5nQ{p{Mn%#C1uufI5G|M$E%V9WI!275_gW)#)f!Cp^Cla*;28n1l6nK0kCK->@gsRNXk@~A3-X#e^_JWdJm{+iCHCZn z5tOgJ2#VfRc7UhnE3<-CRXPCm_h^Jb=yJ?>{F05#wZej1SqysN-2oe2=U3%YGo4j! zO;}gk(L)l!K-jD?;!YhA$sV>~1R}ibeEO~y*H=Im1e!tKbP#Rd`5f1(4_yfdCYn?w zlctK3=H)0!rUH9jYS++LK@Lp7MTFM(96$I^w^82HJQig36^q2C!cN4FkG3C|l~=7J z5>+fz6zY$d1lz8;FGmY}-Z?Ee~g z*Y3y~1?@YQR1f-`CACbW=OrNq?=?|dWyyci&LvNiV3vc6MguB&{1g{u(6s7zZp{EC zPFDw^ECqk<>UOR0QW0^1a)JPr;&)G*oNEldHl~G_BI;vfcY4spf)tm03Ez1ZU_@pH z51AL8msGW>NiH4n=Xxp%nUSMwn`l|!bA0n*;Ztk+Y;P+0o3WSAeyB-U$+w;WNnp^F z@iOMt*!yTPu{$#d67Bdd{7l~Vz){AP@oB=v)|ASje_xojTAzU5zDXIq_vvq%cK80I z0Qv6d+H~b|4BL=0vq*4Z92A^JH}%<`hyz;Z&n*prt|g!Gpg#b-bOOv((#yH^#-WN1 z&hE|vNZfB*=S|CuST44mHPk*XJ7ndDuodjoqJ0NTn-;PFkrH0<)rWOsLufupXh|d9 ziSJ?AUnp!pSq8&khtqG3d*$n2O@!e(&*y%xZWDy{tnyg=&rEbFT1(zbJJtB^?`&2B zS5Q;DoAGpS1YWd|mTs$w&(pUorFQ|er=Wb?&W)#JWbL*Z1@$h;u zT_$fGzxpEY6bV@y3JIqN&B{|;oCC$w8Sp}f1o#vEsHi#i-#f|cd0S_$lp26HD1U}F~MheQC z$Qb0(CUmBSDczbmPmM9oX?5V8YH^Gx6foMKonA^UVDPT&8hrWm%IbJ(o~K{(|5w0` zzP=3<+n8k-pusGm7vc(qYqRd$ANT;9W={+wQ7~i+j(PJC`l$ZKR1A>vyqhEbabeM_ zspE;=z#m+zHa{?^>h9iDXH*J{IA31aQ$K5osr6elTRlghkzh}HW1wewerXfgdcP2) z3Two+pFZRQcyVidjs~&h4g77K#+NC`Qk6a+D=IC@XUo5|hKS0Bn#W{2&SL5o{oOhi zmQAaO$n%mNsFf;EQCcJV))j?bi1@zwgft09G*~tXMFz<~+W*rslcFBypqqJEjk{RB zQco|bnWYV00k^`=o78mwQdUu1g%LC&NAzi`@4K+N`_#EsX*0aiflPv#AQj3ZZ;vOc z>xQQa37pdj>*h+9?HPW`QzxpK@zohm`2mwU8wYmC=9;*-8#SJJ})yiEY z)avCK@cov2R{y`%U$vJvYWE;LvFaYxan?99c-0%j86=t(*t#`nq6_B|nWr=4Tfl#b zP~^wE7fLe-7i)SKGa}XQ9%SnL79Y=%)janWrZRI}WkBa{Scq&eOft9RpdAc*W%=t% zect=`>Hjy;-$@z*ggo{SkHbgOd^99wB;kBsDJgACfx{4GVvO5ZYEd{aX~>Q8_AD-k?vjm0BqYGHP+wsBk@?l!Nc<64X}IPje^M zr2@~NlsdWo5EJFzc^eTI*jIw?w3zRy>=f8K z|2u3n!gX&-GbJ!M#N*32!Z2v%v_*xp`fK+5$H6fpDy~@#V^o3`)Xs(arG2x$GaZMdx_lqUIlJ=gj@U1FgKk?w`_E zH8|RXjF8rSZ#ti2;7gKpC4n!rBodmD6o4_V7;(@y2gg!1_8C*9ojB9Gq!)Mmw0pPtI_{zN zRf*2I-6VUdN^CVB6MIc?ECfaWytEo$D){v`^j{&H1pmJ_UOFt3eIY8Mul4i!zC|V9 zrgtBSv)Q%TwfeLMe_BmMm!m8|*A!dSejjQ9wpS99f;8;8VU04g@D5H|v^|LS^wg-t z6DRK4@bto@VLIAwiSYkX_#6lGKd!YAkrUBS5H~%aui>sOsPbMI@aEVVUgzVP0s^c0 z=219%!;azWbR}nQ?(5oTQl96v?dKz#ls_i37t^^5s3g4}WAv_{1(NPVN&3Bc92xB_e}D#JF}mDvoyTVN0Y&Y;Jkf5fc%Mh- z+3l~0H?|m#^OPU`zxv`Uq(G<N?ML|TKks#L_DbdbB#Zl&mq(zD$1;vmpZ(kW7F?$ePK+!EnTTbwqN{ueuR?{O zzJ0bT1zb1>dnj{jWHrv$M}qtRZ(*j!ouJxnfJZGeeSN=KA3x6Pa-fknx657$pEic6 z-<@8{e8}>hdtUTPYGqC#F3|aAVk)7E_doV8hrZ%y5b^>0_c} zqBFDDYu}gd#^nJLunyw#2mVIz%x*6d3J z!_FL-O*51i8H{`R3C&C>-6_3VYSB#xSzv^97WNm9lsX zo(-v>Cpu8lHmPq+VY7%E)JQVCv>>9_*)vgib^zBkJE7P;h6^oQPcx&gDi-S4_x7#w z5c~(73jcR`xHR{^j(UAneG4;$Fqrf%w))(PSv1IiJraY1Y)8xv!lac7MJmbGW98cw zL1Ua|bc|IZHI^NxY=ycm*}EBs603oHW6>i-wwv5;)a?cTR%NZ#LxVIrgka1#)Up6^k;HhR%Z43 zpx15mJRmm?--XbZXWyx!Xv;J@M-s9CZj#T44VE7zz;N)_w0Q{7@oVU;R!$yFfT4-W z^*GgAifUEx0=aZ;5Vq{mXGPkU(?{ov6Cc6m#hS#NXN2mEa?2wnZ9ip22D8vu%V$}zQlJZ`Y`hiBshFo4}v9tKNp7k z@eRon{SHGf2nIfB^9{yjtXnkBy0|4w+xIaR!!FErJ8oVG{3>9O!({2)isVved1BzeNh2#UdQF=(2JDN0r_y(ZbVf{Y!+h&p5>MzBMYH}scCKTpFpG$G4cT$*8|d6q^oLTr47YDFn5{~n@fPq8 zFGSY{m$t;CF^d@_%&sY-TaFCheq+QN#k}caQ2{b3+uSn$V;W(?&gphy@SR({edYV0 zI-mV>&rXM6=--!@r&`~4t`<#MNxpu7$Az+3`h)WPm zupy@a|60Gv#H+xN9*ZocfLq5K*q|yR6-Hq+X5xWM86t2Kl*m~Q+|Hegel#EDFplDG z58Ty}D`|GoKbU>xMMy3#o*Nq{PM|9$o>}^#Nb+v`SS3N_6Jbg%(gn}8g7~XxIO(nH zXz~d-Z}zcA#qF}_wql%-MRV}CyZtkW61gOh7xDKmKo@VwK#u(nlr1Pjgp(5dU;Oz< zA3zr;f;eB_O7VRx48{x6uo`y_Md^IEc9J=`O_yg7L~13=dl`^BHvCP+Usj^Kz*$%( z+oQa4i}1ST^&y6~XSUE`r_n$V#x+zSW;A|^GC*yZ3+53^^Ty?wWlsEG9abkYJl%*{ zP_zTuS1fexAS+RU(8TZ`Yl$N+HeDz8KT}*40^%wEQN>zskd@Spcyb`q7 z2x?Rocu;@LBg}G87@0)g(I51D&VR`}mj1C#+0}?JT3XUSme%AS%8Cse^U8QmcBxBG zrDKI^oEnL|KY z0ZduQiN=ATt7N=H3@PC+oXTkX7fPAp>&8qCN0x64559J$+n??^$`oY7IoS)mc*!>? zD{I2OWIh<)HJ;W%{>38Bb)0^**f}i!dIz}<5ZTvvmU8acb`2KG_eHO%yH~sRBa^KC zjDv$Ev_az}WwS{pqwkMekkQpl-1o19sn_O=dTB+R>u3kYr&0VW=guTC5t#oaxl)n* z>-4x%MLK2&a<0uJAsFbh%bFyAPmqQf@zz^?k(_SHJ&2!jB4+IiPHX*frm7RJlXR;y zlhia#6rFe&SCU&}vXG(FJ^r75Km{~fvz6P{{ZwD}`Oq)p%J%K73nLEYY_V}TNIS`? zlnG;gma{l0b*^>zYsy)Ml`(`JB@5E*7$?jXppkWMQj_D)MKwshOs-#X?pV>GLTY@b zGEzV(Yyq0F?yZpRpgm|`v%e!x|2Zik+XVsCRC8vbm_XHXtKjBIB1jNb3Bl%Hb-o=C zthgUw6vST0g4?Z#x)Fd7*0S(TI;sU|Qc46{iy=h$1RoD?Ce5JnC4YLfOlu^A zNSoGD_CQ#>Qg^X#Qov@~(yxvJIRzhR(_sO+9O`1M$Ge>ld_=J9QnUefFMjIgie}3T~cU{>`>q<5O`2q{fJ|qKWsJyj7bvk*vm} z7}lK1H~Z-<-lLbsi5+h|L1q?1dWeXLSOI?0D1IN6#I!>#1nk@A;!zu%NH#8rIR9!U_Tj?=b|IG85BJ`J z`rE``&LUN}zdp&(|2(Nzw1|a-x2(7F)1u0U2EX5y895jVxeSr<5C+)bqgLtv2=fc? zP<+{OyJaVsebrTa6~0yMy8Zlv+J1*%u&A?3qs|G%dK+T_UMJh^1}&9Sy`CTr|DsRl zh5ogVVRqkmf6EDt?BRQ78;$*2ffSUY1{g(mL^b**hwCt1eeMi~mrOd&_G|l;s z#BCE}vyg6^=>1$%9l6aA{zv0q$3K#JQ1DyEAB#cVQur@jX}(koQQxzsNa&FaL;<5) zN$?ZXh@sQJgT7Vl{ukNSviV5|;WZSz#;s~81N@@V*w1Ij#TGkKI!UU?os9q6E%gKA zh~)_brr5uv<4ArQ@_4))6p(oTH-`UX>wW>T`I|Bxq|E$u09xBp9KX4m;`9E@p~8%0 zLGVAqbmQOF@j4lm`d=cxd^R5Hwf{+=)@|^=^j4VI|EIm{jB0B6_7bBAC?E<*mo8nY(v=>n0!koEH}+x<*amdKnL;7>(4Bir|j#>I-Ybc&XH zYdmV+Gy=kRxO3Z644`IHVV~O}Vq6!Z_fo}KhXTB%Ztr8z35bv8k3>lcj-vvmHT*Qs zhX?1w^2=)5t(Tg|B3>^q6!0DfArZVBqI$v(ork0^j_;i?T<&WOmbc6VOCNJ_Xf}2PgO%oFK*{Gp6-WC{BM9V8 z0wkJAmX_1~spY zgih@ueGNRl$=;LWg?ryKmmQqdX5z`$=m{OUyH8{-?`R5@Yj!q0fdlM`w?NxkVU<{% z5}GpGTU9+JNy+`yqD+Pz)|}V-yRo)*HKC9A!nbB$IpH;#(f(PepM0|@^w#V%5I~^z z_;cj1rWyKAkS}A`u04Etg@YyP1wauV>{gLc(lJV(4!pBCJKT_~4hKv9*_H)T-Ot6< z+UQcYE=Mkz4LsJ3zmIPZ1LUQk+}vFLAG6l91wfLCj9G5W0=s{-CTQxtZm)Rln=QrN zqswg)_!kc)c=>VQcWXy2XkYfP8MB%DS?ORahzZMfR!*zj|HDTyn^U4_AoqbItyj_uFyq-*uFmg1zDs;k=u5B z@pA^fd*pv~G$<~}NwT8Qp1jc*UAnh7uh4EbMUpj?2@VarnO|HbE5a7jC0>qxMDQeT z7mD~}KKoS%)3JCp*1vW8N=NA5+XpJV4cVt6sBzoiy( zh66;8S(kGcTR1hOK44E?V!ml_~0rMGo_%12(Zjo4c=M5(ADUAJv7jTWh4Oik&gr4t6kX zojb?ma`*O78jVeUwJgrFw(8j|c8Uc+Dk#<4vJNI@kYc(#?14fc@{J++P>zof z8WusZSD;|#4L+ct`XL@A+4~2pu>)mQOnymZPYro!_J)>$1xo25Ow&)1qrh?b8{Nhi2ay&?$mrSgR8=;sutR5oDwrKGJ5ucQaWEHC1wDOvX_~H zf^|io>h$2FWhuQSv=K28Moc;w+jvaNJsSsGes`5sSh2+i;2!zfvU8(umgxkBFt z4EA+bPgnfG8ZPzJyQrurWhkE)1>=x*eQoS}@~WszlJBJpL42i{qOko0H>B$;pUS2c zyoE^N1K`0t+&Isz77)0E|Bfqvn$@`10QvFWzHY>p1 zdO$`_!?&_e#3T4e9aP2=;O=pNs@k1NFIU!3EM8c$g7_Mi;JDA5xPCVR?|3WQS2Hb5 zf8motRfu9oPfGKyP*)#xiSTI#n1GuV^jP><%*b2S*mJjzUsUX2-L!YH{bcPcetv!y z&bJ(!kzUd;yR`)khV`T&YtB5aMh@13ye@j4=M{GDhi2-LOKf;Lcb^un+_?%G?jN~z z{#HxRvoZf@|HzQ93lg;e?WGwRoZ!sQ&xdbs6L?+D&kst`@)XfHriADHY0IGKR|AaU zOAvi?MR}1nb&MqI%9C99QD$8?mTnRtic3s)$_jxVe+$b#MNUh`1DxC?~3EjnBo=yRMM3s%H3PX?%9Gg2&d39|5!2WwJcwVGO+_=UAoZNs^AGnf!;T-4UO> zXRr-qNT~8NfOcD__bz0PpY^dAKa)*{>4a1BpOTOp_?_2hQP4(+V@hYI5`mRgnml^1 zwA4Y`C9@E%a7hcF7*njuEwz!zoAp5x*>!8Oed6x=Nav1$_%=*~#=55ysil3CF4y%+| z5pBb{kHm9NZmN!7s9>uc)(96@_Y6@#{qzIHWaa9G2=75L(fbpXg3 zX=A2$7fa@YGt=xq8Rhe}+xH=>-EKe`bSw1MQia7WH}_f<_ckg~9b-0Y_2ka(M7a|j z8enRV_9nP}`@XbvGpj34%DfytK7M{FT9%mh>7#IM`t;od@n#nw{rnE`H-(#&?J4tq zOG14yn9l8reYZDMVJm$)*$JH$CMRVcdoz`O=^?OSsP{M+ zwr`lvq4D$tQ?~P>US+g1kuJ-K7*4v3fY zF{o_o=33-~dvH0vsFUC6EGd1%zCN;FGd4iY+adr-ZV}WIeTqc30i4f7f){CEUsa60 zb#{4C@^ywwNV-;53aOEG`H1^ric^yw+;w^=W01VX*xWpUR@7TAhK?(4o?Th36wPN_ ziuT=zg)v17szu`y#A15E?d=Md&L+=bYji0h#t3z`j?jey8mCx?w3blydKVWLf#$g8 z0}i}lD~vR8p&;E6k`fxPJlz={9+3io3v- zdTGGR6A(~?l|s57s`Ph%w_aN>jW5i}N?$MH3PE37Aha46E5stpZA#IBaZkqG7Z1L8 zv?^4Y-@QvE-;tHgL-t+)!})7`XoSOEC@Br$BNM}LDo19PyO=)vCv?TLeZqU_18T10 z^z`Pq|6D-EBT0^>&oCCvHZ)iHP-pkoni8JXptDwv<7CY8a`^52qptBG_L}sgHLE8L zDnyNxu)g@9AP_`Vb$=tovo?|}pcywcRl&_j&T zLf@ShzA+xs)H#U%u(p2CfNk2GfUzhFC{NX5J&+aFrcUmY;jDplxuukvo zXZ>nCq7B(y2qN^Ki3bTCaqIqRvL{eYP4m7tkzb*spPsH-6~%mZq>OnBL7DZ_130&X zr`{)I3=GZoRH&1d9g-Uj{>jP7T>|}5Tq8A0tcI!jG#`L(`p@ zmxMJy$G30aA@1}ly`vBJY!-w${2)&-kIzows+&cm<9NH4LT4nX_!cHmuq|tvZ1p%*y6kC&%(z#aS*03`VYV8^c}1v>fZY%tU^^3KtLn7wl3L#oiQ}zz5yFm z0dAE%Y9xkO!L zV(7&kbB_LiWN+N06-IhwPsUuj1td;xtTLPH9;D_=@Uwu zsL+EYFQlchMh(8%nhtBmjgPZ-D0eR`>deLt3b^w>y~`N*L=<2gQfZmz*K_Sh*S!;H z?7j7&fPPi#piT$i5K{alZOF$p53(EzHAJn1uDRV^-60AN29BdUIN)9}Z}c-GnG5es z%x8AyC?7I9{u~d{fW;=$IMy3)*fy0_TKEMk78Z%u(8NpCXAuldmUW z9xC5bjE?jxe>#e9&* zWE(X_MrIN#0Az*2~-qay7da}Gs>bi6*sz;2miHR(DX`!1-g|-TA_!>{-C=`5noiO7L;I7o(T^i?q4gGyT<0yUuEkFWdcraL?!v;E z8mnSHo)u6Lc}>fWG>P%AdK*Q!;mAnWBe6-KXyQhFWhG>2I3`toi=z}Jk~a<48WQLm zRVF_)o4?1(&8;d9358Tpj1dzOQuwP49qb*M@BHprzFud@+jiQRt^E!EvOd>fp`t+i zBk>64Su=2_RGvQ(E0up$ctfIJ#Z5b8^Sk2^9e9Jj$djirgJFiE_4JxrS zkJbnYS)L5|2^|#7O6w5%yl0uq)pFF)sf(}?eWTvW$c2TRq!O-dZX~@hT43T4@f)#& zsP+7)uF;N&h!%;Qec0!9s1?|M-!Ti(_9qo=2v`yus*z@&2UKp6_Ku%K6n^^})$Dt` zS;aTkB)j)3?-{efZgr zl?RrNCOtO%(*wQy%x^>d{H&}zM!$aDV=+WZO*u`VBI6caBcEfwn2#kc1GHk;7~$zC zo7kH*rk{w!N@{u7#FY65hlXG3bD``y_;oDPj9$1=w*IDlmoI)&k+d zh_Z%ld)p4&)@CD#!=ERUjGFIbWibmko0?3nCJTTM2nu?&O!TY>@2!u@T8wKg5HEB` zWzy2%5+|QiQ7tEa+q_A&T9)#yOe~dpBl!i-USewULj#e;o=|tM_{@zRqTK340CpET z&0*ay+x$Z>i#8hk4701Tv9``ug)JLN>mL~$nzqD^6YMl#al?~By4I~(Y=uhN!-Ext zW#otJ{y}UbxkV6pv;cf=jxy-Hn9|pK3jpzn*LC_T5n8E@!!4Y8vIIs!EY)rBq4T1Jl(e6in znH%fC8RkZ9nKCSj{;Bf|>?bq~cfXFnNU#G2hDIqoYQ-U8-*B<9FSN{u6UvmtMGV?~ zr(5nF_r|1#&p$WssNle(!OcwMMI%5$nT(3KQiH;1{@TvZuc)kcA|7eet7HcN62}cs z3G8+?U5A$`4}tMREj^i(*g(JJg3K)Couf)_Mg52-AV2uC-<0c9_9KSpys@9lISZ4g zXQOWG8r2nXH?9&9H=wTf5sEOUBcm_52rLL)EbkIxy;DIN9T{m3p;&VIo)cwrg8U>& z(oDK$(g+Rjt$B1n#QdT&8}Tz|iaE<|h4i>rdW0?U(A`5vvC-{KwKs13No9P(nx1uT zoWa+(zACnvXqFJkb;F0=<#jr(EaEjT0SdK(ML$OKj93Q@bJ1wohK5EVqnE9N?k*Aq zE7C743)u>ju}~W8hIN8xdC#krKM)Rui$xEU&Uw23h!@-!VHJfi@5QUR8Ecy5xJ;FR z_M)6o5*KbInJi7DzP^pA|(Y@3{-vK>6bqC_>Zci*D4 zDuWwvF15Ft;ZdL1&NH4ODdC2nrN_@@O8gB9PLK|O(*wOtB=pea_F>}MY`%A+=++H3 zlaoTc_N-MggAG<;nG$#5KUx;wSU)$vo#m7FX1}78ZF|XdZnc5H+mnIXJgVi@S&hq%B$AzT9rQbsD z@4MZ>x4OHV57$@{=r`sOeF25?J(7K)@1Y5RO$xP@${?E*R#sL9+#E_PWyM7pzWZ*0 zNGsNLJw59c1{I0yUkBpYo`=w{E?e53TodfX>`CzYgSJOW9pK76M6{BxKi_5F9<|F? zJFf{0l$sXO@pmvY6fs*VH@fh@^R)#<_KdFkntgmoh%G_Pqj^duPN(UHnu!CCV2&=C zX6G{Pl+88kU84TX;>WDTfZuxhtUL7#h*f9jNC($ov0Yt4?yE$6_W+3?C1Mq&6)2i1q3aULG=%#TqE#g1ri>(sHL9&{XBumZ(j94AB74q|HCK`@ydZr126ej z_J5u#8t8#KnsK)JTWMVVIhO=vfRx)Dga2`=^#F{tWPsxi=G%E#%cz~74<7=2qAPyro5-=Ke->c2|z zpW1@?S4sX;TmDs&f0g9lo#a35&i^5Eb9{*>q}EsX1a>id4fsLi)#b`$OoILkce#o9 diff --git a/arbiter/arbiter.rules.yml b/arbiter/arbiter.rules.yml deleted file mode 100644 index 8ce1c44ca..000000000 --- a/arbiter/arbiter.rules.yml +++ /dev/null @@ -1,25 +0,0 @@ -groups: -- name: alert.rules - rules: - - alert: binlog_arbiter_checkpoint_high_delay - expr: (time() - binlog_arbiter_checkpoint_tso / 1000) > 3600 - for: 1m - labels: - env: test-cluster - level: warning - expr: (time() - binlog_arbiter_checkpoint_tso / 1000) > 3600 - annotations: - description: 'cluster: test-cluster, instance: {{ $labels.instance }}, values: {{ $value }}' - value: '{{ $value }}' - summary: arbiter arbiter checkpoint delay more than 1 hour - - - alert: binlog_arbiter_checkpoint_tso_no_change_for_1m - expr: changes(binlog_arbiter_checkpoint_tso[1m]) < 1 - labels: - env: test-cluster - level: warning - expr: changes(binlog_arbiter_checkpoint_tso[1m]) < 1 - annotations: - description: 'cluster: test-cluster, instance: {{ $labels.instance }}, values: {{ $value }}' - value: '{{ $value }}' - summary: binlog arbiter checkpoint tso no change for 1m diff --git a/arbiter/checkpoint.go b/arbiter/checkpoint.go deleted file mode 100644 index a3378affe..000000000 --- a/arbiter/checkpoint.go +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2019 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package arbiter - -import ( - gosql "database/sql" - "fmt" - - pkgsql "github.com/pingcap/tidb-binlog/pkg/sql" - - "github.com/pingcap/errors" -) - -const ( - // StatusNormal means server quit normally, data <= ts is synced to downstream - StatusNormal int = 0 - // StatusRunning means server running or quit abnormally, part of data may or may not been synced to downstream - StatusRunning int = 1 -) - -// Checkpoint is able to save and load checkpoints -type Checkpoint interface { - Save(ts int64, status int) error - Load() (ts int64, status int, err error) -} - -type dbCheckpoint struct { - database string - table string - db *gosql.DB - topicName string -} - -// NewCheckpoint creates a Checkpoint -func NewCheckpoint(db *gosql.DB, topicName string) (Checkpoint, error) { - cp := &dbCheckpoint{ - db: db, - database: "tidb_binlog", - table: "arbiter_checkpoint", - topicName: topicName, - } - - if err := cp.createSchemaIfNeed(); err != nil { - return nil, errors.Trace(err) - } - - return cp, nil -} - -func (c *dbCheckpoint) createSchemaIfNeed() error { - sql := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", pkgsql.QuoteName(c.database)) - _, err := c.db.Exec(sql) - if err != nil { - return errors.Trace(err) - } - - sql = fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s( - topic_name VARCHAR(255) PRIMARY KEY, ts BIGINT NOT NULL, status INT NOT NULL)`, - pkgsql.QuoteSchema(c.database, c.table)) - _, err = c.db.Exec(sql) - if err != nil { - return errors.Trace(err) - } - - return nil -} - -// Save saves the ts and status -func (c *dbCheckpoint) Save(ts int64, status int) error { - sql := fmt.Sprintf("REPLACE INTO %s(topic_name, ts, status) VALUES(?,?,?)", - pkgsql.QuoteSchema(c.database, c.table)) - _, err := c.db.Exec(sql, c.topicName, ts, status) - if err != nil { - return errors.Annotatef(err, "exec fail: '%s', args: %s %d, %d", sql, c.topicName, ts, status) - } - - return nil -} - -// Load return ts and status, if no record in checkpoint, return err = errors.NotFoundf -func (c *dbCheckpoint) Load() (ts int64, status int, err error) { - sql := fmt.Sprintf("SELECT ts, status FROM %s WHERE topic_name = ?", - pkgsql.QuoteSchema(c.database, c.table)) - - row := c.db.QueryRow(sql, c.topicName) - - err = row.Scan(&ts, &status) - if err != nil { - if errors.Cause(err) == gosql.ErrNoRows { - return 0, 0, errors.NotFoundf("no checkpoint for: %s", c.topicName) - } - return 0, 0, errors.Trace(err) - } - - return -} diff --git a/arbiter/checkpoint_test.go b/arbiter/checkpoint_test.go deleted file mode 100644 index bd84a791a..000000000 --- a/arbiter/checkpoint_test.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2019 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package arbiter - -import ( - "fmt" - "testing" - - gosql "database/sql" - sqlmock "github.com/DATA-DOG/go-sqlmock" - check "github.com/pingcap/check" - "github.com/pingcap/errors" - pkgsql "github.com/pingcap/tidb-binlog/pkg/sql" -) - -func Test(t *testing.T) { check.TestingT(t) } - -type CheckpointSuite struct { -} - -var _ = check.Suite(&CheckpointSuite{}) - -func setNewExpect(mock sqlmock.Sqlmock) { - mock.ExpectExec("CREATE DATABASE IF NOT EXISTS").WillReturnResult(sqlmock.NewResult(0, 1)) - mock.ExpectExec("CREATE TABLE IF NOT EXISTS").WillReturnResult(sqlmock.NewResult(0, 1)) -} - -func (cs *CheckpointSuite) TestNewCheckpoint(c *check.C) { - db, mock, err := sqlmock.New() - c.Assert(err, check.IsNil) - - setNewExpect(mock) - - _, err = createDbCheckpoint(db) - c.Assert(err, check.IsNil) - - c.Assert(mock.ExpectationsWereMet(), check.IsNil) -} - -func (cs *CheckpointSuite) TestSaveAndLoad(c *check.C) { - db, mock, err := sqlmock.New() - c.Assert(err, check.IsNil) - - setNewExpect(mock) - cp, err := createDbCheckpoint(db) - c.Assert(err, check.IsNil) - sql := fmt.Sprintf("SELECT (.+) FROM %s WHERE topic_name = ?", - pkgsql.QuoteSchema(cp.database, cp.table)) - mock.ExpectQuery(sql).WithArgs(cp.topicName). - WillReturnError(errors.NotFoundf("no checkpoint for: %s", cp.topicName)) - - _, _, err = cp.Load() - c.Log(err) - c.Assert(errors.IsNotFound(err), check.IsTrue) - - var saveTS int64 = 10 - saveStatus := 1 - mock.ExpectExec("REPLACE INTO"). - WithArgs(cp.topicName, saveTS, saveStatus). - WillReturnResult(sqlmock.NewResult(0, 1)) - err = cp.Save(saveTS, saveStatus) - c.Assert(err, check.IsNil) - - rows := sqlmock.NewRows([]string{"ts", "status"}). - AddRow(saveTS, saveStatus) - mock.ExpectQuery("SELECT ts, status FROM").WillReturnRows(rows) - ts, status, err := cp.Load() - c.Assert(err, check.IsNil) - c.Assert(ts, check.Equals, saveTS) - c.Assert(status, check.Equals, saveStatus) -} - -func createDbCheckpoint(db *gosql.DB) (*dbCheckpoint, error) { - cp, err := NewCheckpoint(db, "topic_name") - return cp.(*dbCheckpoint), err -} diff --git a/arbiter/config.go b/arbiter/config.go deleted file mode 100644 index e4fba5dc6..000000000 --- a/arbiter/config.go +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2019 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package arbiter - -import ( - "encoding/json" - "flag" - "fmt" - "os" - - "github.com/pingcap/errors" - "github.com/pingcap/log" - "github.com/pingcap/tidb-binlog/pkg/flags" - "github.com/pingcap/tidb-binlog/pkg/util" - "github.com/pingcap/tidb-binlog/pkg/version" - "go.uber.org/zap" -) - -const ( - defaultKafkaAddrs = "127.0.0.1:9092" - defaultKafkaVersion = "0.8.2.0" -) - -var ( - errUpTopicNotSpecified = errors.Errorf("up.topic not config, please config the topic name") -) - -// Config is the configuration of Server -type Config struct { - *flag.FlagSet `json:"-"` - LogLevel string `toml:"log-level" json:"log-level"` - ListenAddr string `toml:"addr" json:"addr"` - LogFile string `toml:"log-file" json:"log-file"` - OpenSaramaLog bool `toml:"open-sarama-log" json:"open-sarama-log"` - - Up UpConfig `toml:"up" json:"up"` - Down DownConfig `toml:"down" json:"down"` - - Metrics Metrics `toml:"metrics" json:"metrics"` - configFile string - printVersion bool -} - -// Metrics is configuration of metrics -type Metrics struct { - Addr string `toml:"addr" json:"addr"` - Interval int `toml:"interval" json:"interval"` -} - -// UpConfig is configuration of upstream -type UpConfig struct { - KafkaAddrs string `toml:"kafka-addrs" json:"kafka-addrs"` - KafkaVersion string `toml:"kafka-version" json:"kafka-version"` - - InitialCommitTS int64 `toml:"initial-commit-ts" json:"initial-commit-ts"` - Topic string `toml:"topic" json:"topic"` - MessageBufferSize int `toml:"message-buffer-size" json:"message-buffer-size"` - SaramaBufferSize int `toml:"sarama-buffer-size" json:"sarama-buffer-size"` -} - -// DownConfig is configuration of downstream -type DownConfig struct { - Host string `toml:"host" json:"host"` - Port int `toml:"port" json:"port"` - User string `toml:"user" json:"user"` - Password string `toml:"password" json:"password"` - - WorkerCount int `toml:"worker-count" json:"worker-count"` - BatchSize int `toml:"batch-size" json:"batch-size"` - SafeMode bool `toml:"safe-mode" json:"safe-mode"` -} - -// NewConfig return an instance of configuration -func NewConfig() *Config { - cfg := &Config{} - cfg.FlagSet = flag.NewFlagSet("arbiter", flag.ContinueOnError) - fs := cfg.FlagSet - fs.Usage = func() { - fmt.Fprintln(os.Stderr, "Usage of arbiter:") - fs.PrintDefaults() - } - - fs.StringVar(&cfg.ListenAddr, "addr", "127.0.0.1:8251", "addr (i.e. 'host:port') to listen on for arbiter connections") - fs.StringVar(&cfg.LogLevel, "L", "info", "log level: debug, info, warn, error, fatal") - fs.StringVar(&cfg.configFile, "config", "", "path to the configuration file") - fs.BoolVar(&cfg.printVersion, "V", false, "print version information and exit") - fs.StringVar(&cfg.Metrics.Addr, "metrics.addr", "", "prometheus pushgateway address, leaves it empty will disable prometheus push") - fs.IntVar(&cfg.Metrics.Interval, "metrics.interval", 15, "prometheus client push interval in second, set \"0\" to disable prometheus push") - fs.StringVar(&cfg.LogFile, "log-file", "", "log file path") - fs.BoolVar(&cfg.OpenSaramaLog, "open-sarama-log", true, "save the logs from sarama (https://github.com/Shopify/sarama), a client of Kafka") - - fs.Int64Var(&cfg.Up.InitialCommitTS, "up.initial-commit-ts", 0, "if arbiter doesn't have checkpoint, use initial commitTS to initial checkpoint") - fs.StringVar(&cfg.Up.Topic, "up.topic", "", "topic name of kafka") - - fs.IntVar(&cfg.Down.WorkerCount, "down.worker-count", 16, "concurrency write to downstream") - fs.IntVar(&cfg.Down.BatchSize, "down.batch-size", 64, "batch size write to downstream") - fs.BoolVar(&cfg.Down.SafeMode, "safe-mode", false, "enable safe mode to make reentrant") - - return cfg -} - -func (cfg *Config) String() string { - data, err := json.MarshalIndent(cfg, "\t", "\t") - if err != nil { - log.Error("marshal Config failed", zap.Error(err)) - } - - return string(data) -} - -// Parse parses all config from command-line flags, environment vars or the configuration file -func (cfg *Config) Parse(args []string) error { - // parse first to get config file - perr := cfg.FlagSet.Parse(args) - switch perr { - case nil: - case flag.ErrHelp: - os.Exit(0) - default: - os.Exit(2) - } - if cfg.printVersion { - fmt.Println(version.GetRawVersionInfo()) - os.Exit(0) - } - - // load config file if specified - if cfg.configFile != "" { - if err := cfg.configFromFile(cfg.configFile); err != nil { - return errors.Trace(err) - } - } - - // parse again to replace with command line options - if err := cfg.FlagSet.Parse(args); err != nil { - return errors.Trace(err) - } - if len(cfg.FlagSet.Args()) > 0 { - return errors.Errorf("'%s' is not a valid flag", cfg.FlagSet.Arg(0)) - } - - // replace with environment vars - err := flags.SetFlagsFromEnv("BINLOG_SERVER", cfg.FlagSet) - if err != nil { - return errors.Trace(err) - } - - if err = cfg.adjustConfig(); err != nil { - return errors.Trace(err) - } - - return cfg.validate() -} - -// validate checks whether the configuration is valid -func (cfg *Config) validate() error { - if len(cfg.Up.Topic) == 0 { - return errUpTopicNotSpecified - } - - return nil -} - -func (cfg *Config) adjustConfig() error { - // cfg.Up - if len(cfg.Up.KafkaAddrs) == 0 { - cfg.Up.KafkaAddrs = defaultKafkaAddrs - } - if len(cfg.Up.KafkaVersion) == 0 { - cfg.Up.KafkaVersion = defaultKafkaVersion - } - - // cfg.Down - if len(cfg.Down.Host) == 0 { - cfg.Down.Host = "localhost" - } - if cfg.Down.Port == 0 { - cfg.Down.Port = 3306 - } - if len(cfg.Down.User) == 0 { - cfg.Down.User = "root" - } - - return nil -} - -func (cfg *Config) configFromFile(path string) error { - return util.StrictDecodeFile(path, "arbiter", cfg) -} diff --git a/arbiter/config_test.go b/arbiter/config_test.go deleted file mode 100644 index 9b765839a..000000000 --- a/arbiter/config_test.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2019 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package arbiter - -import ( - "bytes" - "fmt" - "os" - "path" - "runtime" - "strings" - - "github.com/BurntSushi/toml" - "github.com/pingcap/check" -) - -type TestConfigSuite struct { -} - -var _ = check.Suite(&TestConfigSuite{}) - -func (t *TestConfigSuite) TestAdjustConfig(c *check.C) { - config := Config{ - Up: UpConfig{}, - Down: DownConfig{}, - } - err := config.adjustConfig() - c.Assert(err, check.IsNil) - c.Assert(config.Up.KafkaAddrs, check.Equals, defaultKafkaAddrs) - c.Assert(config.Up.KafkaVersion, check.Equals, defaultKafkaVersion) - c.Assert(config.Down.Host, check.Equals, "localhost") - c.Assert(config.Down.Port, check.Equals, 3306) - c.Assert(config.Down.User, check.Equals, "root") -} - -func (t *TestConfigSuite) TestParseConfig(c *check.C) { - args := make([]string, 0, 10) - - // not set `up.topic`, invalid - config := NewConfig() - configFile := getTemplateConfigFilePath() - args = append(args, fmt.Sprintf("-config=%s", configFile)) - err := config.Parse(args) - c.Assert(err, check.Equals, errUpTopicNotSpecified) - - // set `up.topic` through command line args, valid - config = NewConfig() - upTopic := "topic-test" - args = append(args, fmt.Sprintf("-up.topic=%s", upTopic)) - err = config.Parse(args) - c.Assert(err, check.IsNil) - // check config item - c.Assert(config.LogLevel, check.Equals, "info") - c.Assert(config.LogFile, check.Equals, "") - c.Assert(config.ListenAddr, check.Equals, "127.0.0.1:8251") - c.Assert(config.configFile, check.Equals, configFile) - c.Assert(config.Up.KafkaAddrs, check.Equals, defaultKafkaAddrs) - c.Assert(config.Up.KafkaVersion, check.Equals, defaultKafkaVersion) - c.Assert(config.Up.InitialCommitTS, check.Equals, int64(0)) - c.Assert(config.Up.Topic, check.Equals, upTopic) - c.Assert(config.Down.Host, check.Equals, "localhost") - c.Assert(config.Down.Port, check.Equals, 3306) - c.Assert(config.Down.User, check.Equals, "root") - c.Assert(config.Down.Password, check.Equals, "") - c.Assert(config.Down.WorkerCount, check.Equals, 16) - c.Assert(config.Down.BatchSize, check.Equals, 64) - c.Assert(config.Metrics.Addr, check.Equals, "") - c.Assert(config.Metrics.Interval, check.Equals, 15) - - // overwrite with more command line args - listenAddr := "127.0.0.1:8252" - args = append(args, fmt.Sprintf("-addr=%s", listenAddr)) - logLevel := "error" - args = append(args, fmt.Sprintf("-L=%s", logLevel)) - logFile := "arbiter.log" - args = append(args, fmt.Sprintf("-log-file=%s", logFile)) - upInitCTS := int64(123) - args = append(args, fmt.Sprintf("-up.initial-commit-ts=%d", upInitCTS)) - downWC := 456 - args = append(args, fmt.Sprintf("-down.worker-count=%d", downWC)) - downBS := 789 - args = append(args, fmt.Sprintf("-down.batch-size=%d", downBS)) - - // parse again - config = NewConfig() - err = config.Parse(args) - c.Assert(err, check.IsNil) - // check again - c.Assert(config.ListenAddr, check.Equals, listenAddr) - c.Assert(config.LogLevel, check.Equals, logLevel) - c.Assert(config.LogFile, check.Equals, logFile) - c.Assert(config.Up.InitialCommitTS, check.Equals, upInitCTS) - c.Assert(config.Down.WorkerCount, check.Equals, downWC) - c.Assert(config.Down.BatchSize, check.Equals, downBS) - - // simply verify json string - c.Assert(strings.Contains(config.String(), listenAddr), check.IsTrue) -} - -func (t *TestConfigSuite) TestParseConfigFileWithInvalidArgs(c *check.C) { - yc := struct { - LogLevel string `toml:"log-level" json:"log-level"` - ListenAddr string `toml:"addr" json:"addr"` - LogFile string `toml:"log-file" json:"log-file"` - UnrecognizedOptionTest bool `toml:"unrecognized-option-test" json:"unrecognized-option-test"` - }{ - "debug", - "127.0.0.1:8251", - "/tmp/arbiter", - true, - } - - var buf bytes.Buffer - e := toml.NewEncoder(&buf) - err := e.Encode(yc) - c.Assert(err, check.IsNil) - - configFilename := path.Join(c.MkDir(), "arbiter_config_invalid.toml") - err = os.WriteFile(configFilename, buf.Bytes(), 0644) - c.Assert(err, check.IsNil) - - args := []string{ - "--config", - configFilename, - } - - cfg := NewConfig() - err = cfg.Parse(args) - c.Assert(err, check.ErrorMatches, ".*contained unknown configuration options: unrecognized-option-test.*") -} - -func getTemplateConfigFilePath() string { - // we put the template config file in "cmd/arbiter/arbiter.toml" - _, filename, _, _ := runtime.Caller(0) - return path.Join(path.Dir(filename), "../cmd/arbiter/arbiter.toml") -} diff --git a/arbiter/metrics.go b/arbiter/metrics.go deleted file mode 100644 index 7d8a70bd3..000000000 --- a/arbiter/metrics.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2019 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package arbiter - -import ( - "fmt" - "os" - - "github.com/pingcap/log" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" - "go.uber.org/zap" -) - -var ( - checkpointTSOGauge = prometheus.NewGauge( - prometheus.GaugeOpts{ - Namespace: "binlog", - Subsystem: "arbiter", - Name: "checkpoint_tso", - Help: "save checkpoint tso of arbiter.", - }) - - queryHistogramVec = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: "binlog", - Subsystem: "arbiter", - Name: "query_duration_time", - Help: "Bucketed histogram of processing time (s) of a query to sync data to downstream.", - Buckets: prometheus.ExponentialBuckets(0.00005, 2, 18), - }, []string{"type"}) - - eventCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: "binlog", - Subsystem: "arbiter", - Name: "event", - Help: "the count of sql event(dml, ddl).", - }, []string{"type"}) - - queueSizeGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "binlog", - Subsystem: "arbiter", - Name: "queue_size", - Help: "the size of queue", - }, []string{"name"}) - - txnLatencySecondsHistogram = prometheus.NewHistogram( - prometheus.HistogramOpts{ - Namespace: "binlog", - Subsystem: "arbiter", - Name: "txn_latency_seconds", - Help: "Bucketed histogram of seconds of a txn between loaded to downstream and committed at upstream.", - Buckets: prometheus.ExponentialBuckets(0.00005, 2, 20), - }) -) - -// Registry is the metrics registry of server -var Registry = prometheus.NewRegistry() - -func init() { - Registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) - Registry.MustRegister(collectors.NewGoCollector()) - - Registry.MustRegister(checkpointTSOGauge) - Registry.MustRegister(queryHistogramVec) - Registry.MustRegister(eventCounter) - Registry.MustRegister(queueSizeGauge) - Registry.MustRegister(txnLatencySecondsHistogram) -} - -var getHostname = os.Hostname - -func instanceName(port int) string { - hostname, err := getHostname() - if err != nil { - log.Error("Failed to get hostname", zap.Error(err)) - return "unknown" - } - return fmt.Sprintf("%s_%d", hostname, port) -} diff --git a/arbiter/metrics_test.go b/arbiter/metrics_test.go deleted file mode 100644 index c9f83eaf9..000000000 --- a/arbiter/metrics_test.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2019 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package arbiter - -import ( - . "github.com/pingcap/check" - "github.com/pingcap/errors" -) - -type instanceNameSuite struct{} - -var _ = Suite(&instanceNameSuite{}) - -func (s *instanceNameSuite) TestShouldRetUnknown(c *C) { - orig := getHostname - defer func() { - getHostname = orig - }() - getHostname = func() (string, error) { - return "", errors.New("host") - } - - n := instanceName(9090) - c.Assert(n, Equals, "unknown") -} - -func (s *instanceNameSuite) TestShouldUseHostname(c *C) { - orig := getHostname - defer func() { - getHostname = orig - }() - getHostname = func() (string, error) { - return "kendoka", nil - } - - n := instanceName(9090) - c.Assert(n, Equals, "kendoka_9090") -} diff --git a/arbiter/server.go b/arbiter/server.go deleted file mode 100644 index 2e8f2a91c..000000000 --- a/arbiter/server.go +++ /dev/null @@ -1,317 +0,0 @@ -// Copyright 2019 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package arbiter - -import ( - "context" - "database/sql" - "net" - "strconv" - "strings" - "sync" - "time" - - "github.com/pingcap/errors" - "github.com/pingcap/log" - "github.com/pingcap/tidb-binlog/pkg/loader" - "github.com/pingcap/tidb-binlog/pkg/util" - "github.com/pingcap/tidb/tidb-binlog/driver/reader" - "github.com/tikv/client-go/v2/oracle" - "go.uber.org/zap" -) - -var ( - initSafeModeDuration = time.Minute * 5 - - // Make it possible to mock the following functions - createDB = loader.CreateDB - newReader = reader.NewReader - newLoader = loader.NewLoader -) - -// Server is the server to load data to mysql -type Server struct { - cfg *Config - port int - - load loader.Loader - - checkpoint Checkpoint - kafkaReader *reader.Reader - downDB *sql.DB - - // all txn commitTS <= finishTS has loaded to downstream - finishTS int64 - - metrics *util.MetricClient - - closed bool - mu sync.Mutex -} - -// NewServer creates a Server -func NewServer(cfg *Config) (srv *Server, err error) { - srv = new(Server) - srv.cfg = cfg - - _, port, err := net.SplitHostPort(cfg.ListenAddr) - if err != nil { - return nil, errors.Annotatef(err, "wrong ListenAddr: %s", cfg.ListenAddr) - } - - srv.port, err = strconv.Atoi(port) - if err != nil { - return nil, errors.Annotatef(err, "ListenAddr: %s", cfg.ListenAddr) - } - - up := cfg.Up - down := cfg.Down - - srv.downDB, err = createDB(down.User, down.Password, down.Host, down.Port, nil) - if err != nil { - return nil, errors.Trace(err) - } - - // set checkpoint - srv.checkpoint, err = NewCheckpoint(srv.downDB, up.Topic) - if err != nil { - return nil, errors.Trace(err) - } - - srv.finishTS = up.InitialCommitTS - - status, err := srv.loadStatus() - if err != nil { - return nil, errors.Trace(err) - } - - // set reader to read binlog from kafka - readerCfg := &reader.Config{ - KafkaAddr: strings.Split(up.KafkaAddrs, ","), - CommitTS: srv.finishTS, - Topic: up.Topic, - SaramaBufferSize: up.SaramaBufferSize, - MessageBufferSize: up.MessageBufferSize, - } - - log.Info("use kafka binlog reader", zap.Reflect("cfg", readerCfg)) - - srv.kafkaReader, err = newReader(readerCfg) - if err != nil { - return nil, errors.Trace(err) - } - - log.Info("new kafka reader success") - - // set loader - srv.load, err = newLoader(srv.downDB, - loader.WorkerCount(cfg.Down.WorkerCount), - loader.BatchSize(cfg.Down.BatchSize), - loader.Metrics(&loader.MetricsGroup{ - EventCounterVec: eventCounter, - QueryHistogramVec: queryHistogramVec, - })) - if err != nil { - return nil, errors.Trace(err) - } - - if down.SafeMode { - srv.load.SetSafeMode(true) - } else { - // set safe mode in first 5 min if abnormal quit last time - if status == StatusRunning { - log.Info("set safe mode to be true") - srv.load.SetSafeMode(true) - go func() { - time.Sleep(initSafeModeDuration) - srv.load.SetSafeMode(false) - log.Info("set safe mode to be false") - }() - } - } - - // set metrics - if cfg.Metrics.Addr != "" && cfg.Metrics.Interval != 0 { - srv.metrics = util.NewMetricClient( - cfg.Metrics.Addr, - time.Duration(cfg.Metrics.Interval)*time.Second, - Registry, - ) - } - - return -} - -// Close closes the Server -func (s *Server) Close() error { - s.mu.Lock() - defer s.mu.Unlock() - - if s.closed { - return nil - } - - s.kafkaReader.Close() - - s.closed = true - return nil -} - -// Run runs the Server, will quit once encounter error or Server is closed -func (s *Server) Run() error { - defer s.downDB.Close() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // push metrics if need - if s.metrics != nil { - go s.metrics.Start(ctx, map[string]string{"instance": instanceName(s.port)}) - } - - var wg sync.WaitGroup - - wg.Add(1) - go func() { - s.trackTS(ctx, time.Second) - wg.Done() - }() - - var syncErr error - - syncCtx, syncCancel := context.WithCancel(ctx) - wg.Add(1) - go func() { - defer wg.Done() - syncErr = syncBinlogs(syncCtx, s.kafkaReader.Messages(), s.load) - if syncErr != nil { - s.Close() - } - }() - - err := s.load.Run() - if err != nil { - syncCancel() - s.Close() - } - - wg.Wait() - - syncCancel() - if err != nil { - return errors.Trace(err) - } - - if syncErr != nil { - return errors.Trace(syncErr) - } - - if err = s.saveFinishTS(StatusNormal); err != nil { - return errors.Trace(err) - } - - return nil -} - -func (s *Server) updateFinishTS(msg *reader.Message) { - s.finishTS = msg.Binlog.CommitTs - - ms := time.Now().UnixNano()/1000000 - oracle.ExtractPhysical(uint64(s.finishTS)) - txnLatencySecondsHistogram.Observe(float64(ms) / 1000.0) -} - -func (s *Server) saveFinishTS(status int) error { - err := s.checkpoint.Save(s.finishTS, status) - if err != nil { - return err - } - checkpointTSOGauge.Set(float64(oracle.ExtractPhysical(uint64(s.finishTS)))) - return nil -} - -func (s *Server) trackTS(ctx context.Context, saveInterval time.Duration) { - saveTick := time.NewTicker(saveInterval) - defer saveTick.Stop() - -L: - for { - select { - case txn, ok := <-s.load.Successes(): - if !ok { - log.Info("load successes channel closed") - break L - } - msg := txn.Metadata.(*reader.Message) - log.Debug("get success binlog", zap.Int64("ts", msg.Binlog.CommitTs), zap.Int64("offset", msg.Offset)) - s.updateFinishTS(msg) - case <-saveTick.C: - if err := s.saveFinishTS(StatusRunning); err != nil { - log.Error("save finish ts failed", zap.Error(err)) - } - case <-ctx.Done(): - break L - } - } - - if err := s.saveFinishTS(StatusRunning); err != nil { - log.Error("save finish ts failed", zap.Error(err)) - } -} - -func (s *Server) loadStatus() (int, error) { - ts, status, err := s.checkpoint.Load() - if err != nil { - if !errors.IsNotFound(err) { - return 0, errors.Trace(err) - } - log.Info("no checkpoint found") - err = nil - } else { - log.Info("load checkpoint", zap.Int64("ts", ts), zap.Int("status", status)) - s.finishTS = ts - } - return status, errors.Trace(err) -} - -func syncBinlogs(ctx context.Context, source <-chan *reader.Message, ld loader.Loader) (err error) { - dest := ld.Input() - defer ld.Close() - var receivedTs int64 - for msg := range source { - log.Debug("recv msg from kafka reader", zap.Int64("ts", msg.Binlog.CommitTs), zap.Int64("offset", msg.Offset)) - - if msg.Binlog.CommitTs <= receivedTs { - log.Info("skip repeated binlog", zap.Int64("ts", msg.Binlog.CommitTs), zap.Int64("offset", msg.Offset)) - continue - } - receivedTs = msg.Binlog.CommitTs - - txn, err := loader.SecondaryBinlogToTxn(msg.Binlog, nil, false) - if err != nil { - log.Error("transfer binlog failed, program will stop handling data from loader", zap.Error(err)) - return err - } - txn.Metadata = msg - // avoid block when no process is handling ld.input - select { - case dest <- txn: - case <-ctx.Done(): - return nil - } - - queueSizeGauge.WithLabelValues("kafka_reader").Set(float64(len(source))) - queueSizeGauge.WithLabelValues("loader_input").Set(float64(len(dest))) - } - return nil -} diff --git a/arbiter/server_test.go b/arbiter/server_test.go deleted file mode 100644 index 6a547f44f..000000000 --- a/arbiter/server_test.go +++ /dev/null @@ -1,468 +0,0 @@ -// Copyright 2019 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package arbiter - -import ( - "context" - "crypto/tls" - "database/sql" - "fmt" - "sync" - "time" - - "github.com/DATA-DOG/go-sqlmock" - . "github.com/pingcap/check" - "github.com/pingcap/errors" - "github.com/pingcap/tidb-binlog/pkg/loader" - "github.com/pingcap/tidb/tidb-binlog/driver/reader" - pb "github.com/pingcap/tidb/tidb-binlog/proto/go-binlog" -) - -type dummyLoader struct { - loader.Loader - successes chan *loader.Txn - safe bool - input chan *loader.Txn - closed bool -} - -func (l *dummyLoader) SetSafeMode(safe bool) { - l.safe = safe -} - -func (l *dummyLoader) Successes() <-chan *loader.Txn { - return l.successes -} - -func (l *dummyLoader) Input() chan<- *loader.Txn { - return l.input -} - -func (l *dummyLoader) Close() { - l.closed = true -} - -type testNewServerSuite struct { - db *sql.DB - dbMock sqlmock.Sqlmock - origCreateDB func(string, string, string, int, *tls.Config) (*sql.DB, error) - origNewReader func(*reader.Config) (*reader.Reader, error) - origNewLoader func(*sql.DB, ...loader.Option) (loader.Loader, error) -} - -var _ = Suite(&testNewServerSuite{}) - -func (s *testNewServerSuite) SetUpTest(c *C) { - db, mock, err := sqlmock.New() - if err != nil { - c.Fatalf("Failed to create mock db: %s", err) - } - s.db = db - s.dbMock = mock - - s.origCreateDB = createDB - createDB = func(user string, password string, host string, port int, _ *tls.Config) (*sql.DB, error) { - return s.db, nil - } - - s.origNewReader = newReader - newReader = func(cfg *reader.Config) (r *reader.Reader, err error) { - return &reader.Reader{}, nil - } - - s.origNewLoader = newLoader - newLoader = func(db *sql.DB, opt ...loader.Option) (loader.Loader, error) { - return &dummyLoader{}, nil - } -} - -func (s *testNewServerSuite) TearDownTest(c *C) { - s.db.Close() - - createDB = s.origCreateDB - newReader = s.origNewReader - newLoader = s.origNewLoader -} - -func (s *testNewServerSuite) TestRejectInvalidAddr(c *C) { - cfg := Config{ListenAddr: "whatever"} - _, err := NewServer(&cfg) - c.Assert(err, ErrorMatches, ".*wrong ListenAddr.*") - - cfg.ListenAddr = "whatever:invalid" - _, err = NewServer(&cfg) - c.Assert(err, ErrorMatches, "ListenAddr.*") -} - -func (s *testNewServerSuite) TestStopIfFailedtoConnectDownStream(c *C) { - createDB = func(user string, password string, host string, port int, _ *tls.Config) (*sql.DB, error) { - return nil, fmt.Errorf("Can't create db") - } - - cfg := Config{ListenAddr: "localhost:8080"} - _, err := NewServer(&cfg) - c.Assert(err, ErrorMatches, "Can't create db") -} - -func (s *testNewServerSuite) TestStopIfCannotCreateCheckpoint(c *C) { - s.dbMock.ExpectExec("CREATE DATABASE IF NOT EXISTS `tidb_binlog`").WillReturnError( - fmt.Errorf("cannot create")) - - cfg := Config{ListenAddr: "localhost:8080"} - _, err := NewServer(&cfg) - c.Assert(err, ErrorMatches, "cannot create") -} - -func (s *testNewServerSuite) TestStopIfCannotLoadStatus(c *C) { - s.dbMock.ExpectExec("CREATE DATABASE.*").WillReturnResult(sqlmock.NewResult(0, 0)) - s.dbMock.ExpectExec("CREATE TABLE.*").WillReturnResult(sqlmock.NewResult(0, 0)) - s.dbMock.ExpectQuery("SELECT ts, status.*"). - WithArgs("test_topic"). - WillReturnError(errors.New("Failed load")) - - cfg := Config{ - ListenAddr: "localhost:8080", - Up: UpConfig{ - Topic: "test_topic", - }, - } - _, err := NewServer(&cfg) - c.Assert(err, ErrorMatches, "Failed load") -} - -func (s *testNewServerSuite) TestStopIfCannotCreateReader(c *C) { - s.dbMock.ExpectExec("CREATE DATABASE.*").WillReturnResult(sqlmock.NewResult(0, 0)) - s.dbMock.ExpectExec("CREATE TABLE.*").WillReturnResult(sqlmock.NewResult(0, 0)) - s.dbMock.ExpectQuery("SELECT ts, status.*"). - WithArgs("test_topic"). - WillReturnError(errors.NotFoundf("")) - newReader = func(cfg *reader.Config) (r *reader.Reader, err error) { - return nil, errors.New("no reader") - } - - cfg := Config{ - ListenAddr: "localhost:8080", - Up: UpConfig{ - Topic: "test_topic", - }, - } - _, err := NewServer(&cfg) - c.Assert(err, ErrorMatches, "no reader") -} - -func (s *testNewServerSuite) TestStopIfCannotCreateLoader(c *C) { - s.dbMock.ExpectExec("CREATE DATABASE.*").WillReturnResult(sqlmock.NewResult(0, 0)) - s.dbMock.ExpectExec("CREATE TABLE.*").WillReturnResult(sqlmock.NewResult(0, 0)) - s.dbMock.ExpectQuery("SELECT ts, status.*"). - WithArgs("test_topic"). - WillReturnError(errors.New("not found")) - newLoader = func(db *sql.DB, opt ...loader.Option) (loader.Loader, error) { - return nil, errors.New("no loader") - } - - cfg := Config{ - ListenAddr: "localhost:8080", - Up: UpConfig{ - Topic: "test_topic", - }, - } - _, err := NewServer(&cfg) - c.Assert(err, ErrorMatches, "no loader") -} - -func (s *testNewServerSuite) TestSetSafeMode(c *C) { - s.dbMock.ExpectExec("CREATE DATABASE.*").WillReturnResult(sqlmock.NewResult(0, 0)) - s.dbMock.ExpectExec("CREATE TABLE.*").WillReturnResult(sqlmock.NewResult(0, 0)) - rows := sqlmock.NewRows([]string{"ts", "status"}).AddRow(42, StatusRunning) - s.dbMock.ExpectQuery("SELECT ts, status.*"). - WithArgs("test_topic"). - WillReturnRows(rows) - var ld dummyLoader - newLoader = func(db *sql.DB, opt ...loader.Option) (loader.Loader, error) { - return &ld, nil - } - - origDuration := initSafeModeDuration - defer func() { - initSafeModeDuration = origDuration - }() - initSafeModeDuration = 10 * time.Millisecond - - cfg := Config{ - ListenAddr: "localhost:8080", - Up: UpConfig{ - Topic: "test_topic", - }, - } - _, err := NewServer(&cfg) - c.Assert(err, IsNil) - c.Assert(ld.safe, IsTrue) - time.Sleep(2 * initSafeModeDuration) - c.Assert(ld.safe, IsFalse) -} - -func (s *testNewServerSuite) TestCreateMetricCli(c *C) { - s.dbMock.ExpectExec("CREATE DATABASE.*").WillReturnResult(sqlmock.NewResult(0, 0)) - s.dbMock.ExpectExec("CREATE TABLE.*").WillReturnResult(sqlmock.NewResult(0, 0)) - s.dbMock.ExpectQuery("SELECT ts, status.*"). - WithArgs("test_topic"). - WillReturnError(errors.New("not found")) - - cfg := Config{ - ListenAddr: "localhost:8080", - Up: UpConfig{ - Topic: "test_topic", - }, - Metrics: Metrics{ - Addr: "testing", - Interval: 10, - }, - } - srv, err := NewServer(&cfg) - c.Assert(err, IsNil) - c.Assert(srv.metrics, NotNil) -} - -type updateFinishTSSuite struct{} - -var _ = Suite(&updateFinishTSSuite{}) - -func (s *updateFinishTSSuite) TestShouldSetFinishTS(c *C) { - server := Server{} - msg := reader.Message{ - Binlog: &pb.Binlog{ - CommitTs: 1024, - }, - } - c.Assert(server.finishTS, Equals, int64(0)) - server.updateFinishTS(&msg) - c.Assert(server.finishTS, Equals, int64(1024)) -} - -type trackTSSuite struct{} - -var _ = Suite(&trackTSSuite{}) - -type dummyCp struct { - Checkpoint - timestamps []int64 - status []int -} - -func (cp *dummyCp) Save(ts int64, status int) error { - cp.timestamps = append(cp.timestamps, ts) - cp.status = append(cp.status, status) - return nil -} - -func (s *trackTSSuite) TestShouldUpdateFinishTS(c *C) { - cp := dummyCp{} - successes := make(chan *loader.Txn, 1) - ld := dummyLoader{successes: successes} - server := Server{ - load: &ld, - checkpoint: &cp, - } - - var wg sync.WaitGroup - wg.Add(1) - go func() { - server.trackTS(context.Background(), 50*time.Millisecond) - wg.Done() - }() - - for i := 0; i < 42; i++ { - successes <- &loader.Txn{Metadata: &reader.Message{Binlog: &pb.Binlog{CommitTs: int64(i)}}} - } - close(successes) - - wg.Wait() - c.Assert(server.finishTS, Equals, int64(41)) -} - -func (s *trackTSSuite) TestShouldSaveFinishTS(c *C) { - db, _, err := sqlmock.New() - if err != nil { - c.Fatalf("Failed to create mock db: %s", err) - } - ld, err := loader.NewLoader(db) - c.Assert(err, IsNil) - cp := dummyCp{} - server := Server{ - load: ld, - checkpoint: &cp, - } - - ctx, cancel := context.WithCancel(context.Background()) - - stop := make(chan struct{}) - go func() { - server.trackTS(ctx, 50*time.Millisecond) - close(stop) - }() - - for i := 0; i < 42; i++ { - server.finishTS = int64(i) - time.Sleep(2 * time.Millisecond) - } - - cancel() - - select { - case <-stop: - case <-time.After(time.Second): - c.Fatal("Doesn't stop in time") - } - - c.Assert(len(cp.status), Greater, 1) - c.Assert(len(cp.timestamps), Greater, 1) - c.Assert(cp.status[len(cp.status)-1], Equals, StatusRunning) - c.Assert(cp.timestamps[len(cp.timestamps)-1], Equals, int64(41)) -} - -type loadStatusSuite struct{} - -var _ = Suite(&loadStatusSuite{}) - -type configurableCp struct { - Checkpoint - ts int64 - status int - err error -} - -func (c *configurableCp) Load() (ts int64, status int, err error) { - return c.ts, c.status, c.err -} - -func (s *loadStatusSuite) TestShouldIgnoreNotFound(c *C) { - cp := configurableCp{status: StatusNormal, err: errors.NotFoundf("")} - server := Server{ - checkpoint: &cp, - } - status, err := server.loadStatus() - c.Assert(err, IsNil) - c.Assert(status, Equals, cp.status) -} - -func (s *loadStatusSuite) TestShouldSetFinishTS(c *C) { - cp := configurableCp{status: StatusRunning, ts: 1984} - server := Server{ - checkpoint: &cp, - } - status, err := server.loadStatus() - c.Assert(err, IsNil) - c.Assert(status, Equals, cp.status) - c.Assert(server.finishTS, Equals, cp.ts) -} - -func (s *loadStatusSuite) TestShouldRetErr(c *C) { - cp := configurableCp{status: StatusNormal, err: errors.New("other")} - server := Server{ - checkpoint: &cp, - } - _, err := server.loadStatus() - c.Assert(err, NotNil) - c.Assert(err, ErrorMatches, "other") -} - -type syncBinlogsSuite struct{} - -var _ = Suite(&syncBinlogsSuite{}) - -func (s *syncBinlogsSuite) createMsg(schema, table, sql string, commitTs int64) *reader.Message { - return &reader.Message{ - Binlog: &pb.Binlog{ - DdlData: &pb.DDLData{ - SchemaName: &schema, - TableName: &table, - DdlQuery: []byte(sql), - }, - CommitTs: commitTs, - }, - } -} - -func (s *syncBinlogsSuite) TestShouldSendBinlogToLoader(c *C) { - source := make(chan *reader.Message, 1) - msgs := []*reader.Message{ - s.createMsg("test42", "users", "alter table users add column gender smallint", 1), - s.createMsg("test42", "users", "alter table users add column gender smallint", 1), - s.createMsg("test42", "operations", "alter table operations drop column seq", 2), - s.createMsg("test42", "users", "alter table users add column gender smallint", 1), - s.createMsg("test42", "operations", "alter table operations drop column seq", 2), - } - expectMsgs := []*reader.Message{ - s.createMsg("test42", "users", "alter table users add column gender smallint", 1), - s.createMsg("test42", "operations", "alter table operations drop column seq", 2), - } - dest := make(chan *loader.Txn, len(msgs)) - go func() { - for _, m := range msgs { - source <- m - } - close(source) - }() - ld := dummyLoader{input: dest} - - err := syncBinlogs(context.Background(), source, &ld) - c.Assert(err, IsNil) - - c.Assert(len(dest), Equals, len(expectMsgs)) - for _, m := range expectMsgs { - txn := <-dest - c.Assert(txn.Metadata.(*reader.Message), DeepEquals, m) - } - - c.Assert(ld.closed, IsTrue) -} - -func (s *syncBinlogsSuite) TestShouldQuitWhenSomeErrorOccurs(c *C) { - readerMsgs := make(chan *reader.Message, 1024) - dummyLoaderImpl := &dummyLoader{ - successes: make(chan *loader.Txn), - // input is set small to trigger blocking easily - input: make(chan *loader.Txn, 1), - } - msg := s.createMsg("test42", "users", "alter table users add column gender smallint", 1) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - // start a routine keep sending msgs to kafka reader - go func() { - // make sure there will be some msgs in readerMsgs for test - for i := 0; i < 3; i++ { - readerMsgs <- msg - } - defer close(readerMsgs) - for { - select { - case <-ctx.Done(): - return - case readerMsgs <- msg: - } - } - }() - errCh := make(chan error) - go func() { - errCh <- syncBinlogs(ctx, readerMsgs, dummyLoaderImpl) - }() - - cancel() - select { - case err := <-errCh: - c.Assert(err, IsNil) - case <-time.After(time.Second): - c.Fatal("server doesn't quit in 1s when some error occurs in loader") - } -} diff --git a/cmd/arbiter/arbiter.toml b/cmd/arbiter/arbiter.toml deleted file mode 100644 index d72dfee16..000000000 --- a/cmd/arbiter/arbiter.toml +++ /dev/null @@ -1,24 +0,0 @@ -# Arbiter Configuration. - -# addr (i.e. 'host:port') to listen on for Arbiter connections -# addr = "127.0.0.1:8251" - -[up] -# if arbiter donesn't have checkpoint, use initial commitTS to initial checkpoint -initial-commit-ts = 0 -kafka-addrs = "127.0.0.1:9092" -kafka-version = "0.8.2.0" -# topic name of kafka to consume binlog -#topic = "" - - -[down] -host = "localhost" -port = 3306 -user = "root" -password = "" -# max concurrent write to downstream -# worker-count = 16 -# max DML operation in a transaction when write to downstream -# batch-size = 64 -# safe-mode = false diff --git a/cmd/arbiter/main.go b/cmd/arbiter/main.go deleted file mode 100644 index d7aeefe9f..000000000 --- a/cmd/arbiter/main.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2019 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "io" - stdlog "log" - "net/http" - _ "net/http/pprof" - "os" - - "github.com/Shopify/sarama" - _ "github.com/go-sql-driver/mysql" - "github.com/pingcap/log" - "github.com/pingcap/tidb-binlog/arbiter" - "github.com/pingcap/tidb-binlog/pkg/util" - "github.com/pingcap/tidb-binlog/pkg/version" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" - "go.uber.org/zap" -) - -func main() { - cfg := arbiter.NewConfig() - if err := cfg.Parse(os.Args[1:]); err != nil { - log.Fatal("verifying flags failed. See 'arbiter --help'.", zap.Error(err)) - } - - if err := util.InitLogger(cfg.LogLevel, cfg.LogFile); err != nil { - log.Fatal("Failed to initialize log", zap.Error(err)) - } - - // We have set sarama.Logger in util.InitLogger. - if !cfg.OpenSaramaLog { - // may too many noise, discard sarama log now - sarama.Logger = stdlog.New(io.Discard, "[Sarama] ", stdlog.LstdFlags) - } - - log.Info("start arbiter...", zap.Reflect("config", cfg)) - version.PrintVersionInfo("Arbiter") - - go startHTTPServer(cfg.ListenAddr) - - srv, err := arbiter.NewServer(cfg) - if err != nil { - log.Error("new server failed", zap.Error(err)) - return - } - - util.SetupSignalHandler(func(_ os.Signal) { - srv.Close() - }) - - log.Info("start run server...") - err = srv.Run() - if err != nil { - log.Error("run server failed", zap.Error(err)) - } - - log.Info("server exit") -} - -func startHTTPServer(addr string) { - prometheus.DefaultGatherer = arbiter.Registry - http.Handle("/metrics", promhttp.Handler()) - - err := http.ListenAndServe(addr, nil) - if err != nil { - log.Fatal("listen and server http failed", zap.String("addr", addr), zap.Error(err)) - } -}