Skip to content
This repository has been archived by the owner on Jan 6, 2022. It is now read-only.

Commit

Permalink
Merge branch 'rickyhuo.clickhoust.logger'
Browse files Browse the repository at this point in the history
  • Loading branch information
@RickyHuo
RickyHuo committed Jan 11, 2018
2 parents e01e37b + 2c1dfbc commit 58a5250
Show file tree
Hide file tree
Showing 2 changed files with 34 additions and 19 deletions.
10 changes: 10 additions & 0 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,16 @@
<artifactId>clickhouse-jdbc</artifactId>
<version>0.1.34</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.7</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.7</version>
</dependency>
</dependencies>

<build>
Expand Down
43 changes: 24 additions & 19 deletions src/main/java/com/sina/bip/hangout/outputs/Clickhouse.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@

import com.ctrip.ops.sysdev.baseplugin.BaseOutput;
import com.ctrip.ops.sysdev.render.TemplateRender;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import ru.yandex.clickhouse.BalancedClickhouseDataSource;
import ru.yandex.clickhouse.ClickHouseDataSource;
import ru.yandex.clickhouse.settings.ClickHouseProperties;
Expand All @@ -20,6 +22,7 @@

public class Clickhouse extends BaseOutput {

private static final Logger log = LogManager.getLogger(Clickhouse.class);
private final static int BULKSIZE = 1000;

private String host;
Expand All @@ -45,19 +48,19 @@ protected void prepare() {
this.events = new ArrayList<Map>();

if (!this.config.containsKey("host")) {
System.out.println("hostname must be included in config");
log.error("hostname must be included in config");
System.exit(1);
}
this.host = (String) this.config.get("host");

if (!this.config.containsKey("database")) {
System.out.println("database must be included in config");
log.error("database must be included in config");
System.exit(1);
}
this.database = (String) this.config.get("database");

if (!this.config.containsKey("table")) {
System.out.println("table must be included in config");
log.error("table must be included in config");
System.exit(1);
}
this.table = (String) this.config.get("table");
Expand All @@ -68,7 +71,7 @@ protected void prepare() {
this.withCredit = true;

} else if (this.config.containsKey("user") || this.config.containsKey("password")) {
System.out.println("User and password must be included in config at same time");
log.warn("user and password must be included in config at same time");
} else {
this.withCredit = false;
}
Expand All @@ -82,7 +85,8 @@ protected void prepare() {
if (this.config.containsKey("fields")) {
this.fields = (List<String>) this.config.get("fields");
} else {
System.out.println("fields must be included in config");
log.error("fields must be included in config");
System.exit(1);
}

if (this.config.containsKey("replace_include_fields")) {
Expand All @@ -94,8 +98,8 @@ protected void prepare() {
}

if (this.config.containsKey("replace_include_fields") && this.config.containsKey("replace_exclude_fields")) {
System.out.println("Replace_include_fields and replace_exclude_fields exist at the same time," +
" please use one of them.");
log.error("Replace_include_fields and replace_exclude_fields exist at the same time, " +
"please use one of them.");
System.exit(1);
}

Expand All @@ -114,22 +118,22 @@ protected void prepare() {
try {
this.schema = ClickhouseUtils.getSchema(dataSource, this.table);
} catch (SQLException e) {
System.out.println("table is not vaild");
log.error("input table is not vaild");
System.exit(1);
}

this.templateRenderMap = new HashMap<>();
for (String field: fields) {
if (!this.schema.containsKey(ClickhouseUtils.realField(field))) {
String msg = String.format("table [%s] doesn't contain field '%s'", this.table, field);
System.out.println(msg);
log.error(msg);
System.exit(1);
}
try {
this.templateRenderMap.put(field, TemplateRender.getRender(field, false));
} catch (Exception e) {
String msg = String.format("cannot get templateRender of field [%s]", field);
System.out.println(msg);
log.warn(msg);
}
}

Expand All @@ -143,17 +147,18 @@ private String initSql() {
}

String init = String.format("insert into %s (%s) values", this.table, String.join(" ,", realFields));
log.debug("init sql: "+ init);
return init;
}

private String dealWithQuote(String str) {
/*
* 因为Clickhouse SQL语句必须用单引号'
* 因为Clickhouse SQL语句必须用单引号'， 例如：
* insert into test.test (date, value) values ('2017-10-29', '23')
* SQL语句需要将数值中的单引号'转义
* */

if (str.indexOf("'") < 0) {
if (!str.contains("'")) {
return str;
} else if (str.indexOf("\\'") > 0) {
// deal with "\'"
Expand Down Expand Up @@ -220,14 +225,14 @@ private void bulkInsert(List<Map> events) throws Exception {
conn.createStatement().execute(sqls.toString());
conn.close();
} catch (SQLException e){
System.out.println(sqls.toString());
System.out.println(e.toString());
log.error(e.toString());
log.debug(sqls.toString());

for (int i = 0; i < this.events.size(); i++) {
System.out.println(events.get(i));
log.debug(events.get(i));
}
} catch (Exception e) {
System.out.println(e.toString());
System.out.println("error");
log.error("error");
}
conn.close();
}
Expand All @@ -249,8 +254,8 @@ protected void emit(Map event) {
try {
eventInsert(event);
} catch (Exception e) {
System.out.println(e.toString());
System.out.println("insert error");
log.error(e.toString());
log.warn("insert error");
}
}

Expand Down

0 comments on commit 58a5250

Please sign in to comment.