forked from Xue-Alex/commoncrawl_notebooks
-
Notifications
You must be signed in to change notification settings - Fork 0
/
run_webgraph.sh
37 lines (24 loc) · 1.05 KB
/
run_webgraph.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
#!/bin/bash
export LC_ALL=C
source "$(dirname $0)"/webgraph_config.sh
DIR=$(dirname $0)/webgraph-$WEBGRAPH_VERSION
_CLASSPATH=$DIR/webgraph-$WEBGRAPH_VERSION.jar";"$(ls $DIR/deps/*.jar | tr '\n' ';')
if [ -n "$CLASSPATH" ]; then
_CLASSPATH=$CLASSPATH:$_CLASSPATH
fi
JAVA_OPTS="-Xmx3g"
if ! echo "$JAVA_OPTS" | grep -qE -e "-Xmx[0-9]+"; then
# heuristics to run webgraph with 80% of available RAM (or all RAM - 8 GB if this is larger)
MEMMB=$(free -m | perl -ne 'do { $p80 = int($1*.8); $a8 = int($1-8192); $m = $p80; $m = $a8 if $a8 > $p80; print $m; last } if /(\d+)/')
JAVA_OPTS="$JAVA_OPTS -Xmx${MEMMB}m"
fi
case "$1" in
it.unimi.dsi.webgraph.algo.HyperBall )
JAVA_OPTS="$JAVA_OPTS -server -Xss256K -XX:PretenureSizeThreshold=512M -XX:MaxNewSize=$(($MEMMB/3))m \
-XX:+UseNUMA -XX:+UseTLAB -XX:+ResizeTLAB \
-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=99 -XX:+UseCMSInitiatingOccupancyOnly \
-verbose:gc -Xloggc:gc.log"
;;
esac
set -x
time java $JAVA_OPTS -cp "$_CLASSPATH" "$@"