Skip to content

Commit

Permalink
releaseb gzip stored data
Browse files Browse the repository at this point in the history
  • Loading branch information
@Sammers21
Sammers21 committed Mar 4, 2024
1 parent 981cdb8 commit c8b1a98
Show file tree
Hide file tree
Showing 7 changed files with 102 additions and 109 deletions.
2 changes: 1 addition & 1 deletion settings/logback.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,4 @@
<root level="INFO">
<appender-ref ref="STDOUT"/>
</root>
</configuration>
</configuration>
5 changes: 3 additions & 2 deletions src/io/github/sammers/pla/blizzard/PvpLeaderBoard.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import io.github.sammers.pla.db.Character;
import io.github.sammers.pla.http.JsonConvertable;
import io.github.sammers.pla.logic.CharacterCache;
import io.vertx.core.json.JsonArray;
import io.vertx.core.json.JsonObject;

Expand Down Expand Up @@ -59,7 +60,7 @@ public PvpLeaderBoard combine(PvpLeaderBoard other) {
return new PvpLeaderBoard(links, season, name, bracket, newEntities);
}

public Set<Character> toCharacters(Map<String, WowAPICharacter> characterCache, String bracketId) {
public Set<Character> toCharacters(CharacterCache characterCache, String bracketId) {
return entities.stream()
.map(entity -> (JsonObject) entity)
.flatMap(entity -> {
Expand All @@ -73,7 +74,7 @@ public Set<Character> toCharacters(Map<String, WowAPICharacter> characterCache,
JsonObject seasonMatchStatistics = entity.getJsonObject("season_match_statistics");
Long won = seasonMatchStatistics.getLong("won");
Long lost = seasonMatchStatistics.getLong("lost");
WowAPICharacter wowAPICharacter = characterCache.get(Character.fullNameByRealmAndName(name, slug));
WowAPICharacter wowAPICharacter = characterCache.getByFullName(Character.fullNameByRealmAndName(name, slug));
if(wowAPICharacter == null) {
return Stream.empty();
} else {
Expand Down
13 changes: 12 additions & 1 deletion src/io/github/sammers/pla/blizzard/WowAPICharacter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -335,6 +335,17 @@ public WowAPICharacter updatePvpBracketData(CharAndDiff diff, BracketType bracke
return new WowAPICharacter(id, hidden, name, realm, gender, fraction, race, activeSpec, level, clazz, itemLevel, region, newBrackets, lastUpdatedUTCms, achievements, petHash, media, talents);
}

public byte[] toGzippedJson() {
return Calculator.gzipCompress(toJson().encode().getBytes());
}

public static WowAPICharacter fromGzippedJson(byte[] gzippedJson) {
if (gzippedJson == null) {
return null;
}
return fromJson(new JsonObject(new String(Calculator.gzipUncompress(gzippedJson))));
}

@Override
public int hashCode() {
return fullName().hashCode();
Expand Down Expand Up @@ -363,4 +374,4 @@ public JsonObject toJson() {
.put("talents", talents);
}

}
}
65 changes: 56 additions & 9 deletions src/io/github/sammers/pla/logic/Calculator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.time.Duration;
import java.time.ZoneId;
import java.time.ZonedDateTime;
Expand All @@ -22,6 +25,8 @@
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

public class Calculator {

Expand Down Expand Up @@ -271,6 +276,12 @@ private static void roleGroup(int pplInTheGroup, long maxOfThatRoleInGroup, Pred


public static SnapshotDiff calculateDiff(Snapshot oldChars, Snapshot newChars, String bracket, boolean newIsZero) {
// if (o)
if ((oldChars != null && newChars != null) && oldChars.timestamp() > newChars.timestamp()) {
Snapshot temp = oldChars;
oldChars = newChars;
newChars = temp;
}
ArrayList<CharAndDiff> res = new ArrayList<>(newChars.characters().size());
Function<Character, String> idF = getIdFunction(bracket);
Map<String, Character> oldMap;
Expand All @@ -279,15 +290,17 @@ public static SnapshotDiff calculateDiff(Snapshot oldChars, Snapshot newChars, S
} else {
oldMap = oldChars.characters().stream().collect(Collectors.toMap(idF, c -> c, (a, b) -> a));
}
for (Character newCharx : newChars.characters()) {
Character newChar;
Character oldChar = oldMap.get(idF.apply(newCharx));
if (oldChar != null && (newCharx.wins() + newCharx.losses() < oldChar.wins() + oldChar.losses())) {
newChar = oldChar;
oldChar = newCharx;
} else {
newChar = newCharx;
}
for (Character newChar : newChars.characters()) {
Character oldChar = oldMap.get(idF.apply(newChar));
// if (oldChar != null && ((newChar.wins() + newChar.losses()) < (oldChar.wins() + oldChar.losses()))) {
// if (newChar.name().equals("Lôôny") && bracket.equals("shuffle")) {
// log.debug("Lôôny: {}", newChar);
// }
// newChar = oldChar;
// oldChar = newChar;
// } else {
// newChar = newChar;
// }
CharAndDiff e;
if (oldChar == null) {
if (newIsZero) {
Expand Down Expand Up @@ -451,6 +464,40 @@ public static Long totalPages(long itemsTotal, long pageSize) {
return result;
}

public static byte[] gzipCompress(byte[] uncompressedData) {
byte[] result = new byte[]{};
try (ByteArrayOutputStream bos = new ByteArrayOutputStream(uncompressedData.length);
GZIPOutputStream gzipOS = new GZIPOutputStream(bos)) {
gzipOS.write(uncompressedData);
// You need to close it before using bos
gzipOS.close();
result = bos.toByteArray();
} catch (IOException e) {
e.printStackTrace();
}
return result;
}

public static byte[] gzipUncompress(byte[] compressedData) {
if (compressedData ==null ) {
return null;
}
byte[] result = new byte[]{};
try (ByteArrayInputStream bis = new ByteArrayInputStream(compressedData);
ByteArrayOutputStream bos = new ByteArrayOutputStream();
GZIPInputStream gzipIS = new GZIPInputStream(bis)) {
byte[] buffer = new byte[1024];
int len;
while ((len = gzipIS.read(buffer)) != -1) {
bos.write(buffer, 0, len);
}
result = bos.toByteArray();
} catch (IOException e) {
e.printStackTrace();
}
return result;
}

public static String realmCalc(String realm) {
String withCapFirst = realm.substring(0, 1).toUpperCase() + realm.substring(1).toLowerCase();
return withCapFirst.replace(" ", "-");
Expand Down
66 changes: 3 additions & 63 deletions src/io/github/sammers/pla/logic/CharUpdater.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,9 @@ public Completable updateCharacters(String region,
.toList();
List<Pair<String, String>> randomNotUpdatedChars = new ArrayList<>(
characterCache.values().stream()
.filter(wowAPICharacter -> tick - wowAPICharacter.lastUpdatedUTCms() > units.toMillis(timeWithoutUpdateMin))
.map(wowAPICharacter -> Pair.with(wowAPICharacter.name(), wowAPICharacter.realm()))
.filter(bts -> tick - WowAPICharacter.fromGzippedJson(bts).lastUpdatedUTCms() > units.toMillis(timeWithoutUpdateMin))
.map(WowAPICharacter::fromGzippedJson)
.map(btw -> Pair.with(btw.name(), btw.realm()))
.toList()
);

Expand Down Expand Up @@ -143,67 +144,6 @@ public Completable updateCharacters(String region,
}).onErrorComplete().subscribeOn(Main.VTHREAD_SCHEDULER);
}

public Completable updateCharsInfinite(String region) {
return Completable.defer(() -> {
log.info("Updating chars in region " + region);
List<Snapshot> snapshots = new ArrayList<>(Stream.of(TWO_V_TWO, THREE_V_THREE, RBG, SHUFFLE)
.map(bracket -> refs.refByBracket(bracket, region).get()).toList());
Map<String, Long> nickAndMaxRating = new ConcurrentHashMap<>();
return Maybe.just(snapshots)
.map((List<Snapshot> list) -> {
Set<String> charsToUpdate = new HashSet<>();
for (Snapshot board : list) {
for (Character charOnLeaderBoard : board.characters()) {
if (characterCache.getByFullName(charOnLeaderBoard.fullName()) == null) {
charsToUpdate.add(charOnLeaderBoard.fullName());
nickAndMaxRating.compute(charOnLeaderBoard.fullName(), (nick, maxRating) -> {
if (maxRating == null) {
return charOnLeaderBoard.rating();
} else {
return Math.max(maxRating, charOnLeaderBoard.rating());
}
});
}
}
}
int newChars = charsToUpdate.size();
long dayAgo = Instant.now().minus(1, ChronoUnit.DAYS).toEpochMilli();
characterCache.values().stream().flatMap(wowAPICharacter -> {
if (wowAPICharacter.lastUpdatedUTCms() < dayAgo) {
nickAndMaxRating.compute(wowAPICharacter.fullName(), (nick, maxRating) -> {
Long chMax = wowAPICharacter.brackets().stream().max(Comparator.comparingLong(PvpBracket::rating)).map(PvpBracket::rating).orElse(0L);
if (maxRating == null) {
return chMax;
} else {
return Math.max(maxRating, chMax);
}
});
return Stream.of(wowAPICharacter);
} else {
return Stream.of();
}
}).map(WowAPICharacter::fullName).forEach(charsToUpdate::add);
log.info("Completely new chars required to be updated: " + newChars + ", old chars: " + (charsToUpdate.size() - newChars));
return charsToUpdate;
}).flatMapCompletable(charsToUpdate -> {
log.info("Updating " + charsToUpdate.size() + " chars");
// first we update 300 highest rated characters
// and everything else is after
List<String> charsToUpdateList = new ArrayList<>(charsToUpdate);
charsToUpdateList.sort(Comparator.comparingLong(nickAndMaxRating::get).reversed());
List<String> firstPortion = charsToUpdateList.subList(0, Math.min(charsToUpdateList.size(), 300));
List<String> rest = charsToUpdateList.subList(firstPortion.size(), charsToUpdateList.size());
return updateChars(firstPortion, region)
.doOnSubscribe(d -> log.info("Updating first portion of chars: " + firstPortion.size()))
.doOnComplete(() -> log.info("First portion of chars has been updated"))
.andThen(updateChars(rest, region))
.doOnSubscribe(d -> log.info("Updating second portion of chars: " + rest.size()))
.doOnComplete(() -> log.info("Second portion of chars has been updated"))
.onErrorComplete();
});
}).subscribeOn(Main.VTHREAD_SCHEDULER).doOnError(e -> log.error("Update chars infinite error in region " + region, e));
}

public Completable updateChars(List<String> nickNames, String region) {
return Completable.concat(nickNames.stream().map(nickName -> updateChar(region, nickName)).toList())
.subscribeOn(Main.VTHREAD_SCHEDULER);
Expand Down
56 changes: 25 additions & 31 deletions src/io/github/sammers/pla/logic/CharacterCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,33 +11,35 @@

public class CharacterCache {

private final Map<Long, WowAPICharacter> idCache;
private final Map<String, WowAPICharacter> nameCache;
private final Map<Integer, Set<WowAPICharacter>> alts = new ConcurrentHashMap<>();
private final Map<Long, byte[]> idCache;
private final Map<String, byte[]> nameCache;
private final Map<Integer, Set<Long>> alts;

public CharacterCache() {
nameCache = new ConcurrentHashMap<>();
idCache = new ConcurrentHashMap<>();
alts = new ConcurrentHashMap<>();
}

public WowAPICharacter getByFullName(String name) {
return nameCache.get(name);
return WowAPICharacter.fromGzippedJson(nameCache.get(name));
}

public WowAPICharacter getById(Long id) {
return idCache.get(id);
return WowAPICharacter.fromGzippedJson(idCache.get(id));
}

public void upsert(WowAPICharacter character) {
nameCache.put(character.fullName(), character);
idCache.put(character.id(), character);
indexCharAlts(alts, character);
byte[] gzip = character.toGzippedJson();
nameCache.put(character.fullName(), gzip);
idCache.put(character.id(), gzip);
indexCharAlts(alts, character.id(), character.petHash());
}

public Optional<WowAPICharacter> upsertDiff(CharAndDiff diff, String bracket) {
Character character = diff.character();
WowAPICharacter wowAPICharacter = nameCache.get(character.fullName());
if(wowAPICharacter == null) {
WowAPICharacter wowAPICharacter = this.getByFullName(character.fullName());
if (wowAPICharacter == null) {
return Optional.empty();
}
WowAPICharacter updated = wowAPICharacter.updatePvpBracketData(diff, BracketType.fromType(bracket), List.of());
Expand All @@ -50,54 +52,46 @@ public List<WowAPICharacter> upsertGroupDiff(List<CharAndDiff> groupDiff, String
for (int i = 0; i < groupDiff.size(); i++) {
CharAndDiff diff = groupDiff.get(i);
Character character = diff.character();
WowAPICharacter wowAPICharacter = nameCache.get(character.fullName());
if(wowAPICharacter == null) {
WowAPICharacter wowAPICharacter = getByFullName(character.fullName());
if (wowAPICharacter == null) {
continue;
}
List<CharAndDiff> withWho = new ArrayList<>(groupDiff.subList(0, i));
withWho.addAll(groupDiff.subList(i + 1, groupDiff.size()));
WowAPICharacter updated = wowAPICharacter.updatePvpBracketData(
diff,
BracketType.fromType(bracket),
withWho.stream().map(CharAndDiff::character).toList()
diff,
BracketType.fromType(bracket),
withWho.stream().map(CharAndDiff::character).toList()
);
upsert(updated);
res.add(updated);
}
return res;
}

public Collection<WowAPICharacter> values() {
public Collection<byte[]> values() {
return idCache.values();
}

public Map<String, WowAPICharacter> nameCache() {
return nameCache;
}

public Map<Long, WowAPICharacter> idCache() {
return idCache;
}

public Set<WowAPICharacter> altsFor(WowAPICharacter character) {
return Optional.ofNullable(alts.get(character.petHash()))
return Optional.of(alts.get(character.petHash()).stream().map(idCache::get).map(WowAPICharacter::fromGzippedJson
).collect(Collectors.toSet()))
.orElse(Set.of())
.stream()
.filter(c -> !c.hidden())
.collect(Collectors.toSet());
}

public static void indexCharAlts(Map<Integer, Set<WowAPICharacter>> alts, WowAPICharacter character) {
int hash = character.petHash();
alts.compute(hash, (key, value) -> {
public static void indexCharAlts(Map<Integer, Set<Long>> alts, Long charId, int petHash) {
alts.compute(petHash, (key, value) -> {
if (key == -1) {
return null;
}
if (value == null) {
value = new TreeSet<>(Comparator.comparing(WowAPICharacter::id));
value = new TreeSet<>();
}
value.remove(character);
value.add(character);
value.remove(charId);
value.add(charId);
return value;
});
}
Expand Down
4 changes: 2 additions & 2 deletions src/io/github/sammers/pla/logic/Ladder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ public Single<List<Character>> pureBlizzardApiFetch(String bracket, String regio
Single<List<Character>> sh = Single.just(new ArrayList<>(shuffleSpecs.size()));
String specForBlizApi = shuffleSpec.replaceAll("/", "-");
sh = sh.flatMap(thisSpecChars -> blizzardAPI.pvpLeaderboard(specForBlizApi, region).flatMapSingle(leaderboard -> {
Set<Character> chrs = leaderboard.toCharacters(characterCache.nameCache(), shuffleSpec);
Set<Character> chrs = leaderboard.toCharacters(characterCache, shuffleSpec);
thisSpecChars.addAll(chrs);
return Single.just(thisSpecChars);
}));
Expand All @@ -226,7 +226,7 @@ public Single<List<Character>> pureBlizzardApiFetch(String bracket, String regio
} else {
Single<List<Character>> res = Single.just(new ArrayList<>(5000));
resCharList = res.flatMap(s -> blizzardAPI.pvpLeaderboard(bracket, region).flatMapSingle(leaderboard -> {
Set<Character> chrs = leaderboard.toCharacters(characterCache.nameCache(), bracket);
Set<Character> chrs = leaderboard.toCharacters(characterCache, bracket);
s.addAll(chrs);
return Single.just(s);
}));
Expand Down

0 comments on commit c8b1a98

Please sign in to comment.