From c8b1a98d9629263b7ba1965c441d79b77f1eba58 Mon Sep 17 00:00:00 2001 From: Sammers21 Date: Tue, 5 Mar 2024 00:18:15 +0300 Subject: [PATCH] releaseb gzip stored data --- settings/logback.xml | 2 +- .../sammers/pla/blizzard/PvpLeaderBoard.java | 5 +- .../sammers/pla/blizzard/WowAPICharacter.java | 13 +++- .../github/sammers/pla/logic/Calculator.java | 65 +++++++++++++++--- .../github/sammers/pla/logic/CharUpdater.java | 66 +------------------ .../sammers/pla/logic/CharacterCache.java | 56 +++++++--------- src/io/github/sammers/pla/logic/Ladder.java | 4 +- 7 files changed, 102 insertions(+), 109 deletions(-) diff --git a/settings/logback.xml b/settings/logback.xml index 0cb9e071..974a2733 100644 --- a/settings/logback.xml +++ b/settings/logback.xml @@ -8,4 +8,4 @@ - \ No newline at end of file + diff --git a/src/io/github/sammers/pla/blizzard/PvpLeaderBoard.java b/src/io/github/sammers/pla/blizzard/PvpLeaderBoard.java index 730442d3..925aafda 100644 --- a/src/io/github/sammers/pla/blizzard/PvpLeaderBoard.java +++ b/src/io/github/sammers/pla/blizzard/PvpLeaderBoard.java @@ -2,6 +2,7 @@ import io.github.sammers.pla.db.Character; import io.github.sammers.pla.http.JsonConvertable; +import io.github.sammers.pla.logic.CharacterCache; import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; @@ -59,7 +60,7 @@ public PvpLeaderBoard combine(PvpLeaderBoard other) { return new PvpLeaderBoard(links, season, name, bracket, newEntities); } - public Set toCharacters(Map characterCache, String bracketId) { + public Set toCharacters(CharacterCache characterCache, String bracketId) { return entities.stream() .map(entity -> (JsonObject) entity) .flatMap(entity -> { @@ -73,7 +74,7 @@ public Set toCharacters(Map characterCache, JsonObject seasonMatchStatistics = entity.getJsonObject("season_match_statistics"); Long won = seasonMatchStatistics.getLong("won"); Long lost = seasonMatchStatistics.getLong("lost"); - WowAPICharacter wowAPICharacter = characterCache.get(Character.fullNameByRealmAndName(name, slug)); + WowAPICharacter wowAPICharacter = characterCache.getByFullName(Character.fullNameByRealmAndName(name, slug)); if(wowAPICharacter == null) { return Stream.empty(); } else { diff --git a/src/io/github/sammers/pla/blizzard/WowAPICharacter.java b/src/io/github/sammers/pla/blizzard/WowAPICharacter.java index c1af124b..7d88e12f 100644 --- a/src/io/github/sammers/pla/blizzard/WowAPICharacter.java +++ b/src/io/github/sammers/pla/blizzard/WowAPICharacter.java @@ -335,6 +335,17 @@ public WowAPICharacter updatePvpBracketData(CharAndDiff diff, BracketType bracke return new WowAPICharacter(id, hidden, name, realm, gender, fraction, race, activeSpec, level, clazz, itemLevel, region, newBrackets, lastUpdatedUTCms, achievements, petHash, media, talents); } + public byte[] toGzippedJson() { + return Calculator.gzipCompress(toJson().encode().getBytes()); + } + + public static WowAPICharacter fromGzippedJson(byte[] gzippedJson) { + if (gzippedJson == null) { + return null; + } + return fromJson(new JsonObject(new String(Calculator.gzipUncompress(gzippedJson)))); + } + @Override public int hashCode() { return fullName().hashCode(); @@ -363,4 +374,4 @@ public JsonObject toJson() { .put("talents", talents); } -} \ No newline at end of file +} diff --git a/src/io/github/sammers/pla/logic/Calculator.java b/src/io/github/sammers/pla/logic/Calculator.java index 027a3804..0a2ddd4e 100644 --- a/src/io/github/sammers/pla/logic/Calculator.java +++ b/src/io/github/sammers/pla/logic/Calculator.java @@ -14,6 +14,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.time.Duration; import java.time.ZoneId; import java.time.ZonedDateTime; @@ -22,6 +25,8 @@ import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; public class Calculator { @@ -271,6 +276,12 @@ private static void roleGroup(int pplInTheGroup, long maxOfThatRoleInGroup, Pred public static SnapshotDiff calculateDiff(Snapshot oldChars, Snapshot newChars, String bracket, boolean newIsZero) { +// if (o) + if ((oldChars != null && newChars != null) && oldChars.timestamp() > newChars.timestamp()) { + Snapshot temp = oldChars; + oldChars = newChars; + newChars = temp; + } ArrayList res = new ArrayList<>(newChars.characters().size()); Function idF = getIdFunction(bracket); Map oldMap; @@ -279,15 +290,17 @@ public static SnapshotDiff calculateDiff(Snapshot oldChars, Snapshot newChars, S } else { oldMap = oldChars.characters().stream().collect(Collectors.toMap(idF, c -> c, (a, b) -> a)); } - for (Character newCharx : newChars.characters()) { - Character newChar; - Character oldChar = oldMap.get(idF.apply(newCharx)); - if (oldChar != null && (newCharx.wins() + newCharx.losses() < oldChar.wins() + oldChar.losses())) { - newChar = oldChar; - oldChar = newCharx; - } else { - newChar = newCharx; - } + for (Character newChar : newChars.characters()) { + Character oldChar = oldMap.get(idF.apply(newChar)); +// if (oldChar != null && ((newChar.wins() + newChar.losses()) < (oldChar.wins() + oldChar.losses()))) { +// if (newChar.name().equals("Lôôny") && bracket.equals("shuffle")) { +// log.debug("Lôôny: {}", newChar); +// } +// newChar = oldChar; +// oldChar = newChar; +// } else { +// newChar = newChar; +// } CharAndDiff e; if (oldChar == null) { if (newIsZero) { @@ -451,6 +464,40 @@ public static Long totalPages(long itemsTotal, long pageSize) { return result; } + public static byte[] gzipCompress(byte[] uncompressedData) { + byte[] result = new byte[]{}; + try (ByteArrayOutputStream bos = new ByteArrayOutputStream(uncompressedData.length); + GZIPOutputStream gzipOS = new GZIPOutputStream(bos)) { + gzipOS.write(uncompressedData); + // You need to close it before using bos + gzipOS.close(); + result = bos.toByteArray(); + } catch (IOException e) { + e.printStackTrace(); + } + return result; + } + + public static byte[] gzipUncompress(byte[] compressedData) { + if (compressedData ==null ) { + return null; + } + byte[] result = new byte[]{}; + try (ByteArrayInputStream bis = new ByteArrayInputStream(compressedData); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + GZIPInputStream gzipIS = new GZIPInputStream(bis)) { + byte[] buffer = new byte[1024]; + int len; + while ((len = gzipIS.read(buffer)) != -1) { + bos.write(buffer, 0, len); + } + result = bos.toByteArray(); + } catch (IOException e) { + e.printStackTrace(); + } + return result; + } + public static String realmCalc(String realm) { String withCapFirst = realm.substring(0, 1).toUpperCase() + realm.substring(1).toLowerCase(); return withCapFirst.replace(" ", "-"); diff --git a/src/io/github/sammers/pla/logic/CharUpdater.java b/src/io/github/sammers/pla/logic/CharUpdater.java index b25bed61..e8297c05 100644 --- a/src/io/github/sammers/pla/logic/CharUpdater.java +++ b/src/io/github/sammers/pla/logic/CharUpdater.java @@ -104,8 +104,9 @@ public Completable updateCharacters(String region, .toList(); List> randomNotUpdatedChars = new ArrayList<>( characterCache.values().stream() - .filter(wowAPICharacter -> tick - wowAPICharacter.lastUpdatedUTCms() > units.toMillis(timeWithoutUpdateMin)) - .map(wowAPICharacter -> Pair.with(wowAPICharacter.name(), wowAPICharacter.realm())) + .filter(bts -> tick - WowAPICharacter.fromGzippedJson(bts).lastUpdatedUTCms() > units.toMillis(timeWithoutUpdateMin)) + .map(WowAPICharacter::fromGzippedJson) + .map(btw -> Pair.with(btw.name(), btw.realm())) .toList() ); @@ -143,67 +144,6 @@ public Completable updateCharacters(String region, }).onErrorComplete().subscribeOn(Main.VTHREAD_SCHEDULER); } - public Completable updateCharsInfinite(String region) { - return Completable.defer(() -> { - log.info("Updating chars in region " + region); - List snapshots = new ArrayList<>(Stream.of(TWO_V_TWO, THREE_V_THREE, RBG, SHUFFLE) - .map(bracket -> refs.refByBracket(bracket, region).get()).toList()); - Map nickAndMaxRating = new ConcurrentHashMap<>(); - return Maybe.just(snapshots) - .map((List list) -> { - Set charsToUpdate = new HashSet<>(); - for (Snapshot board : list) { - for (Character charOnLeaderBoard : board.characters()) { - if (characterCache.getByFullName(charOnLeaderBoard.fullName()) == null) { - charsToUpdate.add(charOnLeaderBoard.fullName()); - nickAndMaxRating.compute(charOnLeaderBoard.fullName(), (nick, maxRating) -> { - if (maxRating == null) { - return charOnLeaderBoard.rating(); - } else { - return Math.max(maxRating, charOnLeaderBoard.rating()); - } - }); - } - } - } - int newChars = charsToUpdate.size(); - long dayAgo = Instant.now().minus(1, ChronoUnit.DAYS).toEpochMilli(); - characterCache.values().stream().flatMap(wowAPICharacter -> { - if (wowAPICharacter.lastUpdatedUTCms() < dayAgo) { - nickAndMaxRating.compute(wowAPICharacter.fullName(), (nick, maxRating) -> { - Long chMax = wowAPICharacter.brackets().stream().max(Comparator.comparingLong(PvpBracket::rating)).map(PvpBracket::rating).orElse(0L); - if (maxRating == null) { - return chMax; - } else { - return Math.max(maxRating, chMax); - } - }); - return Stream.of(wowAPICharacter); - } else { - return Stream.of(); - } - }).map(WowAPICharacter::fullName).forEach(charsToUpdate::add); - log.info("Completely new chars required to be updated: " + newChars + ", old chars: " + (charsToUpdate.size() - newChars)); - return charsToUpdate; - }).flatMapCompletable(charsToUpdate -> { - log.info("Updating " + charsToUpdate.size() + " chars"); - // first we update 300 highest rated characters - // and everything else is after - List charsToUpdateList = new ArrayList<>(charsToUpdate); - charsToUpdateList.sort(Comparator.comparingLong(nickAndMaxRating::get).reversed()); - List firstPortion = charsToUpdateList.subList(0, Math.min(charsToUpdateList.size(), 300)); - List rest = charsToUpdateList.subList(firstPortion.size(), charsToUpdateList.size()); - return updateChars(firstPortion, region) - .doOnSubscribe(d -> log.info("Updating first portion of chars: " + firstPortion.size())) - .doOnComplete(() -> log.info("First portion of chars has been updated")) - .andThen(updateChars(rest, region)) - .doOnSubscribe(d -> log.info("Updating second portion of chars: " + rest.size())) - .doOnComplete(() -> log.info("Second portion of chars has been updated")) - .onErrorComplete(); - }); - }).subscribeOn(Main.VTHREAD_SCHEDULER).doOnError(e -> log.error("Update chars infinite error in region " + region, e)); - } - public Completable updateChars(List nickNames, String region) { return Completable.concat(nickNames.stream().map(nickName -> updateChar(region, nickName)).toList()) .subscribeOn(Main.VTHREAD_SCHEDULER); diff --git a/src/io/github/sammers/pla/logic/CharacterCache.java b/src/io/github/sammers/pla/logic/CharacterCache.java index 04345f30..6f1ba7e6 100644 --- a/src/io/github/sammers/pla/logic/CharacterCache.java +++ b/src/io/github/sammers/pla/logic/CharacterCache.java @@ -11,33 +11,35 @@ public class CharacterCache { - private final Map idCache; - private final Map nameCache; - private final Map> alts = new ConcurrentHashMap<>(); + private final Map idCache; + private final Map nameCache; + private final Map> alts; public CharacterCache() { nameCache = new ConcurrentHashMap<>(); idCache = new ConcurrentHashMap<>(); + alts = new ConcurrentHashMap<>(); } public WowAPICharacter getByFullName(String name) { - return nameCache.get(name); + return WowAPICharacter.fromGzippedJson(nameCache.get(name)); } public WowAPICharacter getById(Long id) { - return idCache.get(id); + return WowAPICharacter.fromGzippedJson(idCache.get(id)); } public void upsert(WowAPICharacter character) { - nameCache.put(character.fullName(), character); - idCache.put(character.id(), character); - indexCharAlts(alts, character); + byte[] gzip = character.toGzippedJson(); + nameCache.put(character.fullName(), gzip); + idCache.put(character.id(), gzip); + indexCharAlts(alts, character.id(), character.petHash()); } public Optional upsertDiff(CharAndDiff diff, String bracket) { Character character = diff.character(); - WowAPICharacter wowAPICharacter = nameCache.get(character.fullName()); - if(wowAPICharacter == null) { + WowAPICharacter wowAPICharacter = this.getByFullName(character.fullName()); + if (wowAPICharacter == null) { return Optional.empty(); } WowAPICharacter updated = wowAPICharacter.updatePvpBracketData(diff, BracketType.fromType(bracket), List.of()); @@ -50,16 +52,16 @@ public List upsertGroupDiff(List groupDiff, String for (int i = 0; i < groupDiff.size(); i++) { CharAndDiff diff = groupDiff.get(i); Character character = diff.character(); - WowAPICharacter wowAPICharacter = nameCache.get(character.fullName()); - if(wowAPICharacter == null) { + WowAPICharacter wowAPICharacter = getByFullName(character.fullName()); + if (wowAPICharacter == null) { continue; } List withWho = new ArrayList<>(groupDiff.subList(0, i)); withWho.addAll(groupDiff.subList(i + 1, groupDiff.size())); WowAPICharacter updated = wowAPICharacter.updatePvpBracketData( - diff, - BracketType.fromType(bracket), - withWho.stream().map(CharAndDiff::character).toList() + diff, + BracketType.fromType(bracket), + withWho.stream().map(CharAndDiff::character).toList() ); upsert(updated); res.add(updated); @@ -67,37 +69,29 @@ public List upsertGroupDiff(List groupDiff, String return res; } - public Collection values() { + public Collection values() { return idCache.values(); } - public Map nameCache() { - return nameCache; - } - - public Map idCache() { - return idCache; - } - public Set altsFor(WowAPICharacter character) { - return Optional.ofNullable(alts.get(character.petHash())) + return Optional.of(alts.get(character.petHash()).stream().map(idCache::get).map(WowAPICharacter::fromGzippedJson + ).collect(Collectors.toSet())) .orElse(Set.of()) .stream() .filter(c -> !c.hidden()) .collect(Collectors.toSet()); } - public static void indexCharAlts(Map> alts, WowAPICharacter character) { - int hash = character.petHash(); - alts.compute(hash, (key, value) -> { + public static void indexCharAlts(Map> alts, Long charId, int petHash) { + alts.compute(petHash, (key, value) -> { if (key == -1) { return null; } if (value == null) { - value = new TreeSet<>(Comparator.comparing(WowAPICharacter::id)); + value = new TreeSet<>(); } - value.remove(character); - value.add(character); + value.remove(charId); + value.add(charId); return value; }); } diff --git a/src/io/github/sammers/pla/logic/Ladder.java b/src/io/github/sammers/pla/logic/Ladder.java index 1950d7c6..c592acb2 100644 --- a/src/io/github/sammers/pla/logic/Ladder.java +++ b/src/io/github/sammers/pla/logic/Ladder.java @@ -209,7 +209,7 @@ public Single> pureBlizzardApiFetch(String bracket, String regio Single> sh = Single.just(new ArrayList<>(shuffleSpecs.size())); String specForBlizApi = shuffleSpec.replaceAll("/", "-"); sh = sh.flatMap(thisSpecChars -> blizzardAPI.pvpLeaderboard(specForBlizApi, region).flatMapSingle(leaderboard -> { - Set chrs = leaderboard.toCharacters(characterCache.nameCache(), shuffleSpec); + Set chrs = leaderboard.toCharacters(characterCache, shuffleSpec); thisSpecChars.addAll(chrs); return Single.just(thisSpecChars); })); @@ -226,7 +226,7 @@ public Single> pureBlizzardApiFetch(String bracket, String regio } else { Single> res = Single.just(new ArrayList<>(5000)); resCharList = res.flatMap(s -> blizzardAPI.pvpLeaderboard(bracket, region).flatMapSingle(leaderboard -> { - Set chrs = leaderboard.toCharacters(characterCache.nameCache(), bracket); + Set chrs = leaderboard.toCharacters(characterCache, bracket); s.addAll(chrs); return Single.just(s); }));