From 45b2eaae84f338bbd28ac17256ab1d29b1081f75 Mon Sep 17 00:00:00 2001 From: Dalibor Duric Date: Sun, 6 May 2018 17:07:27 +1000 Subject: [PATCH 1/7] initial LiveLogParser commit --- live/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 live/__init__.py diff --git a/live/__init__.py b/live/__init__.py new file mode 100644 index 0000000..e69de29 From 317b6560d04935665501bff3677b6ca037d0603e Mon Sep 17 00:00:00 2001 From: Dalibor Duric Date: Sun, 6 May 2018 17:34:38 +1000 Subject: [PATCH 2/7] first prototype of LiveLogParser --- .gitignore | 13 ++++ example_live_parser.py | 40 +++++++++++ {live => hslog/live}/__init__.py | 0 hslog/live/entities.py | 68 ++++++++++++++++++ hslog/live/export.py | 30 ++++++++ hslog/live/packets.py | 13 ++++ hslog/live/parser.py | 114 +++++++++++++++++++++++++++++++ 7 files changed, 278 insertions(+) create mode 100644 .gitignore create mode 100644 example_live_parser.py rename {live => hslog/live}/__init__.py (100%) create mode 100644 hslog/live/entities.py create mode 100644 hslog/live/export.py create mode 100644 hslog/live/packets.py create mode 100644 hslog/live/parser.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..db1c371 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +__pycache__/ +node_modules/ +*.log +celerybeat.pid +*.pyc +.project +.pydevproject +.settings +celerybeat-schedule +creds.py +*.retry +.vscode/ +public/ diff --git a/example_live_parser.py b/example_live_parser.py new file mode 100644 index 0000000..29380af --- /dev/null +++ b/example_live_parser.py @@ -0,0 +1,40 @@ +from time import sleep +import traceback + +from hslog.live.parser import LiveLogParser + +''' + ---------------------------------------------------------------------- + LiveLogParser assumes that you've configured Power.log to be a symlink + + in "SOME_PATH/Hearthstone/Logs" folder: + ln -s Power.log /tmp/hearthstone-redirected.log + + this will redirect all data coming into Power.log + so we can access it from a RAM disk + ---------------------------------------------------------------------- + For better performance make /tmp of type tmpfs (or another location) + + in /etc/fstab add line: + tmpfs /tmp tmpfs nodev,nosuid,size=1G 0 0 + + this will create in-memory storage which is faster then SSD + you need to restart the computer for this to take effect + ---------------------------------------------------------------------- +''' +def main(): + try: + file = '/tmp/hearthstone-redirected.log' + liveParser = LiveLogParser(file) + liveParser.start() + + while True: + sleep(1) + + except: + print(traceback.format_exc()) + liveParser.stop() + + +if __name__ == "__main__": + main() diff --git a/live/__init__.py b/hslog/live/__init__.py similarity index 100% rename from live/__init__.py rename to hslog/live/__init__.py diff --git a/hslog/live/entities.py b/hslog/live/entities.py new file mode 100644 index 0000000..5751c97 --- /dev/null +++ b/hslog/live/entities.py @@ -0,0 +1,68 @@ +from hearthstone.entities import Card, Game, Player, Entity +from hearthstone.enums import GameTag + +''' + * Card is called on export from game + * LiveCard replaces Card and inserts update_callback + * The point is to become able to route update events towards an API end-point +''' +class LiveEntity(Entity): + + def __init__(self, entity_id, parent, **kwargs): + ''' Entity requires an ID, store everything else in kwargs ''' + self.parent = parent + self.game_index = self.parent.parser.games.index(self.parent) + super(LiveEntity, self).__init__(entity_id, **kwargs) + + # push data to an end-point + print(f'GAME {self.game_index} --- ENTITY CREATED:', self) + + def tag_change(self, tag, value): + if tag == GameTag.CONTROLLER and not self._initial_controller: + self._initial_controller = self.tags.get(GameTag.CONTROLLER, value) + self.tags[tag] = value + + # update notify + self.update_callback() + + def update_callback(self): + # push data to an end-point + print(f'GAME {self.game_index} --- ENTITY UPDATED:', self) + + +class LiveCard(Card, LiveEntity): + + def __init__(self, entity_id, card_id, parent): + super(LiveCard, self).__init__( + entity_id=entity_id, + card_id=card_id, + parent=parent) + + ''' if card_id doesn't change, there's no need to pass it as the argument. + we can use self.card_id instead as it is set by Card class ''' + def reveal(self, card_id, tags): + self.revealed = True + self.card_id = card_id + if self.initial_card_id is None: + self.initial_card_id = card_id + self.tags.update(tags) + + # update notify + self.update_callback() + + def hide(self): + self.revealed = False + + # update notify + self.update_callback() + + ''' same comment as for reveal ''' + def change(self, card_id, tags): + if self.initial_card_id is None: + self.initial_card_id = card_id + self.card_id = card_id + self.tags.update(tags) + + # update notify + self.update_callback() + \ No newline at end of file diff --git a/hslog/live/export.py b/hslog/live/export.py new file mode 100644 index 0000000..f37d24d --- /dev/null +++ b/hslog/live/export.py @@ -0,0 +1,30 @@ +from hslog.export import EntityTreeExporter +from hslog.live.entities import LiveCard + + +class LiveEntityTreeExporter(EntityTreeExporter): + card_class = LiveCard + + def __init__(self, packet_tree): + super(LiveEntityTreeExporter, self).__init__(packet_tree) + + def handle_full_entity(self, packet): + entity_id = packet.entity + + # Check if the entity already exists in the game first. + # This prevents creating it twice. + # This can legitimately happen in case of GAME_RESET + if entity_id <= len(self.game.entities): + # That first if check is an optimization to prevent always looping over all of + # the game's entities every single FULL_ENTITY packet... + # FIXME: Switching to a dict for game.entities would simplify this. + existing_entity = self.game.find_entity_by_id(entity_id) + if existing_entity is not None: + existing_entity.card_id = packet.card_id + existing_entity.tags = dict(packet.tags) + return existing_entity + + entity = self.card_class(entity_id, packet.card_id, self.packet_tree) + entity.tags = dict(packet.tags) + self.game.register_entity(entity) + return entity \ No newline at end of file diff --git a/hslog/live/packets.py b/hslog/live/packets.py new file mode 100644 index 0000000..53745e4 --- /dev/null +++ b/hslog/live/packets.py @@ -0,0 +1,13 @@ +from hslog.live.export import LiveEntityTreeExporter +from hslog.packets import PacketTree + + +class LivePacketTree(PacketTree): + + def __init__(self, ts, parser): + self.parser = parser + self.liveExporter = LiveEntityTreeExporter(self) + super(LivePacketTree, self).__init__(ts) + + def live_export(self, packet): + return self.liveExporter.export_packet(packet) \ No newline at end of file diff --git a/hslog/live/parser.py b/hslog/live/parser.py new file mode 100644 index 0000000..8e8dfff --- /dev/null +++ b/hslog/live/parser.py @@ -0,0 +1,114 @@ +from collections import deque +from threading import Thread +import time + +from hslog.live.packets import LivePacketTree +from hslog.parser import LogParser +from hslog.player import PlayerManager, LazyPlayer +from hslog.utils import parse_tag +from hslog import tokens, packets + + +class LiveLogParser(LogParser): + + def __init__(self, filepath): + super(LiveLogParser, self).__init__() + self.running = False + self.filepath = filepath + self.lines_deque = deque([]) + + def new_packet_tree(self, ts): + self._packets = LivePacketTree(ts, self) + self._packets.spectator_mode = self.spectator_mode + self._packets.manager = PlayerManager() + self.current_block = self._packets + self.games.append(self._packets) + + ''' why is this return important? ''' + return self._packets + + def tag_change(self, ts, e, tag, value, def_change): + entity_id = self.parse_entity_or_player(e) + tag, value = parse_tag(tag, value) + self._check_for_mulligan_hack(ts, tag, value) + + ''' skipping LazyPlayer here because it doesn't have data ''' + skip = False + if isinstance(entity_id, LazyPlayer): + entity_id = self._packets.manager.register_player_name_on_tag_change(entity_id, tag, value) + skip = True + has_change_def = def_change == tokens.DEF_CHANGE + packet = packets.TagChange(ts, entity_id, tag, value, has_change_def) + + if not skip: self.register_packet(packet) + return packet + + def register_packet(self, packet, node=None): + ''' make sure we're registering packets to the current game''' + if not self._packets or self._packets != self.games[-1]: + self._packets = self.games[-1] + + if node is None: + node = self.current_block.packets + node.append(packet) + + ''' line below triggers packet export which will + run update_callback for entity being + updated by the packet. + + self._packets == EntityTreeExporter + ''' + self._packets.live_export(packet) + + self._packets._packet_counter += 1 + packet.packet_id = self._packets._packet_counter + + def file_worker(self): + file = open(self.filepath, 'r') + while self.running: + line = file.readline() + if line: + self.lines_deque.append(line) + else: + time.sleep(0.2) + + def parse_worker(self): + while self.running: + if len(self.lines_deque): + line = self.lines_deque.popleft() + self.read_line(line) + else: + time.sleep(0.2) + + def start_file_worker(self): + file_thread = Thread(target=self.file_worker) + file_thread.setDaemon(True) + file_thread.start() + + def start_parse_worker(self): + parse_thread = Thread(target=self.parse_worker) + parse_thread.setDaemon(True) + parse_thread.start() + + def start(self): + self.running = True + self.start_file_worker() + self.start_parse_worker() + + def stop(self): + self.running = False + + def flask_endpoint(self, line): + ''' + !!! EXPERIMENTAL: not really required for LiveLogParser to work + + #!/bin/bash + # used with run-server.sh + # can be used instead of file_worker to populate lines_deque + + nohup python server/server.py >/dev/null 2>&1 & + sleep 1 + echo "running tail > flask server" + tail /tmp/hearthstone-redirected.log --follow | python server/proxy.py + ''' + self.lines_deque.append(line) \ No newline at end of file From 823c47a97718adfc0ad87d635753b2efb1c05065 Mon Sep 17 00:00:00 2001 From: Dalibor Duric Date: Sun, 6 May 2018 17:44:13 +1000 Subject: [PATCH 3/7] flake8 cleanup --- example_live_parser.py | 22 +++++++++++--------- hslog/live/entities.py | 23 +++++++++++---------- hslog/live/export.py | 6 +++--- hslog/live/packets.py | 6 +++--- hslog/live/parser.py | 47 ++++++++++++++++++++++-------------------- 5 files changed, 55 insertions(+), 49 deletions(-) diff --git a/example_live_parser.py b/example_live_parser.py index 29380af..93bd92f 100644 --- a/example_live_parser.py +++ b/example_live_parser.py @@ -6,35 +6,37 @@ ''' ---------------------------------------------------------------------- LiveLogParser assumes that you've configured Power.log to be a symlink - + in "SOME_PATH/Hearthstone/Logs" folder: ln -s Power.log /tmp/hearthstone-redirected.log - + this will redirect all data coming into Power.log so we can access it from a RAM disk ---------------------------------------------------------------------- - For better performance make /tmp of type tmpfs (or another location) - + For better performance make /tmp of type tmpfs (or another location) + in /etc/fstab add line: tmpfs /tmp tmpfs nodev,nosuid,size=1G 0 0 - + this will create in-memory storage which is faster then SSD you need to restart the computer for this to take effect ---------------------------------------------------------------------- ''' + + def main(): try: file = '/tmp/hearthstone-redirected.log' liveParser = LiveLogParser(file) liveParser.start() - + while True: sleep(1) - - except: + + except Exception as e: print(traceback.format_exc()) liveParser.stop() - - + + if __name__ == "__main__": main() diff --git a/hslog/live/entities.py b/hslog/live/entities.py index 5751c97..ae73d01 100644 --- a/hslog/live/entities.py +++ b/hslog/live/entities.py @@ -1,4 +1,4 @@ -from hearthstone.entities import Card, Game, Player, Entity +from hearthstone.entities import Card, Entity from hearthstone.enums import GameTag ''' @@ -6,32 +6,34 @@ * LiveCard replaces Card and inserts update_callback * The point is to become able to route update events towards an API end-point ''' + + class LiveEntity(Entity): - + def __init__(self, entity_id, parent, **kwargs): ''' Entity requires an ID, store everything else in kwargs ''' self.parent = parent self.game_index = self.parent.parser.games.index(self.parent) super(LiveEntity, self).__init__(entity_id, **kwargs) - + # push data to an end-point print(f'GAME {self.game_index} --- ENTITY CREATED:', self) - + def tag_change(self, tag, value): if tag == GameTag.CONTROLLER and not self._initial_controller: self._initial_controller = self.tags.get(GameTag.CONTROLLER, value) self.tags[tag] = value - + # update notify self.update_callback() - + def update_callback(self): # push data to an end-point print(f'GAME {self.game_index} --- ENTITY UPDATED:', self) class LiveCard(Card, LiveEntity): - + def __init__(self, entity_id, card_id, parent): super(LiveCard, self).__init__( entity_id=entity_id, @@ -46,13 +48,13 @@ def reveal(self, card_id, tags): if self.initial_card_id is None: self.initial_card_id = card_id self.tags.update(tags) - + # update notify self.update_callback() def hide(self): self.revealed = False - + # update notify self.update_callback() @@ -62,7 +64,6 @@ def change(self, card_id, tags): self.initial_card_id = card_id self.card_id = card_id self.tags.update(tags) - + # update notify self.update_callback() - \ No newline at end of file diff --git a/hslog/live/export.py b/hslog/live/export.py index f37d24d..5955cce 100644 --- a/hslog/live/export.py +++ b/hslog/live/export.py @@ -4,10 +4,10 @@ class LiveEntityTreeExporter(EntityTreeExporter): card_class = LiveCard - + def __init__(self, packet_tree): super(LiveEntityTreeExporter, self).__init__(packet_tree) - + def handle_full_entity(self, packet): entity_id = packet.entity @@ -27,4 +27,4 @@ def handle_full_entity(self, packet): entity = self.card_class(entity_id, packet.card_id, self.packet_tree) entity.tags = dict(packet.tags) self.game.register_entity(entity) - return entity \ No newline at end of file + return entity diff --git a/hslog/live/packets.py b/hslog/live/packets.py index 53745e4..3fecf7f 100644 --- a/hslog/live/packets.py +++ b/hslog/live/packets.py @@ -3,11 +3,11 @@ class LivePacketTree(PacketTree): - + def __init__(self, ts, parser): self.parser = parser self.liveExporter = LiveEntityTreeExporter(self) super(LivePacketTree, self).__init__(ts) - + def live_export(self, packet): - return self.liveExporter.export_packet(packet) \ No newline at end of file + return self.liveExporter.export_packet(packet) diff --git a/hslog/live/parser.py b/hslog/live/parser.py index 8e8dfff..b98b9ee 100644 --- a/hslog/live/parser.py +++ b/hslog/live/parser.py @@ -10,23 +10,23 @@ class LiveLogParser(LogParser): - + def __init__(self, filepath): super(LiveLogParser, self).__init__() self.running = False self.filepath = filepath self.lines_deque = deque([]) - + def new_packet_tree(self, ts): self._packets = LivePacketTree(ts, self) self._packets.spectator_mode = self.spectator_mode self._packets.manager = PlayerManager() self.current_block = self._packets self.games.append(self._packets) - + ''' why is this return important? ''' return self._packets - + def tag_change(self, ts, e, tag, value, def_change): entity_id = self.parse_entity_or_player(e) tag, value = parse_tag(tag, value) @@ -35,43 +35,46 @@ def tag_change(self, ts, e, tag, value, def_change): ''' skipping LazyPlayer here because it doesn't have data ''' skip = False if isinstance(entity_id, LazyPlayer): - entity_id = self._packets.manager.register_player_name_on_tag_change(entity_id, tag, value) + entity_id = self._packets.manager.register_player_name_on_tag_change( + entity_id, tag, value + ) skip = True has_change_def = def_change == tokens.DEF_CHANGE packet = packets.TagChange(ts, entity_id, tag, value, has_change_def) - - if not skip: self.register_packet(packet) + + if not skip: + self.register_packet(packet) return packet - + def register_packet(self, packet, node=None): ''' make sure we're registering packets to the current game''' if not self._packets or self._packets != self.games[-1]: self._packets = self.games[-1] - + if node is None: node = self.current_block.packets node.append(packet) - + ''' line below triggers packet export which will run update_callback for entity being updated by the packet. - + self._packets == EntityTreeExporter ''' self._packets.live_export(packet) - + self._packets._packet_counter += 1 packet.packet_id = self._packets._packet_counter - + def file_worker(self): file = open(self.filepath, 'r') - while self.running: + while self.running: line = file.readline() if line: self.lines_deque.append(line) else: time.sleep(0.2) - + def parse_worker(self): while self.running: if len(self.lines_deque): @@ -79,36 +82,36 @@ def parse_worker(self): self.read_line(line) else: time.sleep(0.2) - + def start_file_worker(self): file_thread = Thread(target=self.file_worker) file_thread.setDaemon(True) file_thread.start() - + def start_parse_worker(self): parse_thread = Thread(target=self.parse_worker) parse_thread.setDaemon(True) parse_thread.start() - + def start(self): self.running = True self.start_file_worker() self.start_parse_worker() - + def stop(self): self.running = False def flask_endpoint(self, line): ''' !!! EXPERIMENTAL: not really required for LiveLogParser to work - + #!/bin/bash # used with run-server.sh # can be used instead of file_worker to populate lines_deque - + nohup python server/server.py >/dev/null 2>&1 & sleep 1 echo "running tail > flask server" tail /tmp/hearthstone-redirected.log --follow | python server/proxy.py ''' - self.lines_deque.append(line) \ No newline at end of file + self.lines_deque.append(line) From 1ffd9a240fce3236f0358ee499e812227fc33e40 Mon Sep 17 00:00:00 2001 From: Dalibor Duric Date: Sun, 6 May 2018 17:54:34 +1000 Subject: [PATCH 4/7] replaced single quotes with double quotes project-wide --- .gitignore | 2 ++ example_live_parser.py | 8 ++++---- hslog/live/entities.py | 16 ++++++++-------- hslog/live/export.py | 2 +- hslog/live/parser.py | 16 ++++++++-------- 5 files changed, 23 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index db1c371..4c9d245 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ creds.py *.retry .vscode/ public/ +.tox/ +hslog.egg-info/ diff --git a/example_live_parser.py b/example_live_parser.py index 93bd92f..eac0d51 100644 --- a/example_live_parser.py +++ b/example_live_parser.py @@ -3,9 +3,9 @@ from hslog.live.parser import LiveLogParser -''' +""" ---------------------------------------------------------------------- - LiveLogParser assumes that you've configured Power.log to be a symlink + LiveLogParser assumes that you"ve configured Power.log to be a symlink in "SOME_PATH/Hearthstone/Logs" folder: ln -s Power.log /tmp/hearthstone-redirected.log @@ -21,12 +21,12 @@ this will create in-memory storage which is faster then SSD you need to restart the computer for this to take effect ---------------------------------------------------------------------- -''' +""" def main(): try: - file = '/tmp/hearthstone-redirected.log' + file = "/tmp/hearthstone-redirected.log" liveParser = LiveLogParser(file) liveParser.start() diff --git a/hslog/live/entities.py b/hslog/live/entities.py index ae73d01..30d5a6a 100644 --- a/hslog/live/entities.py +++ b/hslog/live/entities.py @@ -1,23 +1,23 @@ from hearthstone.entities import Card, Entity from hearthstone.enums import GameTag -''' +""" * Card is called on export from game * LiveCard replaces Card and inserts update_callback * The point is to become able to route update events towards an API end-point -''' +""" class LiveEntity(Entity): def __init__(self, entity_id, parent, **kwargs): - ''' Entity requires an ID, store everything else in kwargs ''' + """ Entity requires an ID, store everything else in kwargs """ self.parent = parent self.game_index = self.parent.parser.games.index(self.parent) super(LiveEntity, self).__init__(entity_id, **kwargs) # push data to an end-point - print(f'GAME {self.game_index} --- ENTITY CREATED:', self) + print(f"GAME {self.game_index} --- ENTITY CREATED:", self) def tag_change(self, tag, value): if tag == GameTag.CONTROLLER and not self._initial_controller: @@ -29,7 +29,7 @@ def tag_change(self, tag, value): def update_callback(self): # push data to an end-point - print(f'GAME {self.game_index} --- ENTITY UPDATED:', self) + print(f"GAME {self.game_index} --- ENTITY UPDATED:", self) class LiveCard(Card, LiveEntity): @@ -40,8 +40,8 @@ def __init__(self, entity_id, card_id, parent): card_id=card_id, parent=parent) - ''' if card_id doesn't change, there's no need to pass it as the argument. - we can use self.card_id instead as it is set by Card class ''' + """ if card_id doesn"t change, there"s no need to pass it as the argument. + we can use self.card_id instead as it is set by Card class """ def reveal(self, card_id, tags): self.revealed = True self.card_id = card_id @@ -58,7 +58,7 @@ def hide(self): # update notify self.update_callback() - ''' same comment as for reveal ''' + """ same comment as for reveal """ def change(self, card_id, tags): if self.initial_card_id is None: self.initial_card_id = card_id diff --git a/hslog/live/export.py b/hslog/live/export.py index 5955cce..74364ef 100644 --- a/hslog/live/export.py +++ b/hslog/live/export.py @@ -16,7 +16,7 @@ def handle_full_entity(self, packet): # This can legitimately happen in case of GAME_RESET if entity_id <= len(self.game.entities): # That first if check is an optimization to prevent always looping over all of - # the game's entities every single FULL_ENTITY packet... + # the game"s entities every single FULL_ENTITY packet... # FIXME: Switching to a dict for game.entities would simplify this. existing_entity = self.game.find_entity_by_id(entity_id) if existing_entity is not None: diff --git a/hslog/live/parser.py b/hslog/live/parser.py index b98b9ee..f435d57 100644 --- a/hslog/live/parser.py +++ b/hslog/live/parser.py @@ -24,7 +24,7 @@ def new_packet_tree(self, ts): self.current_block = self._packets self.games.append(self._packets) - ''' why is this return important? ''' + """ why is this return important? """ return self._packets def tag_change(self, ts, e, tag, value, def_change): @@ -32,7 +32,7 @@ def tag_change(self, ts, e, tag, value, def_change): tag, value = parse_tag(tag, value) self._check_for_mulligan_hack(ts, tag, value) - ''' skipping LazyPlayer here because it doesn't have data ''' + """ skipping LazyPlayer here because it doesn"t have data """ skip = False if isinstance(entity_id, LazyPlayer): entity_id = self._packets.manager.register_player_name_on_tag_change( @@ -47,7 +47,7 @@ def tag_change(self, ts, e, tag, value, def_change): return packet def register_packet(self, packet, node=None): - ''' make sure we're registering packets to the current game''' + """ make sure we"re registering packets to the current game""" if not self._packets or self._packets != self.games[-1]: self._packets = self.games[-1] @@ -55,19 +55,19 @@ def register_packet(self, packet, node=None): node = self.current_block.packets node.append(packet) - ''' line below triggers packet export which will + """ line below triggers packet export which will run update_callback for entity being updated by the packet. self._packets == EntityTreeExporter - ''' + """ self._packets.live_export(packet) self._packets._packet_counter += 1 packet.packet_id = self._packets._packet_counter def file_worker(self): - file = open(self.filepath, 'r') + file = open(self.filepath, "r") while self.running: line = file.readline() if line: @@ -102,7 +102,7 @@ def stop(self): self.running = False def flask_endpoint(self, line): - ''' + """ !!! EXPERIMENTAL: not really required for LiveLogParser to work #!/bin/bash @@ -113,5 +113,5 @@ def flask_endpoint(self, line): sleep 1 echo "running tail > flask server" tail /tmp/hearthstone-redirected.log --follow | python server/proxy.py - ''' + """ self.lines_deque.append(line) From 1d49e1bd9315bb6cf81fc6321b99708dc699dcf8 Mon Sep 17 00:00:00 2001 From: Dalibor Duric Date: Sun, 6 May 2018 17:59:17 +1000 Subject: [PATCH 5/7] additional formatting fix --- .gitignore | 15 ---- example_live_parser.py | 49 +++++----- hslog/live/entities.py | 111 +++++++++++------------ hslog/live/export.py | 42 ++++----- hslog/live/packets.py | 12 +-- hslog/live/parser.py | 199 +++++++++++++++++++---------------------- 6 files changed, 200 insertions(+), 228 deletions(-) delete mode 100644 .gitignore diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 4c9d245..0000000 --- a/.gitignore +++ /dev/null @@ -1,15 +0,0 @@ -__pycache__/ -node_modules/ -*.log -celerybeat.pid -*.pyc -.project -.pydevproject -.settings -celerybeat-schedule -creds.py -*.retry -.vscode/ -public/ -.tox/ -hslog.egg-info/ diff --git a/example_live_parser.py b/example_live_parser.py index eac0d51..63f7d4c 100644 --- a/example_live_parser.py +++ b/example_live_parser.py @@ -1,42 +1,43 @@ -from time import sleep import traceback +from time import sleep from hslog.live.parser import LiveLogParser + """ - ---------------------------------------------------------------------- - LiveLogParser assumes that you"ve configured Power.log to be a symlink + ---------------------------------------------------------------------- + LiveLogParser assumes that you"ve configured Power.log to be a symlink - in "SOME_PATH/Hearthstone/Logs" folder: - ln -s Power.log /tmp/hearthstone-redirected.log + in "SOME_PATH/Hearthstone/Logs" folder: + ln -s Power.log /tmp/hearthstone-redirected.log - this will redirect all data coming into Power.log - so we can access it from a RAM disk - ---------------------------------------------------------------------- - For better performance make /tmp of type tmpfs (or another location) + this will redirect all data coming into Power.log + so we can access it from a RAM disk + ---------------------------------------------------------------------- + For better performance make /tmp of type tmpfs (or another location) - in /etc/fstab add line: - tmpfs /tmp tmpfs nodev,nosuid,size=1G 0 0 + in /etc/fstab add line: + tmpfs /tmp tmpfs nodev,nosuid,size=1G 0 0 - this will create in-memory storage which is faster then SSD - you need to restart the computer for this to take effect - ---------------------------------------------------------------------- + this will create in-memory storage which is faster then SSD + you need to restart the computer for this to take effect + ---------------------------------------------------------------------- """ def main(): - try: - file = "/tmp/hearthstone-redirected.log" - liveParser = LiveLogParser(file) - liveParser.start() + try: + file = "/tmp/hearthstone-redirected.log" + liveParser = LiveLogParser(file) + liveParser.start() - while True: - sleep(1) + while True: + sleep(1) - except Exception as e: - print(traceback.format_exc()) - liveParser.stop() + except Exception as e: + print(traceback.format_exc()) + liveParser.stop() if __name__ == "__main__": - main() + main() diff --git a/hslog/live/entities.py b/hslog/live/entities.py index 30d5a6a..4409911 100644 --- a/hslog/live/entities.py +++ b/hslog/live/entities.py @@ -1,69 +1,70 @@ from hearthstone.entities import Card, Entity from hearthstone.enums import GameTag -""" - * Card is called on export from game - * LiveCard replaces Card and inserts update_callback - * The point is to become able to route update events towards an API end-point -""" - class LiveEntity(Entity): - def __init__(self, entity_id, parent, **kwargs): - """ Entity requires an ID, store everything else in kwargs """ - self.parent = parent - self.game_index = self.parent.parser.games.index(self.parent) - super(LiveEntity, self).__init__(entity_id, **kwargs) + def __init__(self, entity_id, parent, **kwargs): + """ Entity requires an ID, store everything else in kwargs """ + self.parent = parent + self.game_index = self.parent.parser.games.index(self.parent) + super(LiveEntity, self).__init__(entity_id, **kwargs) + + # push data to an end-point + print(f"GAME {self.game_index} --- ENTITY CREATED:", self) - # push data to an end-point - print(f"GAME {self.game_index} --- ENTITY CREATED:", self) + def tag_change(self, tag, value): + if tag == GameTag.CONTROLLER and not self._initial_controller: + self._initial_controller = self.tags.get(GameTag.CONTROLLER, value) + self.tags[tag] = value - def tag_change(self, tag, value): - if tag == GameTag.CONTROLLER and not self._initial_controller: - self._initial_controller = self.tags.get(GameTag.CONTROLLER, value) - self.tags[tag] = value + # update notify + self.update_callback() - # update notify - self.update_callback() + def update_callback(self): + # push data to an end-point + print(f"GAME {self.game_index} --- ENTITY UPDATED:", self) - def update_callback(self): - # push data to an end-point - print(f"GAME {self.game_index} --- ENTITY UPDATED:", self) + +""" + * Card is called on export from game + * LiveCard replaces Card and inserts update_callback + * The point is to become able to route update events towards an API end-point +""" class LiveCard(Card, LiveEntity): - def __init__(self, entity_id, card_id, parent): - super(LiveCard, self).__init__( - entity_id=entity_id, - card_id=card_id, - parent=parent) - - """ if card_id doesn"t change, there"s no need to pass it as the argument. - we can use self.card_id instead as it is set by Card class """ - def reveal(self, card_id, tags): - self.revealed = True - self.card_id = card_id - if self.initial_card_id is None: - self.initial_card_id = card_id - self.tags.update(tags) - - # update notify - self.update_callback() - - def hide(self): - self.revealed = False - - # update notify - self.update_callback() - - """ same comment as for reveal """ - def change(self, card_id, tags): - if self.initial_card_id is None: - self.initial_card_id = card_id - self.card_id = card_id - self.tags.update(tags) - - # update notify - self.update_callback() + def __init__(self, entity_id, card_id, parent): + super(LiveCard, self).__init__( + entity_id=entity_id, + card_id=card_id, + parent=parent) + + """ if card_id doesn"t change, there"s no need to pass it as the argument. + we can use self.card_id instead as it is set by Card class """ + def reveal(self, card_id, tags): + self.revealed = True + self.card_id = card_id + if self.initial_card_id is None: + self.initial_card_id = card_id + self.tags.update(tags) + + # update notify + self.update_callback() + + def hide(self): + self.revealed = False + + # update notify + self.update_callback() + + """ same comment as for reveal """ + def change(self, card_id, tags): + if self.initial_card_id is None: + self.initial_card_id = card_id + self.card_id = card_id + self.tags.update(tags) + + # update notify + self.update_callback() diff --git a/hslog/live/export.py b/hslog/live/export.py index 74364ef..0b32387 100644 --- a/hslog/live/export.py +++ b/hslog/live/export.py @@ -3,28 +3,28 @@ class LiveEntityTreeExporter(EntityTreeExporter): - card_class = LiveCard + card_class = LiveCard - def __init__(self, packet_tree): - super(LiveEntityTreeExporter, self).__init__(packet_tree) + def __init__(self, packet_tree): + super(LiveEntityTreeExporter, self).__init__(packet_tree) - def handle_full_entity(self, packet): - entity_id = packet.entity + def handle_full_entity(self, packet): + entity_id = packet.entity - # Check if the entity already exists in the game first. - # This prevents creating it twice. - # This can legitimately happen in case of GAME_RESET - if entity_id <= len(self.game.entities): - # That first if check is an optimization to prevent always looping over all of - # the game"s entities every single FULL_ENTITY packet... - # FIXME: Switching to a dict for game.entities would simplify this. - existing_entity = self.game.find_entity_by_id(entity_id) - if existing_entity is not None: - existing_entity.card_id = packet.card_id - existing_entity.tags = dict(packet.tags) - return existing_entity + # Check if the entity already exists in the game first. + # This prevents creating it twice. + # This can legitimately happen in case of GAME_RESET + if entity_id <= len(self.game.entities): + # That first if check is an optimization to prevent always looping over all of + # the game"s entities every single FULL_ENTITY packet... + # FIXME: Switching to a dict for game.entities would simplify this. + existing_entity = self.game.find_entity_by_id(entity_id) + if existing_entity is not None: + existing_entity.card_id = packet.card_id + existing_entity.tags = dict(packet.tags) + return existing_entity - entity = self.card_class(entity_id, packet.card_id, self.packet_tree) - entity.tags = dict(packet.tags) - self.game.register_entity(entity) - return entity + entity = self.card_class(entity_id, packet.card_id, self.packet_tree) + entity.tags = dict(packet.tags) + self.game.register_entity(entity) + return entity diff --git a/hslog/live/packets.py b/hslog/live/packets.py index 3fecf7f..8c7a486 100644 --- a/hslog/live/packets.py +++ b/hslog/live/packets.py @@ -4,10 +4,10 @@ class LivePacketTree(PacketTree): - def __init__(self, ts, parser): - self.parser = parser - self.liveExporter = LiveEntityTreeExporter(self) - super(LivePacketTree, self).__init__(ts) + def __init__(self, ts, parser): + self.parser = parser + self.liveExporter = LiveEntityTreeExporter(self) + super(LivePacketTree, self).__init__(ts) - def live_export(self, packet): - return self.liveExporter.export_packet(packet) + def live_export(self, packet): + return self.liveExporter.export_packet(packet) diff --git a/hslog/live/parser.py b/hslog/live/parser.py index f435d57..bfad575 100644 --- a/hslog/live/parser.py +++ b/hslog/live/parser.py @@ -1,117 +1,102 @@ +import time from collections import deque from threading import Thread -import time +from hslog import packets, tokens from hslog.live.packets import LivePacketTree from hslog.parser import LogParser -from hslog.player import PlayerManager, LazyPlayer +from hslog.player import LazyPlayer, PlayerManager from hslog.utils import parse_tag -from hslog import tokens, packets class LiveLogParser(LogParser): - def __init__(self, filepath): - super(LiveLogParser, self).__init__() - self.running = False - self.filepath = filepath - self.lines_deque = deque([]) - - def new_packet_tree(self, ts): - self._packets = LivePacketTree(ts, self) - self._packets.spectator_mode = self.spectator_mode - self._packets.manager = PlayerManager() - self.current_block = self._packets - self.games.append(self._packets) - - """ why is this return important? """ - return self._packets - - def tag_change(self, ts, e, tag, value, def_change): - entity_id = self.parse_entity_or_player(e) - tag, value = parse_tag(tag, value) - self._check_for_mulligan_hack(ts, tag, value) - - """ skipping LazyPlayer here because it doesn"t have data """ - skip = False - if isinstance(entity_id, LazyPlayer): - entity_id = self._packets.manager.register_player_name_on_tag_change( - entity_id, tag, value - ) - skip = True - has_change_def = def_change == tokens.DEF_CHANGE - packet = packets.TagChange(ts, entity_id, tag, value, has_change_def) - - if not skip: - self.register_packet(packet) - return packet - - def register_packet(self, packet, node=None): - """ make sure we"re registering packets to the current game""" - if not self._packets or self._packets != self.games[-1]: - self._packets = self.games[-1] - - if node is None: - node = self.current_block.packets - node.append(packet) - - """ line below triggers packet export which will - run update_callback for entity being - updated by the packet. - - self._packets == EntityTreeExporter - """ - self._packets.live_export(packet) - - self._packets._packet_counter += 1 - packet.packet_id = self._packets._packet_counter - - def file_worker(self): - file = open(self.filepath, "r") - while self.running: - line = file.readline() - if line: - self.lines_deque.append(line) - else: - time.sleep(0.2) - - def parse_worker(self): - while self.running: - if len(self.lines_deque): - line = self.lines_deque.popleft() - self.read_line(line) - else: - time.sleep(0.2) - - def start_file_worker(self): - file_thread = Thread(target=self.file_worker) - file_thread.setDaemon(True) - file_thread.start() - - def start_parse_worker(self): - parse_thread = Thread(target=self.parse_worker) - parse_thread.setDaemon(True) - parse_thread.start() - - def start(self): - self.running = True - self.start_file_worker() - self.start_parse_worker() - - def stop(self): - self.running = False - - def flask_endpoint(self, line): - """ - !!! EXPERIMENTAL: not really required for LiveLogParser to work - - #!/bin/bash - # used with run-server.sh - # can be used instead of file_worker to populate lines_deque - - nohup python server/server.py >/dev/null 2>&1 & - sleep 1 - echo "running tail > flask server" - tail /tmp/hearthstone-redirected.log --follow | python server/proxy.py - """ - self.lines_deque.append(line) + def __init__(self, filepath): + super(LiveLogParser, self).__init__() + self.running = False + self.filepath = filepath + self.lines_deque = deque([]) + + def new_packet_tree(self, ts): + self._packets = LivePacketTree(ts, self) + self._packets.spectator_mode = self.spectator_mode + self._packets.manager = PlayerManager() + self.current_block = self._packets + self.games.append(self._packets) + + """ why is this return important? """ + return self._packets + + def tag_change(self, ts, e, tag, value, def_change): + entity_id = self.parse_entity_or_player(e) + tag, value = parse_tag(tag, value) + self._check_for_mulligan_hack(ts, tag, value) + + """ skipping LazyPlayer here because it doesn"t have data """ + skip = False + if isinstance(entity_id, LazyPlayer): + entity_id = self._packets.manager.register_player_name_on_tag_change( + entity_id, tag, value + ) + skip = True + has_change_def = def_change == tokens.DEF_CHANGE + packet = packets.TagChange(ts, entity_id, tag, value, has_change_def) + + if not skip: + self.register_packet(packet) + return packet + + def register_packet(self, packet, node=None): + """ make sure we"re registering packets to the current game""" + if not self._packets or self._packets != self.games[-1]: + self._packets = self.games[-1] + + if node is None: + node = self.current_block.packets + node.append(packet) + + """ line below triggers packet export which will + run update_callback for entity being + updated by the packet. + + self._packets == EntityTreeExporter + """ + self._packets.live_export(packet) + + self._packets._packet_counter += 1 + packet.packet_id = self._packets._packet_counter + + def file_worker(self): + file = open(self.filepath, "r") + while self.running: + line = file.readline() + if line: + self.lines_deque.append(line) + else: + time.sleep(0.2) + + def parse_worker(self): + while self.running: + if len(self.lines_deque): + line = self.lines_deque.popleft() + self.read_line(line) + else: + time.sleep(0.2) + + def start_file_worker(self): + file_thread = Thread(target=self.file_worker) + file_thread.setDaemon(True) + file_thread.start() + + def start_parse_worker(self): + parse_thread = Thread(target=self.parse_worker) + parse_thread.setDaemon(True) + parse_thread.start() + + def start(self): + self.running = True + self.start_file_worker() + self.start_parse_worker() + + def stop(self): + self.running = False From 60fc9c86bfb5bff9bed7478e2f187b1dc91af0e8 Mon Sep 17 00:00:00 2001 From: Dalibor Duric Date: Sun, 6 May 2018 21:23:52 +1000 Subject: [PATCH 6/7] improved docstrings and removed parent dependency for LiveEntities --- example_live_parser.py | 34 +++++++------ hslog/live/entities.py | 81 ++++++++++++++++++------------- hslog/live/export.py | 37 ++++++++------- hslog/live/packets.py | 4 ++ hslog/live/parser.py | 105 ++++++++++++++++++++++++++++++++--------- hslog/live/player.py | 29 ++++++++++++ hslog/live/utils.py | 50 ++++++++++++++++++++ 7 files changed, 252 insertions(+), 88 deletions(-) create mode 100644 hslog/live/player.py create mode 100644 hslog/live/utils.py diff --git a/example_live_parser.py b/example_live_parser.py index 63f7d4c..4564455 100644 --- a/example_live_parser.py +++ b/example_live_parser.py @@ -4,28 +4,26 @@ from hslog.live.parser import LiveLogParser -""" - ---------------------------------------------------------------------- - LiveLogParser assumes that you"ve configured Power.log to be a symlink - - in "SOME_PATH/Hearthstone/Logs" folder: - ln -s Power.log /tmp/hearthstone-redirected.log - - this will redirect all data coming into Power.log - so we can access it from a RAM disk - ---------------------------------------------------------------------- - For better performance make /tmp of type tmpfs (or another location) +def main(): + """ + ---------------------------------------------------------------------- + LiveLogParser assumes that you"ve configured Power.log to be a symlink. - in /etc/fstab add line: - tmpfs /tmp tmpfs nodev,nosuid,size=1G 0 0 + In "SOME_PATH/Hearthstone/Logs" folder: + ln -s Power.log /tmp/hearthstone-redirected.log - this will create in-memory storage which is faster then SSD - you need to restart the computer for this to take effect - ---------------------------------------------------------------------- -""" + This will redirect all data coming into Power.log + so we can access it from a RAM disk. + ---------------------------------------------------------------------- + For better performance make /tmp of type tmpfs (or another location) + In /etc/fstab add line: + tmpfs /tmp tmpfs nodev,nosuid,size=1G 0 0 -def main(): + This will create in-memory storage which is faster then SSD. + You need to restart the computer for this to take effect. + ---------------------------------------------------------------------- + """ try: file = "/tmp/hearthstone-redirected.log" liveParser = LiveLogParser(file) diff --git a/hslog/live/entities.py b/hslog/live/entities.py index 4409911..e0e089e 100644 --- a/hslog/live/entities.py +++ b/hslog/live/entities.py @@ -1,48 +1,53 @@ -from hearthstone.entities import Card, Entity +from hearthstone.entities import Card, Entity, Game, Player from hearthstone.enums import GameTag +from hslog.live.utils import terminal_output + class LiveEntity(Entity): - def __init__(self, entity_id, parent, **kwargs): - """ Entity requires an ID, store everything else in kwargs """ - self.parent = parent - self.game_index = self.parent.parser.games.index(self.parent) - super(LiveEntity, self).__init__(entity_id, **kwargs) + def __init__(self, entity_id): + super(LiveEntity, self).__init__(entity_id) + self._game = None - # push data to an end-point - print(f"GAME {self.game_index} --- ENTITY CREATED:", self) + @property + def game(self): + return self._game + + @game.setter + def game(self, value): + # this happens when game calls register_entity and entity sets self.game + self._game = value + if value is not None: + terminal_output("ENTITY CREATED", self) + # push data to an end-point def tag_change(self, tag, value): if tag == GameTag.CONTROLLER and not self._initial_controller: self._initial_controller = self.tags.get(GameTag.CONTROLLER, value) self.tags[tag] = value - - # update notify - self.update_callback() - - def update_callback(self): + terminal_output("TAG UPDATED", self, tag, value) # push data to an end-point - print(f"GAME {self.game_index} --- ENTITY UPDATED:", self) - -""" - * Card is called on export from game - * LiveCard replaces Card and inserts update_callback - * The point is to become able to route update events towards an API end-point -""" + def update_callback(self, caller): + terminal_output("ENTITY UPDATED", self) + # push data to an end-point class LiveCard(Card, LiveEntity): - - def __init__(self, entity_id, card_id, parent): - super(LiveCard, self).__init__( - entity_id=entity_id, - card_id=card_id, - parent=parent) - - """ if card_id doesn"t change, there"s no need to pass it as the argument. - we can use self.card_id instead as it is set by Card class """ + """ + Card is called on export from game + LiveCard replaces Card and inserts update_callback + The point is to become able to route update events towards an API end-point + """ + + def __init__(self, entity_id, card_id): + super(LiveCard, self).__init__(entity_id, card_id) + + """ + if card_id doesn"t change, there"s no need to pass it as the argument. + we can use self.card_id instead as it is set by Card class + """ def reveal(self, card_id, tags): self.revealed = True self.card_id = card_id @@ -51,13 +56,13 @@ def reveal(self, card_id, tags): self.tags.update(tags) # update notify - self.update_callback() + self.update_callback(self) def hide(self): self.revealed = False # update notify - self.update_callback() + self.update_callback(self) """ same comment as for reveal """ def change(self, card_id, tags): @@ -67,4 +72,16 @@ def change(self, card_id, tags): self.tags.update(tags) # update notify - self.update_callback() + self.update_callback(self) + + +class LivePlayer(Player, LiveEntity): + + def __init__(self, packet_id, player_id, hi, lo, name=None): + super(LivePlayer, self).__init__(packet_id, player_id, hi, lo, name) + + +class LiveGame(Game, LiveEntity): + + def __init__(self, entity_id): + super(LiveGame, self).__init__(entity_id) diff --git a/hslog/live/export.py b/hslog/live/export.py index 0b32387..b6d0581 100644 --- a/hslog/live/export.py +++ b/hslog/live/export.py @@ -1,30 +1,33 @@ +from hearthstone.enums import GameTag + from hslog.export import EntityTreeExporter -from hslog.live.entities import LiveCard +from hslog.live.entities import LiveCard, LiveGame, LivePlayer +from hslog.live.utils import ACCESS_DEBUG class LiveEntityTreeExporter(EntityTreeExporter): + """ + Inherits EntityTreeExporter to provide Live entities + """ + + game_class = LiveGame + player_class = LivePlayer card_class = LiveCard def __init__(self, packet_tree): super(LiveEntityTreeExporter, self).__init__(packet_tree) - def handle_full_entity(self, packet): - entity_id = packet.entity - - # Check if the entity already exists in the game first. - # This prevents creating it twice. - # This can legitimately happen in case of GAME_RESET - if entity_id <= len(self.game.entities): - # That first if check is an optimization to prevent always looping over all of - # the game"s entities every single FULL_ENTITY packet... - # FIXME: Switching to a dict for game.entities would simplify this. - existing_entity = self.game.find_entity_by_id(entity_id) - if existing_entity is not None: - existing_entity.card_id = packet.card_id - existing_entity.tags = dict(packet.tags) - return existing_entity + def handle_player(self, packet): + ACCESS_DEBUG(self.__class__, "handle_player") + entity_id = int(packet.entity) - entity = self.card_class(entity_id, packet.card_id, self.packet_tree) + if hasattr(self.packet_tree, "manager"): + # If we have a PlayerManager, first we mutate the CreateGame.Player packet. + # This will have to change if we"re ever able to immediately get the names. + player = self.packet_tree.manager.get_player_by_id(entity_id) + packet.name = player.name + entity = self.player_class(entity_id, packet.player_id, packet.hi, packet.lo, packet.name) entity.tags = dict(packet.tags) self.game.register_entity(entity) + entity.initial_hero_entity_id = entity.tags.get(GameTag.HERO_ENTITY, 0) return entity diff --git a/hslog/live/packets.py b/hslog/live/packets.py index 8c7a486..d2226b0 100644 --- a/hslog/live/packets.py +++ b/hslog/live/packets.py @@ -10,4 +10,8 @@ def __init__(self, ts, parser): super(LivePacketTree, self).__init__(ts) def live_export(self, packet): + """ + Triggers packet export which will run the proper handler for the packet. + This will also run update_callback for entity being updated by the packet. + """ return self.liveExporter.export_packet(packet) diff --git a/hslog/live/parser.py b/hslog/live/parser.py index bfad575..802ecd2 100644 --- a/hslog/live/parser.py +++ b/hslog/live/parser.py @@ -2,14 +2,28 @@ from collections import deque from threading import Thread +from hearthstone.enums import FormatType, GameType + from hslog import packets, tokens +from hslog.exceptions import RegexParsingError from hslog.live.packets import LivePacketTree +from hslog.live.player import LivePlayerManager from hslog.parser import LogParser -from hslog.player import LazyPlayer, PlayerManager -from hslog.utils import parse_tag +from hslog.player import LazyPlayer +from hslog.utils import parse_enum, parse_tag class LiveLogParser(LogParser): + """ + LiveLogParser adds live log translation into useful data. + + Lines are read and pushed into a deque by a separate thread. + Deque is emptied by parse_worker which replaces the read() + function of LogParser and it"s also in a separate thread. + + This approach is non-blocking and allows for live parsing + of incoming lines. + """ def __init__(self, filepath): super(LiveLogParser, self).__init__() @@ -18,55 +32,101 @@ def __init__(self, filepath): self.lines_deque = deque([]) def new_packet_tree(self, ts): + """ + LivePacketTree is introduced here because it instantiates LiveEntityTreeExporter + and keeps track of the parser parent. It also contains a function that + utilizes the liveExporter instance across all the games. + + self.parser = parser + self.liveExporter = LiveEntityTreeExporter(self) + """ self._packets = LivePacketTree(ts, self) self._packets.spectator_mode = self.spectator_mode - self._packets.manager = PlayerManager() + self._packets.manager = LivePlayerManager() self.current_block = self._packets self.games.append(self._packets) - """ why is this return important? """ + """ + why is this return important? + it"s called only here: + + def create_game(self, ts): + self.new_packet_tree(ts) + """ return self._packets + def handle_game(self, ts, data): + if data.startswith("PlayerID="): + sre = tokens.GAME_PLAYER_META.match(data) + if not sre: + raise RegexParsingError(data) + player_id, player_name = sre.groups() + + # set the name of the player + players = self.games[-1].liveExporter.game.players + for p in players: + if p.player_id == int(player_id): + p.name = player_name + + player_id = int(player_id) + else: + key, value = data.split("=") + key = key.strip() + value = value.strip() + if key == "GameType": + value = parse_enum(GameType, value) + elif key == "FormatType": + value = parse_enum(FormatType, value) + else: + value = int(value) + + self.game_meta[key] = value + def tag_change(self, ts, e, tag, value, def_change): entity_id = self.parse_entity_or_player(e) tag, value = parse_tag(tag, value) self._check_for_mulligan_hack(ts, tag, value) - """ skipping LazyPlayer here because it doesn"t have data """ - skip = False if isinstance(entity_id, LazyPlayer): entity_id = self._packets.manager.register_player_name_on_tag_change( - entity_id, tag, value - ) - skip = True + entity_id, tag, value) + has_change_def = def_change == tokens.DEF_CHANGE packet = packets.TagChange(ts, entity_id, tag, value, has_change_def) - - if not skip: + if entity_id: self.register_packet(packet) return packet def register_packet(self, packet, node=None): - """ make sure we"re registering packets to the current game""" + """ + LogParser.register_packet override + + This uses the live_export functionality introduces by LivePacketTree + It also keeps track of which LivePacketTree is being used when there + are multiple in parser.games + + A better naming for a PacketTree/LivePacketTree would be HearthstoneGame? + Then parser.games would contain HearthstoneGame instances and would + be more obvious what the purpose is. + """ + + # make sure we"re registering packets to the current game if not self._packets or self._packets != self.games[-1]: self._packets = self.games[-1] if node is None: node = self.current_block.packets node.append(packet) - - """ line below triggers packet export which will - run update_callback for entity being - updated by the packet. - - self._packets == EntityTreeExporter - """ - self._packets.live_export(packet) - self._packets._packet_counter += 1 packet.packet_id = self._packets._packet_counter + self._packets.live_export(packet) def file_worker(self): + """ + File reader thread. (Naive implementation) + Reads the log file continuously and appends to deque. + """ + file = open(self.filepath, "r") while self.running: line = file.readline() @@ -76,6 +136,9 @@ def file_worker(self): time.sleep(0.2) def parse_worker(self): + """ + If deque contains lines, this initiates parsing. + """ while self.running: if len(self.lines_deque): line = self.lines_deque.popleft() diff --git a/hslog/live/player.py b/hslog/live/player.py new file mode 100644 index 0000000..277dfd4 --- /dev/null +++ b/hslog/live/player.py @@ -0,0 +1,29 @@ +from hearthstone.enums import GameTag + +from hslog.exceptions import ParsingError +from hslog.player import PlayerManager + + +class LivePlayerManager(PlayerManager): + + def register_player_name_on_tag_change(self, player, tag, value): + """ + Triggers on every TAG_CHANGE where the corresponding entity is a LazyPlayer. + Will attempt to return a new value instead + """ + if tag == GameTag.ENTITY_ID: + # This is the simplest check. When a player entity is declared, + # its ENTITY_ID is not available immediately (in pre-6.0). + # If we get a matching ENTITY_ID, then we can use that to match it. + return self.register_player_name(player.name, value) + elif tag == GameTag.LAST_CARD_PLAYED: + # This is a fallback to register_player_name_mulligan in case the mulligan + # phase is not available in this game (spectator mode, reconnects). + if value not in self._entity_controller_map: + raise ParsingError("Unknown entity ID on TAG_CHANGE: %r" % (value)) + player_id = self._entity_controller_map[value] + entity_id = int(self._players_by_player_id[player_id]) + return self.register_player_name(player.name, entity_id) + elif tag == GameTag.MULLIGAN_STATE: + return None + return player diff --git a/hslog/live/utils.py b/hslog/live/utils.py new file mode 100644 index 0000000..b2222d2 --- /dev/null +++ b/hslog/live/utils.py @@ -0,0 +1,50 @@ +def align(text, size, char=" "): + """ + Format text to fit into a predefined amount of space + + Positive size aligns text to the left + Negative size aligns text to the right + """ + text = str(text).strip() + text_len = len(text) + if text_len > abs(size): + text = f"{text[:size-3]}..." + offset = "".join(char * (abs(size) - text_len)) + if size < 0: + return f"{offset}{text}" + else: + return f"{text}{offset}" + + +def color(): + def color_decorator(func): + colors = { + "LivePlayer": 93, + "red": 31, + "green": 32, + "LiveGame": 33, + "blue": 34, + "purple": 35, + "cyan": 36, + "grey": 37, + } + + def func_wrapper(msg_type, obj, *args): + class_name = obj.__class__.__name__ + color_key = class_name if class_name in colors else "green" + line = "\033[{}m{}\033[0m".format(colors[color_key], func(msg_type, obj, *args)) + print(line) + + return func_wrapper + + return color_decorator + + +@color() +def terminal_output(msg_type, obj, attr=None, value=None): + return "{} | {} | {} | {}".format( + align(msg_type, -20), + align(repr(obj), 120), + align(repr(attr), 40), + align(value, 30), + ) From e1859a5729d8fc0c30579c4fa0b1587d42f4e06f Mon Sep 17 00:00:00 2001 From: Dalibor Duric Date: Sun, 6 May 2018 21:23:52 +1000 Subject: [PATCH 7/7] improved docstrings and removed parent dependency for LiveEntities --- example_live_parser.py | 34 +++++++------ hslog/live/entities.py | 81 ++++++++++++++++++------------- hslog/live/export.py | 36 +++++++------- hslog/live/packets.py | 4 ++ hslog/live/parser.py | 105 ++++++++++++++++++++++++++++++++--------- hslog/live/player.py | 29 ++++++++++++ hslog/live/utils.py | 50 ++++++++++++++++++++ 7 files changed, 251 insertions(+), 88 deletions(-) create mode 100644 hslog/live/player.py create mode 100644 hslog/live/utils.py diff --git a/example_live_parser.py b/example_live_parser.py index 63f7d4c..4564455 100644 --- a/example_live_parser.py +++ b/example_live_parser.py @@ -4,28 +4,26 @@ from hslog.live.parser import LiveLogParser -""" - ---------------------------------------------------------------------- - LiveLogParser assumes that you"ve configured Power.log to be a symlink - - in "SOME_PATH/Hearthstone/Logs" folder: - ln -s Power.log /tmp/hearthstone-redirected.log - - this will redirect all data coming into Power.log - so we can access it from a RAM disk - ---------------------------------------------------------------------- - For better performance make /tmp of type tmpfs (or another location) +def main(): + """ + ---------------------------------------------------------------------- + LiveLogParser assumes that you"ve configured Power.log to be a symlink. - in /etc/fstab add line: - tmpfs /tmp tmpfs nodev,nosuid,size=1G 0 0 + In "SOME_PATH/Hearthstone/Logs" folder: + ln -s Power.log /tmp/hearthstone-redirected.log - this will create in-memory storage which is faster then SSD - you need to restart the computer for this to take effect - ---------------------------------------------------------------------- -""" + This will redirect all data coming into Power.log + so we can access it from a RAM disk. + ---------------------------------------------------------------------- + For better performance make /tmp of type tmpfs (or another location) + In /etc/fstab add line: + tmpfs /tmp tmpfs nodev,nosuid,size=1G 0 0 -def main(): + This will create in-memory storage which is faster then SSD. + You need to restart the computer for this to take effect. + ---------------------------------------------------------------------- + """ try: file = "/tmp/hearthstone-redirected.log" liveParser = LiveLogParser(file) diff --git a/hslog/live/entities.py b/hslog/live/entities.py index 4409911..e0e089e 100644 --- a/hslog/live/entities.py +++ b/hslog/live/entities.py @@ -1,48 +1,53 @@ -from hearthstone.entities import Card, Entity +from hearthstone.entities import Card, Entity, Game, Player from hearthstone.enums import GameTag +from hslog.live.utils import terminal_output + class LiveEntity(Entity): - def __init__(self, entity_id, parent, **kwargs): - """ Entity requires an ID, store everything else in kwargs """ - self.parent = parent - self.game_index = self.parent.parser.games.index(self.parent) - super(LiveEntity, self).__init__(entity_id, **kwargs) + def __init__(self, entity_id): + super(LiveEntity, self).__init__(entity_id) + self._game = None - # push data to an end-point - print(f"GAME {self.game_index} --- ENTITY CREATED:", self) + @property + def game(self): + return self._game + + @game.setter + def game(self, value): + # this happens when game calls register_entity and entity sets self.game + self._game = value + if value is not None: + terminal_output("ENTITY CREATED", self) + # push data to an end-point def tag_change(self, tag, value): if tag == GameTag.CONTROLLER and not self._initial_controller: self._initial_controller = self.tags.get(GameTag.CONTROLLER, value) self.tags[tag] = value - - # update notify - self.update_callback() - - def update_callback(self): + terminal_output("TAG UPDATED", self, tag, value) # push data to an end-point - print(f"GAME {self.game_index} --- ENTITY UPDATED:", self) - -""" - * Card is called on export from game - * LiveCard replaces Card and inserts update_callback - * The point is to become able to route update events towards an API end-point -""" + def update_callback(self, caller): + terminal_output("ENTITY UPDATED", self) + # push data to an end-point class LiveCard(Card, LiveEntity): - - def __init__(self, entity_id, card_id, parent): - super(LiveCard, self).__init__( - entity_id=entity_id, - card_id=card_id, - parent=parent) - - """ if card_id doesn"t change, there"s no need to pass it as the argument. - we can use self.card_id instead as it is set by Card class """ + """ + Card is called on export from game + LiveCard replaces Card and inserts update_callback + The point is to become able to route update events towards an API end-point + """ + + def __init__(self, entity_id, card_id): + super(LiveCard, self).__init__(entity_id, card_id) + + """ + if card_id doesn"t change, there"s no need to pass it as the argument. + we can use self.card_id instead as it is set by Card class + """ def reveal(self, card_id, tags): self.revealed = True self.card_id = card_id @@ -51,13 +56,13 @@ def reveal(self, card_id, tags): self.tags.update(tags) # update notify - self.update_callback() + self.update_callback(self) def hide(self): self.revealed = False # update notify - self.update_callback() + self.update_callback(self) """ same comment as for reveal """ def change(self, card_id, tags): @@ -67,4 +72,16 @@ def change(self, card_id, tags): self.tags.update(tags) # update notify - self.update_callback() + self.update_callback(self) + + +class LivePlayer(Player, LiveEntity): + + def __init__(self, packet_id, player_id, hi, lo, name=None): + super(LivePlayer, self).__init__(packet_id, player_id, hi, lo, name) + + +class LiveGame(Game, LiveEntity): + + def __init__(self, entity_id): + super(LiveGame, self).__init__(entity_id) diff --git a/hslog/live/export.py b/hslog/live/export.py index 0b32387..7b059bf 100644 --- a/hslog/live/export.py +++ b/hslog/live/export.py @@ -1,30 +1,32 @@ +from hearthstone.enums import GameTag + from hslog.export import EntityTreeExporter -from hslog.live.entities import LiveCard +from hslog.live.entities import LiveCard, LiveGame, LivePlayer +from hslog.live.utils import ACCESS_DEBUG class LiveEntityTreeExporter(EntityTreeExporter): + """ + Inherits EntityTreeExporter to provide Live entities + """ + + game_class = LiveGame + player_class = LivePlayer card_class = LiveCard def __init__(self, packet_tree): super(LiveEntityTreeExporter, self).__init__(packet_tree) - def handle_full_entity(self, packet): - entity_id = packet.entity - - # Check if the entity already exists in the game first. - # This prevents creating it twice. - # This can legitimately happen in case of GAME_RESET - if entity_id <= len(self.game.entities): - # That first if check is an optimization to prevent always looping over all of - # the game"s entities every single FULL_ENTITY packet... - # FIXME: Switching to a dict for game.entities would simplify this. - existing_entity = self.game.find_entity_by_id(entity_id) - if existing_entity is not None: - existing_entity.card_id = packet.card_id - existing_entity.tags = dict(packet.tags) - return existing_entity + def handle_player(self, packet): + entity_id = int(packet.entity) - entity = self.card_class(entity_id, packet.card_id, self.packet_tree) + if hasattr(self.packet_tree, "manager"): + # If we have a PlayerManager, first we mutate the CreateGame.Player packet. + # This will have to change if we"re ever able to immediately get the names. + player = self.packet_tree.manager.get_player_by_id(entity_id) + packet.name = player.name + entity = self.player_class(entity_id, packet.player_id, packet.hi, packet.lo, packet.name) entity.tags = dict(packet.tags) self.game.register_entity(entity) + entity.initial_hero_entity_id = entity.tags.get(GameTag.HERO_ENTITY, 0) return entity diff --git a/hslog/live/packets.py b/hslog/live/packets.py index 8c7a486..d2226b0 100644 --- a/hslog/live/packets.py +++ b/hslog/live/packets.py @@ -10,4 +10,8 @@ def __init__(self, ts, parser): super(LivePacketTree, self).__init__(ts) def live_export(self, packet): + """ + Triggers packet export which will run the proper handler for the packet. + This will also run update_callback for entity being updated by the packet. + """ return self.liveExporter.export_packet(packet) diff --git a/hslog/live/parser.py b/hslog/live/parser.py index bfad575..802ecd2 100644 --- a/hslog/live/parser.py +++ b/hslog/live/parser.py @@ -2,14 +2,28 @@ from collections import deque from threading import Thread +from hearthstone.enums import FormatType, GameType + from hslog import packets, tokens +from hslog.exceptions import RegexParsingError from hslog.live.packets import LivePacketTree +from hslog.live.player import LivePlayerManager from hslog.parser import LogParser -from hslog.player import LazyPlayer, PlayerManager -from hslog.utils import parse_tag +from hslog.player import LazyPlayer +from hslog.utils import parse_enum, parse_tag class LiveLogParser(LogParser): + """ + LiveLogParser adds live log translation into useful data. + + Lines are read and pushed into a deque by a separate thread. + Deque is emptied by parse_worker which replaces the read() + function of LogParser and it"s also in a separate thread. + + This approach is non-blocking and allows for live parsing + of incoming lines. + """ def __init__(self, filepath): super(LiveLogParser, self).__init__() @@ -18,55 +32,101 @@ def __init__(self, filepath): self.lines_deque = deque([]) def new_packet_tree(self, ts): + """ + LivePacketTree is introduced here because it instantiates LiveEntityTreeExporter + and keeps track of the parser parent. It also contains a function that + utilizes the liveExporter instance across all the games. + + self.parser = parser + self.liveExporter = LiveEntityTreeExporter(self) + """ self._packets = LivePacketTree(ts, self) self._packets.spectator_mode = self.spectator_mode - self._packets.manager = PlayerManager() + self._packets.manager = LivePlayerManager() self.current_block = self._packets self.games.append(self._packets) - """ why is this return important? """ + """ + why is this return important? + it"s called only here: + + def create_game(self, ts): + self.new_packet_tree(ts) + """ return self._packets + def handle_game(self, ts, data): + if data.startswith("PlayerID="): + sre = tokens.GAME_PLAYER_META.match(data) + if not sre: + raise RegexParsingError(data) + player_id, player_name = sre.groups() + + # set the name of the player + players = self.games[-1].liveExporter.game.players + for p in players: + if p.player_id == int(player_id): + p.name = player_name + + player_id = int(player_id) + else: + key, value = data.split("=") + key = key.strip() + value = value.strip() + if key == "GameType": + value = parse_enum(GameType, value) + elif key == "FormatType": + value = parse_enum(FormatType, value) + else: + value = int(value) + + self.game_meta[key] = value + def tag_change(self, ts, e, tag, value, def_change): entity_id = self.parse_entity_or_player(e) tag, value = parse_tag(tag, value) self._check_for_mulligan_hack(ts, tag, value) - """ skipping LazyPlayer here because it doesn"t have data """ - skip = False if isinstance(entity_id, LazyPlayer): entity_id = self._packets.manager.register_player_name_on_tag_change( - entity_id, tag, value - ) - skip = True + entity_id, tag, value) + has_change_def = def_change == tokens.DEF_CHANGE packet = packets.TagChange(ts, entity_id, tag, value, has_change_def) - - if not skip: + if entity_id: self.register_packet(packet) return packet def register_packet(self, packet, node=None): - """ make sure we"re registering packets to the current game""" + """ + LogParser.register_packet override + + This uses the live_export functionality introduces by LivePacketTree + It also keeps track of which LivePacketTree is being used when there + are multiple in parser.games + + A better naming for a PacketTree/LivePacketTree would be HearthstoneGame? + Then parser.games would contain HearthstoneGame instances and would + be more obvious what the purpose is. + """ + + # make sure we"re registering packets to the current game if not self._packets or self._packets != self.games[-1]: self._packets = self.games[-1] if node is None: node = self.current_block.packets node.append(packet) - - """ line below triggers packet export which will - run update_callback for entity being - updated by the packet. - - self._packets == EntityTreeExporter - """ - self._packets.live_export(packet) - self._packets._packet_counter += 1 packet.packet_id = self._packets._packet_counter + self._packets.live_export(packet) def file_worker(self): + """ + File reader thread. (Naive implementation) + Reads the log file continuously and appends to deque. + """ + file = open(self.filepath, "r") while self.running: line = file.readline() @@ -76,6 +136,9 @@ def file_worker(self): time.sleep(0.2) def parse_worker(self): + """ + If deque contains lines, this initiates parsing. + """ while self.running: if len(self.lines_deque): line = self.lines_deque.popleft() diff --git a/hslog/live/player.py b/hslog/live/player.py new file mode 100644 index 0000000..277dfd4 --- /dev/null +++ b/hslog/live/player.py @@ -0,0 +1,29 @@ +from hearthstone.enums import GameTag + +from hslog.exceptions import ParsingError +from hslog.player import PlayerManager + + +class LivePlayerManager(PlayerManager): + + def register_player_name_on_tag_change(self, player, tag, value): + """ + Triggers on every TAG_CHANGE where the corresponding entity is a LazyPlayer. + Will attempt to return a new value instead + """ + if tag == GameTag.ENTITY_ID: + # This is the simplest check. When a player entity is declared, + # its ENTITY_ID is not available immediately (in pre-6.0). + # If we get a matching ENTITY_ID, then we can use that to match it. + return self.register_player_name(player.name, value) + elif tag == GameTag.LAST_CARD_PLAYED: + # This is a fallback to register_player_name_mulligan in case the mulligan + # phase is not available in this game (spectator mode, reconnects). + if value not in self._entity_controller_map: + raise ParsingError("Unknown entity ID on TAG_CHANGE: %r" % (value)) + player_id = self._entity_controller_map[value] + entity_id = int(self._players_by_player_id[player_id]) + return self.register_player_name(player.name, entity_id) + elif tag == GameTag.MULLIGAN_STATE: + return None + return player diff --git a/hslog/live/utils.py b/hslog/live/utils.py new file mode 100644 index 0000000..b2222d2 --- /dev/null +++ b/hslog/live/utils.py @@ -0,0 +1,50 @@ +def align(text, size, char=" "): + """ + Format text to fit into a predefined amount of space + + Positive size aligns text to the left + Negative size aligns text to the right + """ + text = str(text).strip() + text_len = len(text) + if text_len > abs(size): + text = f"{text[:size-3]}..." + offset = "".join(char * (abs(size) - text_len)) + if size < 0: + return f"{offset}{text}" + else: + return f"{text}{offset}" + + +def color(): + def color_decorator(func): + colors = { + "LivePlayer": 93, + "red": 31, + "green": 32, + "LiveGame": 33, + "blue": 34, + "purple": 35, + "cyan": 36, + "grey": 37, + } + + def func_wrapper(msg_type, obj, *args): + class_name = obj.__class__.__name__ + color_key = class_name if class_name in colors else "green" + line = "\033[{}m{}\033[0m".format(colors[color_key], func(msg_type, obj, *args)) + print(line) + + return func_wrapper + + return color_decorator + + +@color() +def terminal_output(msg_type, obj, attr=None, value=None): + return "{} | {} | {} | {}".format( + align(msg_type, -20), + align(repr(obj), 120), + align(repr(attr), 40), + align(value, 30), + )