Last active
September 4, 2018 17:12
-
-
Save gidim/4e1d76a4f9934769d8d1e8db347a2100 to your computer and use it in GitHub Desktop.
The following script listens to new lines on a file and reports them to Comet.ml
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import json | |
import logging | |
import time | |
from collections import defaultdict | |
import os | |
from multiprocessing import Lock | |
from watchdog.observers import Observer | |
from watchdog.events import FileSystemEventHandler, LoggingEventHandler | |
from comet_ml import Experiment | |
import argparse | |
""" | |
The following script listens to new lines on a file and reports them to Comet.ml | |
The start the script run: | |
$ python comet-listen.py -f/--file, -k/--api_key, -w/--workspace, -p/--project | |
For example you can run it as follows: | |
python comet-listen.py -f /path/to/text/file/log2.txt -k 7bEAnfh9UoEAqz3QyNLpt36lY66 -p our-project -w your_workspace | |
The script will run until explicitly killed using Ctrl-C. | |
How to write lines to file from another process | |
------------------------------------------------ | |
From your training script or executable write single JSON objects in each line + an EOL symbol. | |
For example: | |
{"type": "metric", "name": "name", "value": 100} \n | |
Make sure to append the file in append mode ("a"). | |
The supported JSON objects are as follows: | |
{"type":"other", "name":"name", "value": INTEGER/FLOAT/STRING} | |
{"type":"parameter", "name":"name", "value": INTEGER/FLOAT/STRING , step=INTEGER} | |
{"type":"metric", "name":"name", "value":INTEGER/FLOAT/STRING , step=INTEGER} | |
Note that the step property is optional | |
Here's an example of a "training" script: | |
``` | |
for i in range(100): | |
with open("./my_experiment.txt", "a") as f: | |
# Your training happens here | |
metric_msg = {"type":"metric", "name":"name", "value":i} | |
f.write(json.dumps(metric_msg) + "\n") | |
metric_msg = {"type":"other", "name":"name", "value":"some_value"} | |
f.write(json.dumps(metric_msg) + "\n") | |
metric_msg = {"type":"parameter", "name":"name", "value":"parameter_1"} | |
f.write(json.dumps(metric_msg) + "\n") | |
f.flush() | |
``` | |
""" | |
class CometEventHandler(FileSystemEventHandler): | |
"""Logs all the events captured.""" | |
def __init__(self, api_key, project_name, worksapce, file_path): | |
self.api_key = api_key | |
self.project_name = project_name | |
self.workspace = worksapce | |
self.file_path = file_path | |
self.experiment = None | |
self.line_number = -1 | |
self.lock = Lock() | |
def on_moved(self, event): | |
super(CometEventHandler, self).on_moved(event) | |
what = 'directory' if event.is_directory else 'file' | |
logging.info("Moved %s: %s", what, event.src_path) | |
def on_created(self, event): | |
super(CometEventHandler, self).on_created(event) | |
what = 'directory' if event.is_directory else 'file' | |
logging.info("Created %s: %s", what, event.src_path) | |
if event.src_path == self.file_path: | |
if self.experiment is None: | |
self._start_experiment() | |
def on_deleted(self, event): | |
super(CometEventHandler, self).on_deleted(event) | |
what = 'directory' if event.is_directory else 'file' | |
logging.info("Deleted %s: %s", what, event.src_path) | |
def on_modified(self, event): | |
super(CometEventHandler, self).on_modified(event) | |
what = 'directory' if event.is_directory else 'file' | |
logging.info("Modified %s: %s", what, event.src_path) | |
if event.src_path == self.file_path: | |
if self.experiment is None: | |
self._start_experiment() | |
with self.lock: | |
with open(event.src_path, 'r', encoding="utf-8") as f: | |
count = 0 | |
for line in f: | |
if count > self.line_number: | |
self._process_line(line) | |
self.line_number = count | |
count += 1 | |
def _start_experiment(self): | |
self.experiment = Experiment(api_key=self.api_key, project_name=self.project_name, workspace=self.workspace, | |
log_code=False, log_graph=False, auto_param_logging=False, | |
auto_metric_logging=False, | |
parse_args=False, auto_output_logging=None, | |
log_env_details=False, log_git_metadata=False, log_git_patch=False | |
) | |
self.experiment.disable_mp() | |
def _process_line(self, line): | |
""" | |
{"type":"metric", "name":"something", "value":"some value", "step": 10} | |
:param line: | |
:return: | |
""" | |
try: | |
message = json.loads(line, object_pairs_hook=lambda obj: defaultdict(lambda: None, obj)) | |
should_print = True | |
if message["type"] == "metric": | |
self.experiment.log_metric(message["name"], message["value"], step=message["step"]) | |
elif message["type"] == "parameter": | |
self.experiment.log_parameter(message["name"], message["value"], step=message["step"]) | |
elif message["type"] == "other": | |
self.experiment.log_other(message["name"], message["value"]) | |
else: | |
should_print = False | |
if should_print: | |
logging.info("processed line %s", message) | |
except Exception as e: | |
print(e) | |
if __name__ == "__main__": | |
logging.basicConfig(level=logging.INFO, | |
format='%(asctime)s - %(message)s', | |
datefmt='%Y-%m-%d %H:%M:%S') | |
parser = argparse.ArgumentParser() | |
parser.add_argument("-f", "--file", required=True, | |
help="path to file with logs") | |
parser.add_argument("-k", "--api_key", required=True, | |
help="Comet.ml api key") | |
parser.add_argument("-w", "--workspace", required=True, | |
help="Comet.ml workspace name") | |
parser.add_argument("-p", "--project", required=True, | |
help="Comet.ml project name") | |
args = vars(parser.parse_args()) | |
print(args) | |
event_handler = CometEventHandler(args["api_key"], args["project"], args["workspace"], args['file']) | |
observer = Observer() | |
observer.schedule(event_handler, os.path.dirname(args["file"]), recursive=True) | |
observer.start() | |
try: | |
while True: | |
time.sleep(1) | |
except KeyboardInterrupt: | |
observer.stop() | |
observer.join() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
The following script listens to new lines on a file and reports them to Comet.ml
The start the script run:
$ python comet-listen.py -f/--file, -k/--api_key, -w/--workspace, -p/--project
For example you can run it as follows:
python comet-listen.py -f /path/to/text/file/log2.txt -k 7bEAnfh9UoEAqz3QyNLpt36lY66 -p our-project -w your_workspace
The script will run until explicitly killed using Ctrl-C.
How to write lines to file from another process
From your training script or executable write single JSON objects in each line + an EOL symbol.
For example:
{"type": "metric", "name": "name", "value": 100} \n
Make sure to append the file in append mode ("a").
The supported JSON objects are as follows:
{"type":"other", "name":"name", "value": INTEGER/FLOAT/STRING}
{"type":"parameter", "name":"name", "value": INTEGER/FLOAT/STRING , step=INTEGER}
{"type":"metric", "name":"name", "value":INTEGER/FLOAT/STRING , step=INTEGER}
Note that the step property is optional
Here's an example of a "training" script: