Last active
March 7, 2024 18:45
-
-
Save deepaksood619/99e790959f5eba6ba0815e056a8067d7 to your computer and use it in GitHub Desktop.
Setting up Logging in Python and Flask Application the right way
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import json | |
import requests | |
import logging | |
import os | |
from logging.config import dictConfig | |
# debug settings | |
debug = eval(os.environ.get("DEBUG", "False")) | |
from flask import Flask, make_response, request | |
# for sending error logs to slack | |
class HTTPSlackHandler(logging.Handler): | |
def emit(self, record): | |
log_entry = self.format(record) | |
json_text = json.dumps({"text": log_entry}) | |
url = 'https://hooks.slack.com/services/<org_id>/<api_key>' | |
return requests.post(url, json_text, headers={"Content-type": "application/json"}).content | |
dictConfig({ | |
"version": 1, | |
"disable_existing_loggers": True, | |
"formatters": { | |
"default": { | |
"format": "[%(asctime)s] %(levelname)s in %(module)s: %(message)s", | |
}, | |
"access": { | |
"format": "%(message)s", | |
} | |
}, | |
"handlers": { | |
"console": { | |
"level": "INFO", | |
"class": "logging.StreamHandler", | |
"formatter": "default", | |
"stream": "ext://sys.stdout", | |
}, | |
"email": { | |
"class": "logging.handlers.SMTPHandler", | |
"formatter": "default", | |
"level": "ERROR", | |
"mailhost": ("smtp.example.com", 587), | |
"fromaddr": "[email protected]", | |
"toaddrs": ["[email protected]", "[email protected]"], | |
"subject": "Error Logs", | |
"credentials": ("username", "password"), | |
}, | |
"slack": { | |
"class": "app.HTTPSlackHandler", | |
"formatter": "default", | |
"level": "ERROR", | |
}, | |
"error_file": { | |
"class": "logging.handlers.RotatingFileHandler", | |
"formatter": "default", | |
"filename": "/var/log/gunicorn.error.log", | |
"maxBytes": 10000, | |
"backupCount": 10, | |
"delay": "True", | |
}, | |
"access_file": { | |
"class": "logging.handlers.RotatingFileHandler", | |
"formatter": "access", | |
"filename": "/var/log/gunicorn.access.log", | |
"maxBytes": 10000, | |
"backupCount": 10, | |
"delay": "True", | |
} | |
}, | |
"loggers": { | |
"gunicorn.error": { | |
"handlers": ["console"] if debug else ["console", "slack", "error_file"], | |
"level": "INFO", | |
"propagate": False, | |
}, | |
"gunicorn.access": { | |
"handlers": ["console"] if debug else ["console", "access_file"], | |
"level": "INFO", | |
"propagate": False, | |
} | |
}, | |
"root": { | |
"level": "DEBUG" if debug else "INFO", | |
"handlers": ["console"] if debug else ["console", "slack"], | |
} | |
}) | |
app = Flask(__name__) | |
@app.route("/status", methods=["GET"]) | |
def health_check(): | |
logging.debug("debug log") | |
logging.info("info log") | |
logging.warning("warning log") | |
logging.error("error log") | |
# logging.exception("exception log") | |
return make_response("OK", 200) | |
if __name__ == "__main__": | |
app.run(debug=debug, host="0.0.0.0", port="5000") | |
# Run with Gunicorn | |
# gunicorn app:app -b 0.0.0.0:5000 --workers 2 -k gevent --timeout 300 --worker-connections 1000 --max-requests 1000000 --limit-request-line 8190 --access-logfile '-' --error-logfile '-' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Flask==1.1.2 | |
requests==2.23.0 | |
gunicorn[gevent]==20.0.4 |
Hello
I finally solved the problem I got .
I defined my own dictConfig :
import logging
from defaultapp.config import Config
from logging.config import dictConfig
import os
root = os.path.dirname(os.path.abspath(__file__))
logdir = os.path.join(root, 'logs')
if not os.path.exists(logdir):
os.mkdir(logdir)
# debug settings
debug = Config.DEBUG
dictConfig({
"version": 1,
"disable_existing_loggers": True,
"formatters": {
"default": {
"format": "[%(asctime)s] %(levelname)s in %(module)s - %(lineno)d: %(message)s ",
}
},
"handlers": {
"console": {
"level": Config.LOG_LEVEL,
"class": "logging.StreamHandler",
"formatter": "default",
"stream": "ext://flask.logging.wsgi_errors_stream",
},
"diarios_file": {
"level": Config.LOG_LEVEL,
"class": "logging.handlers.TimedRotatingFileHandler",
"formatter": "default",
"filename": os.path.join(logdir,"diarios.log"),
"when": "m",
"interval": 1,
"backupCount": 5,
"encoding":"utf8"
},
"patrimonio_file": {
"level": Config.LOG_LEVEL,
"class": "logging.handlers.TimedRotatingFileHandler",
"formatter": "default",
"filename": os.path.join(logdir,"patrimonio.log"),
"when": "m",
"interval": 1,
"backupCount": 5,
"encoding":"utf8"
},
"app_file": {
"level": Config.LOG_LEVEL,
"class": "logging.handlers.TimedRotatingFileHandler",
"formatter": "default",
"filename": os.path.join(logdir,"app.log"),
"when": "m",
"interval": 1,
"backupCount": 5,
"encoding":"utf8"
},
},
"loggers": {
"defaultapp.diarios": {
"handlers": ["console"] if debug else ["console", "diarios_file"],
"level": Config.LOG_LEVEL,
"propagate": False
},
"defaultapp.patrimonio": {
"handlers": ["console"] if debug else ["console", "patrimonio_file"],
"level": Config.LOG_LEVEL,
"propagate": False
},
"defaultapp": {
"handlers": ["console"] if debug else ["console", "app_file"],
"level": Config.LOG_LEVEL,
"propagate": False
}
},
"root": {
"level": "DEBUG" if debug else "INFO",
"handlers": ["console"]
}
})
and I named my loggers following my flask python module names and inside these I take care of importing logging and calling getLogger(name):
import logging
logger = logging.getLogger(__name__)
so any log produced inside a particular module (.py file) will be written into the corresponding LOG file defined by in my handlers.
rgds
@collinsmarra - Is it possibly because in the error_file handler, you specify a formatter called 'precise', but there is no such formatter in your formatters section?
@collinsmarra inside formatters you need to define precise or use default formatter rather than precise.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Hello .
What will be the setup in order to log messages to two different TimeRotating files ( each file will log messages from different "functional modules" ). I defined the following loggers.py file :
and import either diarios_logger, or patrimonio_logger or common_logger in other modules where I want to log messages to , but got the following error:
I have searched this error message in the web and found the reason is because flask-wsgi is runs multiprocessing workers and they compete to handle and rotate the log file , creating a mess!!
I also found one possible solution to this concurrency lock issue is to use QueueHandlers and QueueListeners...!
Thanks in advance for any hint or advice you could provide.
Rgds