background
The most common way to use logs in Python is to output logs in the console and files, and the logging module also provides a good corresponding class, which is also very convenient to use, but sometimes we may have some requirements, such as the need to send logs to the far end, or write directly It is reported that how to achieve this requirement?
StreamHandler and FileHandler
# -*- coding: utf-8 -*-
"""
-------------------------------------------------
File Name： loger
Description :
Author : yangyanxing
date： 2020/9/23
-------------------------------------------------
"""
import logging
import sys
import os
# 初始化logger
logger = logging.getLogger("yyx")
logger.setLevel(logging.DEBUG)
# 设置日志格式
fmt = logging.Formatter('[%(asctime)s] [%(levelname)s] %(message)s', '%Y-%m-%d
%H:%M:%S')
# 添加cmd handler
cmd_handler = logging.StreamHandler(sys.stdout)
cmd_handler.setLevel(logging.DEBUG)
cmd_handler.setFormatter(fmt)
# 添加文件的handler
logpath = os.path.join(os.getcwd(), 'debug.log')
file_handler = logging.FileHandler(logpath)
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(fmt)
# 将cmd和file handler添加到logger中
logger.addHandler(cmd_handler)
logger.addHandler(file_handler)
logger.debug("今天天气不错")
First initialize a logger, and set its log level to DEBUG, then initialize the cmd_handler and file_handler, and finally add them to the logger, run the script, and print it out in cmd
[2020-09-23 10:45:56] [DEBUG] 今天天气不错
Add HTTPHandler
# 添加一个httphandler
import logging.handlers
http_handler = logging.handlers.HTTPHandler(r"127.0.0.1:1987", '/api/log/get')
http_handler.setLevel(logging.DEBUG)
http_handler.setFormatter(fmt)
logger.addHandler(http_handler)
logger.debug("今天天气不错")
As a result, we received a lot of information on the server side
{
'name': [b 'yyx'],
'msg': [b
'\xe4\xbb\x8a\xe5\xa4\xa9\xe5\xa4\xa9\xe6\xb0\x94\xe4\xb8\x8d\xe9\x94\x99'],
'args': [b '()'],
'levelname': [b 'DEBUG'],
'levelno': [b '10'],
'pathname': [b 'I:/workplace/yangyanxing/test/loger.py'],
'filename': [b 'loger.py'],
'module': [b 'loger'],
'exc_info': [b 'None'],
'exc_text': [b 'None'],
'stack_info': [b 'None'],
'lineno': [b '41'],
'funcName': [b '<module>'],
'created': [b '1600831054.8881223'],
'msecs': [b '888.1223201751709'],
'relativeCreated': [b '22.99976348876953'],
'thread': [b '14876'],
'threadName': [b 'MainThread'],
'processName': [b 'MainProcess'],
'process': [b '8648'],
'message': [b
'\xe4\xbb\x8a\xe5\xa4\xa9\xe5\xa4\xa9\xe6\xb0\x94\xe4\xb8\x8d\xe9\x94\x99'],
'asctime': [b '2020-09-23 11:17:34']
}
It can be said that there is a lot of information, but it is not what we want, we just want to be similar
[2020-09-23 10:45:56][DEBUG] 今天天气不错
logging.handlers.HTTPHandler simply sends all the information of the log to the server, and the server is up to the server to organize the content. So we can have two methods, one is to change the server code, according to the log information passed over to reorganize the log content, the second is to rewrite a class, so that it sends the reformatted log content to the server when it is sent.
We take the second approach, because this method is more flexible, the server side is only used for recording, and what content is sent should be decided by the client.
We need to redefine a class, we can refer to the logging.handlers.HTTPHandler class and rewrite an httpHandler class
class CustomHandler(logging.Handler):
def __init__(self, host, uri, method="POST"):
logging.Handler.__init__(self)
self.url = "%s/%s" % (host, uri)
method = method.upper()
if method not in ["GET", "POST"]:
raise ValueError("method must be GET or POST")
self.method = method
def emit(self, record):
'''
重写emit方法，这里主要是为了把初始化时的baseParam添加进来
:param record:
:return:
'''
msg = self.format(record)
if self.method == "GET":
if (self.url.find("?") >= 0):
sep = '&'
else:
sep = '?'
url = self.url + "%c%s" % (sep, urllib.parse.urlencode({"log":
msg}))
requests.get(url, timeout=1)
else:
headers = {
"Content-type": "application/x-www-form-urlencoded",
"Content-length": str(len(msg))
}
requests.post(self.url, data={'log': msg}, headers=headers,
timeout=1)
This line of code indicates that the corresponding content will be returned according to the format set by the log object.
{'log': [b'[2020-09-23 11:39:45] [DEBUG]
\xe4\xbb\x8a\xe5\xa4\xa9\xe5\xa4\xa9\xe6\xb0\x94\xe4\xb8\x8d\xe9\x94\x99']}
Turn the bytes type around and you'll get it
[2020-09-23 11:43:50] [DEBUG] 今天天气不错
Send remote logs asynchronously
async def post(self):
print(self.getParam('log'))
await asyncio.sleep(5)
self.write({"msg": 'ok'})
At this point we will print the above log
logger.debug("今天天气不错")
logger.debug("是风和日丽的")
The resulting output is
[2020-09-23 11:47:33] [DEBUG] 今天天气不错
[2020-09-23 11:47:38] [DEBUG] 是风和日丽的
Now the problem is that what was originally just a log is now a burden that drags down the entire script, so we need to process the remote write log asynchronously.
1
Use multithreading
def emit(self, record):
msg = self.format(record)
if self.method == "GET":
if (self.url.find("?") >= 0):
sep = '&'
else:
sep = '?'
url = self.url + "%c%s" % (sep, urllib.parse.urlencode({"log": msg}))
t = threading.Thread(target=requests.get, args=(url,))
t.start()
else:
headers = {
"Content-type": "application/x-www-form-urlencoded",
"Content-length": str(len(msg))
}
t = threading.Thread(target=requests.post, args=(self.url,), kwargs=
{"data":{'log': msg},
2
Use thread pool processing
Python's concurrent.futures have ThreadPoolExecutor, ProcessPoolExecutor class, which is a thread pool and a process pool, that is, at the initialization of a few threads, and then let these threads process the corresponding functions, so that you do not have to recreate the threads every time
exector = ThreadPoolExecutor(max_workers=1) # 初始化一个线程池,只有一个线程
exector.submit(fn, args, kwargs) # 将函数submit到线程池中
exector = ThreadPoolExecutor(max_workers=1)
def emit(self, record):
msg = self.format(record)
timeout = aiohttp.ClientTimeout(total=6)
if self.method == "GET":
if (self.url.find("?") >= 0):
sep = '&'
else:
sep = '?'
url = self.url + "%c%s" % (sep, urllib.parse.urlencode({"log": msg}))
exector.submit(requests.get, url, timeout=6)
else:
headers = {
"Content-type": "application/x-www-form-urlencoded",
"Content-length": str(len(msg))
}
exector.submit(requests.post, self.url, data={'log': msg},
headers=headers, timeout=6)
3
Use the asynchronous aiohttp library to send requests
class CustomHandler(logging.Handler):
def __init__(self, host, uri, method="POST"):
logging.Handler.__init__(self)
self.url = "%s/%s" % (host, uri)
method = method.upper()
if method not in ["GET", "POST"]:
raise ValueError("method must be GET or POST")
self.method = method
async def emit(self, record):
msg = self.format(record)
timeout = aiohttp.ClientTimeout(total=6)
if self.method == "GET":
if (self.url.find("?") >= 0):
sep = '&'
else:
sep = '?'
url = self.url + "%c%s" % (sep, urllib.parse.urlencode({"log":
msg}))
async with aiohttp.ClientSession(timeout=timeout) as session:
async with session.get(self.url) as resp:
print(await resp.text())
else:
headers = {
"Content-type": "application/x-www-form-urlencoded",
"Content-length": str(len(msg))
}
async with aiohttp.ClientSession(timeout=timeout, headers=headers)
as session:
async with session.post(self.url, data={'log': msg}) as resp:
print(await resp.text())
At this point the code execution crashes
C:\Python37\lib\logging\__init__.py:894: RuntimeWarning: coroutine
'CustomHandler.emit' was never awaited
self.emit(record)
RuntimeWarning: Enable tracemalloc to get the object allocation traceback
The reason for this is that because the async with session.post function is used in the emit method, it needs to be a function that uses async decoration Because of execution, modify the emit function, use async to decorate, here the equal function becomes an asynchronous function, returning a coroutine object, to execute the coroutine object, you need to use await, but the script is not there to call await emit(), so the crash message shows coroutine 'CustomHandler.emit' was never awaited.
async def main():
await logger.debug("今天天气不错")
await logger.debug("是风和日丽的")
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
Execution still reports errors
raise TypeError('An asyncio.Future, a coroutine or an awaitable is '
There seems to be no way out of it, wanting to use an asynchronous library to send, but there is no place to call await.
import asyncio
async def test(n):
while n > 0:
await asyncio.sleep(1)
print("test {}".format(n))
n -= 1
return n
async def test2(n):
while n >0:
await asyncio.sleep(1)
print("test2 {}".format(n))
n -= 1
def stoploop(task):
print("执行结束, task n is {}".format(task.result()))
loop.stop()
loop = asyncio.get_event_loop()
task = loop.create_task(test(5))
task2 = loop.create_task(test2(3))
task.add_done_callback(stoploop)
task2 = loop.create_task(test2(3))
loop.run_forever()
Notice that in the code above, we didn't use await somewhere to execute the coroutine, but by registering the coroutine on an event loop object and then calling the loop's method method run_forever() function, so that the coroutine object on the loop could execute normally.
test 5
test2 3
test 4
test2 2
test 3
test2 1
test 2
test 1
执行结束, task n is 0
As you can see, a task created with an event loop object can be executed after the loop executes run_forever() and if the loop.run_forever() function is not executed, the coroutine registered on it will not be executed
loop = asyncio.get_event_loop()
task = loop.create_task(test(5))
task.add_done_callback(stoploop)
task2 = loop.create_task(test2(3))
time.sleep(5)
# loop.run_forever()
loop = asyncio.get_event_loop()
class CustomHandler(logging.Handler):
def __init__(self, host, uri, method="POST"):
logging.Handler.__init__(self)
self.url = "%s/%s" % (host, uri)
method = method.upper()
if method not in ["GET", "POST"]:
raise ValueError("method must be GET or POST")
self.method = method
# 使用aiohttp封装发送数据函数
async def submit(self, data):
timeout = aiohttp.ClientTimeout(total=6)
if self.method == "GET":
if self.url.find("?") >= 0:
sep = '&'
else:
sep = '?'
url = self.url + "%c%s" % (sep, urllib.parse.urlencode({"log":
data}))
async with aiohttp.ClientSession(timeout=timeout) as session:
async with session.get(url) as resp:
print(await resp.text())
else:
headers = {
"Content-type": "application/x-www-form-urlencoded",
}
async with aiohttp.ClientSession(timeout=timeout, headers=headers)
as session:
async with session.post(self.url, data={'log': data}) as resp:
print(await resp.text())
return True
def emit(self, record):
msg = self.format(record)
loop.create_task(self.submit(msg))
# 添加一个httphandler
http_handler = CustomHandler(r"http://127.0.0.1:1987", 'api/log/get')
http_handler.setLevel(logging.DEBUG)
http_handler.setFormatter(fmt)
logger.addHandler(http_handler)
logger.debug("今天天气不错")
logger.debug("是风和日丽的")
loop.run_forever()
loop.create_task (self.submit(msg)) can also be used
asyncio.ensure_future(self.submit(msg), loop=loop) instead, all of which register coroutine objects in the event loop.
But one thing to note this way is that loop.run_forever() will always be blocked, so there needs to be a place to call the loop.stop() method. You can register to a callback on a task.