django+tornado實(shí)現(xiàn)實(shí)時(shí)查看遠(yuǎn)程日志的方法

更新時(shí)間：2019年08月12日 09:50:17 作者：tuxinhang

今天小編就為大家分享一篇django+tornado實(shí)現(xiàn)實(shí)時(shí)查看遠(yuǎn)程日志的方法，具有很好的參考價(jià)值，希望對(duì)大家有所幫助。一起跟隨小編過來看看吧

大致思路：

1.利用tornado提供的websocket功能與瀏覽器建立長連接，讀取實(shí)時(shí)日志并輸出到瀏覽器

2.寫一個(gè)實(shí)時(shí)讀取日志的腳本，利用saltstack遠(yuǎn)程執(zhí)行，并把實(shí)時(shí)日志發(fā)往redis中。

3.tornado讀取redis中的信息，發(fā)往瀏覽器。

此過程用到了redis的發(fā)布和訂閱功能。

先看一下tornado中是如何處理的：

import os
import sys
import tornado.websocket
import tornado.web
import tornado.ioloop
import redis
import salt.client

from tornado import gen
from tornado.escape import to_unicode

from logs.utility import get_last_lines
from logs import settings


class SubWebSocket(tornado.websocket.WebSocketHandler):
 """
 此handler處理遠(yuǎn)程日志查看
 """
 def open(self, *args, **kwargs):
  print("opened")

 @gen.coroutine
 def on_message(self, message):
  # 主機(jī)名，要查看的日志路徑，運(yùn)行腳本的命令這些信息從瀏覽器傳過來
  hostname, log_path, cmd = message.split("||")
  local = salt.client.LocalClient()
  r = redis.StrictRedis(host=settings.REDIS_HOST, port=settings.REDIS_PORT,
        password=settings.REDIS_PASSWD, db=5)
  # 訂閱頻道，服務(wù)器和日志路徑確定一個(gè)頻道
  key = settings.LOG_KEY.format(server=hostname.strip(), log_path=log_path.strip())
  channel = r.pubsub()
  channel.subscribe(key)
  # 異步方式執(zhí)行命令，遠(yuǎn)程運(yùn)行腳本
  local.cmd_async(hostname, "cmd.run", [cmd])
  try:
   while True:
    data = channel.get_message()
    if not data:
     # 如果讀取不到消息，間隔一定時(shí)間，避免無謂的CPU消耗
     yield gen.sleep(0.05)
     continue
    if data["type"] == "message":
     line = format_line(data["data"])
     self.write_message(line)
  except tornado.websocket.WebSocketClosedError:
   self.close()

 def on_close(self):
  global FLAG
  FLAG = False
  print("closed")


def format_line(line):
 line = to_unicode(line)
 if "INFO" in line:
  color = "#46A3FF"
 elif "WARN" in line:
  color = "#FFFF37"
 elif "ERROR" in line:
  color = "red"
 elif "CRITICAL" in line:
  color = "red"
 else:
  color = "#FFFFFF"

 return "<span style='color:{}'>{}</span>".format(color, line)


class EchoWebSocket(tornado.websocket.WebSocketHandler):
 def open(self):
  print("WebSocket opened")

 @gen.coroutine
 def on_message(self, message):
  log = message
  print "log file: ", log

  try:
   with open(log, 'r') as f:
    for line in get_last_lines(f):
     line1 = format_line(line)
     self.write_message(line1)
    while True:
     line = f.readline()
     if not line:
      yield gen.sleep(0.05)
      continue
     self.write_message(format_line(line.strip()))
  except tornado.websocket.WebSocketClosedError as e:
   print e
   self.close()

 # def check_origin(self, origin):
 #  print origin, self.request.headers.get("Host")
 #  # super(EchoWebSocket, self).check_origin()
 #  return True

 def on_close(self):
  print("WebSocket closed")


class Application(tornado.web.Application):
 def __init__(self):
  handlers = [
   (r'/log/', MainHandler), # 提供瀏覽頁面，頁面中的JS與服務(wù)器建立連接
   (r'/log/local', EchoWebSocket), # 處理本地日志實(shí)時(shí)查看，比較簡單
   (r'/log/remote', SubWebSocket), # 處理遠(yuǎn)程日志實(shí)時(shí)查看，稍微復(fù)雜
  ]
  settings = {
   "debug": True,
   "template_path": os.path.join(os.path.dirname(__file__), "templates"),
   "static_path": os.path.join(os.path.dirname(__file__), "static"),
  }
  super(Application, self).__init__(handlers, **settings)


class MainHandler(tornado.web.RequestHandler):
 def get(self):
  # 要查看的日志路徑
  log = self.get_argument("log", None)
  # hostname實(shí)際上是saltstack中這臺(tái)機(jī)器對(duì)應(yīng)的minion id
  hostname = self.get_argument("hostname", None)
  # 本地日志還是遠(yuǎn)程日志
  type = self.get_argument("type", "local")
  # 運(yùn)行讀取實(shí)時(shí)日志的腳本，參數(shù)比較多，后面會(huì)有
  cmd = self.get_argument("cmd", "")
  context = {
   "log": log,
   "hostname": hostname,
   "type": type,
   "cmd": cmd,
  }
  self.render("index.html", **context)

配置文件中主要記錄了redis服務(wù)器的地址等信息

# encoding: utf-8

LOG_KEY = "logs:{server}:{log_path}"

LOG_NAME = "catalina.out"
TAIL_LINE_NUM = 20

REDIS_HOST = "127.0.0.1"
REDIS_PORT = "6379"
REDIS_PASSWD = None
REDIS_EXPIRE = 300

try:
 from local_settings import *
except ImportError:
 pass

index.html的內(nèi)容如下：

<html>
<head>
<link href="{{ static_url('public/css/public.css') }}" rel="external nofollow" rel="stylesheet" />
<link href="{{ static_url('kylin/css/style.css') }}" rel="external nofollow" rel="stylesheet" />
</head>
<body style="background:#000000">
<div style="margin-left:10px;">
 <pre id="id-content">
 </pre>
 <div id="id-bottom"></div>
 <input type="hidden" id="id-log" value="{{ log }}" />
 <input type="hidden" id="id-type" value="{{ type }}" />
 <input type="hidden" id="id-hostname" value="{{ hostname }}" />
 <input type="hidden" id="id-cmd" value="{{ cmd }}" />
 <div class="btns btns_big">
  <button type="button" class="query_btn cancle" id="id-stop">Stop</button>
  <button type="button" class="query_btn commit" id="id-start">Start</button>
 </div>
</div>
<script type="text/javascript" src="{{ static_url('js/jquery-1.11.3.min.js') }}"></script>
<script type="text/javascript">
 var log_name = $("#id-log").val();
 var type = $("#id-type").val();
 var hostname = $("#id-hostname").val();
 var cmd = $("#id-cmd").val();
 // 初始化websocket對(duì)象
 var ws = new WebSocket("ws://{{ request.host }}/log/" + type);
 ws.onopen = function(){
  if (type === "local"){
   ws.send(log_name);
  } else {
   // 建立連接后把相關(guān)信息發(fā)往服務(wù)器，對(duì)應(yīng)上面的SubWebSocket
   ws.send(hostname + "||" + log_name + "||" + cmd);
  }
 };
 var get_message = function(evt){
  $("#id-content").append(evt.data + "\n");
  document.getElementById("id-bottom").scrollIntoView()
 };
 ws.onmessage = get_message;
 // 兩個(gè)按鈕控制日志的輸出，如果看到需要的日志信息，可以暫停日志的輸出，
 // 之后可以繼續(xù)啟動(dòng)日志的輸出
 $("#id-stop").click(function(){
  ws.onmessage = function(){};
 })
 $("#id-start").click(function(){
  ws.onmessage = get_message;
 })
</script>
</body>
</html>

這個(gè)tornado僅僅是提供了實(shí)時(shí)日志的服務(wù)，實(shí)際項(xiàng)目使用的是django，django中要做的其實(shí)很簡單，提供log_name,hostname,type,cmd等四個(gè)參數(shù)。

下面看一個(gè)實(shí)例：

class LogView(KylinView):
 # 實(shí)時(shí)讀取日志的腳本，事先使用saltstack批量傳到各臺(tái)服務(wù)器上
 client_path = "/tmp/logtail.py"

 def get(self, request):
  minion_id = request.GET.get("minion_id")
  context = {
   "minion_id": minion_id,
   "tail_log_url": settings.TAIL_LOG_URL,
  }
  return render(request, "cmdb/log_view.html", context)

 def post(self, request):
  minion_id = request.POST.get("minion_id")
  log_path = request.POST.get("log_path")
  if not log_path:
   return JsonResponse({"success": False, "message": "請(qǐng)?zhí)顚懭罩韭窂?})
  try:
   # 制定一開始讀取的行數(shù)
   line_count = request.POST.get("line_count")
  except (TypeError, ValueError):
   return JsonResponse({"success": False, "message": "請(qǐng)輸入正確的行數(shù)"})
  local = salt.client.LocalClient()
  # 確保saltstack能連通并且日志文件存在
  ret = local.cmd(minion_id, "file.file_exists", [log_path])
  if minion_id not in ret:
   return JsonResponse({"success": False, "message": "服務(wù)器無法連通"})
  if not ret[minion_id]:
   return JsonResponse({"success": False, "message": "日志文件不存在"})
  # 組成命令的各個(gè)參數(shù)，redis信息需要和tornado配置文件中的redis信息一致
  cmd = "{} {} {} {} {} {} {} {}".format(
   settings.PYTHON_BIN, self.client_path, minion_id, log_path, line_count, settings.REDIS_HOST,
   settings.REDIS_PORT, settings.REDIS_PASSWD)
  # settings.TAIL_LOG_URL是tornado中MainHandler對(duì)應(yīng)的url,把其它幾個(gè)
  # 參數(shù)組合成最終的URL，直接訪問這個(gè)URL就可以在瀏覽器中實(shí)時(shí)讀取日志了。
  url = "{}?type=remote&log={}&hostname={}&cmd={}".format(
   settings.TAIL_LOG_URL, log_path, minion_id, cmd)
  # 這一步的操作確保同一個(gè)日志文件只有一個(gè)腳本在讀取，避免日志信息重復(fù)，這一步
  # 也很重要，必不可少
  local.cmd(minion_id, "cmd.run",
     ["kill `ps aux|grep logtail.py|grep %s|grep -v grep|awk '{print $2}'`" % (log_path,)])
  return JsonResponse({"success": True, "url": url})

下面來看看logtail.py的實(shí)現(xiàn)：

# encoding: utf-8
from __future__ import unicode_literals, division

import math
import time
import sys
import socket
import signal
import redis

FLAG = True


def get_last_lines(f, num=10):
 """讀取文件的最后幾行
 """
 size = 1000
 try:
  f.seek(-size, 2)
 except IOError: # 文件內(nèi)容不足size
  f.seek(0)
  return f.readlines()[-num:]

 data = f.read()
 lines = data.splitlines()
 n = len(lines)
 while n < num:
  size *= int(math.ceil(num / n))
  try:
   f.seek(-size, 2)
  except IOError:
   f.seek(0)
   return f.readlines()[-num:]
  data = f.read()
  lines = data.splitlines()
  n = len(lines)

 return lines[-num:]


def process_line(r, channel, line):
 r.publish(channel, line.strip())


def sig_handler(signum, frame):
 global FLAG
 FLAG = False


# 收到退出信號(hào)后，以比較優(yōu)雅的方式終止腳本
signal.signal(signal.SIGTERM, sig_handler)
# 為了避免日志輸出過多，瀏覽器承受不住，設(shè)置5分鐘后腳本自動(dòng)停止
signal.signal(signal.SIGALRM, sig_handler)
signal.alarm(300)


def get_hostname():
 return socket.gethostname()


def force_str(s):
 if isinstance(s, unicode):
  s = s.encode("utf-8")
 return s


def tail():
 password = sys.argv[6]
 if password == "None":
  password = None
 r = redis.StrictRedis(host=sys.argv[4], port=sys.argv[5], password=password, db=5)
 log_path = sys.argv[2]
 line_count = int(sys.argv[3])
 # 往redis頻道發(fā)送實(shí)時(shí)日志
 channel = "logs:{hostname}:{log_path}".format(hostname=sys.argv[1], log_path=log_path)

 with open(log_path, 'r') as f:
  last_lines = get_last_lines(f, line_count)
  for line in last_lines:
   process_line(r, channel, force_str(line))
  try:
   while FLAG: # 通過信號(hào)控制這個(gè)變量，實(shí)現(xiàn)優(yōu)雅退出循環(huán)
    line = f.readline()
    if not line:
     time.sleep(0.05)
     continue
    process_line(r, channel, line)
  except KeyboardInterrupt:
   pass
 print("Exiting...")

if __name__ == "__main__":
 if len(sys.argv) < 6:
  print "Usage: %s minion_id log_path host port redis_pass"
  exit(1)

 tail()

到此為止，整個(gè)實(shí)時(shí)讀取遠(yuǎn)程日志的流程就講完了。

github: https://github.com/tuxinhang1989/logs

以上這篇django+tornado實(shí)現(xiàn)實(shí)時(shí)查看遠(yuǎn)程日志的方法就是小編分享給大家的全部內(nèi)容了，希望能給大家一個(gè)參考，也希望大家多多支持腳本之家。

您可能感興趣的文章: