1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
|
#!/usr/bin/python3
import re, os, gzip
from datetime import datetime
from pathlib import Path
from dataclasses import dataclass
from jinja2 import Environment
from time import time
LINE_PATTERN = r"^([^ ]+) ([^ ]+) - \[(.*)\] \"([A-Z]+) ([^ ]+) HTTP/\d\.\d\""
TEMPLATE="""
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Most visited pages</title>
<style type="text/css">
.stats {
text-align: center
}
</style>
</head>
<body>
<div class="stats">
<h1>Stats</h1>
<p>
Logs start at {{ logs_start_date }} ({{ logs_days_ago }} days ago)
</p>
<p>
Stats generated in {{ seconds }}s
</p>
<p>
Total visits: {{ total_visits }}
</p>
<p>
Unique visits: {{ unique_visits }}
</p>
<p>
Most visited url: {{ most_visited_site }}
</p>
</div>
</body>
</html>
"""
@dataclass
class Line:
source: str
dest: str
time: str
kind: str
resource: str
start_time = time()
logs_dir = Path(os.environ.get("LOGS_DIR", "/var/log/lighttpd"))
log_files = [log for log in logs_dir.iterdir()
if log.is_file()
and log.name.startswith("access.log")]
urls = {}
seen = set()
total_visits = 0
for file in log_files:
if file.suffix == ".gz":
reader = gzip.open(file, "rt")
else:
reader = file.open()
for line in reader:
if (match := re.match(LINE_PATTERN, line)):
line = Line(
match.group(1),
match.group(2),
match.group(3),
match.group(4),
match.group(5)
)
if line.kind == "GET":
try:
count = urls[line.resource]
except KeyError:
count = 0
urls[line.resource] = count + 1
total_visits += 1
seen.add(line.source)
first_log_file = log_files[0]
logs_start_date = datetime.fromtimestamp(first_log_file.stat().st_ctime)
logs_days_ago = (datetime.now() - logs_start_date).days
unique_visits = len(seen)
most_visited_site = sorted(urls.items(), key=lambda item: item[1])[-1][0]
seconds = round(time() - start_time, 3)
environment = Environment()
template = environment.from_string(TEMPLATE)
print(template.render(
seconds=seconds,
logs_start_date=logs_start_date.ctime(),
logs_days_ago=logs_days_ago,
total_visits=total_visits,
unique_visits=unique_visits,
most_visited_site=most_visited_site
))
|
