-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
0 parents
commit c0891d1
Showing
3 changed files
with
203 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
# nginwho | ||
|
||
nginwho continuously parses nginx logs and writes them to a sqlite3 database for further analysis. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,187 @@ | ||
import std/[strutils, os, strformat, re] | ||
import db_connector/db_sqlite | ||
from parseopt import CmdLineKind, initOptParser, next | ||
|
||
|
||
const | ||
NGINX_DEFAULT_LOG_PATH = "/var/log/nginx/access.log" | ||
DB_FILE = "/var/log/nginwho.db" | ||
TEN_SECONDS = 10000 | ||
VERSION = "0.3.0" | ||
|
||
|
||
type Log = object | ||
remoteIP: string | ||
remoteUser: string | ||
authenticatedUser: string | ||
date: string | ||
httpMethod: string | ||
requestURI: string | ||
statusCode: string | ||
responseSize: string | ||
referrer: string | ||
userAgent: string | ||
nonStandard: string | ||
|
||
|
||
type | ||
Flags = tuple[ | ||
nginxLogPath: string, | ||
dbPath: string, | ||
interval: int | ||
] | ||
Logs = seq[Log] | ||
|
||
|
||
proc usage() = | ||
echo """ | ||
--help, -h : show help | ||
--version : Display version and quit | ||
--dbPath, : Path to SQLite database to log reports (default: /var/log/nginwho.db) | ||
--nginxLogPath, : Path to nginx access logs (default: /var/log/nginx/access.log) | ||
--interval : Refresh interval in seconds (default: 10) | ||
""" | ||
quit() | ||
|
||
|
||
proc getArgs(): Flags = | ||
var flags: Flags = (nginxLogPath: NGINX_DEFAULT_LOG_PATH, dbPath:DB_FILE, interval: TEN_SECONDS) | ||
|
||
var p = initOptParser() | ||
|
||
while true: | ||
p.next() | ||
case p.kind | ||
of cmdEnd: break | ||
of cmdShortOption, cmdLongOption: | ||
case p.key | ||
of "help", "h": usage() | ||
of "version": | ||
echo VERSION | ||
quit(0) | ||
of "nginxLogPath": flags.nginxLogPath = p.val | ||
of "dbPath": flags.dbPath = p.val | ||
of "interval": flags.interval = parseInt(p.val) | ||
of cmdArgument: discard | ||
|
||
return flags | ||
|
||
|
||
proc writeToDatabase(logs: var seq[Log], databaseName: string) = | ||
let db = open(databaseName, "", "", "") | ||
|
||
db.exec(sql"""CREATE TABLE IF NOT EXISTS nginwho | ||
( | ||
id INTEGER PRIMARY KEY, | ||
remoteIP TEXT NOT NULL, | ||
remoteUser TEXT NOT NULL, | ||
authenticatedUser TEXT NOT NULL, | ||
date TEXT NOT NULL, | ||
httpMethod TEXT NOT NULL, | ||
requestURI TEXT NOT NULL, | ||
statusCode TEXT NOT NULL, | ||
responseSize TEXT NOT NULL, | ||
referrer TEXT NOT NULL, | ||
userAgent TEXT NOT NULL, | ||
nonStandard TEXT | ||
)""" | ||
) | ||
|
||
let lastEntryDate = db.getRow(sql"SELECT date FROM nginwho WHERE TRIM(date) <> '' ORDER BY rowid DESC LIMIT 1;") | ||
var lastEntryDateValue: string | ||
|
||
if lastEntryDate[0] == "": | ||
echo "First time fetcing date from DB" | ||
else: | ||
lastEntryDateValue = lastEntryDate[0] | ||
|
||
#TODO: Expand the conditional check (perhaps on user-agent and URL) to | ||
# avoid missing entries on busy servers | ||
if logs[^1].date == lastEntryDateValue: | ||
echo "Rows are already written to DB" | ||
return | ||
|
||
db.exec(sql"BEGIN") | ||
|
||
for log in logs: | ||
db.exec(sql"""INSERT INTO nginwho | ||
( | ||
date, | ||
remoteIP, | ||
httpMethod, | ||
requestURI, | ||
statusCode, | ||
responseSize, | ||
referrer, | ||
userAgent, | ||
remoteUser, | ||
authenticatedUser, | ||
nonStandard) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", | ||
log.date, | ||
log.remoteIP, | ||
log.httpMethod, | ||
log.requestURI, | ||
log.statusCode, | ||
log.responseSize, | ||
log.referrer, | ||
log.userAgent, | ||
log.nonStandard, | ||
log.remoteUser, | ||
log.authenticatedUser | ||
) | ||
|
||
db.exec(sql"COMMIT") | ||
db.close() | ||
|
||
|
||
proc parseLogEntry(logLine: string): Log = | ||
var log: Log | ||
|
||
let matches = logLine.split(re"[ ]+") | ||
|
||
if matches.len >= 12: | ||
log.remoteIP = matches[0].replace("\"", "") | ||
log.date = matches[3].replace("\"", "").replace("[", "").replace("/", "-") | ||
log.httpMethod = matches[5].replace("\"", "") | ||
log.requestURI = matches[6].replace("\"", "") | ||
log.statusCode = matches[8].replace("\"", "") | ||
log.responseSize = matches[9].replace("\"", "") | ||
|
||
var referrer = matches[10].replace("\"", "") | ||
#TODO: Try `contains` instead | ||
# if referrer.contains("thegraynode.io"): | ||
if referrer.startsWith("https://thegraynode.io") or referrer.startsWith("https://www.thegraynode.io"): | ||
referrer = "" | ||
elif referrer == "-": | ||
referrer = "" | ||
else: | ||
log.referrer = referrer | ||
|
||
log.userAgent = matches[11..^1].join(" ").replace("\"", "") | ||
log.nonStandard = "" | ||
else: | ||
echo fmt"Could not parse: {logLine}" | ||
log.nonStandard = logLine | ||
|
||
return log | ||
|
||
|
||
proc main() = | ||
let args = getArgs() | ||
|
||
while true: | ||
var logs: Logs | ||
|
||
for line in lines(args.nginxLogPath): | ||
let log = parseLogEntry(line) | ||
logs.add(log) | ||
|
||
writeToDatabase(logs, args.dbPath) | ||
# logs = newSeq[Log]() | ||
sleep args.interval | ||
|
||
|
||
when is_main_module: | ||
main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
[Unit] | ||
Description=nginx log analyzer | ||
After=Network.target | ||
|
||
[Service] | ||
Type=simple | ||
RemainAfterExit=yes | ||
ExecStart=/usr/local/bin/nginwho --nginxLogPath:/var/log/nginx/access.log --dbPath:/var/log/nginwho.db | ||
Restart=on-failure | ||
RestartSec=5 | ||
|
||
[Install] | ||
WantedBy=multi-user.target |