diff options
author | 2013-07-29 21:55:51 +0300 | |
---|---|---|
committer | 2013-07-29 21:55:51 +0300 | |
commit | e5bdc5bc099096186e00e61604484fc6a1ecdeb2 (patch) | |
tree | 0e31d855e720459ad438216c47898b5080fb8c92 | |
parent | More basename() defensiveness (diff) | |
download | log-analysis-e5bdc5bc099096186e00e61604484fc6a1ecdeb2.tar.gz log-analysis-e5bdc5bc099096186e00e61604484fc6a1ecdeb2.tar.bz2 log-analysis-e5bdc5bc099096186e00e61604484fc6a1ecdeb2.zip |
Database (SQL) class for functionality common across all processors
Currently uses MySQLdb. Schema included.
-rw-r--r-- | database.py | 22 | ||||
-rw-r--r-- | flask_app.py | 18 | ||||
-rw-r--r-- | portage_processor.py | 9 | ||||
-rw-r--r-- | schema.sql | 12 | ||||
-rw-r--r-- | simple_client.py | 1 | ||||
-rw-r--r-- | submission.proto | 3 |
6 files changed, 56 insertions, 9 deletions
diff --git a/database.py b/database.py new file mode 100644 index 0000000..5202876 --- /dev/null +++ b/database.py @@ -0,0 +1,22 @@ +from contextlib import closing +import MySQLdb + +class DatabaseConnection: + def __init__(self, conn): + self.conn = conn + + def insert_file(self, path, group_id): + with closing(self.conn.cursor()) as c: + c.execute("insert into `files` (`path`, `group_id`) values (%s, %s)", (path, group_id)) + self.conn.commit() + return c.lastrowid + + def insert_group(self, name, provider, date): + with closing(self.conn.cursor()) as c: + c.execute("insert into `groups` (`name`, `provider`, `date`) values (%s, %s, %s)", (name, provider, date)) + self.conn.commit() + return c.lastrowid + +def get_connection(user, passwd, db): + conn = MySQLdb.connect(user=user, passwd=passwd, db=db) + return DatabaseConnection(conn) diff --git a/flask_app.py b/flask_app.py index 832702c..5356bc4 100644 --- a/flask_app.py +++ b/flask_app.py @@ -4,14 +4,24 @@ When run as a script, the Flask development server is started. """ import os, socket -import submission_pb2, storage -from flask import Flask, request +import submission_pb2, storage, database +from flask import Flask, request, g from portage_processor import PortageProcessor app = Flask(__name__) store = storage.FilesystemStorage('logs/') -processors = {'portage' : PortageProcessor(None, store)} # TODO: initialise from config file +processors = {'portage' : PortageProcessor(store)} # TODO: initialise from config file + +@app.before_request +def before_request(): + g.db = database.get_connection('gsoc', 'gsocpasswd', 'loganalysis') + +@app.teardown_request +def teardown_request(exception): + db = getattr(g, 'db', None) + if db is not None: + db.conn.close() @app.route('/') def index(): @@ -23,7 +33,7 @@ def submit(): submission.ParseFromString(request.data) source = socket.getfqdn(request.remote_addr) # TODO: is this ok? - processors[submission.provider].process(submission, source) + processors[submission.provider].process(submission, source, g.db) return '' if __name__ == '__main__': diff --git a/portage_processor.py b/portage_processor.py index 2403cdf..66fb970 100644 --- a/portage_processor.py +++ b/portage_processor.py @@ -1,4 +1,4 @@ -import re, StringIO +import os, re, StringIO, time class PortageProcessor: _r = { @@ -10,11 +10,11 @@ class PortageProcessor: 'escapes' : re.compile(r"\x1b\[[^\x40-\x7e]*[\x40-\x7e]") } - def __init__(self, db, storage): - self.db = db + def __init__(self, storage): self.storage = storage - def process(self, request, source): + def process(self, request, source, db): + group_id = db.insert_group(request.group_name, 'portage', int(time.time())) for f in request.files: matches = 0 pkg_failed = False @@ -72,3 +72,4 @@ class PortageProcessor: ''') self.storage.save_file(source, f.filename, output.getvalue()) + file_id = db.insert_file(os.path.join(source, f.filename), group_id) diff --git a/schema.sql b/schema.sql new file mode 100644 index 0000000..564385e --- /dev/null +++ b/schema.sql @@ -0,0 +1,12 @@ +create table if not exists `files` ( + `id` int primary key auto_increment, + `path` text not null, + `group_id` int not null +); + +create table if not exists `groups` ( + `id` int primary key auto_increment, + `name` text not null, + `provider` varchar(16) not null, + `date` int not null +); diff --git a/simple_client.py b/simple_client.py index ab4bccf..c89b6c1 100644 --- a/simple_client.py +++ b/simple_client.py @@ -7,6 +7,7 @@ import submission_pb2, sys, urllib2, os def send_submission(filenames): submission = submission_pb2.Submission() submission.provider = "portage" + submission.group_name = "Manual submission" for f in filenames: new_file = submission.files.add() diff --git a/submission.proto b/submission.proto index 42cf97c..3cbf474 100644 --- a/submission.proto +++ b/submission.proto @@ -5,5 +5,6 @@ message Submission { } required string provider = 1; - repeated File files = 2; + optional string group_name = 2; + repeated File files = 3; } |