Commit e0f58919 authored by Inkhey's avatar Inkhey

using threads

parent 501704c8
......@@ -11,6 +11,7 @@
import sqlite3
import os
from threading import Thread
from subprocess import Popen, PIPE
from walkdir import filtered_walk
from timeit import default_timer as timer
......@@ -31,6 +32,7 @@ def calctime(method):
return calctime
class sqlGit(object):
''' Transform Git data into sqlite database to
......@@ -40,21 +42,22 @@ class sqlGit(object):
def __init__(self, db):
''' Initialise sqlite database file'''
self.conn = sqlite3.connect(db)
self.conn = sqlite3.connect(db, check_same_thread=False)
@calctime
def create_table(self):
''' create SQL tables : author, commits, file, link_author
commit and link commit_file .'''
cmds=[]
c = self.conn.cursor()
c.execute('''CREATE TABLE author(
cmds=[]
cmds.append('''CREATE TABLE author(
name text,
email text,
primary key(name,email));''')
c.execute('''CREATE TABLE commits(
cmds.append('''CREATE TABLE commits(
subject text,
hash text primary key,
abbrev_hash text,
......@@ -63,13 +66,16 @@ class sqlGit(object):
# root -> "."
# folder -> "./dossier"
# subfolders-> "./dossier1/dossier2"
c.execute('''CREATE TABLE file(
cmds.append('''CREATE TABLE file(
name text,
size integer,
dir text )''')
threads = [Thread(target=self._execCommit, args=(cmd,)) for cmd in cmds]
[t.start() for t in threads]
[t.join() for t in threads]
# Link
c.execute('''CREATE TABLE link_author_commits(
cmds=[]
cmds.append('''CREATE TABLE link_author_commits(
author_name text,
author_mail text,
commits text
......@@ -79,7 +85,7 @@ class sqlGit(object):
# TODO : chercher à voir si l'on trouve des données
# ajout/suppressions de lignes sur le commit pour un fichier défini.
c.execute('''CREATE TABLE link_file_commits(
cmds.append('''CREATE TABLE link_file_commits(
file text,
dir text,
commits text,
......@@ -89,7 +95,7 @@ class sqlGit(object):
references commits(hash));''')
# save last and first commit date for file and author
c.execute('''CREATE TABLE date_file(
cmds.append('''CREATE TABLE date_file(
file text,
dir text,
deb_date integer,
......@@ -98,7 +104,7 @@ class sqlGit(object):
references file(name,dir)
);''')
c.execute('''CREATE TABLE date_author(
cmds.append('''CREATE TABLE date_author(
name text,
email text,
deb_date integer,
......@@ -106,8 +112,9 @@ class sqlGit(object):
foreign key (name,email)
references author(name,email)
);''')
self.conn.commit()
threads = [Thread(target=self._execCommit, args=(cmd,)) for cmd in cmds]
[t.start() for t in threads]
[t.join() for t in threads]
# INSERT
# DATA
def _gInsert(self,l):
......@@ -144,12 +151,32 @@ class sqlGit(object):
def insert_all(self, path, depth):
''' insert all content and delete unused file'''
os.chdir(path)
self.insert_author()
self.insert_commit()
self.insert_file(depth)
self.insert_link_A_C()
self.insert_link_F_C()
self.insert_dates()
t1 = Thread(target=self.insert_author())
t2 = Thread(target=self.insert_commit())
t3 = Thread(target=self.insert_file(depth))
t4 = Thread(target=self.insert_link_A_C())
t5 = Thread(target=self.insert_link_F_C())
t6 = Thread(target=self.insert_dates_author())
t7 = Thread(target=self.insert_dates_file())
# insert data
t1.start()
t2.start()
t3.start()
t1.join()
t2.join()
# insert file is long, begin link A_C before
t4.start()
t3.join()
#insert link and dates
t5.start()
# lac libéré,date auteurs possible
t4.join()
t6.start()
t5.join()
t7.start()
t6.join()
t7.join()
self.delete_unused_file()
@calctime
......@@ -237,30 +264,35 @@ class sqlGit(object):
self.conn.commit()
@calctime
def insert_dates(self):
c = self.conn.cursor()
c.execute(
'''INSERT INTO date_file(deb_date,fin_date,file,dir)
def insert_dates_file(self):
cmd='''INSERT INTO date_file(deb_date,fin_date,file,dir)
Select min(c.date),max(c.date),lfc.file,lfc.dir
from link_file_commits lfc join commits c
on c.hash=lfc.commits
group by lfc.file,lfc.dir;'''
)
c.execute(
'''INSERT INTO date_author(deb_date,fin_date,name,email)
self._execCommit(cmd)
@calctime
def insert_dates_author(self):
cmd='''INSERT INTO date_author(deb_date,fin_date,name,email)
Select min(c.date),max(c.date),lac.author_name,lac.author_mail
from link_author_commits lac join commits c
on c.hash=lac.commits
group by lac.author_name,lac.author_mail;'''
)
self._execCommit(cmd)
def _execCommit(self,cmd):
c = self.conn.cursor()
c.execute(cmd)
self.conn.commit()
@calctime
def view_3nodestype(self):
'''View with 3 types of nodes : author,commit,file'''
c = self.conn.cursor()
c.execute(
cmds=[]
cmds.append(
'''create table nodes as
select f.dir || f.name as id ,
f.size as taille,
......@@ -305,7 +337,7 @@ class sqlGit(object):
abbrev_hash as label
from commits;
''')
c.execute(
cmds.append(
'''create table edges as
select author_name || author_mail as source,
commits as target,
......@@ -319,14 +351,15 @@ class sqlGit(object):
dir
from link_file_commits;
''')
self.conn.commit()
threads = [Thread(target=self._execCommit, args=(cmd,)) for cmd in cmds]
[t.start() for t in threads]
[t.join() for t in threads]
@calctime
def view_2nodestype(self):
'''View with 2 types of nodes : author,file'''
c = self.conn.cursor()
c.execute('''create table nodes as
cmds=[]
cmds.append('''create table nodes as
select f.dir || f.name as id ,
f.size as taille,
f.dir as dir,
......@@ -353,7 +386,7 @@ class sqlGit(object):
where a.email=da.email
and a.name=da.name''')
c.execute(''' create table edges as
cmds.append(''' create table edges as
select
lac.author_name || lac.author_mail as source,
lfc.dir || lfc.file as target,
......@@ -371,14 +404,14 @@ class sqlGit(object):
on c.hash = lfc.commits
group by lac.author_name || lac.author_mail,lfc.dir || lfc.file;
''')
self.conn.commit()
threads = [Thread(target=self._execCommit, args=(cmd,)) for cmd in cmds]
[t.start() for t in threads]
[t.join() for t in threads]
@calctime
def view_files(self):
'''View of relation between files'''
c = self.conn.cursor()
c.execute('''create table nodes as
sNodes='''create table nodes as
select f.dir || f.name as id ,
f.size as taille,
f.dir as dir,
......@@ -387,18 +420,18 @@ class sqlGit(object):
f.dir || f.name as label
from file f join date_file df
where f.dir=df.dir
and f.name=df.file;''')
and f.name=df.file;'''
#tmp table to gain time
c.execute('''
sTmp='''
create table tmp as
select
a.author_name || a.author_mail as author,
f.dir || f.file as file
from link_file_commits f, link_author_commits a
where ( f.commits=a.commits);
''')
self.conn.commit()
c.execute('''
'''
sEdge='''
create table edges as
select f1.file AS source , f2.file AS target,
count(distinct(f1.author)) as nbAuthors,
......@@ -410,14 +443,21 @@ class sqlGit(object):
on f1.author = f2.author
and f1.file != f2.file
)
group by f1.file , f2.file;''')
self.conn.commit()
group by f1.file , f2.file;'''
t1 = Thread(target=self._execCommit, args=(sNodes,))
t2 = Thread(target=self._execCommit, args=(sTmp,))
t3 = Thread(target=self._execCommit, args=(sEdges,))
t1.start()
t2.start()
t2.join()
t3.start()
t3.join()
t1.join()
@calctime
def view_authors(self):
'''View of relation between authors'''
c = self.conn.cursor()
c.execute('''
sNodes='''
create table nodes as
select distinct( a.name || a.email) as id,
a.name as name,
......@@ -427,18 +467,18 @@ class sqlGit(object):
datetime(da.fin_date,'unixepoch') as fin_date
from author a join date_author da
where a.email=da.email
and a.name=da.name;''')
and a.name=da.name;'''
c.execute('''
sTmp='''
create table tmp as
select
a.author_name || a.author_mail as author,
f.dir || f.file as file
from link_file_commits f, link_author_commits a
where ( f.commits=a.commits);
''')
self.conn.commit()
c.execute('''
'''
sEdges='''
create table edges as
select a1.author AS source , a2.author AS target,
count(a1.file) as nbfile,
......@@ -453,9 +493,16 @@ class sqlGit(object):
and a1.author != a2.author
)
group by a1.author,a2.author
;''')
self.conn.commit()
;'''
t1 = Thread(target=self._execCommit, args=(sNodes,))
t2 = Thread(target=self._execCommit, args=(sTmp,))
t3 = Thread(target=self._execCommit, args=(sEdges,))
t1.start()
t2.start()
t2.join()
t3.start()
t3.join()
t1.join()
@calctime
def view_file_commits(self):
pass
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment