Commit 31578fc4 authored by Doriane's avatar Doriane 💬
Browse files

optimized time of extraction

parent 5019e38f
......@@ -16,6 +16,8 @@ WHERE type='COMMENT'
GROUP BY potelet_id
HAVING c > 3;
SELECT potelet_id, count(*) AS c FROM attachments WHERE type='COMMENT' GROUP BY potelet_id HAVING c > 3;
# => got interesting result with this method (the one with a lot of comments)! look at 254080
# the one marked as duplicates
......
......@@ -19,8 +19,8 @@ mobilierurbain_catid = 1007
potelet_catid = 2030
# ratio for the number of items got by request
itemsbypages = 12
poteletbycategory = 100
itemsbypages = 24
poteletbycategory = 48
#---- DB WRITING ----
......@@ -158,7 +158,6 @@ def addPotelet(conn, potelet):
VALUES(?,?,?,?,?,?,?,?,?) '''
cur = conn.cursor()
cur.execute(sql, value_list)
conn.commit()
def addAttachment(conn, attachment):
id = attachment['id']
......@@ -186,7 +185,6 @@ def addAttachment(conn, attachment):
VALUES(?,?,?,?,?) '''
cur = conn.cursor()
cur.execute(sql, value_list)
conn.commit()
def addStory(conn, story):
id = story['id']
......@@ -207,7 +205,6 @@ def addStory(conn, story):
VALUES(?,?,?,?) '''
cur = conn.cursor()
cur.execute(sql, value_list)
conn.commit()
def addActor(conn, value_list):
cur = conn.cursor()
......@@ -223,7 +220,6 @@ def addActor(conn, value_list):
sql = ''' INSERT INTO actors(id,name,type,contact)
VALUES(?,?,?,?) '''
cur.execute(sql, value_list)
conn.commit()
else:
if exists[0]['name'] != value_list[1]:
print("Error: two organisations/department with same id and different names")
......@@ -406,7 +402,6 @@ if __name__ == '__main__':
start_time = time.time()
print('~!~ POTELETS ~!~')
print('Total number of incidents: ' + str(getNumberOfIncidents()[0]))
print('')
# --- CREATE DB
......@@ -421,7 +416,13 @@ if __name__ == '__main__':
# --- EXTRACT AND FILL DB
print('Total number of incidents: ' + str(getNumberOfIncidents()[0]))
potelet_cat = getPoteletCat()
total_number_of_potelet = 0
for potelet_subcat in potelet_cat['subCategories']:
total_number_of_potelet += getNumberOfIncidents(potelet_subcat['id'])[0]
print('Total number of potelets: ' + str(total_number_of_potelet))
potelets = getPotelets(poteletbycategory)
for potelet in potelets:
......@@ -440,9 +441,18 @@ if __name__ == '__main__':
for story in history:
addStory(conn, story)
# we commit the change to the db at only the end of every potelets
# to not loose too much time
conn.commit()
print("(+) potelets added in db")
print('')
conn.close()
print('process finished in: ' + str(time.time() - start_time) + ' seconds')
# printing the time it tooks
duration = time.time() - start_time
ratio = duration / (poteletbycategory*2)
total_duration = ratio * total_number_of_potelet
print('process finished in: ' + time.strftime('%H:%M:%S', time.gmtime(duration)))
print('rate of ' + str(ratio) + ' seconds per potelets')
print('estimated for whole process: ' + time.strftime('%H:%M:%S', time.gmtime(total_duration)))
No preview for this file type
File added
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment