pjht/buku
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

check that all records are processed

Browse Source
This commit is contained in:
Arun Prakash Jana 2016-12-01 08:30:17 +05:30
parent 321fc05377
commit 3e4f735bb7
No known key found for this signature in database
GPG Key ID: A75979F35C080412
1 changed files with 14 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
buku.py
View File

@ -771,6 +771,8 @@ class BukuDb:
query = 'UPDATE bookmarks SET metadata = ? WHERE id = ?' query = 'UPDATE bookmarks SET metadata = ? WHERE id = ?'
done = {'value': 0} # count threads completed done = {'value': 0} # count threads completed
processed = {'value': 0} # count number of records processed
cond = threading.Condition() cond = threading.Condition()
cond.acquire() cond.acquire()
@ -826,6 +828,7 @@ class BukuDb:
logdbg('Thread %d: processed %d', threading.get_ident(), count) logdbg('Thread %d: processed %d', threading.get_ident(), count)
with cond: with cond:
done['value'] += 1 done['value'] += 1
processed['value'] += count
cond.notify() cond.notify()
if recs < NUM_THREADS: if recs < NUM_THREADS:
@ -839,6 +842,10 @@ class BukuDb:
cond.wait() cond.wait()
logdbg('%d threads completed', done['value']) logdbg('%d threads completed', done['value'])
# Guard: records found == total records processed
if recs != processed['value']:
logerr('Records: %d, processed: %d !!!', recs, processed['value'])
cond.release() cond.release()
self.conn.commit() self.conn.commit()
return True return True
@ -1560,12 +1567,13 @@ def get_PoolManager():
:return: ProxyManager if https_proxy is defined, else PoolManager. :return: ProxyManager if https_proxy is defined, else PoolManager.
''' '''
headers={'Accept-Encoding': 'gzip,deflate', headers = {
'User-Agent': USER_AGENT, 'Accept-Encoding': 'gzip,deflate',
'Accept': '*/*', 'User-Agent': USER_AGENT,
'Cookie': '', 'Accept': '*/*',
'DNT': '1' 'Cookie': '',
} 'DNT': '1'
}
proxy = os.environ.get('https_proxy') proxy = os.environ.get('https_proxy')
if proxy: if proxy: