ADD(buku) tests for firefox json import
This commit is contained in:
Chris Drexler
parent
6931a91d17
commit
da7cd0b1a4
114
buku
114
buku
@ -2376,15 +2376,28 @@ class BukuDb:
|
||||
items = []
|
||||
if filepath.endswith('.md'):
|
||||
items = import_md(filepath=filepath, newtag=newtag)
|
||||
|
||||
elif filepath.endswith('org'):
|
||||
items = import_org(filepath=filepath, newtag=newtag)
|
||||
|
||||
elif filepath.endswith('json'):
|
||||
if not tacit:
|
||||
resp = input('Add Bookmark folder name as tag? (y/n): ')
|
||||
else:
|
||||
resp = 'y'
|
||||
add_bookmark_folder_as_tag = (resp == 'y')
|
||||
items = import_firefox_json(filepath, add_bookmark_folder_as_tag, newtag )
|
||||
try:
|
||||
with open(filepath, 'r', encoding='utf-8') as datafile:
|
||||
data = json.load(datafile)
|
||||
|
||||
items = import_firefox_json(data, add_bookmark_folder_as_tag, newtag )
|
||||
|
||||
except json.decoder.JSONDecodeError as e:
|
||||
logerr("JSONDecodeError: " + str(e))
|
||||
return False
|
||||
except Exception as e:
|
||||
logerr(e)
|
||||
return False
|
||||
|
||||
else:
|
||||
try:
|
||||
@ -2872,7 +2885,7 @@ def import_org(filepath, newtag):
|
||||
|
||||
|
||||
|
||||
def import_firefox_json(path, add_bookmark_folder_as_tag, unique_tag):
|
||||
def import_firefox_json(json, add_bookmark_folder_as_tag=False, unique_tag=None):
|
||||
"""Open Firefox json export file and import data.
|
||||
|
||||
Ignore 'SmartBookmark' and 'Separator' entries.
|
||||
@ -2917,72 +2930,79 @@ def import_firefox_json(path, add_bookmark_folder_as_tag, unique_tag):
|
||||
d = [ anno for anno in entry['annos'] if anno['name'] == "bookmarkProperties/description" ]
|
||||
return d[0]['value']
|
||||
except:
|
||||
logdbg("no description found for entry: ", entry['uri'], entry['title'])
|
||||
return ""
|
||||
|
||||
def extract_tags_string(entry, parent_folder):
|
||||
tags = ""
|
||||
try:
|
||||
tags = entry['tags']
|
||||
except:
|
||||
pass
|
||||
|
||||
if add_parent_folder_as_tag:
|
||||
tags += ","+parent_folder
|
||||
|
||||
if unique_tag:
|
||||
tags += ","+unique_tag
|
||||
|
||||
return tags
|
||||
|
||||
def extract_tags(entry):
|
||||
tags = []
|
||||
try:
|
||||
tags = entry['tags'].split(',')
|
||||
except:
|
||||
logdbg("no tags found for entry: ", entry['uri'], entry['title'])
|
||||
pass
|
||||
|
||||
return tags
|
||||
|
||||
def iterate_children(parent_folder, entry_list):
|
||||
for bm_entry in entry_list:
|
||||
if TypeCode.uri.value == bm_entry['typeCode'] :
|
||||
if is_smart(bm_entry):
|
||||
continue
|
||||
if is_nongeneric_url(bm_entry['uri']):
|
||||
continue
|
||||
try:
|
||||
typeCode = bm_entry['typeCode']
|
||||
except:
|
||||
logdbg("item without typeCode found, ignoring: ", bm_entry['title'] )
|
||||
continue
|
||||
|
||||
desc = extract_desc(bm_entry)
|
||||
bookmark_tags = extract_tags(bm_entry)
|
||||
|
||||
if add_bookmark_folder_as_tag:
|
||||
bookmark_tags.append(parent_folder)
|
||||
|
||||
if unique_tag:
|
||||
bookmark_tags.append(unique_tag)
|
||||
|
||||
formatted_tags = [DELIM + tag for tag in bookmark_tags]
|
||||
tags = parse_tags(formatted_tags)
|
||||
|
||||
yield (bm_entry['uri'], bm_entry['title'], tags, desc, 0, True)
|
||||
|
||||
elif TypeCode.folder.value == bm_entry['typeCode'] :
|
||||
if TypeCode.uri.value == typeCode :
|
||||
try:
|
||||
# from python 3.3
|
||||
# yield from iterate_children(bm_entry['title'], bm_entry['children'])
|
||||
if is_smart(bm_entry):
|
||||
logdbg("SmartBookmark found,m ignoring: ", bm_entry['title'] )
|
||||
continue
|
||||
|
||||
for entry in iterate_children(bm_entry['title'], bm_entry['children']):
|
||||
yield entry
|
||||
if is_nongeneric_url(bm_entry['uri']):
|
||||
logdbg("Non-Generic URL found,m ignoring: ", bm_entry['title'] )
|
||||
continue
|
||||
|
||||
except:
|
||||
desc = extract_desc(bm_entry)
|
||||
bookmark_tags = extract_tags(bm_entry)
|
||||
|
||||
if add_bookmark_folder_as_tag:
|
||||
bookmark_tags.append(parent_folder)
|
||||
|
||||
if unique_tag:
|
||||
bookmark_tags.append(unique_tag)
|
||||
|
||||
formatted_tags = [DELIM + tag for tag in bookmark_tags]
|
||||
tags = parse_tags(formatted_tags)
|
||||
|
||||
logdbg("Entry found: ", bm_entry['uri'], bm_entry['title'], tags, desc)
|
||||
yield (bm_entry['uri'], bm_entry['title'], tags, desc, 0, True)
|
||||
|
||||
except Exception as e:
|
||||
logerr(e)
|
||||
pass
|
||||
|
||||
elif TypeCode.separator.value == bm_entry['typeCode'] :
|
||||
pass
|
||||
elif TypeCode.folder.value == typeCode :
|
||||
try:
|
||||
# from python 3.3 on:
|
||||
# yield from iterate_children(bm_entry['title'], bm_entry['children'])
|
||||
|
||||
with open(path, 'r') as datafile:
|
||||
data = json.load(datafile)
|
||||
for entry in iterate_children(parent_folder+"/"+bm_entry['title'], bm_entry['children']):
|
||||
yield entry
|
||||
except Exception as e:
|
||||
# if any of the properties does not exist, bail out silently
|
||||
logerr(e)
|
||||
pass
|
||||
|
||||
yield from iterate_children(data['title'], data['children'])
|
||||
elif TypeCode.separator.value == typeCode :
|
||||
logdbg("Unknonw typeCode found : ", typeCode)
|
||||
pass
|
||||
|
||||
try:
|
||||
entry_list = json['children']
|
||||
except:
|
||||
logerr("No children in Root entry found")
|
||||
return []
|
||||
|
||||
yield from iterate_children("", entry_list )
|
||||
|
||||
|
||||
def import_html(html_soup, add_parent_folder_as_tag, newtag):
|
||||
|
||||
@ -1352,44 +1352,6 @@ def test_load_firefox_database(firefox_db, add_pt):
|
||||
yaml.dump(call_args_list_dict, f)
|
||||
print('call args list dict dumped to:{}'.format(res_yaml_file))
|
||||
|
||||
@pytest.fixture()
|
||||
def firefox_json():
|
||||
# compatibility
|
||||
dir_path = os.path.dirname(os.path.realpath(__file__))
|
||||
res_yaml_file = os.path.join(dir_path, 'test_bukuDb', '')
|
||||
res_nopt_yaml_file = os.path.join(dir_path, 'test_bukuDb', '')
|
||||
json_file = os.path.join(dir_path, 'test_bukuDb', 'firefox_bookmarks.json')
|
||||
return json_file, res_yaml_file, res_nopt_yaml_file
|
||||
|
||||
|
||||
## TODO
|
||||
# * test typeCode (1 = uri, text/x-moz-place = bookmarklet,query,<whatelse?>)
|
||||
# * handle charset propert?
|
||||
@pytest.mark.parametrize('add_pt', [True, False])
|
||||
def test_load_firefox_json(firefox_json, add_pt):
|
||||
"""test method."""
|
||||
# compatibility
|
||||
json_file = firefox_json[0]
|
||||
res_yaml_file = firefox_json[1] if add_pt else firefox_json[2]
|
||||
dump_data = False # NOTE: change this value to dump data
|
||||
if not dump_data:
|
||||
with open(res_yaml_file, 'r') as f:
|
||||
res_yaml = yaml.load(f)
|
||||
# init
|
||||
import buku
|
||||
bdb = buku.BukuDb()
|
||||
bdb.add_rec = mock.Mock()
|
||||
bdb.load_firefox_json(json_file, None, add_pt)
|
||||
call_args_list_dict = dict(bdb.add_rec.call_args_list)
|
||||
# test
|
||||
if not dump_data:
|
||||
assert call_args_list_dict == res_yaml
|
||||
# dump data for new test
|
||||
if dump_data:
|
||||
with open(res_yaml_file, 'w') as f:
|
||||
yaml.dump(call_args_list_dict, f)
|
||||
print('call args list dict dumped to:{}'.format(res_yaml_file))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'keyword_results, stag_results, exp_res',
|
||||
|
||||
200
tests/test_import_firefox_json.py
Normal file
200
tests/test_import_firefox_json.py
Normal file
@ -0,0 +1,200 @@
|
||||
import json
|
||||
from buku import import_firefox_json
|
||||
|
||||
|
||||
def test_load_from_empty():
|
||||
"""test method."""
|
||||
# Arrange
|
||||
data = json.loads("{}")
|
||||
|
||||
# Act
|
||||
items = import_firefox_json(data)
|
||||
|
||||
# Assert
|
||||
count = sum(1 for _ in items)
|
||||
assert 0 == count
|
||||
|
||||
|
||||
def test_load_no_typecode():
|
||||
"""test method."""
|
||||
# Arrange
|
||||
data = json.loads("""
|
||||
{
|
||||
"title" : "title",
|
||||
"children": [
|
||||
{
|
||||
"title" : "title1",
|
||||
"uri" : "http://uri1",
|
||||
"annos" : [{
|
||||
"name": "bookmarkProperties/description",
|
||||
"value": "desc"
|
||||
}]
|
||||
}]
|
||||
}
|
||||
"""
|
||||
)
|
||||
|
||||
# Act
|
||||
items = import_firefox_json(data)
|
||||
|
||||
# Assert
|
||||
result = []
|
||||
for item in items:
|
||||
result.append(item)
|
||||
|
||||
assert 0 == len(result)
|
||||
|
||||
|
||||
def test_load_invalid_typecode():
|
||||
"""test method."""
|
||||
# Arrange
|
||||
data = json.loads("""
|
||||
{
|
||||
"title" : "title",
|
||||
"children": [
|
||||
{
|
||||
"title" : "title1",
|
||||
"typeCode" : 99,
|
||||
"uri" : "http://uri1",
|
||||
"annos" : [{
|
||||
"name": "bookmarkProperties/description",
|
||||
"value": "desc"
|
||||
}]
|
||||
}]
|
||||
}
|
||||
"""
|
||||
)
|
||||
|
||||
# Act
|
||||
items = import_firefox_json(data)
|
||||
|
||||
# Assert
|
||||
result = []
|
||||
for item in items:
|
||||
result.append(item)
|
||||
|
||||
assert 0 == len(result)
|
||||
|
||||
|
||||
def test_load_one_child():
|
||||
"""test method."""
|
||||
|
||||
# Arrange
|
||||
data = json.loads("""
|
||||
{
|
||||
"title" : "title",
|
||||
"typeCode" : 2,
|
||||
"children": [
|
||||
{
|
||||
"title" : "title1",
|
||||
"typeCode" : 1,
|
||||
"uri" : "http://uri1",
|
||||
"annos" : [{
|
||||
"name": "bookmarkProperties/description",
|
||||
"value": "desc"
|
||||
}]
|
||||
}]
|
||||
}
|
||||
"""
|
||||
)
|
||||
|
||||
# Act
|
||||
items = import_firefox_json(data)
|
||||
|
||||
# Assert
|
||||
result = []
|
||||
for item in items:
|
||||
result.append(item)
|
||||
|
||||
assert 1 == len(result)
|
||||
assert 'http://uri1' == result[0][0]
|
||||
assert 'title1' == result[0][1]
|
||||
assert ',' == result[0][2]
|
||||
assert 'desc' == result[0][3]
|
||||
|
||||
|
||||
|
||||
def test_load_one_container_child():
|
||||
"""test method."""
|
||||
|
||||
# Arrange
|
||||
data = json.loads("""
|
||||
{
|
||||
"title" : "title",
|
||||
"typeCode" : 2,
|
||||
"children": [
|
||||
{
|
||||
"guid":"mobile______",
|
||||
"title":"Mobile Lesezeichen",
|
||||
"index":4,
|
||||
"dateAdded":1489602681463000,
|
||||
"lastModified":1519334403977000,
|
||||
"id":330524,
|
||||
"typeCode":2,
|
||||
"annos":[
|
||||
{
|
||||
"name":"mobile/bookmarksRoot",
|
||||
"value":"1",
|
||||
"expires":4,
|
||||
"flags":0
|
||||
}],
|
||||
"type":"text/x-moz-place-container",
|
||||
"root":"mobileFolder"
|
||||
} ]
|
||||
}""")
|
||||
|
||||
|
||||
# Act
|
||||
items = import_firefox_json(data)
|
||||
|
||||
# Assert
|
||||
result = []
|
||||
for item in items:
|
||||
result.append(item)
|
||||
|
||||
assert 0 == len(result)
|
||||
|
||||
|
||||
def test_load_many_children():
|
||||
"""test method."""
|
||||
|
||||
# Arrange
|
||||
data = json.loads("""
|
||||
{
|
||||
"guid":"unfiled_____",
|
||||
"title":"Weitere Lesezeichen",
|
||||
"index":3,
|
||||
"dateAdded":1305446109028000,"lastModified":1544905385223000,
|
||||
"id":5,
|
||||
"typeCode":2,
|
||||
"type":"text/x-moz-place-container",
|
||||
"root":"unfiledBookmarksFolder",
|
||||
"children": [
|
||||
{"guid":"_0yvSg7JN7WD","title":"title1","index":0,"dateAdded":1287656058186000,"lastModified":1305978155312000,"id":169,"typeCode":1,"charset":"UTF-8","annos":[{"name":"bookmarkProperties/description","value":"desc3","expires":4,"flags":0}],"type":"text/x-moz-place","uri":"http://uri1.com/#more-74"},
|
||||
{"guid":"YI8BEgRJ1Hio","title":"title2","index":1,"dateAdded":1232202012000000,"lastModified":1305978155291000,"id":157,"typeCode":1,"charset":"ISO-8859-1","annos":[{"name":"bookmarkProperties/description","value":"desc2","expires":4,"flags":0}],"type":"text/x-moz-place","uri":"http://uri2.com/xyz"},
|
||||
{"guid":"XYewkZ6jVnOt","title":"title3","index":2,"dateAdded":1236767103079000,"lastModified":1305978155293000,"id":158,"typeCode":1,"charset":"ISO-8859-1","type":"text/x-moz-place","uri":"http://uri3.com"}
|
||||
]
|
||||
} """)
|
||||
|
||||
# Act
|
||||
items = import_firefox_json(data)
|
||||
|
||||
# Assert
|
||||
result = []
|
||||
for item in items:
|
||||
result.append(item)
|
||||
|
||||
assert 3 == len(result)
|
||||
|
||||
|
||||
# Tests TODO:
|
||||
# separator type code
|
||||
# multiple tags
|
||||
# hierarchic folders
|
||||
# utf-8 encoded titles
|
||||
# non-generic url
|
||||
# tags with comma(?), UTF chars, ...
|
||||
# endurance test (>200 entries)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
x
Reference in New Issue
Block a user