1250 lines
36 KiB
Python
Executable File
1250 lines
36 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
#
|
|
# Bookmark management utility
|
|
#
|
|
# Copyright (C) 2015 Arun Prakash Jana <engineerarun@gmail.com>
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with buku. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
import sys
|
|
import os
|
|
import sqlite3
|
|
from getopt import getopt, GetoptError
|
|
import readline
|
|
import webbrowser
|
|
import html.parser as HTMLParser
|
|
from http.client import HTTPConnection, HTTPSConnection
|
|
from urllib.parse import urljoin, quote, unquote
|
|
import gzip
|
|
import io
|
|
import signal
|
|
import json
|
|
import shutil
|
|
|
|
# Import libraries needed for encryption
|
|
try:
|
|
import getpass
|
|
import hashlib
|
|
from Crypto.Cipher import AES
|
|
from Crypto import Random
|
|
import struct
|
|
|
|
no_crypto = False
|
|
encrypt = False # Lock database file
|
|
decrypt = False # Unlock database file
|
|
iterations = 8 # Number of hash iterations to generate key
|
|
BLOCKSIZE = 65536
|
|
SALT_SIZE = 32
|
|
CHUNKSIZE = 0x80000 # Read/write 512 KB chunks
|
|
except ImportError:
|
|
no_crypto = True
|
|
|
|
|
|
|
|
# Globals
|
|
addurl = False # Add a URL
|
|
addindex = None # DB index to insert URL into
|
|
online = False # Fetch title from web
|
|
delete = False # Delete bookmark(s)
|
|
empty = False # List all bookmarks with no title or tag
|
|
openurl = None # Open URL in browser
|
|
show = False # Show all details of a bookmark
|
|
showindex = None # Index of bookmark to show
|
|
showOpt = 0 # Modify show. 1: show only URL, 2: show URL and tag
|
|
showTags = False # List all unique tags
|
|
search = False # Search for keywords
|
|
searchAll = False # Match all keywords in search
|
|
entry = None # DB index to update or delete
|
|
update = False # Update a bookmark in DB
|
|
debug = False # Enable debug logs
|
|
titleData = None # Title fetched from a page
|
|
titleManual = None # Manually add a title offline
|
|
refresh = False # Refresh the full DB
|
|
replace = False # Replace a tag
|
|
jsonOutput = False # Output json formatted result
|
|
pipeargs = [] # Holds arguments piped to the program
|
|
_VERSION_ = 1.9 # Program version
|
|
|
|
|
|
class BMHTMLParser(HTMLParser.HTMLParser):
|
|
"""Class to parse and fetch the title from a HTML page, if available"""
|
|
|
|
def __init__(self):
|
|
HTMLParser.HTMLParser.__init__(self)
|
|
self.inTitle = False
|
|
self.data = ""
|
|
self.lasttag = None
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
self.inTitle = False
|
|
if tag == "title":
|
|
self.inTitle = True
|
|
self.lasttag = tag
|
|
|
|
def handle_endtag(self, tag):
|
|
global titleData
|
|
|
|
if tag == "title":
|
|
self.inTitle = False
|
|
if self.data != "":
|
|
titleData = self.data
|
|
self.reset() # We have received title data, exit parsing
|
|
|
|
def handle_data(self, data):
|
|
if self.lasttag == "title" and self.inTitle == True:
|
|
self.data += data
|
|
|
|
def error(self, message):
|
|
pass
|
|
|
|
|
|
|
|
def getDataPath():
|
|
"""Determine the DB file path:
|
|
if $XDG_DATA_HOME is defined, use it
|
|
else if $HOME exists, use it
|
|
else use the current directory
|
|
"""
|
|
|
|
data_home = os.environ.get('XDG_DATA_HOME')
|
|
if data_home is None:
|
|
if os.environ.get('HOME') is None:
|
|
data_home = '.'
|
|
else:
|
|
data_home = os.path.join(os.environ.get('HOME'), '.local', 'share')
|
|
|
|
return os.path.join(data_home, 'buku')
|
|
|
|
|
|
|
|
def moveOldDatabase():
|
|
"""Move database file from earlier path used in versions <= 1.8
|
|
to new path. Errors out if both the old and new DB files exist.
|
|
"""
|
|
|
|
olddbpath = os.path.join(os.environ.get('HOME'), '.cache', 'buku')
|
|
olddbfile = os.path.join(olddbpath, 'bookmarks.db')
|
|
|
|
if not os.path.exists(olddbfile):
|
|
return
|
|
|
|
newdbpath = getDataPath()
|
|
newdbfile = os.path.join(newdbpath, 'bookmarks.db')
|
|
|
|
if os.path.exists(newdbfile):
|
|
print("Both old (%s) and new (%s) databases exist, need manual action" % (olddbfile, newdbfile))
|
|
sys.exit(1)
|
|
|
|
if not os.path.exists(newdbpath):
|
|
os.makedirs(newdbpath)
|
|
|
|
shutil.move(olddbfile, newdbfile)
|
|
print("Database was moved from old (%s) to new (%s) location.\n" % (olddbfile, newdbfile))
|
|
|
|
os.rmdir(olddbpath)
|
|
|
|
|
|
|
|
def initdb():
|
|
"""Initialize the database connection. Create DB file and/or bookmarks table
|
|
if they don't exist. Alert on encryption options on first execution.
|
|
|
|
Returns: connection, cursor
|
|
"""
|
|
|
|
dbpath = getDataPath()
|
|
if not os.path.exists(dbpath):
|
|
os.makedirs(dbpath)
|
|
|
|
dbfile = os.path.join(dbpath, 'bookmarks.db')
|
|
|
|
encpath = os.path.join(dbpath, 'bookmarks.db.enc')
|
|
# Notify if DB file needs to be decrypted first
|
|
if os.path.exists(encpath) and not os.path.exists(dbfile):
|
|
print("Unlock database first")
|
|
sys.exit(1)
|
|
|
|
# Show info on first creation
|
|
if no_crypto == False and not os.path.exists(dbfile):
|
|
print("DB file is being created. You may want to encrypt it later.")
|
|
|
|
# Create a connection
|
|
conn = sqlite3.connect(dbfile)
|
|
cur = conn.cursor()
|
|
|
|
# Create table if it doesn't exist
|
|
cur.execute('''CREATE TABLE if not exists bookmarks \
|
|
(id integer PRIMARY KEY, URL text NOT NULL UNIQUE, metadata text, tags text)''')
|
|
conn.commit()
|
|
return (conn, cur)
|
|
|
|
|
|
|
|
def getPageResp(url, fullurl=False):
|
|
"""Connect to a server and fetch the requested page data.
|
|
Supports gzip compression.
|
|
|
|
Params: URL to fetch, redirection status
|
|
Returns: connection, HTTP(S) GET response
|
|
"""
|
|
|
|
if url.find("%20") != -1:
|
|
url = unquote(url)
|
|
url = url.replace(" ", "%20")
|
|
else:
|
|
url = unquote(url)
|
|
|
|
if debug:
|
|
print("unquoted: %s" % url)
|
|
|
|
if url.find("https://") >= 0: # Secure connection
|
|
server = url[8:]
|
|
marker = server.find("/")
|
|
if marker > 0:
|
|
if fullurl == False:
|
|
url = server[marker:]
|
|
server = server[:marker]
|
|
urlconn = HTTPSConnection(server, timeout=30)
|
|
elif url.find("http://") >= 0: # Insecure connection
|
|
server = url[7:]
|
|
marker = server.find("/")
|
|
if marker > 0:
|
|
if fullurl == False:
|
|
url = server[marker:]
|
|
server = server[:marker]
|
|
urlconn = HTTPConnection(server, timeout=30)
|
|
else:
|
|
printmsg("Not a valid HTTP(S) url", "WARNING")
|
|
return (None, None)
|
|
|
|
if debug:
|
|
print("server [%s] url [%s]" % (server, url))
|
|
|
|
# Handle URLs passed with %xx escape
|
|
try:
|
|
url.encode('ascii')
|
|
except:
|
|
url = quote(url)
|
|
|
|
urlconn.request("GET", url, None, {
|
|
"Accept-encoding": "gzip",
|
|
})
|
|
return (urlconn, urlconn.getresponse())
|
|
|
|
|
|
|
|
def getTitleData(resp):
|
|
"""Invoke HTML parser and extract title from HTTP response
|
|
|
|
Params: GET response
|
|
"""
|
|
|
|
data = None
|
|
charset = resp.headers.get_content_charset()
|
|
|
|
if resp.headers.get('Content-Encoding') == 'gzip':
|
|
if debug:
|
|
print("gzip response")
|
|
data = gzip.GzipFile(fileobj=io.BytesIO(resp.read())).read()
|
|
else:
|
|
data = resp.read()
|
|
|
|
if charset == None:
|
|
charset = 'utf-8'
|
|
if debug:
|
|
printmsg("Charset missing in response", "WARNING")
|
|
|
|
if debug:
|
|
print("charset: %s" % charset)
|
|
|
|
parser = BMHTMLParser()
|
|
try:
|
|
if charset == 'utf-8':
|
|
parser.feed(data.decode(charset, "replace"))
|
|
else:
|
|
parser.feed(data.decode(charset))
|
|
except Exception as e:
|
|
if debug and str(e) != "we should not get here!":
|
|
# Suppress Exception due to intentional self.reset() in HTMLParser
|
|
print("\x1b[1mEXCEPTION\x1b[21m [getTitleData]: (%s) %s" % (type(e).__name__, e))
|
|
|
|
|
|
|
|
def fetchTitle(url):
|
|
"""Handle server connection and redirections
|
|
|
|
Params: URL to fetch
|
|
Returns: page title or empty string, if not found
|
|
"""
|
|
|
|
global titleData
|
|
titleData = None
|
|
urlconn = None
|
|
retry = False
|
|
|
|
try:
|
|
urlconn, resp = getPageResp(url, False)
|
|
|
|
while 1:
|
|
if resp is None:
|
|
break
|
|
elif resp.status == 200:
|
|
getTitleData(resp)
|
|
break
|
|
elif resp.status in [301, 302]:
|
|
redirurl = urljoin(url, resp.getheader('location', ''))
|
|
printmsg(redirurl, "REDIRECTION")
|
|
|
|
if redirurl.find("sorry/IndexRedirect?") >= 0: # gracefully handle Google blocks
|
|
printmsg("Connection blocked due to unusual activity", "ERROR")
|
|
break
|
|
|
|
marker = redirurl.find("redirectUrl=")
|
|
if marker != -1:
|
|
redirurl = redirurl[marker + 12:]
|
|
|
|
# break same URL redirection loop
|
|
if url == redirurl:
|
|
printmsg("Detected repeated redirection to same URL", "ERROR")
|
|
break
|
|
|
|
url = redirurl
|
|
urlconn.close()
|
|
# Try with complete URL on redirection
|
|
urlconn, resp = getPageResp(url, True)
|
|
elif resp.status == 500 and retry == False:
|
|
"""Retry on status 500 (Internal Server Error) with truncated
|
|
URL. Some servers support truncated request URL on redirection.
|
|
"""
|
|
urlconn.close()
|
|
if debug:
|
|
print("Received status 500: retrying.")
|
|
urlconn, resp = getPageResp(url, False)
|
|
retry = True
|
|
else:
|
|
printmsg(("[" + str(resp.status) + "] " + resp.reason), "ERROR")
|
|
break
|
|
except Exception as e:
|
|
print("\x1b[1mEXCEPTION\x1b[21m [fetchTitle]: (%s) %s" % (type(e).__name__, e))
|
|
finally:
|
|
if urlconn is not None:
|
|
urlconn.close()
|
|
if titleData is None:
|
|
return ''
|
|
return titleData.strip().replace("\n","")
|
|
|
|
|
|
|
|
def isBookmarkAdded(cur, url):
|
|
"""Check if URL already exists in DB
|
|
|
|
Params: cursor, URL to search
|
|
Returns: DB index if URL found, else -1
|
|
"""
|
|
|
|
cur.execute("SELECT id FROM bookmarks WHERE URL = ?", (url,))
|
|
resultset = cur.fetchall()
|
|
if len(resultset) == 0:
|
|
return -1
|
|
|
|
return resultset[0][0]
|
|
|
|
|
|
|
|
def AddUpdateEntry(conn, cur, keywords, index):
|
|
"""Add a new bookmark or update an existing record at index
|
|
or insert a new record at addindex (if empty)
|
|
|
|
Params: connection, cursor, index to update
|
|
"""
|
|
|
|
global titleManual
|
|
global online
|
|
tags = ','
|
|
meta = ''
|
|
url = keywords[0]
|
|
|
|
"""In case of an add or insert operation ensure
|
|
that the URL does not exist in DB already
|
|
"""
|
|
if index is None:
|
|
id = isBookmarkAdded(cur, url)
|
|
if id != -1:
|
|
print("URL already exists at index %d" % id)
|
|
return
|
|
|
|
# Cleanse and get the tags
|
|
if len(keywords) > 1:
|
|
for tag in keywords[1:]:
|
|
if tag[-1] == ',':
|
|
tag = tag.strip(',') + ',' # a last token of a tag ends with a comma
|
|
else:
|
|
tag = tag.strip(',') # a token in a multi-word tag
|
|
|
|
if tag == ',':
|
|
continue
|
|
|
|
if tags[-1] == ',':
|
|
tags += tag
|
|
else:
|
|
tags += ' ' + tag
|
|
|
|
if tags[-1] != ',':
|
|
tags += ','
|
|
|
|
if titleManual != None:
|
|
meta = titleManual
|
|
elif online == True:
|
|
meta = fetchTitle(url)
|
|
if meta == '':
|
|
print("\x1B[91mTitle: []\x1B[0m")
|
|
else:
|
|
print("Title: [%s]" % meta)
|
|
|
|
if index == None: # Insert a new entry
|
|
try:
|
|
if addindex == None: # addindex is index number to insert record at
|
|
cur.execute('INSERT INTO bookmarks(URL, metadata, tags) VALUES (?, ?, ?)', (url, meta, tags,))
|
|
else:
|
|
cur.execute('INSERT INTO bookmarks(id, URL, metadata, tags) VALUES (?, ?, ?, ?)', (int(addindex), url, meta, tags,))
|
|
conn.commit()
|
|
print("Added at index %d\n" % cur.lastrowid)
|
|
printdb(cur, str(cur.lastrowid))
|
|
except sqlite3.IntegrityError:
|
|
for row in cur.execute("SELECT id from bookmarks where URL LIKE ?", (url,)):
|
|
print("URL already exists at index %s" % row[0])
|
|
return
|
|
|
|
print("Index %s exists" % addindex)
|
|
else: # Update an existing entry
|
|
try:
|
|
cur.execute("UPDATE bookmarks SET URL = ?, metadata = ?, tags = ? WHERE id = ?", (url, meta, tags, int(index),))
|
|
conn.commit()
|
|
if cur.rowcount == 1:
|
|
print("Updated index %d\n" % int(index))
|
|
printdb(cur, int(index))
|
|
else:
|
|
print("No matching index")
|
|
except sqlite3.IntegrityError:
|
|
print("URL already exists")
|
|
|
|
|
|
|
|
def dbRefresh(conn, cur):
|
|
"""Refresh ALL records in the database. Fetch title for each
|
|
bookmark from the web and update the records. Doesn't udpate
|
|
the record if title is empty.
|
|
This API doesn't change DB index, URL or tags of a bookmark.
|
|
|
|
Params: connection, cursor
|
|
"""
|
|
|
|
cur.execute("SELECT id, url FROM bookmarks ORDER BY id ASC")
|
|
resultset = cur.fetchall()
|
|
for row in resultset:
|
|
title = fetchTitle(row[1])
|
|
if title == '':
|
|
print("\x1B[91mTitle: []")
|
|
print("\x1b[1mNOT updating index %d\x1b[21m\x1B[0m\n" % row[0])
|
|
continue
|
|
else:
|
|
print("Title: [%s]" % title)
|
|
|
|
cur.execute("UPDATE bookmarks SET metadata = ? WHERE id = ?", (title, row[0],))
|
|
conn.commit()
|
|
print("Updated index %d\n" % row[0])
|
|
|
|
|
|
|
|
def searchdb(cur, keywords):
|
|
"""Search the database for an entries with tags or URL
|
|
or title info matching keywords and list those.
|
|
|
|
Params: cursor, keywords to search
|
|
"""
|
|
|
|
global jsonOutput
|
|
|
|
searchtag = ''
|
|
for token in keywords:
|
|
searchtag += token + " "
|
|
searchtag = searchtag[0:-1]
|
|
|
|
arguments = []
|
|
arguments.append(searchtag)
|
|
placeholder = "'%' || ? || '%'"
|
|
query = "SELECT id, url, metadata, tags FROM bookmarks WHERE tags LIKE (%s)" % placeholder
|
|
if searchAll == True: # Match all keywords in URL or Title
|
|
query += " OR ("
|
|
for token in keywords:
|
|
query += "URL LIKE (%s) AND " % (placeholder)
|
|
arguments.append(token)
|
|
|
|
query = query[:-5] + ") OR ("
|
|
|
|
for token in keywords:
|
|
query += "metadata LIKE (%s) AND " % (placeholder)
|
|
arguments.append(token)
|
|
|
|
query = query[:-5] + ")"
|
|
|
|
else: # Match any keyword in URL or Title
|
|
for token in keywords:
|
|
query += " OR URL LIKE (%s) OR metadata LIKE (%s)" % (placeholder, placeholder)
|
|
arguments.append(token)
|
|
arguments.append(token)
|
|
|
|
if debug:
|
|
print("\"%s\", (%s)" % (query, arguments))
|
|
|
|
count = 0
|
|
results = []
|
|
resultset = cur.execute(query, arguments)
|
|
|
|
if jsonOutput == False:
|
|
for row in resultset:
|
|
results.append(row[1])
|
|
count += 1
|
|
print("\x1B[1m\x1B[93m%d. \x1B[0m\x1B[92m%s\x1B[0m (%d)\n\t%s\n\t\x1B[91m[TAGS]\x1B[0m %s" % (count, row[1], row[0], row[2], row[3][1:-1]))
|
|
|
|
if count == 0:
|
|
return
|
|
|
|
print("")
|
|
|
|
while True:
|
|
try:
|
|
nav = input("Result number to open: ")
|
|
except EOFError:
|
|
return
|
|
|
|
if is_int(nav):
|
|
index = int(nav) - 1
|
|
if index < 0 or index >= count:
|
|
print("Index out of bound")
|
|
continue
|
|
|
|
try:
|
|
openurl = unquote(results[int(nav) - 1])
|
|
browser_open(openurl)
|
|
except Exception as e:
|
|
print("\x1b[1mEXCEPTION\x1b[21m [searchdb]: (%s) %s" % (type(e).__name__, e))
|
|
else:
|
|
break
|
|
|
|
else:
|
|
results = cur.fetchall();
|
|
if len(results) > 0:
|
|
print(formatJson(results))
|
|
else:
|
|
return
|
|
|
|
|
|
|
|
def compactDB(conn, cur, index):
|
|
"""When an entry at index is deleted, move the last
|
|
entry in DB to index, if index is lesser.
|
|
|
|
Params: connection, cursor, index of deleted entry
|
|
"""
|
|
|
|
cur.execute('SELECT MAX(id) from bookmarks')
|
|
results = cur.fetchall()
|
|
for row in results:
|
|
if row[0] > index:
|
|
cur.execute('SELECT id, URL, metadata, tags FROM bookmarks WHERE id = ?', (row[0],))
|
|
results = cur.fetchall()
|
|
for row in results:
|
|
cur.execute('DELETE FROM bookmarks WHERE id = ?', (row[0],))
|
|
conn.commit()
|
|
cur.execute('INSERT INTO bookmarks(id, URL, metadata, tags) VALUES (?, ?, ?, ?)', (index, row[1], row[2], row[3],))
|
|
conn.commit()
|
|
print("Index %d moved to %d" % (row[0], index))
|
|
|
|
|
|
|
|
def cleardb(conn, cur, index):
|
|
"""Delete a single record or remove the table if index is None
|
|
|
|
Params: connection, cursor, index to delete
|
|
"""
|
|
|
|
if index == None: # Remove the table
|
|
resp = input("ALL bookmarks will be removed. Enter \x1b[1my\x1b[21m to confirm: ")
|
|
if resp != 'y':
|
|
print("No bookmarks deleted")
|
|
return
|
|
|
|
cur.execute('DROP TABLE if exists bookmarks')
|
|
conn.commit()
|
|
print("All bookmarks deleted")
|
|
else: # Remove a single entry
|
|
try:
|
|
cur.execute('DELETE FROM bookmarks WHERE id = ?', (int(index),))
|
|
conn.commit()
|
|
if cur.rowcount == 1:
|
|
print("Removed index %d" % int(index))
|
|
compactDB(conn, cur, int(index))
|
|
else:
|
|
print("No matching index")
|
|
except IndexError:
|
|
print("Index out of bound")
|
|
|
|
|
|
|
|
def printdb(cur, index, empty=False):
|
|
"""Print bookmark details at index or all bookmarks if index is None
|
|
Print only bookmarks with empty title or tags if empty is True
|
|
|
|
Params: cursor, index to print, flag to show only bookmarks with no title or tags
|
|
"""
|
|
|
|
global showOpt
|
|
global jsonOutput
|
|
|
|
resultset = None
|
|
if index == None: # Show all entries
|
|
if empty == False:
|
|
cur.execute('SELECT * FROM bookmarks')
|
|
resultset = cur.fetchall()
|
|
else:
|
|
cur.execute("SELECT * FROM bookmarks WHERE metadata = '' OR tags = ','")
|
|
resultset = cur.fetchall()
|
|
print("\x1b[1m%d records found\x1b[21m\n" % len(resultset))
|
|
|
|
if jsonOutput == False:
|
|
for row in resultset:
|
|
if showOpt == 1:
|
|
print("%s %s" % (row[0], row[1]))
|
|
elif showOpt == 2:
|
|
print("%s %s %s" % (row[0], row[1], row[3][1:-1]))
|
|
else:
|
|
print("\x1B[1m\x1B[93m%s. \x1B[0m\x1B[92m%s\x1B[0m\n\t%s\n\t\x1B[91m[TAGS]\x1B[0m %s" % (row[0], row[1], row[2], row[3][1:-1]))
|
|
else:
|
|
print(formatJson(resultset))
|
|
else: # Show record at index
|
|
try:
|
|
resultset = cur.execute("SELECT * FROM bookmarks WHERE id = ?", (int(index),))
|
|
except IndexError:
|
|
print("Index out of bound")
|
|
return
|
|
|
|
if jsonOutput == False:
|
|
for row in resultset:
|
|
print("\x1B[1m\x1B[93m%s. \x1B[0m\x1B[92m%s\x1B[0m\n\t%s\n\t\x1B[91m[TAGS]\x1B[0m %s" % (row[0], row[1], row[2], row[3][1:-1]))
|
|
return
|
|
print("No matching index")
|
|
else:
|
|
print(formatJson(resultset, True))
|
|
|
|
|
|
|
|
def formatJson(resultset, single=False):
|
|
global showOpt
|
|
|
|
if single == False:
|
|
marks = []
|
|
for row in resultset:
|
|
if showOpt == 1:
|
|
record = { 'url': row[1] }
|
|
elif showOpt == 2:
|
|
record = { 'url': row[1], 'tags': row[3][1:-1] }
|
|
else:
|
|
record = { 'url': row[1], 'title': row[2], 'tags': row[3][1:-1]}
|
|
|
|
marks.append(record)
|
|
else:
|
|
marks = {}
|
|
for row in resultset:
|
|
if showOpt == 1:
|
|
marks['url'] = row[1]
|
|
elif showOpt == 2:
|
|
marks['title'] = row[2]
|
|
marks['tags'] = row[3][1:-1]
|
|
else:
|
|
marks['url'] = row[1]
|
|
marks['title'] = row[2]
|
|
marks['tags'] = row[3][1:-1]
|
|
|
|
|
|
return json.dumps(marks, sort_keys=True, indent=4)
|
|
|
|
def showUniqueTags(cur):
|
|
"""Print all unique tags ordered alphabetically
|
|
|
|
Params: cursor
|
|
"""
|
|
|
|
count = 1
|
|
Tags = []
|
|
uniqueTags = []
|
|
for row in cur.execute('SELECT DISTINCT tags FROM bookmarks'):
|
|
if row[0] == ',':
|
|
continue
|
|
|
|
Tags.extend(row[0].strip(',').split(','))
|
|
|
|
for tag in Tags:
|
|
if tag not in uniqueTags:
|
|
uniqueTags.append(tag)
|
|
|
|
Tags = sorted(uniqueTags, key=str.lower)
|
|
for tag in Tags:
|
|
print("%6d. %s" % (count, tag))
|
|
count += 1
|
|
|
|
|
|
|
|
def replaceTags(conn, cur, orig, new):
|
|
"""Replace orig tags with new tags in DB for all records.
|
|
Remove orig tag is new tag is empty.
|
|
|
|
Params: connection, cursor, original and new tags
|
|
"""
|
|
|
|
update = False
|
|
delete = False
|
|
|
|
orig = ',' + orig + ','
|
|
new = new.strip(',')
|
|
if new == '':
|
|
new = ','
|
|
delete = True
|
|
else:
|
|
new = ',' + new + ','
|
|
|
|
if orig == new:
|
|
print("Tags are same.")
|
|
return
|
|
|
|
cur.execute("SELECT id, tags FROM bookmarks WHERE tags LIKE ?", ('%' + orig + '%',))
|
|
results = cur.fetchall()
|
|
|
|
for row in results:
|
|
if delete == False:
|
|
if row[1].find(new) >= 0:
|
|
new = ','
|
|
|
|
newtags = row[1].replace(orig, new)
|
|
cur.execute("UPDATE bookmarks SET tags = ? WHERE id = ?", (newtags, row[0],))
|
|
print("Updated index %d" % row[0])
|
|
update = True
|
|
|
|
if update:
|
|
conn.commit()
|
|
|
|
|
|
|
|
def fetchopen(index):
|
|
"""Fetch URL at index and open in browser
|
|
|
|
Params: index
|
|
"""
|
|
|
|
try:
|
|
for row in cur.execute("SELECT URL FROM bookmarks WHERE id = ?", (int(index),)):
|
|
url = unquote(row[0])
|
|
browser_open(url)
|
|
return
|
|
print("No matching index")
|
|
except IndexError:
|
|
print("Index out of bound")
|
|
|
|
|
|
|
|
def is_int(string):
|
|
"""Check if a string is a digit
|
|
|
|
Params: string
|
|
"""
|
|
|
|
try:
|
|
int(string)
|
|
return True
|
|
except:
|
|
return False
|
|
|
|
|
|
|
|
def browser_open(url):
|
|
"""Duplicate stdin, stdout (to suppress showing errors
|
|
on the terminal) and open URL in default browser
|
|
|
|
Params: url to open
|
|
"""
|
|
|
|
url = url.replace("%22", "\"")
|
|
|
|
_stderr = os.dup(2)
|
|
os.close(2)
|
|
_stdout = os.dup(1)
|
|
os.close(1)
|
|
fd = os.open(os.devnull, os.O_RDWR)
|
|
os.dup2(fd, 2)
|
|
os.dup2(fd, 1)
|
|
try:
|
|
webbrowser.open(url)
|
|
except Exception as e:
|
|
print("\x1b[1mEXCEPTION\x1b[21m [browser_open]: (%s) %s" % (type(e).__name__, e))
|
|
finally:
|
|
os.close(fd)
|
|
os.dup2(_stderr, 2)
|
|
os.dup2(_stdout, 1)
|
|
|
|
|
|
|
|
def get_filehash(filepath):
|
|
"""Get the SHA256 hash of a file
|
|
|
|
Params: path to the file
|
|
"""
|
|
|
|
with open(filepath, 'rb') as f:
|
|
hasher = hashlib.sha256()
|
|
buf = f.read(BLOCKSIZE)
|
|
while len(buf) > 0:
|
|
hasher.update(buf)
|
|
buf = f.read(BLOCKSIZE)
|
|
|
|
return hasher.digest()
|
|
|
|
|
|
|
|
def encrypt_file():
|
|
"""Encrypt the bookmarks database file"""
|
|
|
|
dbpath = os.path.join(getDataPath(), 'bookmarks.db')
|
|
encpath = dbpath + '.enc'
|
|
if not os.path.exists(dbpath):
|
|
print("%s missing. Already encrypted?" % dbpath)
|
|
sys.exit(1)
|
|
|
|
# If both encrypted file and flat file exist, error out
|
|
if os.path.exists(dbpath) and os.path.exists(encpath):
|
|
printmsg("Both encrypted and flat DB files exist!", "ERROR")
|
|
sys.exit(1)
|
|
|
|
password = ''
|
|
password = getpass.getpass()
|
|
passconfirm = getpass.getpass()
|
|
if password == '':
|
|
print("Empty password");
|
|
sys.exit(1)
|
|
if password != passconfirm:
|
|
print("Passwords don't match");
|
|
sys.exit(1)
|
|
|
|
# Get SHA256 hash of DB file
|
|
dbhash = get_filehash(dbpath)
|
|
|
|
# Generate random 256-bit salt and key
|
|
salt = Random.get_random_bytes(SALT_SIZE)
|
|
key = (password + salt.decode('utf-8', "replace")).encode('utf-8')
|
|
for i in range(iterations):
|
|
key = hashlib.sha256(key).digest()
|
|
|
|
iv = Random.get_random_bytes(16)
|
|
cipher = AES.new(key, AES.MODE_CBC, iv)
|
|
filesize = os.path.getsize(dbpath)
|
|
|
|
with open(dbpath, 'rb') as infile:
|
|
with open(encpath, 'wb') as outfile:
|
|
outfile.write(struct.pack('<Q', filesize))
|
|
outfile.write(salt)
|
|
outfile.write(iv)
|
|
|
|
# Embed DB file hash in encrypted file
|
|
outfile.write(dbhash)
|
|
|
|
while True:
|
|
chunk = infile.read(CHUNKSIZE)
|
|
if len(chunk) == 0:
|
|
break
|
|
elif len(chunk) % 16 != 0:
|
|
chunk += ' ' * (16 - len(chunk) % 16)
|
|
|
|
outfile.write(cipher.encrypt(chunk))
|
|
|
|
os.remove(dbpath)
|
|
print("File encrypted")
|
|
sys.exit(0)
|
|
|
|
|
|
|
|
def decrypt_file():
|
|
"""Decrypt the bookmarks database file"""
|
|
|
|
dbpath = os.path.join(getDataPath(), 'bookmarks.db')
|
|
encpath = dbpath + '.enc'
|
|
if not os.path.exists(encpath):
|
|
printmsg((encpath + " missing"), "ERROR")
|
|
sys.exit(1)
|
|
|
|
# If both encrypted file and flat file exist, error out
|
|
if os.path.exists(dbpath) and os.path.exists(encpath):
|
|
printmsg("Both encrypted and flat DB files exist!", "ERROR")
|
|
sys.exit(1)
|
|
|
|
password = ''
|
|
password = getpass.getpass()
|
|
if password == '':
|
|
print("Decryption failed");
|
|
sys.exit(1)
|
|
|
|
with open(encpath, 'rb') as infile:
|
|
origsize = struct.unpack('<Q', infile.read(struct.calcsize('Q')))[0]
|
|
|
|
# Read 256-bit salt and generate key
|
|
salt = infile.read(32)
|
|
key = (password + salt.decode('utf-8', "replace")).encode('utf-8')
|
|
for i in range(iterations):
|
|
key = hashlib.sha256(key).digest()
|
|
|
|
iv = infile.read(16)
|
|
cipher = AES.new(key, AES.MODE_CBC, iv)
|
|
|
|
# Get original DB file's SHA256 hash from encrypted file
|
|
enchash = infile.read(32)
|
|
|
|
with open(dbpath, 'wb') as outfile:
|
|
while True:
|
|
chunk = infile.read(CHUNKSIZE)
|
|
if len(chunk) == 0:
|
|
break;
|
|
|
|
outfile.write(cipher.decrypt(chunk))
|
|
|
|
outfile.truncate(origsize)
|
|
|
|
# Match hash of generated file with that of original DB file
|
|
dbhash = get_filehash(dbpath)
|
|
if dbhash != enchash:
|
|
os.remove(dbpath)
|
|
print("Decryption failed");
|
|
else:
|
|
os.remove(encpath)
|
|
print("File decrypted")
|
|
|
|
sys.exit(0)
|
|
|
|
|
|
|
|
def sigint_handler(signum, frame):
|
|
"""Custom SIGINT handler"""
|
|
|
|
print('\nInterrupted.', file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
signal.signal(signal.SIGINT, sigint_handler)
|
|
|
|
|
|
|
|
def printmsg(msg, level=None):
|
|
"""Print a message in 2 parts, with the level in bold
|
|
|
|
Params: msg, level
|
|
"""
|
|
|
|
if level is not None:
|
|
print("\x1b[1m%s:\x1b[21m %s" % (level, msg))
|
|
else:
|
|
print("%s" % msg)
|
|
|
|
|
|
|
|
def usage():
|
|
"""Show buku usage, options, general information and exit"""
|
|
|
|
print("Usage: OPTIONS [URL] [TAGS] [KEYWORDS ...]\n\n"
|
|
|
|
"A private cmdline bookmark manager. Your mini web!\n\n"
|
|
|
|
"General options\n"
|
|
" -a URL [tags] add URL as bookmark with comma separated tags\n"
|
|
" -d N delete entry at DB index N (from -P), move last entry to N\n"
|
|
" -g list all tags alphabetically\n"
|
|
" -m title manually specify the title, for -a, -i, -u\n"
|
|
" -s keyword(s) search all bookmarks for a (partial) tag or any keyword\n"
|
|
" -S keyword(s) search all bookmarks for a (partial) tag or all keywords\n"
|
|
" -u N URL [tags] update all fields of entry at DB index N\n"
|
|
" -w fetch title from web, for -a, -i, -u\n\n"
|
|
|
|
"Power toys\n"
|
|
" -D delete ALL bookmarks\n"
|
|
" -e show bookmarks with empty titles or no tagsi\n"
|
|
" -i N insert new bookmark at free DB index N\n"
|
|
" -j show results in Json format\n"
|
|
" -k decrypt (unlock) database file\n"
|
|
" -l encrypt (lock) database file\n"
|
|
" -o N open URL at DB index N in browser\n"
|
|
" -p N show details of bookmark record at DB index N\n"
|
|
" -P show all bookmarks along with index from DB\n"
|
|
" -r oldtag [newtag] replace oldtag with newtag, delete oldtag if newtag empty\n"
|
|
" -R refresh title from web for all bookmarks, update if non-empty\n"
|
|
" -t N use N (> 0) hash iterations to generate key, for -k, -l\n"
|
|
" -x N modify -P behaviour, N=1: show only URL, N=2: show URL and tag\n"
|
|
" -z show debug information\n\n"
|
|
|
|
"Keys\n"
|
|
" 1-N open Nth search result in browser\n"
|
|
"Enter exit buku\n\n"
|
|
|
|
"Version %.1f\n"
|
|
"Copyright (C) 2015 Arun Prakash Jana <engineerarun@gmail.com>\n"
|
|
"License: GPLv3\n"
|
|
"Webpage: https://github.com/jarun/buku\n" % _VERSION_)
|
|
sys.exit(1)
|
|
|
|
|
|
|
|
"""main starts here"""
|
|
def main(argv = sys.argv):
|
|
# detects whether have pipe line parsing in
|
|
if not sys.stdin.isatty():
|
|
pipeargs.extend(sys.argv)
|
|
for s in sys.stdin.readlines():
|
|
pipeargs.extend(s.split())
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main(sys.argv)
|
|
except KeyboardInterrupt:
|
|
pass
|
|
|
|
optlist = None
|
|
keywords = None
|
|
|
|
if len(sys.argv) < 2:
|
|
usage()
|
|
|
|
# Check cmdline options
|
|
try:
|
|
|
|
if len(pipeargs) > 0:
|
|
optlist, keywords = getopt(pipeargs[1:], "d:i:m:o:p:t:u:x:aDegjklPRrsSwz")
|
|
else:
|
|
optlist, keywords = getopt(sys.argv[1:], "d:i:m:o:p:t:u:x:aDegjklPRrsSwz")
|
|
if len(optlist) < 1:
|
|
usage()
|
|
|
|
for opt in optlist:
|
|
if opt[0] == "-a":
|
|
if update == True or delete == True:
|
|
print("You can either add or update or delete in one instance\n")
|
|
usage()
|
|
|
|
addurl = True
|
|
elif opt[0] == "-d":
|
|
if addurl == True or update == True:
|
|
print("You can either add or update or delete in one instance\n")
|
|
usage()
|
|
|
|
if not opt[1].isdigit():
|
|
usage()
|
|
|
|
entry = opt[1]
|
|
if int(entry) <= 0:
|
|
usage()
|
|
|
|
delete = True
|
|
elif opt[0] == "-D":
|
|
if addurl == True or update == True:
|
|
print("You can either add or update or delete in one instance\n")
|
|
usage()
|
|
|
|
delete = True
|
|
elif opt[0] == "-e":
|
|
empty = True
|
|
elif opt[0] == "-g":
|
|
showTags = True
|
|
elif opt[0] == "-i":
|
|
if update == True or delete == True:
|
|
print("You can either add or update or delete in one instance\n")
|
|
usage()
|
|
|
|
if not opt[1].isdigit():
|
|
usage()
|
|
|
|
addindex = opt[1]
|
|
if int(addindex) <= 0:
|
|
usage()
|
|
|
|
addurl = True
|
|
elif opt[0] == "-j":
|
|
jsonOutput = True
|
|
elif opt[0] == "-k":
|
|
if no_crypto == True:
|
|
printmsg("PyCrypto missing", "ERROR")
|
|
sys.exit(0)
|
|
|
|
decrypt = True
|
|
elif opt[0] == "-l":
|
|
if no_crypto == True:
|
|
printmsg("PyCrypto missing", "ERROR")
|
|
sys.exit(0)
|
|
|
|
encrypt = True
|
|
elif opt[0] == "-m":
|
|
titleManual = opt[1]
|
|
elif opt[0] == "-o":
|
|
if not opt[1].isdigit():
|
|
usage()
|
|
|
|
openurl = opt[1]
|
|
if int(openurl) <= 0:
|
|
usage()
|
|
elif opt[0] == "-p":
|
|
if not opt[1].isdigit():
|
|
usage()
|
|
|
|
showindex = opt[1]
|
|
if int(showindex) <= 0:
|
|
usage()
|
|
|
|
show = True
|
|
elif opt[0] == "-P":
|
|
show = True
|
|
elif opt[0] == "-R":
|
|
if addurl == True or delete == True:
|
|
print("You can either add or update or delete in one instance\n")
|
|
usage()
|
|
|
|
refresh = True
|
|
elif opt[0] == "-r":
|
|
replace = True
|
|
elif opt[0] == "-s":
|
|
search = True
|
|
elif opt[0] == "-S":
|
|
searchAll = True
|
|
search = True
|
|
elif opt[0] == "-t":
|
|
if not opt[1].isdigit():
|
|
usage()
|
|
|
|
iterations = int(opt[1])
|
|
if iterations <= 0:
|
|
usage()
|
|
elif opt[0] == "-u":
|
|
if addurl == True or delete == True:
|
|
print("You can either add or update or delete in one instance\n")
|
|
usage()
|
|
|
|
if not opt[1].isdigit():
|
|
usage()
|
|
|
|
entry = opt[1]
|
|
if int(entry) <= 0:
|
|
usage()
|
|
|
|
update = True
|
|
elif opt[0] == "-w":
|
|
online = True
|
|
elif opt[0] == "-x":
|
|
if not opt[1].isdigit():
|
|
usage()
|
|
|
|
showOpt = int(opt[1])
|
|
if showOpt < 1 or showOpt > 2:
|
|
usage()
|
|
elif opt[0] == "-z":
|
|
debug = True
|
|
except GetoptError as e:
|
|
print("buku:", e)
|
|
sys.exit(1)
|
|
|
|
if debug:
|
|
print("Version %.1f" % _VERSION_)
|
|
|
|
# Update and full DB refresh are mutually exclusive
|
|
if update == True and refresh == True:
|
|
print("You can either update a single index or refresh full DB at once.\n")
|
|
usage()
|
|
|
|
# Online title update conflicts with manual title option
|
|
if online == True and titleManual != None:
|
|
print("You can either fetch title from web or add/update title manually.\n")
|
|
usage()
|
|
|
|
# Move database to new location, if needed
|
|
moveOldDatabase()
|
|
|
|
# Handle encrypt/decrypt options at top priority
|
|
if encrypt == True:
|
|
encrypt_file()
|
|
|
|
if decrypt == True:
|
|
decrypt_file()
|
|
|
|
# Initilize the database and get handles
|
|
conn, cur = initdb()
|
|
|
|
# Replace a tag in DB
|
|
if replace == True:
|
|
numargs = len(keywords)
|
|
|
|
if addurl == True or update == True or delete == True:
|
|
print("Tag replace doesn't work with add or update or delete.\n")
|
|
conn.close()
|
|
usage()
|
|
elif numargs < 1 or numargs > 2:
|
|
print("Tag replace accepts 1 or 2 arguments\n")
|
|
conn.close()
|
|
usage()
|
|
elif numargs == 1:
|
|
replaceTags(conn, cur, keywords[0], "")
|
|
else:
|
|
replaceTags(conn, cur, keywords[0], keywords[1])
|
|
|
|
# Call add or update record
|
|
if addurl == True or update == True:
|
|
if len(keywords) < 1:
|
|
conn.close()
|
|
usage()
|
|
|
|
AddUpdateEntry(conn, cur, keywords, entry)
|
|
|
|
# Refresh full DB
|
|
if refresh == True:
|
|
dbRefresh(conn, cur)
|
|
|
|
# Search tags, URLs, Title info
|
|
if search == True:
|
|
if len(keywords) < 1:
|
|
conn.close()
|
|
usage()
|
|
|
|
searchdb(cur, keywords)
|
|
|
|
# Print all records
|
|
if show == True:
|
|
printdb(cur, showindex)
|
|
|
|
# Show all unique tags
|
|
if showTags == True:
|
|
showUniqueTags(cur)
|
|
|
|
if empty == True:
|
|
printdb(cur, None, empty)
|
|
|
|
# Open URL in browser
|
|
if openurl != None:
|
|
fetchopen(openurl)
|
|
|
|
# Remove a single record of all records
|
|
if delete == True:
|
|
cleardb(conn, cur, entry)
|
|
|
|
# Close the connection before exiting
|
|
conn.close()
|