buku/buku

#!/usr/bin/env python3
#
# Bookmark management utility
#
# Copyright (C) 2015-2016 Arun Prakash Jana <engineerarun@gmail.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with buku.  If not, see <http://www.gnu.org/licenses/>.

import sys
import os
import sqlite3
import argparse
import readline
import webbrowser
import html.parser as HTMLParser
from http.client import HTTPConnection, HTTPSConnection
from urllib.parse import urljoin, quote, unquote
import gzip
import io
import signal

# Import libraries needed for encryption
try:
    import getpass
    import hashlib
    from Crypto.Cipher import AES
    from Crypto import Random
    import struct

    no_crypto = False
    BLOCKSIZE = 65536
    SALT_SIZE = 32
    CHUNKSIZE = 0x80000 # Read/write 512 KB chunks
except ImportError:
    no_crypto = True


# Globals
update = False          # Update a bookmark in DB
tagManual = None        # Tags for update command
titleManual = None      # Manually add a title offline
description = None      # Description of the bookmark
tagsearch = False       # Search bookmarks by tag
titleData = None        # Title fetched from a page
jsonOutput = False      # Output json formatted result
showOpt = 0             # Modify show. 1: show only URL, 2: show URL and tag
debug = False           # Enable debug logs
pipeargs = []           # Holds arguments piped to the program
noninteractive = False  # Show the prompt or not
DELIMITER = ','         # Delimiter used to store tags in DB
_VERSION_ = '2.1'       # Program version


class BMHTMLParser(HTMLParser.HTMLParser):
    """Class to parse and fetch the title from a HTML page, if available"""

    def __init__(self):
        HTMLParser.HTMLParser.__init__(self)
        self.inTitle = False
        self.data = ''
        self.lasttag = None

    def handle_starttag(self, tag, attrs):
        self.inTitle = False
        if tag == 'title':
            self.inTitle = True
            self.lasttag = tag

    def handle_endtag(self, tag):
        global titleData

        if tag == 'title':
            self.inTitle = False
            if self.data != '':
                titleData = self.data
                self.reset()  # We have received title data, exit parsing

    def handle_data(self, data):
        if self.lasttag == 'title' and self.inTitle == True:
            self.data += data

    def error(self, message):
        pass

class BukuDb:

    def __init__(self, *args, **kwargs):
        conn, cur = BukuDb.initdb()
        self.conn = conn
        self.cur = cur
        self.noninteractive = noninteractive

    @staticmethod
    def get_dbdir_path():
        """Determine the directory path where dbfile will be stored:
        if $XDG_DATA_HOME is defined, use it
        else if $HOME exists, use it
        else use the current directory
        """

        data_home = os.environ.get('XDG_DATA_HOME')
        if data_home is None:
            if os.environ.get('HOME') is None:
                data_home = '.'
            else:
                data_home = os.path.join(os.environ.get('HOME'), '.local', 'share')

        return os.path.join(data_home, 'buku')

    @staticmethod
    def move_legacy_dbfile():
        """Move database file from earlier path used in versions <= 1.8
        to new path. Errors out if both the old and new DB files exist.
        """

        olddbpath = os.path.join(os.environ.get('HOME'), '.cache', 'buku')
        olddbfile = os.path.join(olddbpath, 'bookmarks.db')

        if not os.path.exists(olddbfile):
            return

        newdbpath = BukuDb.get_dbdir_path()
        newdbfile = os.path.join(newdbpath, 'bookmarks.db')

        if os.path.exists(newdbfile):
            print('Both old (%s) and new (%s) databases exist, need manual action' % (olddbfile, newdbfile))
            sys.exit(1)

        if not os.path.exists(newdbpath):
            os.makedirs(newdbpath)

        os.rename(olddbfile, newdbfile)
        print('Database was moved from old (%s) to new (%s) location.\n' % (olddbfile, newdbfile))

        os.rmdir(olddbpath)


    @staticmethod
    def initdb():
        """Initialize the database connection. Create DB file and/or bookmarks table
        if they don't exist. Alert on encryption options on first execution.

        Returns: connection, cursor
        """

        dbpath = BukuDb.get_dbdir_path()
        if not os.path.exists(dbpath):
            os.makedirs(dbpath)

        dbfile = os.path.join(dbpath, 'bookmarks.db')

        encpath = os.path.join(dbpath, 'bookmarks.db.enc')
        # Notify if DB file needs to be decrypted first
        if os.path.exists(encpath) and not os.path.exists(dbfile):
            print('Unlock database first')
            sys.exit(1)

        # Show info on first creation
        if not os.path.exists(dbfile):
            print('DB file is being created. You may want to encrypt it later.')

        try:
            # Create a connection
            conn = sqlite3.connect(dbfile)
            cur = conn.cursor()

            # Create table if it doesn't exist
            cur.execute("CREATE TABLE if not exists bookmarks \
                       (id integer PRIMARY KEY, URL text NOT NULL UNIQUE, metadata text default \'\', tags text default \',\', desc text default \'\')")
            conn.commit()
        except Exception as e:
            print('\x1b[1mEXCEPTION\x1b[21m [initdb]: (%s) %s' % (type(e).__name__, e))
            sys.exit(1)

        # Add description column in existing DB (from version 2.1)
        try:
            cur.execute("ALTER TABLE bookmarks ADD COLUMN desc text default \'\'")
            conn.commit()
        except:
            pass

        return (conn, cur)


    def get_bookmark_index(self, url):
        """Check if URL already exists in DB

        Params: URL to search
        Returns: DB index if URL found, else -1
        """

        self.cur.execute('SELECT id FROM bookmarks WHERE URL = ?', (url,))
        resultset = self.cur.fetchall()
        if len(resultset) == 0:
            return -1

        return resultset[0][0]


    def add_bookmark(self, url, title_manual=None, tag_manual=None, desc=None):
        """Add a new bookmark

        :param url: url to bookmark
        :param tag_manual: string of comma-separated tags to add manually
        :param title_manual: string title to add manually
        :param desc: string description
        """

        # Ensure that the URL does not exist in DB already
        id = self.get_bookmark_index(url)
        if id != -1:
            print('URL [%s] already exists at index %d' % (url, id))
            return

        # Process title
        if title_manual is not None:
            meta = title_manual
        else:
            meta = network_handler(url)
            if meta == '':
                print('\x1B[91mTitle: []\x1B[0m\n')
            elif debug:
                print('Title: [%s]\n' % meta)

        # Process tags
        if tag_manual is None:
            tag_manual = DELIMITER
        else:
            if tag_manual[0] != DELIMITER:
                tag_manual = DELIMITER + tag_manual
            if tag_manual[-1] != DELIMITER:
                tag_manual = tag_manual + DELIMITER

        # Process description
        if desc is None:
            desc = ''

        try:
            self.cur.execute('INSERT INTO bookmarks(URL, metadata, tags, desc) VALUES (?, ?, ?, ?)', (url, meta, tag_manual, desc))
            self.conn.commit()
            self.print_bookmark(self.cur.lastrowid)
        except Exception as e:
            print('\x1b[1mEXCEPTION\x1b[21m [add_bookmark]: (%s) %s' % (type(e).__name__, e))


    def update_bookmark(self, index, url='', title_manual=None, tag_manual=None, desc=None):
        """ Update an existing record at index

        :param index: int position to update
        :param url: address
        :param tag_manual: string of comma-separated tags to add manually
        :param title_manual: string title to add manually
        :param desc: string description
        :return:
        """

        arguments = []
        query = 'UPDATE bookmarks SET'
        to_update = False

        # Update URL if passed as argument
        if url != '':
            query += ' URL = ?,'
            arguments.append(url)
            to_update = True

        # Update tags if passed as argument
        if tag_manual is not None:
            query += ' tags = ?,'
            arguments.append(tag_manual)
            to_update = True

        # Update description if passed as an argument
        if desc is not None:
            query += ' desc = ?,'
            arguments.append(desc)
            to_update = True

        # Update title
        #
        # 1. if -t has no arguments, delete existing title
        # 2. if -t has arguments, update existing title
        # 3. if -t option is omitted at cmdline:
        #    if URL is passed, update the title from web using the URL
        # 4. if no other argument (url, tag, comment) passed update title from web using DB URL
        meta = None
        if title_manual is not None:
            meta = title_manual
        elif url != '':
            meta = network_handler(url)
            if meta == '':
                print('\x1B[91mTitle: []\x1B[0m')
            elif debug:
                print('Title: [%s]' % meta)
        elif not to_update:
            self.refreshdb(index)
            self.print_bookmark(index)

        if meta is not None:
            query += ' metadata = ?,'
            arguments.append(meta)
            to_update = True

        if not to_update:       # Nothing to update
            return

        query = query[:-1] + ' WHERE id = ?'
        arguments.append(index)
        if debug:
            print('query: [%s], args: [%s]' % (query, arguments))

        try:
            self.cur.execute(query, arguments)
            self.conn.commit()
            if self.cur.rowcount == 1:
                self.print_bookmark(index)
            else:
                print('No matching index')
        except sqlite3.IntegrityError:
            print('URL already exists')


    def refreshdb(self, index, title_manual=None):
        """Refresh ALL records in the database. Fetch title for each
        bookmark from the web and update the records. Doesn't udpate
        the record if title is empty.
        This API doesn't change DB index, URL or tags of a bookmark.

        :param index: index of record to update, or 0 for all records
        :param title_manual: custom title
        """

        if index == 0:
            self.cur.execute('SELECT id, url FROM bookmarks ORDER BY id ASC')
        else:
            self.cur.execute('SELECT id, url FROM bookmarks WHERE id = ?', (index,))

        resultset = self.cur.fetchall()
        if title_manual is None:
            for row in resultset:
                title = network_handler(row[1])
                if title == '':
                    print('\x1b[1mIndex %d: empty title\x1b[21m\x1B[0m\n' % row[0])
                    continue
                else:
                    print('Title: [%s]' % title)

                self.cur.execute('UPDATE bookmarks SET metadata = ? WHERE id = ?', (title, row[0],))
                self.conn.commit()
                print('Index %d updated\n' % row[0])
        else:
            title = title_manual

            for row in resultset:
                self.cur.execute('UPDATE bookmarks SET metadata = ? WHERE id = ?', (title, row[0],))
                self.conn.commit()
                print('Index %d updated\n' % row[0])


    def searchdb(self, keywords, all_keywords=False, json=False):
        """Search the database for an entries with tags or URL
        or title info matching keywords and list those.

        :param keywords: keywords to search
        :param all_keywords: search any or all keywords
        :param json: json formatted output
        """

        arguments = []
        placeholder = "'%' || ? || '%'"
        query = "SELECT id, url, metadata, tags, desc FROM bookmarks WHERE"

        if all_keywords == True: # Match all keywords in URL or Title
            for token in keywords:
                query += " (tags LIKE (%s) OR URL LIKE (%s) OR metadata LIKE (%s) OR desc LIKE (%s)) AND" % (placeholder, placeholder, placeholder, placeholder)
                arguments.append(token)
                arguments.append(token)
                arguments.append(token)
                arguments.append(token)
            query = query[:-4]
        else: # Match any keyword in URL or Title
            for token in keywords:
                query += " tags LIKE (%s) OR URL LIKE (%s) OR metadata LIKE (%s) OR desc LIKE (%s) OR" % (placeholder, placeholder, placeholder, placeholder)
                arguments.append(token)
                arguments.append(token)
                arguments.append(token)
                arguments.append(token)
            query = query[:-3]

        if debug:
            print("\"%s\", (%s)" % (query, arguments))

        self.cur.execute(query, arguments)
        results = self.cur.fetchall()
        if len(results) == 0:
            return

        if json == False:
            prompt(results, self.noninteractive)
        else:
            print(format_json(results))


    def search_by_tag(self, tag, json=False):
        """Search and list bookmarks with a tag

        :param tag: tag to search
        :param json: print in json format
        """

        self.cur.execute("SELECT id, url, metadata, tags, desc FROM bookmarks WHERE tags LIKE '%' || ? || '%'", (tag,))
        results = self.cur.fetchall()
        if len(results) == 0:
            return

        if json == False:
            prompt(results, self.noninteractive)
        else:
            print(format_json(results))


    def compactdb(self, index):
        """When an entry at index is deleted, move the last
        entry in DB to index, if index is lesser.

        Params: index of deleted entry
        """

        self.cur.execute('SELECT MAX(id) from bookmarks')
        results = self.cur.fetchall()
        if len(results) == 1 and results[0][0] is None: # Return if the last index was just deleted
            return

        for row in results:
            if row[0] > index:
                self.cur.execute('SELECT id, URL, metadata, tags, desc FROM bookmarks WHERE id = ?', (row[0],))
                results = self.cur.fetchall()
                for row in results:
                    self.cur.execute('DELETE FROM bookmarks WHERE id = ?', (row[0],))
                    self.conn.commit()
                    self.cur.execute('INSERT INTO bookmarks(id, URL, metadata, tags, desc) VALUES (?, ?, ?, ?, ?)', (index, row[1], row[2], row[3], row[4],))
                    self.conn.commit()
                    print('Index %d moved to %d' % (row[0], index))


    def delete_bookmark(self, index):
        """Delete a single record or remove the table if index is None

        Params: index to delete
        """

        if index == 0:  # Remove the table
            resp = input('ALL bookmarks will be removed. Enter \x1b[1my\x1b[21m to confirm: ')
            if resp != 'y':
                print('No bookmarks deleted')
                return

            self.cur.execute('DROP TABLE if exists bookmarks')
            self.conn.commit()
            print('All bookmarks deleted')
        else: # Remove a single entry
            try:
                self.cur.execute('DELETE FROM bookmarks WHERE id = ?', (index,))
                self.conn.commit()
                if self.cur.rowcount == 1:
                    print('Removed index %d' % index)
                    self.compactdb(index)
                else:
                    print('No matching index')
            except IndexError:
                print('Index out of bound')


    def print_bookmark(self, index, empty=False):
        """Print bookmark details at index or all bookmarks if index is None
        Print only bookmarks with blank title or tag if empty is True
        Note: URL is printed on top because title may be blank

        Params: index to print, flag to show only bookmarks with no title or tags
        """

        global showOpt
        global jsonOutput

        resultset = None
        if index == 0: # Show all entries
            if empty == False:
                self.cur.execute('SELECT * FROM bookmarks')
                resultset = self.cur.fetchall()
            else:
                self.cur.execute("SELECT * FROM bookmarks WHERE metadata = '' OR tags = ?", (DELIMITER,))
                resultset = self.cur.fetchall()
                print('\x1b[1m%d records found\x1b[21m\n' % len(resultset))

            if jsonOutput == False:
                if showOpt == 0:
                    for row in resultset:
                        print_record(row)
                elif showOpt == 1:
                    for row in resultset:
                        print('%s %s' % (row[0], row[1]))
                elif showOpt == 2:
                    for row in resultset:
                        print('%s %s        %s' % (row[0], row[1], row[3][1:-1]))
            else:
                print(format_json(resultset))
        else:  # Show record at index
            try:
                self.cur.execute('SELECT * FROM bookmarks WHERE id = ?', (index,))
                results = self.cur.fetchall()
                if len(results) == 0:
                    print('No matching index')
                    return
            except IndexError:
                print('Index out of bound')
                return

            if jsonOutput == False:
                for row in results:
                    if showOpt == 0:
                        print_record(row)
                    elif showOpt == 1:
                        print('%s %s' % (row[0], row[1]))
                    elif showOpt == 2:
                        print('%s %s        %s' % (row[0], row[1], row[3][1:-1]))
            else:
                print(format_json(results, True))


    def list_tags(self):
        """Print all unique tags ordered alphabetically
        """

        count = 1
        Tags = []
        uniqueTags = []
        for row in self.cur.execute('SELECT DISTINCT tags FROM bookmarks'):
            if row[0] == DELIMITER:
                continue

            Tags.extend(row[0].strip(DELIMITER).split(DELIMITER))

        for tag in Tags:
            if tag not in uniqueTags:
                uniqueTags.append(tag)

        Tags = sorted(uniqueTags, key=str.lower)
        for tag in Tags:
            print('%6d. %s' % (count, tag))
            count += 1


    def replace_tag(self, orig, new=None):
        """Replace orig tags with new tags in DB for all records.
        Remove orig tag is new tag is empty.

        Params: original and new tags
        """

        update = False
        delete = False
        newtags = DELIMITER

        orig = DELIMITER + orig + DELIMITER
        if new is None:
            delete = True
        else:
            newtags = parse_tags(new)
            if newtags == DELIMITER:
                delete = True

        if orig == newtags:
            print('Tags are same.')
            return

        self.cur.execute("SELECT id, tags FROM bookmarks WHERE tags LIKE ?", ('%' + orig + '%',))
        results = self.cur.fetchall()

        for row in results:
            if delete == False:
                # Check if tag newtags is already added
                if row[1].find(newtags) >= 0:
                    newtags = DELIMITER

            tags = row[1].replace(orig, newtags)
            self.cur.execute('UPDATE bookmarks SET tags = ? WHERE id = ?', (tags, row[0],))
            print('Index %d updated' % row[0])
            update = True

        if update:
            self.conn.commit()


    def browse_by_index(self, index):
        """Open URL at index in browser

        Params: index
        """

        try:
            for row in self.cur.execute('SELECT URL FROM bookmarks WHERE id = ?', (index,)):
                url = unquote(row[0])
                browser_open(url)
                return
            print('No matching index')
        except IndexError:
                print('Index out of bound')


    def close_quit(self, exitval=0):
        """Close a DB connection and exit"""

        if self.conn is not None:
            try:
                self.cur.close()
                self.conn.close()
            except:  # we don't really care about errors, we're closing down anyway
                pass
        sys.exit(exitval)


    def import_bookmark(self, fp):
        """Import bookmarks from a html file.
        Supports Firefox, Google Chrome and IE imports

        Params: Path to file to import
        """

        if not os.path.exists(fp):
            printmsg((fp + ' not found'), 'ERROR')
            self.close_quit(1)

        try:
            import bs4
        except ImportError:
            printmsg('Beautiful Soup not found', 'ERROR')
            self.close_quit(1)

        with open(fp, encoding='utf-8') as f:
            soup = bs4.BeautifulSoup(f, 'html.parser')

        html_tags = soup.findAll('a')
        for tag in html_tags:
            # Extract comment from <dd> tag
            desc = None
            comment_tag = tag.findNextSibling('dd')
            if comment_tag:
                desc = comment_tag.text[0:comment_tag.text.find('\n')]

            self.add_bookmark(tag['href'],
                              tag.string,
                              (DELIMITER + tag['tags'] + DELIMITER) if tag.has_attr('tags') else None,
                              desc)


    def mergedb(self, fp):
        """Merge bookmarks from another Buku database file

        Params: Path to file to merge
        """

        if not os.path.exists(fp):
            printmsg((fp + ' not found'), 'ERROR')
            self.close_quit(1)

        try:
            # Create a connection
            connfp = sqlite3.connect(fp)
            curfp = connfp.cursor()
        except Exception as e:
            print('\x1b[1mEXCEPTION\x1b[21m [mergedb]: (%s) %s' % (type(e).__name__, e))
            sys.exit(1)

        curfp.execute('SELECT * FROM bookmarks')
        resultset = curfp.fetchall()
        for row in resultset:
            self.add_bookmark(row[1], row[2], row[3], row[4])

        try:
            curfp.close()
            connfp.close()
        except:
            pass


# Generic functions

def connect_server(url, fullurl=False, forced=False):
    """Connect to a server and fetch the requested page data.
    Supports gzip compression.

    If forced is True, for URLs like http://www.domain.com
    or http://www.domain.com/ path is www.domain.com or
    www.domain.com/ correspondingly.

    If fullurl is False, for URLs like http://www.domain.com/,
    path is /, else www.domain.com/.

    Params: URL to fetch, use complete url as path, force flag
    Returns: connection, HTTP(S) GET response
    """

    if url.find('%20') != -1:
        url = unquote(url).replace(' ', '%20')
    else:
        url = unquote(url)

    if debug:
        print('unquoted: %s' % url)

    if url.find('https://') >= 0: # Secure connection
        server = url[8:]
        marker = server.find('/')
        if marker > 0:
            if fullurl == False and forced == False:
                url = server[marker:]
            server = server[:marker]
        elif forced == False:           # Handle domain name without trailing /
            url = '/'
        urlconn = HTTPSConnection(server, timeout=30)
    elif url.find('http://') >= 0: # Insecure connection
        server = url[7:]
        marker = server.find('/')
        if marker > 0:
            if fullurl == False and forced == False:
                url = server[marker:]
            server = server[:marker]
        elif forced == False:
            url = '/'
        urlconn = HTTPConnection(server, timeout=30)
    else:
        printmsg('Not a valid HTTP(S) url', 'WARNING')
        if url.find(':') == -1:
            printmsg("Doesn't appear to be a valid url either", 'WARNING')
        return (None, None)

    if debug:
        print('server [%s] url [%s]' % (server, url))

    # Handle URLs passed with %xx escape
    try:
        url.encode('ascii')
    except:
        url = quote(url)

    urlconn.request('GET', url, None, {
        'Accept-encoding': 'gzip',
    })
    return (urlconn, urlconn.getresponse())


def get_page_title(resp):
    """Invoke HTML parser and extract title from HTTP response

    Params: GET response and invoke HTML parser
    """

    data = None
    charset = resp.headers.get_content_charset()

    if resp.headers.get('Content-Encoding') == 'gzip':
        if debug:
            print('gzip response')
        data = gzip.GzipFile(fileobj=io.BytesIO(resp.read())).read()
    else:
        data = resp.read()

    if charset == None:
        charset = 'utf-8'
        if debug:
            printmsg('Charset missing in response', 'WARNING')

    if debug:
        print('charset: %s' % charset)

    parser = BMHTMLParser()
    try:
        if charset == 'utf-8':
            parser.feed(data.decode(charset, 'replace'))
        else:
            parser.feed(data.decode(charset))
    except Exception as e:
        if debug and str(e) != 'we should not get here!':
            # Suppress Exception due to intentional self.reset() in HTMLParser
            print('\x1b[1mEXCEPTION\x1b[21m [get_page_title]: (%s) %s' % (type(e).__name__, e))


def network_handler(url):
    """Handle server connection and redirections

    Params: URL to fetch
    Returns: page title or empty string, if not found
    """

    global titleData
    titleData = None
    urlconn = None
    retry = False

    try:
        urlconn, resp = connect_server(url, False)

        while 1:
            if resp is None:
                break
            elif resp.status == 200:
                get_page_title(resp)
                break
            elif resp.status in [301, 302]:
                redirurl = urljoin(url, resp.getheader('location', ''))
                if debug:
                    printmsg(redirurl, 'REDIRECTION')
                retry = False       # Reset retry, start fresh on redirection

                if redirurl.find('sorry/IndexRedirect?') >= 0:          # gracefully handle Google blocks
                    printmsg('Connection blocked due to unusual activity', 'ERROR')
                    break

                marker = redirurl.find('redirectUrl=')
                if marker != -1:
                    redirurl = redirurl[marker + 12:]

                # break same URL redirection loop
                if url == redirurl:
                    printmsg('Detected repeated redirection to same URL', 'ERROR')
                    break

                url = redirurl
                urlconn.close()
                # Try with complete URL on redirection
                urlconn, resp = connect_server(url, True)
            elif resp.status == 403 and retry == False:
                """Handle URLs of the form https://www.domain.com or
                https://www.domain.com/ which fails when trying to fetch
                resource '/', retry with full path.
                """
                urlconn.close()
                if debug:
                    print('Received status 403: retrying.')
                # Remove trailing /
                if url[-1] == '/':
                    url = url[:-1]
                urlconn, resp = connect_server(url, False, True)
                retry = True
            elif resp.status == 500 and retry == False:
                """Retry on status 500 (Internal Server Error) with truncated
                URL. Some servers support truncated request URL on redirection.
                """
                urlconn.close()
                if debug:
                    print('Received status 500: retrying.')
                urlconn, resp = connect_server(url, False)
                retry = True
            else:
                printmsg(('[' + str(resp.status) + '] ' + resp.reason), 'ERROR')
                break
    except Exception as e:
        print('\x1b[1mEXCEPTION\x1b[21m [network_handler]: (%s) %s' % (type(e).__name__, e))
    finally:
        if urlconn is not None:
            urlconn.close()
        if titleData is None:
            return ''
        return titleData.strip().replace('\n','')


def parse_tags(keywords=[]):
    """Format and get tag string from tokens"""

    tags = DELIMITER
    origTags = []
    uniqueTags = []

    # Cleanse and get the tags
    tagstr = ' '.join(keywords)
    marker = tagstr.find(',')

    while marker >= 0:
        token = tagstr[0:marker]
        tagstr = tagstr[marker+1:]
        marker = tagstr.find(',')
        token = token.strip()
        if token == '':
            continue

        tags += token + ','

    tagstr = tagstr.strip()
    if tagstr != '':
        tags += tagstr + ','

    if debug:
        print(keywords)
        print('tags: [%s]' % tags)

    if tags == DELIMITER:
        return tags

    origTags.extend(tags.strip(DELIMITER).split(DELIMITER))
    for tag in origTags:
        if tag not in uniqueTags:
            uniqueTags.append(tag)  # Select unique tags

    # Sort the tags
    sortedTags = sorted(uniqueTags, key=str.lower)

    # Wrap with delimiter
    return DELIMITER + DELIMITER.join(sortedTags) + DELIMITER


def prompt(results, noninteractive=False):
    """Show each matching result from a search and prompt"""

    count = 0
    for row in results:
        count += 1
        print_record(row, count)

    if noninteractive == True:
        return

    while True:
        try:
            nav = input('Result number to open: ')
            if not nav:
                nav = input('Result number to open: ')
                if not nav:
                    # Quit on double enter
                    break
        except EOFError:
            return

        if is_int(nav):
            index = int(nav) - 1
            if index < 0 or index >= count:
                print('Index out of bound')
                continue

            try:
                browser_open(unquote(results[index][1]))
            except Exception as e:
                print('\x1b[1mEXCEPTION\x1b[21m [searchdb]: (%s) %s' % (type(e).__name__, e))
        else:
            break


def print_record(row, count=0):
    """Print a single DB record
    Handles differently for search and print (count = 0)
    """

    # Print index and URL
    if count != 0:
        print('\x1B[1m\x1B[93m%d. \x1B[0m\x1B[92m%s\x1B[0m\t[%d]' % (count, row[1], row[0]))
    else:
        print('\x1B[1m\x1B[93m%d. \x1B[0m\x1B[92m%s\x1B[0m' % (row[0], row[1]))

    # Print title
    if row[2] != '':
        print('   \x1B[91m>\x1B[0m %s' % row[2])

    # Print description
    if row[4] != '':
        print('   \x1B[91m+\x1B[0m %s' % row[4])

    # Print tags IF not default (DELIMITER)
    if row[3] != DELIMITER:
        print('   \x1B[91m#\x1B[0m %s' % row[3][1:-1])

    print('')


def format_json(resultset, single=False):
    """Return results in Json format"""

    global showOpt

    if single == False:
        marks = []
        for row in resultset:
            if showOpt == 1:
                record = { 'uri': row[1] }
            elif showOpt == 2:
                record = { 'uri': row[1], 'tags': row[3][1:-1] }
            else:
                record = { 'uri': row[1], 'title': row[2], 'description': row[4], 'tags': row[3][1:-1]}

            marks.append(record)
    else:
        marks = {}
        for row in resultset:
            if showOpt == 1:
                marks['uri'] = row[1]
            elif showOpt == 2:
                marks['uri'] = row[1]
                marks['tags']  = row[3][1:-1]
            else:
                marks['uri']   = row[1]
                marks['title'] = row[2]
                marks['description'] = row[4]
                marks['tags']  = row[3][1:-1]

    return json.dumps(marks, sort_keys=True, indent=4)


def is_int(string):
    """Check if a string is a digit

    Params: string
    """

    try:
        int(string)
        return True
    except:
        return False


def browser_open(url):
    """Duplicate stdin, stdout (to suppress showing errors
    on the terminal) and open URL in default browser

    Params: url to open
    """

    url = url.replace('%22', "\"")

    _stderr = os.dup(2)
    os.close(2)
    _stdout = os.dup(1)
    os.close(1)
    fd = os.open(os.devnull, os.O_RDWR)
    os.dup2(fd, 2)
    os.dup2(fd, 1)
    try:
        webbrowser.open(url)
    except Exception as e:
        print('\x1b[1mEXCEPTION\x1b[21m [browser_open]: (%s) %s' % (type(e).__name__, e))
    finally:
        os.close(fd)
        os.dup2(_stderr, 2)
        os.dup2(_stdout, 1)


def get_filehash(filepath):
    """Get the SHA256 hash of a file

    Params: path to the file
    """

    with open(filepath, 'rb') as f:
        hasher = hashlib.sha256()
        buf = f.read(BLOCKSIZE)
        while len(buf) > 0:
            hasher.update(buf)
            buf = f.read(BLOCKSIZE)

        return hasher.digest()


def encrypt_file(iterations):
    """Encrypt the bookmarks database file"""

    dbpath = os.path.join(BukuDb.get_dbdir_path(), 'bookmarks.db')
    encpath = dbpath + '.enc'
    if not os.path.exists(dbpath):
        print('%s missing. Already encrypted?' % dbpath)
        sys.exit(1)

    # If both encrypted file and flat file exist, error out
    if os.path.exists(dbpath) and os.path.exists(encpath):
        printmsg('Both encrypted and flat DB files exist!', 'ERROR')
        sys.exit(1)

    password = ''
    password = getpass.getpass()
    passconfirm = getpass.getpass()
    if password == '':
        print('Empty password');
        sys.exit(1)
    if password != passconfirm:
        print("Passwords don't match");
        sys.exit(1)

    # Get SHA256 hash of DB file
    dbhash = get_filehash(dbpath)

    # Generate random 256-bit salt and key
    salt = Random.get_random_bytes(SALT_SIZE)
    key = (password + salt.decode('utf-8', 'replace')).encode('utf-8')
    for i in range(iterations):
        key = hashlib.sha256(key).digest()

    iv = Random.get_random_bytes(16)
    cipher = AES.new(key, AES.MODE_CBC, iv)
    filesize = os.path.getsize(dbpath)

    with open(dbpath, 'rb') as infile:
        with open(encpath, 'wb') as outfile:
            outfile.write(struct.pack('<Q', filesize))
            outfile.write(salt)
            outfile.write(iv)

            # Embed DB file hash in encrypted file
            outfile.write(dbhash)

            while True:
                chunk = infile.read(CHUNKSIZE)
                if len(chunk) == 0:
                    break
                elif len(chunk) % 16 != 0:
                    chunk += ' ' * (16 - len(chunk) % 16)

                outfile.write(cipher.encrypt(chunk))

    os.remove(dbpath)
    print('File encrypted')
    sys.exit(0)


def decrypt_file(iterations):
    """Decrypt the bookmarks database file"""

    dbpath = os.path.join(BukuDb.get_dbdir_path(), 'bookmarks.db')
    encpath = dbpath + '.enc'
    if not os.path.exists(encpath):
        printmsg((encpath + ' missing'), 'ERROR')
        sys.exit(1)

    # If both encrypted file and flat file exist, error out
    if os.path.exists(dbpath) and os.path.exists(encpath):
        printmsg('Both encrypted and flat DB files exist!', 'ERROR')
        sys.exit(1)

    password = ''
    password = getpass.getpass()
    if password == '':
        printmsg('Decryption failed', 'ERROR');
        sys.exit(1)

    with open(encpath, 'rb') as infile:
        origsize = struct.unpack('<Q', infile.read(struct.calcsize('Q')))[0]

        # Read 256-bit salt and generate key
        salt = infile.read(32)
        key = (password + salt.decode('utf-8', 'replace')).encode('utf-8')
        for i in range(iterations):
            key = hashlib.sha256(key).digest()

        iv = infile.read(16)
        cipher = AES.new(key, AES.MODE_CBC, iv)

        # Get original DB file's SHA256 hash from encrypted file
        enchash = infile.read(32)

        with open(dbpath, 'wb') as outfile:
            while True:
                chunk = infile.read(CHUNKSIZE)
                if len(chunk) == 0:
                    break;

                outfile.write(cipher.decrypt(chunk))

            outfile.truncate(origsize)

    # Match hash of generated file with that of original DB file
    dbhash = get_filehash(dbpath)
    if dbhash != enchash:
        os.remove(dbpath)
        printmsg('Decryption failed', 'ERROR');
        sys.exit(1)
    else:
        os.remove(encpath)
        print('File decrypted')


def sigint_handler(signum, frame):
    """Custom SIGINT handler"""

    print('\nInterrupted.', file=sys.stderr)
    sys.exit(1)

signal.signal(signal.SIGINT, sigint_handler)


def printmsg(msg, level=None):
    """Print a message in 2 parts, with the level in bold

    Params: msg, level
    """

    if level is not None:
        print('\x1b[1m%s: \x1b[21m%s\x1b[0m' % (level, msg))
    else:
        print('%s' % msg)


# Custom Action classes for argparse

class CustomUpdateAction(argparse.Action):
    """Class to capture if optional param 'update'
    is actually used, even if sans arguments
    """

    def __call__(self, parser, args, values, option_string=None):
        global update

        update = True
        # NOTE: the following converts a None argument to an empty array []
        setattr(args, self.dest, values)


class CustomTagAction(argparse.Action):
    """Class to capture if optional param 'tag'
    is actually used, even if sans arguments
    """

    def __call__(self, parser, args, values, option_string=None):
        global tagManual

        tagManual = [DELIMITER,]
        setattr(args, self.dest, values)


class CustomTitleAction(argparse.Action):
    """Class to capture if optional param 'title'
    is actually used, even if sans arguments
    """

    def __call__(self, parser, args, values, option_string=None):
        global titleManual

        titleManual = ''
        setattr(args, self.dest, values)


class CustomDescAction(argparse.Action):
    """Class to capture if optional param 'comment'
    is actually used, even if sans arguments
    """

    def __call__(self, parser, args, values, option_string=None):
        global description

        description = ''
        setattr(args, self.dest, values)


class CustomTagSearchAction(argparse.Action):
    """Class to capture if optional param 'stag'
    is actually used, even if sans arguments
    """

    def __call__(self, parser, args, values, option_string=None):
        global tagsearch

        tagsearch = True
        setattr(args, self.dest, values)


class ExtendedArgumentParser(argparse.ArgumentParser):
    """Extend classic argument parser"""

    # Print additional help and info
    @staticmethod
    def print_extended_help(file=None):
        file.write('''
prompt keys:
  1-N                  open the Nth search result in web browser
  double Enter         exit buku

Version %s
Copyright (C) 2015-2016 Arun Prakash Jana <engineerarun@gmail.com>
License: GPLv3
Webpage: https://github.com/jarun/buku
''' % _VERSION_)

    # Help
    def print_help(self, file=None):
        super(ExtendedArgumentParser, self).print_help(file)
        self.print_extended_help(file)


"""main starts here"""

# Handle piped input
def main(argv = sys.argv):
    if not sys.stdin.isatty():
        pipeargs.extend(sys.argv)
        for s in sys.stdin.readlines():
            pipeargs.extend(s.split())

if __name__ == '__main__':
    try:
        main(sys.argv)
    except KeyboardInterrupt:
        pass

    # If piped input, set argument vector
    if len(pipeargs) > 0:
        sys.argv = pipeargs

    # Setup custom argument parser
    argparser = ExtendedArgumentParser(
        description='A private command-line bookmark manager. Your mini web!',
        formatter_class=argparse.RawTextHelpFormatter,
        usage='''buku [-a URL [tags ...]] [-u [N]] [-i file] [-d [N]]
            [--url keyword] [--tag [...]] [-t [...]] [-c [...]]
            [-s keyword [...]] [-S keyword [...]] [--st [...]]
            [-k [N]] [-l [N]] [-p [N]] [-f N] [-r oldtag [newtag ...]]
            [-j] [--merge file] [--noprompt] [-o N] [-z] [-h]''',
        add_help=False
    )

    # General options
    general_group = argparser.add_argument_group(title='general options',
        description='''-a, --add URL [tags ...]
                     bookmark URL with comma-separated tags
-u, --update [N]     update fields of bookmark at DB index N
                     refresh all titles, if no arguments
                     refresh title of bookmark at N, if only
                     N is specified without any edit options
-d, --delete [N]     delete bookmark at DB index N
                     delete all bookmarks, if no arguments
-i, --import file    import bookmarks from html file; Firefox,
                     Google Chrome and IE formats supported
-h, --help           show this information''')
    general_group.add_argument('-a', '--add', nargs='+', dest='addurl', metavar=('URL', 'tags'), help=argparse.SUPPRESS)
    general_group.add_argument('-u', '--update', nargs='*', dest='update', action=CustomUpdateAction, metavar=('N', 'URL tags'), help=argparse.SUPPRESS)
    general_group.add_argument('-d', '--delete', nargs='?', dest='delete', type=int, const=0, metavar='N', help=argparse.SUPPRESS)
    general_group.add_argument('-i', '--import', nargs=1, dest='imports', metavar='file', help=argparse.SUPPRESS)
    general_group.add_argument('-h', '--help', dest='help', action='store_true', help=argparse.SUPPRESS)

    # Edit options
    edit_group=argparser.add_argument_group(title='edit options',
        description='''--url keyword        specify url, works with -u only
--tag [...]          set comma-separated tags, works with -a, -u
                     clears tags, if no arguments
-t, --title [...]    manually set title, works with -a, -u
                     if no arguments:
                     -a: do not set title, -u: clear title
-c, --comment [...]  description of the bookmark, works with
                     -a, -u; clears comment, if no arguments''')
    edit_group.add_argument('--url', nargs=1, dest='url', metavar='url', help=argparse.SUPPRESS)
    edit_group.add_argument('--tag', nargs='*', dest='tag', action=CustomTagAction, metavar='tag', help=argparse.SUPPRESS)
    edit_group.add_argument('-t', '--title', nargs='*', dest='title', action=CustomTitleAction, metavar='title', help=argparse.SUPPRESS)
    edit_group.add_argument('-c', '--comment', nargs='*', dest='desc', type=str, action=CustomDescAction, metavar='desc', help=argparse.SUPPRESS)

    # Search options
    search_group=argparser.add_argument_group(title='search options',
        description='''-s, --sany keyword [...]
                     search bookmarks for ANY matching keyword
-S, --sall keyword [...]
                     search bookmarks with ALL keywords
                     special keyword -
                     "blank": list entries with empty title/tag
--st, --stag [...]   search bookmarks by tag
                     list all tags alphabetically, if no arguments''')
    search_group.add_argument('-s', '--sany', nargs='+', metavar='keyword', help=argparse.SUPPRESS)
    search_group.add_argument('-S', '--sall', nargs='+', metavar='keyword', help=argparse.SUPPRESS)
    search_group.add_argument('--st', '--stag', nargs='*', dest='stag', action=CustomTagSearchAction, metavar='keyword', help=argparse.SUPPRESS)

    # Encryption options
    crypto_group=argparser.add_argument_group(title='encryption options',
        description='''-l, --lock [N]       encrypt DB file with N (> 0, default 8)
                     hash iterations to generate key
-k, --unlock [N]     decrypt DB file with N (> 0, default 8)
                     hash iterations to generate key''')
    crypto_group.add_argument('-k', '--unlock', nargs='?', dest='decrypt', type=int, const=8, metavar='N', help=argparse.SUPPRESS)
    crypto_group.add_argument('-l', '--lock', nargs='?', dest='encrypt', type=int, const=8, metavar='N', help=argparse.SUPPRESS)

    # Power toys
    power_group=argparser.add_argument_group(title='power toys',
        description='''-p, --print [N]      show details of bookmark at DB index N
                     show all bookmarks, if no arguments
-f, --format N       modify -p output
                     N=1: show only URL, N=2: show URL and tag
-r, --replace oldtag [newtag ...]
                     replace oldtag with newtag everywhere
                     delete oldtag, if no newtag
-j, --json           Json formatted output for -p, -s, -S, --st
--merge file         merge bookmarks from another buku database
--noprompt           do not show the prompt, run and exit
-o, --open N         open bookmark at DB index N in web browser
-z, --debug          show debug information and additional logs''')
    power_group.add_argument('-p', '--print', nargs='?', dest='printindex', type=int, const=0, metavar='N', help=argparse.SUPPRESS)
    power_group.add_argument('-f', '--format', dest='showOpt', type=int, choices=[1, 2], metavar='N', help=argparse.SUPPRESS)
    power_group.add_argument('-r', '--replace', nargs='+', dest='replace', metavar=('oldtag', 'newtag'), help=argparse.SUPPRESS)
    power_group.add_argument('-j', '--json', dest='jsonOutput', action='store_true', help=argparse.SUPPRESS)
    general_group.add_argument('--merge', nargs=1, dest='merge', metavar='file', help=argparse.SUPPRESS)
    power_group.add_argument('--noprompt', dest='noninteractive', action='store_true', help=argparse.SUPPRESS)
    power_group.add_argument('-o', '--open', dest='openurl', type=int, metavar='N', help=argparse.SUPPRESS)
    power_group.add_argument('-z', '--debug', dest='debug', action='store_true', help=argparse.SUPPRESS)

    # Show help and exit if no arguments
    if len(sys.argv) < 2:
        argparser.print_help(sys.stderr)
        sys.exit(1)

    # Parse the arguments
    args = argparser.parse_args()

    # Show help and exit if help requested
    if args.help == True:
        argparser.print_help(sys.stderr)
        sys.exit(0)

    # Assign the values to globals
    if args.showOpt is not None:
        showOpt = args.showOpt
    if tagManual is not None and len(args.tag) > 0:
        tagManual = args.tag
    if titleManual is not None and len(args.title) > 0:
        titleManual = ' '.join(args.title)
    if description is not None and len(args.desc) > 0:
        description = ' '.join(args.desc)
    if args.jsonOutput:
        import json
        jsonOutput = args.jsonOutput
    noninteractive = args.noninteractive
    debug = args.debug

    # Show version in debug logs
    if debug:
        print('Version %s' % _VERSION_)

    # Move pre-1.9 database to new location
    BukuDb.move_legacy_dbfile()

    # Handle encrypt/decrypt options at top priority
    if args.encrypt is not None:
        if no_crypto:
            printmsg('PyCrypto missing', 'ERROR')
            sys.exit(1)
        if args.encrypt < 1:
            printmsg('Iterations must be >= 1', 'ERROR')
            sys.exit(1)
        encrypt_file(args.encrypt)

    if args.decrypt is not None:
        if no_crypto:
            printmsg('PyCrypto missing', 'ERROR')
            sys.exit(1)
        if args.decrypt < 1:
            printmsg('Decryption failed', 'ERROR');
            sys.exit(1)
        decrypt_file(args.decrypt)

    # Initialize the database and get handles
    bdb = BukuDb()

    # Import bookmarks
    if args.imports is not None:
        bdb.import_bookmark(args.imports[0])

    # Merge a database file and exit
    if args.merge is not None:
        bdb.mergedb(args.merge[0])

    # Add a record
    if args.addurl is not None:
        # Parse tags into a comma-separated string
        tags = DELIMITER
        keywords = args.addurl
        if tagManual is not None and not (tagManual[0] == DELIMITER and len(tagManual) == 1):
            keywords = args.addurl + [DELIMITER] + tagManual

        if len(keywords) > 1:
            tags = parse_tags(keywords[1:])

        bdb.add_bookmark(args.addurl[0], titleManual, tags, description)

    # Update record
    if update == True:
        if len(args.update) == 0:
            bdb.refreshdb(0, titleManual)
        elif not args.update[0].isdigit():
            printmsg('Index must be a number >= 0', 'ERROR')
            bdb.close_quit(1)
        elif int(args.update[0]) == 0:
            bdb.refreshdb(0, titleManual)
        else:
            if args.url is not None:
                new_url = args.url[0]
            else:
                new_url = ''

            # Parse tags into a comma-separated string
            tags = None
            if tagManual is not None and not (tagManual[0] == DELIMITER and len(tagManual) == 1):
                tags = parse_tags(tagManual)

            bdb.update_bookmark(int(args.update[0]), new_url, titleManual, tags, description)

    # Delete record(s)
    if args.delete is not None:
        if args.delete < 0:
            printmsg('Index must be >= 0', 'ERROR')
            bdb.close_quit(1)
        bdb.delete_bookmark(args.delete)

    # Search URLs, titles, tags for any keyword
    if args.sany is not None:
        bdb.searchdb(args.sany, False, jsonOutput)

    # Search URLs, titles, tags with all keywords
    if args.sall is not None:
        if args.sall[0] == 'blank' and len(args.sall) == 1:
            bdb.print_bookmark(0, True)
        else:
            bdb.searchdb(args.sall, True, jsonOutput)

    # Search bookmarks by tag
    if tagsearch == True:
        if len(args.stag) > 0:
            tag = DELIMITER + ' '.join(args.stag) + DELIMITER
            bdb.search_by_tag(tag, jsonOutput)
        else:
            bdb.list_tags()

    # Print all records
    if args.printindex is not None:
        if args.printindex < 0:
            printmsg('Index must be >= 0', 'ERROR')
            bdb.close_quit(1)
        bdb.print_bookmark(args.printindex)

    # Replace a tag in DB
    if args.replace is not None:
        if len(args.replace) == 1:
            bdb.replace_tag(args.replace[0])
        else:
            bdb.replace_tag(args.replace[0], args.replace[1:])

    # Open URL in browser
    if args.openurl is not None:
        if args.openurl < 1:
            printmsg('Index must be >= 1', 'ERROR')
            bdb.close_quit(1)
        bdb.browse_by_index(args.openurl)

    # Close DB connection and quit
    bdb.close_quit(0)