diff --git a/.travis.yml b/.travis.yml index 3988c07..b280081 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,5 +4,24 @@ python: - "3.4" - "3.5" install: "pip install -r requirements.txt" +addons: + apt: + packages: + - debhelper + - devscripts + - zsh script: - - ./tests/run_tests.sh + - ./tests/ci-test-wrapper --watch .travis.yml + - git fetch --unshallow --tags origin + - ./tools/makedeb +deploy: + provider: releases + api-key: + secure: Zf+3StERDV9B0knxNj9UdiMv9kmrE9d80a27/e7IioZv6CUvCqbIpgzN5bD3yoTlJsHq3hY6BHF8OQpkH0B0pj3xwcxgcicwDdpGA9o43aIA+zqNSb6w1VHm784KZ+Z+z1NcVNEzCyIONXEIV0KRe73NUU/7Re6heA46lPDIMFF0EL8Fjv5tPb5VLq3z0jvA8mNlXfqiwtiWT/Zz7y6PvbKQZ5nSebK0WVBdGhuaQLj9EKNwdnxkgH3gsA1gAtiuaQdgDUxF69Xf5VY6hZPhdK5LSLl/5HDpandX9nLu5j3ZuSHn1pJWgdKw72aeWYSpKtgnBQ/uS5JLamqK31kHXfRVebp0uB2I1RBiLYhb5T0MO8BnFc6O+/f2qS7nQHGKZ9M+Mo+I+ceharLmCt7KfDA1yBP+AnwjsHYe1zgnGZfwSm+/ny1R1NoVmuyXPHkEDviOsT5JLSfLvuzCUstY4gsAYyXKHLDbHfMLxXQRRfK1RoJzR4taMntmsWsl2fIshzKujeck1o4wRu/FQIlq2ANYQVNrrcDSO+C5lZkSA8iivg7lIXk/n9Lxk7QcJkvrZkzOg0y9EKAejY87vejpessG1t2OD7GwUqWZMBBlPJXnbfTiUzTJqC+b8brwnAhu/QI8jMUvxWkTMO7XOiyZBpQljv2U9MwFNH8Ge4fwIag= + file_glob: true + file: dist/*.deb + on: + repo: jarun/Buku + tags: true + # Upload from only one job (doesn't matter which one because we're packaging the same thing throughout the matrix) + python: "3.5" diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 0000000..70603b8 --- /dev/null +++ b/CHANGELOG @@ -0,0 +1,167 @@ +------------------------------------------------------------------------------- + +Buku v2.1 +2016-05-28 + +Modifications +- Import bookmarks from Firefox, Google Chrome or IE html bookmark exports +- Support comments on bookmarks +- Prettier output using symbols (`>` title, `+` comments, `#` tags) +- New option (`--st`, `--stag`) to search by tag +- New option (`--noprompt`) for noninteractive mode +- New options (`--url` and `--tag`) +- `--update` now handles each option (url, tag, title, comment) independently +- Several messages removed or moved to debug + +------------------------------------------------------------------------------- + +Buku v2.0 +2016-05-15 + +Modifications +To begin with, 2.0 is a significant release with respect to options. `Buku` now has fewer options with more (and merged) functionality. Please go through the program help at least once to understand the changes. + +- Replace getopt with argparse for parsing arguments +- Long options for each short option +- Options changed + - insert: removed as automatic DB compaction serves the purpose (previously `-i`) + - iterations: removed as optional argument to `-l` and `-k` (previously `-t`) + - title: `-t` is now the short option to set title manually (previously `-m`) + - Special search keywords for ALL search (`-S`): + - tags: show all tags (previously `-g`) + - blank: show bookmarks with empty tags (previously `-e`) + - lock/unlock: now accepts number of hash iterations to generate key + - format: print formatting option changed to `-f` (previously `-x`) + - help: option added to show program help +- Following options apply to ALL bookmarks without arguments + - `-u`, `--update` + - `-d`, `--delete` + - `-p`, `--print` +- Shell-completion scripts for Bash, Fish and Zsh +- Warn if URL is not HTTP(S) +- More comprehensive help +- Fix a bug with deletion when only one entry in DB +- Some import dependencies removed or late loaded (if optional) +- Handle exception if DB file is encrypted or invalid + +------------------------------------------------------------------------------- + +Buku 1.9 +2016-04-23 + +Modifications +- **New location for database file** (refer to README or man page). The old database file, if exists, is migrated automatically. +- **Removed options** + - `-P`: (print all) is now `-p 0` + - `-D`: (delete all) is now `-d 0` + - `-R`: (update all) is now `-u 0` + - `-w`: title web fetch is now the default behaviour, override with `-m title` option +- **Change in search behaviour** + - `-s`: search bookmarks for ANY keyword in URL, title or tags + - `-S`: search bookmarks for ALL keywords in URL, title or tags +- Update only title of a bookmark (`-u N`) +- Set empty title (`-m none`) +- Support HTTP(S) gzip compression +- Optional JSON output for `-p` and `-s` options (thanks @CaptainQuirk) +- Reformatted help and man page with general options on top +- Optimize add and insert: ensure URL is not in DB already +- Handle URLs passed with %xx escape +- Retry with truncated resource path on HTTP error 500 +- Several code optimizations +- Catchier errors and warnings +- Version added to debug logs + +------------------------------------------------------------------------------- + +Buku 1.8 +2016-03-26 + +Modifications +- Auto compact DB on single record removal +- Handle piped input +- Better tag management + - Tag modify or delete support + - Show unique tags alphabetically +- Full DB refresh + - Fix stuff broken earlier + - Optimize to update titles only + - Update titles only if non-empty to preserve earlier data +- Redirection + - Handle multiple redirections + - Detect redirection loop and break + - Show redirected link in bold +- List all bookmarks with no title or tags (for manual bookkeeping) +- Confirm full DB removal +- Better comma (`,`) separator handling for tags +- Help + - Place regular options before power options in program help + - Help added in man page for quick reference + - Additional examples for new features +- Errors & warnings + - Error out if both encrypted and flat DB files exist + - Catchier error and warning messages + +------------------------------------------------------------------------------- + +Buku 1.7 +2016-03-15 + +Modifications +- Add title manually using option `-m` +- Unquote redirected URL +- Quit on `Ctrl-d` at prompt +- More dynamic shebang for python3 + +------------------------------------------------------------------------------- + +Buku 1.6 +2016-01-22 + +Modifications +- Stronger encryption: 256-bit salt, multi-hash key. +- Allow user to specify number of iterations to generate key (check option `-t`). + +------------------------------------------------------------------------------- + +Buku 1.5 +2015-12-20 + +Modifications +- Project name changed to `Buku` to avoid any copyright issues. This also means old users have to move the database file. Run: +
$ mkdir ~/.cache/buku/ +$ mv ~/.cache/markit/bookmarks.db ~/.cache/buku/bookmarks.db +$ rm -rf ~/.cache/markit/bookmarks.db+- Manual AES256 encryption and decryption support (password protection) implemented. This adds dependency on PyCrypto module. Installation instructions updated in README. +- Some typos fixed (thanks @GuilhermeHideki) + +------------------------------------------------------------------------------- + +MarkIt v1.4 +2015-11-13 + +Modifications +- Refresh full bookmark database. Fetch titles from the web, retain tags. +- Notify empty titles in red during online add or update. + +------------------------------------------------------------------------------- + +MarkIt v1.2 +2015-11-11 + +Modifications +- Introduced `-S` search option to match ALL keywords in URL or title +- Introduced `-x` option to show unformatted selective output (for creating batch scripts) +- Added examples on batch add and update (refresh) scripts +- Handle multiple title tags in page +- Handle title data within another tag (e.g. head) +- Show DB index in search results, removal and update confirmation message + +------------------------------------------------------------------------------- + +MarkIt v1.1 +2015-11-10 + +Modifications +- Replace Unicode chars in title data before UTF-8 decoding (for parser to succeed). + +------------------------------------------------------------------------------- diff --git a/README.md b/README.md index 5bb15e6..c2d4052 100644 --- a/README.md +++ b/README.md @@ -199,7 +199,7 @@ Shell completion scripts for Bash, Fish and Zsh can be found in respective subdi - -S : match all the keywords in URL, title or tags. - --st : search bookmarks by tag, or show all tags alphabetically. - You can search bookmarks by tag (see [examples](#examples)). - - Search results are indexed serially. This index is different from actual database index of a bookmark record which is shown within `[]` after the URL. + - Search results are indexed serially. This index is different from actual database index of a bookmark record which is shown in bold within `[]` after the URL. - Auto DB compaction: when a record is deleted, the last record is moved to the index. - **Encryption** is optional and manual. AES256 algorithm is used. If you choose to use encryption, the database file should be unlocked (-k) before using buku and locked (-l) afterwards. Between these 2 operations, the database file lies unencrypted on the disk, and NOT in memory. Also, note that the database file is *unencrypted on creation*. diff --git a/buku b/buku index 83e5637..1458372 100755 --- a/buku +++ b/buku @@ -438,6 +438,8 @@ class BukuDb: def update_bookmark(self, index, url='', title_manual=None, tag_manual=None, desc=None): """ Update an existing record at index + Update all records if index is 0 and url is not specified. + URL is an exception because URLs are unique in DB. :param index: int position to update :param url: address @@ -487,7 +489,9 @@ class BukuDb: print('Title: [%s]' % meta) elif not to_update: self.refreshdb(index) - self.print_bookmark(index) + if index > 0: + self.print_bookmark(index) + return if meta is not None: query = '%s metadata = ?,' % query @@ -497,8 +501,20 @@ class BukuDb: if not to_update: # Nothing to update return - query = '%s WHERE id = ?' % query[:-1] - arguments += (index,) + if index == 0: # Update all records + if url != '': + printmsg('All URLs cannot be same', 'ERROR') + return + + resp = input('ALL bookmarks will be updated. Enter \x1b[1my\x1b[21m to confirm: ') + if resp != 'y': + return + + query = query[:-1] + else: + query = '%s WHERE id = ?' % query[:-1] + arguments += (index,) + if debug: print('query: [%s], args: [%s]' % (query, arguments)) @@ -507,19 +523,18 @@ class BukuDb: self.conn.commit() if self.cur.rowcount == 1: self.print_bookmark(index) - else: + elif self.cur.rowcount == 0: print('No matching index') except sqlite3.IntegrityError: print('URL already exists') - def refreshdb(self, index, title_manual=None): + def refreshdb(self, index): """Refresh ALL records in the database. Fetch title for each bookmark from the web and update the records. Doesn't udpate the record if title is empty. This API doesn't change DB index, URL or tags of a bookmark. :param index: index of record to update, or 0 for all records - :param title_manual: custom title """ if index == 0: @@ -528,26 +543,19 @@ class BukuDb: self.cur.execute('SELECT id, url FROM bookmarks WHERE id = ?', (index,)) resultset = self.cur.fetchall() - if title_manual is None: - for row in resultset: - title = network_handler(row[1]) - if title == '': - print('\x1b[1mIndex %d: empty title\x1b[21m\x1B[0m\n' % row[0]) - continue - else: - print('Title: [%s]' % title) + for row in resultset: + title = network_handler(row[1]) + if title == '': + print('\x1b[1mIndex %d: empty title\x1b[21m\x1B[0m\n' % row[0]) + continue + else: + print('Title: [%s]' % title) - self.cur.execute('UPDATE bookmarks SET metadata = ? WHERE id = ?', (title, row[0],)) - print('Index %d updated\n' % row[0]) - if interrupted: - printmsg('Aborting refreshdb ...', 'WARNING') - break - else: - title = title_manual - - for row in resultset: - self.cur.execute('UPDATE bookmarks SET metadata = ? WHERE id = ?', (title, row[0],)) - print('Index %d updated\n' % row[0]) + self.cur.execute('UPDATE bookmarks SET metadata = ? WHERE id = ?', (title, row[0],)) + print('Index %d updated\n' % row[0]) + if interrupted: + printmsg('Aborting refreshdb ...', 'WARNING') + break self.conn.commit() @@ -728,7 +736,7 @@ class BukuDb: def replace_tag(self, orig, new=None): """Replace orig tags with new tags in DB for all records. - Remove orig tag is new tag is empty. + Remove orig tag if new tag is empty. Params: original and new tags """ @@ -759,6 +767,7 @@ class BukuDb: newtags = DELIMITER tags = row[1].replace(orig, newtags) + tags = parse_tags([tags]) self.cur.execute('UPDATE bookmarks SET tags = ? WHERE id = ?', (tags, row[0],)) print('Index %d updated' % row[0]) update = True @@ -1183,7 +1192,7 @@ def print_record(row, count=0): # Print index and URL if count != 0: - printstr = '\x1B[1m\x1B[93m%d. \x1B[0m\x1B[92m%s\x1B[0m\t[%d]\n' % (count, row[1], row[0]) + printstr = '\x1B[1m\x1B[93m%d. \x1B[0m\x1B[92m%s\x1B[0m \x1B[1m[%s]\x1B[0m\n' % (count, row[1], row[0]) else: printstr = '\x1B[1m\x1B[93m%d. \x1B[0m\x1B[92m%s\x1B[0m\n' % (row[0], row[1]) @@ -1565,21 +1574,22 @@ if __name__ == '__main__': # Update record if update: if len(args.update) == 0: - bdb.refreshdb(0, titleManual) + update_index = 0 elif not args.update[0].isdigit(): printmsg('Index must be a number >= 0', 'ERROR') bdb.close_quit(1) - elif int(args.update[0]) == 0: - bdb.refreshdb(0, titleManual) else: - if args.url is not None: - new_url = args.url[0] - else: - new_url = '' + update_index = int(args.update[0]) - # Parse tags into a comma-separated string - tags = parse_tags(tagManual) - bdb.update_bookmark(int(args.update[0]), new_url, titleManual, tags, description) + if args.url is not None: + new_url = args.url[0] + else: + new_url = '' + + # Parse tags into a comma-separated string + tags = parse_tags(tagManual) + + bdb.update_bookmark(update_index, new_url, titleManual, tags, description) # Delete record(s) if args.delete is not None: diff --git a/buku.1 b/buku.1 index 02a4ff9..b9b5924 100644 --- a/buku.1 +++ b/buku.1 @@ -45,7 +45,7 @@ URLs are unique in DB. The same URL cannot be added twice. You can update tags a - -S : match all the keywords in URL, title or tags. - --st : search bookmarks by tag, or show all tags alphabetically. - You can search bookmarks by tag (see examples below). - - Search results are indexed serially. This index is different from actual database index of a bookmark record which is shown within '[]' after the URL. + - Search results are indexed serially. This index is different from actual database index of a bookmark record which is shown in bold within '[]' after the URL. .PP Auto DB compaction: when a record is deleted, the last record is moved to the index. .PP diff --git a/tests/ci-test-wrapper b/tests/ci-test-wrapper new file mode 100755 index 0000000..17fb500 --- /dev/null +++ b/tests/ci-test-wrapper @@ -0,0 +1,93 @@ +#!/usr/bin/env bash + +set -e + +declare here repo_root +here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +repo_root="$here/.." +export GIT_DIR="$here/../.git" + +declare -a watchlist +watchlist=(buku tests) +while [[ $1 == -* ]]; do + case $1 in + -h|--help) + cat <<'EOF' +Usage: ci-test-wrapper [-h|--help] [--monitor PATH [PATH ...]] + +buku(1) testing wrapper for CIs. + +Options: + -h, --help + Print this help and exit. + --watch PATH [PATH ...] + Additional paths (relative to repository root) to watch. Only run tests + when watched paths have been modified. By default only buku and + tests/ are watched, but sometimes additional paths should be watched + depending on circumstances, e.g., for Travis, .travis.yml should also + be watched. Note that this option consumes all of the remaining command + line arguments. +EOF + exit 1 + ;; + --watch) + shift + watchlist=( "${watchlist[@]}" "$@" ) + shift $# + break + ;; + *) + printf '\033[31mError: Unrecognized option %q.\033[0m\n' "$1" >&2 + exit 1 + ;; + esac + shift +done +(( $# > 0 )) && { + printf '\033[31mError: Unrecognized arguments %s.\033[0m\n' "$*" >&2 + exit 1 +} + +# Abort if the CI_SKIP_TEST environment variable is detected. +if [[ -n $CI_SKIP_TEST ]]; then + printf 'Detected $CI_SKIP_TEST. Skipping tests.' >&2 + exit +fi + +# Diff HEAD against a base commit to see if the changes are worth +# testing. (This check is skipped entirely if the CI_FORCE_TEST environment +# variable is set and non-nil.) +# +# * For a regular branch, diff against HEAD^; +# * For a PR branch, diff against the merge base of HEAD and master. +# +# Currently we use $TRAVIS_PULL_REQUEST to determine whether we're building a +# PR branch. Other criteria may be added if we ever expand to other CIs. + +if [[ -z $CI_FORCE_TEST ]]; then + printf 'We are watching the following paths:\n' >&2 + printf ' - %s\n' "${watchlist[@]}" >&2 + printf '\n' >&2 + + declare diff_commits diff + if [[ -z ${TRAVIS_PULL_REQUEST+x} || $TRAVIS_PULL_REQUEST == false ]]; then + diff_commits='HEAD^..HEAD' + else + diff_commits='master...HEAD' + fi + diff=$(git -C "$repo_root" diff "$diff_commits" -- "${watchlist[@]}") + if [[ -z $diff ]]; then + printf 'None of the watchlist items changed, skipping tests.\n' >&2 + printf 'You may set the $CI_FORCE_TEST environment variable to force testing.\n' >&2 + exit 0 + else + printf 'Changes to watchlist item(s) detected. Will test.\n\n' >&2 + fi +else + printf 'Detected $CI_FORCE_TEST. Skipping necessity checks.\n\n' >&2 +fi + +# Test buku(1) with $repo_root at the beginning of $PATH (so that buku +# from this repo is picked up). +cd $here +PATH="$repo_root:$PATH" python -m pytest test_*.py diff --git a/tests/run_tests.sh b/tests/run_tests.sh deleted file mode 100755 index 9245b0e..0000000 --- a/tests/run_tests.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -tests_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd $tests_dir -python -m pytest test_*.py diff --git a/tools/makedeb b/tools/makedeb new file mode 100755 index 0000000..be1fc6d --- /dev/null +++ b/tools/makedeb @@ -0,0 +1,276 @@ +#!/usr/bin/env zsh + +# Automatically make .deb package from a commit or a tag. +# +# Prerequisites: +# zsh, build-essential, devscripts, debhelper (>= 9) +# +# Reference: https://wiki.debian.org/IntroDebianPackaging. + +setopt errexit noshwordsplit nobashrematch +[[ -n $DEBUG ]] && setopt xtrace + +################### SET UP ENVIRONMENT AND BASE DIRECTORIES #################### + +[[ -z $DEBFULLNAME ]] && DEBFULLNAME='Arun Prakash Jana' +[[ -z $DEBEMAIL ]] && DEBEMAIL='engineerarun@gmail.com' +[[ -z $TZ ]] && TZ='Asia/Kolkata' +export DEBFULLNAME DEBEMAIL TZ + +here=$0:A:h +repodir=$here/.. +builddir=$here/../build +distdir=$here/../dist + +export GIT_DIR=$repodir/.git + +################################# SET UP TRAPS ################################# + +# Trap SIGUSR1: Abort program when functions called from within command +# substitutions in heredocs fail. +trap 'print_error "Encountered problem inside cmdsubst at line $LINENO."; exit 1' SIGUSR1 +export NOTIFY_PID=$$ + +############################### HELPER FUNCTIONS ############################### + +print_error () print -R $'\e[31m'"Error: $*"$'\e[0m' >&2 + +print_warning () print -R $'\e[33m'"Warning: $*"$'\e[0m' >&2 + +# Usage: apt_package_version