diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml new file mode 100644 index 0000000..af87459 --- /dev/null +++ b/.github/workflows/ci_docker.yml @@ -0,0 +1,56 @@ +name: "Docker: build and publish" + +# How to setup: https://event-driven.io/en/how_to_buid_and_push_docker_image_with_github_actions/ +# How to run: https://aschmelyun.com/blog/using-docker-run-inside-of-github-actions/ + +on: + # run it on push to the default repository branch + push: + # branches: [master] + # run it during pull request + # pull_request: + +jobs: + # define job to build and publish docker image + build-and-push-docker-image: + name: Build Docker image and push to repositories + # run only when code is compiling and tests are passing + runs-on: ubuntu-latest + + # steps to perform in job + steps: + - name: Checkout code + uses: actions/checkout@v3 + + # setup Docker build action + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to Github Packages + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GHCR_IO_TOKEN }} + + - name: Build image and push to Docker Hub and GitHub Container Registry + uses: docker/build-push-action@v2 + with: + # relative path to the place where source code with Dockerfile is located + context: . + # Note: tags has to be all lower-case + tags: | + scito/extract_otp_secret_keys:latest + ghcr.io/scito/extract_otp_secret_keys:latest + # build on feature branches, push only on master branch + push: ${{ github.ref == 'refs/heads/master' }} + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml deleted file mode 100644 index d80bbcb..0000000 --- a/.github/workflows/nightly.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: nightly tests - -on: - schedule: - - cron: '47 3 * * *' - -jobs: - build: - - strategy: - matrix: - python-version: ["3.x", "3.11", "3.10", "3.9", "pypy-3.9", "3.8", "pypy-3.8", "3.7", "pypy-3.7"] - platform: [ubuntu-latest, macos-latest, windows-latest] - exclude: - - platform: windows-latest - - python-version: [pypy-3.9] - - runs-on: ${{ matrix.platform }} - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install zbar shared lib for QReader (Linux) - if: runner.os == 'Linux' - run: | - sudo apt-get install -y libzbar0 - - name: Install zbar shared lib for QReader (macOS) - if: runner.os == 'macOS' - run: | - brew install zbar - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 pytest - pip install --use-pep517 -r requirements.txt - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=200 --statistics - - name: Test with pytest - run: pytest diff --git a/Dockerfile b/Dockerfile index 94d0c26..f62aea2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,9 +4,10 @@ WORKDIR /extract COPY . . -RUN apt-get update && apt-get install -y libzbar0 python3-opencv \ - && pip install -r requirements.txt +RUN apt-get update && apt-get install -y libzbar0 python3-opencv nano \ + && pip install -r requirements.txt \ + && /extract/run_pytest.sh WORKDIR /files -ENTRYPOINT [ "python", "/extract/extract_otp_secret_keys.py" ] +ENTRYPOINT ["python", "/extract/extract_otp_secret_keys.py"] diff --git a/Dockerfile_no_qr_reader b/Dockerfile_no_qr_reader index dac895d..e443733 100644 --- a/Dockerfile_no_qr_reader +++ b/Dockerfile_no_qr_reader @@ -4,8 +4,9 @@ WORKDIR /extract COPY . . -RUN pip install protobuf qrcode Pillow +RUN pip install protobuf qrcode Pillow \ + && /extract/run_pytest.sh test_extract_otp_secret_keys_pytest.py -k "not qreader" --relaxed WORKDIR /files -ENTRYPOINT [ "python", "/extract/extract_otp_secret_keys.py" ] +ENTRYPOINT ["python", "/extract/extract_otp_secret_keys.py"] diff --git a/Pipfile.lock b/Pipfile.lock index ff5c753..6a11ca0 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -18,37 +18,37 @@ "default": { "numpy": { "hashes": [ - "sha256:0104d8adaa3a4cc60c2777cab5196593bf8a7f416eda133be1f3803dd0838886", - "sha256:0885d9a7666cafe5f9876c57bfee34226e2b2847bfb94c9505e18d81011e5401", - "sha256:12bba5561d8118981f2f1ff069ecae200c05d7b6c78a5cdac0911f74bc71cbd1", - "sha256:2f8e0df2ecc1928ef7256f18e309c9d6229b08b5be859163f5caa59c93d53646", - "sha256:4445f472b246cad6514cc09fbb5ecb7aab09ca2acc3c16f29f8dca6c468af501", - "sha256:4d01f7832fa319a36fd75ba10ea4027c9338ede875792f7bf617f4b45056fc3a", - "sha256:4f5e78b8b710cd7cd1a8145994cfffc6ddd5911669a437777d8cedfce6c83a98", - "sha256:667b5b1f6a352419e340f6475ef9930348ae5cb7fca15f2cc3afcb530823715e", - "sha256:6e73a1f4f5b74a42abb55bc2b3d869f1b38cbc8776da5f8b66bf110284f7a437", - "sha256:73cf2c5b5a07450f20a0c8e04d9955491970177dce8df8d6903bf253e53268e0", - "sha256:7ad6a024a32ee61d18f5b402cd02e9c0e22c0fb9dc23751991b3a16d209d972e", - "sha256:8b1ddfac6a82d4f3c8e99436c90b9c2c68c0bb14658d1684cdd00f05fab241f5", - "sha256:90075ef2c6ac6397d0035bcd8b298b26e481a7035f7a3f382c047eb9c3414db0", - "sha256:9387c7d6d50e8f8c31e7bfc034241e9c6f4b3eb5db8d118d6487047b922f82af", - "sha256:9af91f794d2d3007d91d749ebc955302889261db514eb24caef30e03e8ec1e41", - "sha256:ab11f6a7602cf8ea4c093e091938207de3068c5693a0520168ecf4395750f7ea", - "sha256:ac4fe68f1a5a18136acebd4eff91aab8bed00d1ef2fdb34b5d9192297ffbbdfc", - "sha256:ada6c1e9608ceadaf7020e1deea508b73ace85560a16f51bef26aecb93626a72", - "sha256:c4ab7c9711fe6b235e86487ca74c1b092a6dd59a3cb45b63241ea0a148501853", - "sha256:cec79ff3984b2d1d103183fc4a3361f5b55bbb66cb395cbf5a920a4bb1fd588d", - "sha256:cf8960f72997e56781eb1c2ea256a70124f92a543b384f89e5fb3503a308b1d3", - "sha256:d7f223554aba7280e6057727333ed357b71b7da7422d02ff5e91b857888c25d1", - "sha256:dbb0490f0a880700a6cc4d000384baf19c1f4df59fff158d9482d4dbbca2b239", - "sha256:e63d2157f9fc98cc178870db83b0e0c85acdadd598b134b00ebec9e0db57a01f", - "sha256:ec3e5e8172a0a6a4f3c2e7423d4a8434c41349141b04744b11a90e017a95bad5", - "sha256:f3c4a9a9f92734a4728ddbd331e0124eabbc968a0359a506e8e74a9b0d2d419b", - "sha256:f9168790149f917ad8e3cf5047b353fefef753bd50b07c547da0bdf30bc15d91", - "sha256:fe44e925c68fb5e8db1334bf30ac1a1b6b963b932a19cf41d2e899cf02f36aab" + "sha256:0044f7d944ee882400890f9ae955220d29b33d809a038923d88e4e01d652acd9", + "sha256:0e3463e6ac25313462e04aea3fb8a0a30fb906d5d300f58b3bc2c23da6a15398", + "sha256:179a7ef0889ab769cc03573b6217f54c8bd8e16cef80aad369e1e8185f994cd7", + "sha256:2386da9a471cc00a1f47845e27d916d5ec5346ae9696e01a8a34760858fe9dd2", + "sha256:26089487086f2648944f17adaa1a97ca6aee57f513ba5f1c0b7ebdabbe2b9954", + "sha256:28bc9750ae1f75264ee0f10561709b1462d450a4808cd97c013046073ae64ab6", + "sha256:28e418681372520c992805bb723e29d69d6b7aa411065f48216d8329d02ba032", + "sha256:442feb5e5bada8408e8fcd43f3360b78683ff12a4444670a7d9e9824c1817d36", + "sha256:6ec0c021cd9fe732e5bab6401adea5a409214ca5592cd92a114f7067febcba0c", + "sha256:7094891dcf79ccc6bc2a1f30428fa5edb1e6fb955411ffff3401fb4ea93780a8", + "sha256:84e789a085aabef2f36c0515f45e459f02f570c4b4c4c108ac1179c34d475ed7", + "sha256:87a118968fba001b248aac90e502c0b13606721b1343cdaddbc6e552e8dfb56f", + "sha256:8e669fbdcdd1e945691079c2cae335f3e3a56554e06bbd45d7609a6cf568c700", + "sha256:ad2925567f43643f51255220424c23d204024ed428afc5aad0f86f3ffc080086", + "sha256:b0677a52f5d896e84414761531947c7a330d1adc07c3a4372262f25d84af7bf7", + "sha256:b07b40f5fb4fa034120a5796288f24c1fe0e0580bbfff99897ba6267af42def2", + "sha256:b09804ff570b907da323b3d762e74432fb07955701b17b08ff1b5ebaa8cfe6a9", + "sha256:b162ac10ca38850510caf8ea33f89edcb7b0bb0dfa5592d59909419986b72407", + "sha256:b31da69ed0c18be8b77bfce48d234e55d040793cebb25398e2a7d84199fbc7e2", + "sha256:caf65a396c0d1f9809596be2e444e3bd4190d86d5c1ce21f5fc4be60a3bc5b36", + "sha256:cfa1161c6ac8f92dea03d625c2d0c05e084668f4a06568b77a25a89111621566", + "sha256:dae46bed2cb79a58d6496ff6d8da1e3b95ba09afeca2e277628171ca99b99db1", + "sha256:ddc7ab52b322eb1e40521eb422c4e0a20716c271a306860979d450decbb51b8e", + "sha256:de92efa737875329b052982e37bd4371d52cabf469f83e7b8be9bb7752d67e51", + "sha256:e274f0f6c7efd0d577744f52032fdd24344f11c5ae668fe8d01aac0422611df1", + "sha256:ed5fb71d79e771ec930566fae9c02626b939e37271ec285e9efaf1b5d4370e7d", + "sha256:ef85cf1f693c88c1fd229ccd1055570cb41cdf4875873b7728b6301f12cd05bf", + "sha256:f1b739841821968798947d3afcefd386fa56da0caf97722a5de53e07c4ccedc7" ], "markers": "python_version >= '3.10'", - "version": "==1.24.0" + "version": "==1.24.1" }, "opencv-python": { "hashes": [ diff --git a/extract_otp_secret_keys.py b/extract_otp_secret_keys.py index ff3995e..f467783 100644 --- a/extract_otp_secret_keys.py +++ b/extract_otp_secret_keys.py @@ -45,6 +45,7 @@ import argparse import base64 import csv import fileinput +import importlib import json import os import re @@ -63,12 +64,13 @@ def sys_main(): def main(sys_args): - global verbose, quiet + global verbose, quiet, qreader_available # allow to use sys.stdout with with (avoid closing) sys.stdout.close = lambda: None # sys.stdout.reconfigure(encoding='utf-8') + args = parse_args(sys_args) verbose = args.verbose if args.verbose else 0 quiet = args.quiet @@ -80,8 +82,15 @@ def main(sys_args): def parse_args(sys_args): - formatter = lambda prog: argparse.HelpFormatter(prog, max_help_position=52) - arg_parser = argparse.ArgumentParser(formatter_class=formatter) + formatter = lambda prog: argparse.RawDescriptionHelpFormatter(prog, max_help_position=52) + example_text = '''examples: +python extract_otp_secret_keys.py example_*.txt +python extract_otp_secret_keys.py - < example_export.txt +python extract_otp_secret_keys.py --csv - example_*.png | tail -n+2 +python extract_otp_secret_keys.py = < example_export.png''' + + arg_parser = argparse.ArgumentParser(formatter_class=formatter, + epilog=example_text) arg_parser.add_argument('infile', help='1) file or - for stdin with "otpauth-migration://..." URLs separated by newlines, lines starting with # are ignored; or 2) image file containing a QR code or = for stdin for an image containing a QR code', nargs='+') arg_parser.add_argument('--json', '-j', help='export json file or - for stdout', metavar=('FILE')) arg_parser.add_argument('--csv', '-c', help='export csv file or - for stdout', metavar=('FILE')) @@ -145,6 +154,7 @@ def extract_otps(args): def get_lines_from_file(filename): + global qreader_available # stdin stream cannot be rewinded, thus distinguish, use - for utf-8 stdin and = for binary image stdin if filename != '=': check_file_exists(filename) @@ -422,6 +432,11 @@ def is_binary(line): return True +def check_module_available(module_name): + module_spec = importlib.util.find_spec(module_name) + return module_spec is not None + + def eprint(*args, **kwargs): '''Print to stderr.''' print(*args, file=sys.stderr, **kwargs) diff --git a/test_extract_otp_secret_keys_pytest.py b/test_extract_otp_secret_keys_pytest.py index 4a2648f..0031971 100644 --- a/test_extract_otp_secret_keys_pytest.py +++ b/test_extract_otp_secret_keys_pytest.py @@ -28,6 +28,8 @@ import pytest import extract_otp_secret_keys from utils import * +qreader_available = extract_otp_secret_keys.check_module_available('cv2') + def test_extract_stdout(capsys): # Act @@ -100,20 +102,31 @@ def test_extract_stdin_empty(capsys, monkeypatch): assert captured.err == 'WARN: stdin is empty\n' -def test_extract_empty_file(capsys): - # Act - with pytest.raises(SystemExit) as e: +# @pytest.mark.skipif(not qreader_available, reason='Test if cv2 and qreader are not available.') +def test_extract_empty_file_no_qreader(capsys): + if qreader_available: + # Act + with pytest.raises(SystemExit) as e: + extract_otp_secret_keys.main(['test/empty_file.txt']) + + # Assert + captured = capsys.readouterr() + + expected_stderr = 'WARN: test/empty_file.txt is empty\n\nERROR: Unable to open file for reading.\ninput file: test/empty_file.txt\n' + + assert captured.err == expected_stderr + assert captured.out == '' + assert e.value.code == 1 + assert e.type == SystemExit + else: + # Act extract_otp_secret_keys.main(['test/empty_file.txt']) - # Assert - captured = capsys.readouterr() + # Assert + captured = capsys.readouterr() - expected_stderr = 'WARN: test/empty_file.txt is empty\n\nERROR: Unable to open file for reading.\ninput file: test/empty_file.txt\n' - - assert captured.err == expected_stderr - assert captured.out == '' - assert e.value.code == 1 - assert e.type == SystemExit + assert captured.err == '' + assert captured.out == '' @pytest.mark.qreader diff --git a/upgrade_deps.sh b/upgrade_deps.sh index 65fe5d1..339edc2 100755 --- a/upgrade_deps.sh +++ b/upgrade_deps.sh @@ -128,45 +128,45 @@ if [ "$OLDVERSION" != "$VERSION" ]; then mkdir -p $DOWNLOADS # https://github.com/protocolbuffers/protobuf/releases/download/v21.6/protoc-21.6-linux-x86_64.zip cmd="wget --trust-server-names https://github.com/protocolbuffers/protobuf/releases/download/v$VERSION/protoc-$VERSION-linux-x86_64.zip -O $DOWNLOADS/$ARCHIVE" - if $interactive ; then askContinueYn "$cmd"; fi + if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" cmd="echo -e '\nSize [Byte]'; stat --printf='%s\n' $DOWNLOADS/$ARCHIVE; echo -e '\nMD5'; md5sum $DOWNLOADS/$ARCHIVE; echo -e '\nSHA256'; sha256sum $DOWNLOADS/$ARCHIVE;" - if $interactive ; then askContinueYn "$cmd"; fi + if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" cmd="mkdir -p $BIN/$NAME; unzip $DOWNLOADS/$ARCHIVE -d $BIN/$NAME" - if $interactive ; then askContinueYn "$cmd"; fi + if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" cmd="echo $VERSION > $BIN/$NAME/.VERSION.txt; echo $VERSION > $BIN/$NAME/.VERSION_$VERSION.txt" - if $interactive ; then askContinueYn "$cmd"; fi + if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" cmd="[ -d $BIN/$DEST.old ] && rm -rf $BIN/$DEST.old || echo 'No old dir to delete'" - if $interactive ; then askContinueYn "$cmd"; fi + if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" cmd="[ -d $BIN/$DEST ] && mv -iT $BIN/$DEST $BIN/$DEST.old || echo 'No previous dir to keep'" - if $interactive ; then askContinueYn "$cmd"; fi + if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" cmd="mv -iT $BIN/$NAME $BIN/$DEST" - if $interactive ; then askContinueYn "$cmd"; fi + if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" cmd="rm $DOWNLOADS/$ARCHIVE" - if $interactive ; then askContinueYn "$cmd"; fi + if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" cmd="$BIN/$DEST/bin/protoc --python_out=protobuf_generated_python google_auth.proto" - if $interactive ; then askContinueYn "$cmd"; fi + if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" # Update README.md cmd="perl -i -pe 's%proto(buf|c)([- ])(\d\.)?$OLDVERSION%proto\$1\$2\${3}$VERSION%g' README.md && perl -i -pe 's%(protobuf/releases/tag/v)$OLDVERSION%\${1}$VERSION%g' README.md" - if $interactive ; then askContinueYn "$cmd"; fi + if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" else echo -e "\nVersion has not changed. Quit" @@ -176,27 +176,27 @@ fi # Upgrade pip requirements cmd="sudo pip install --upgrade pip" -if $interactive ; then askContinueYn "$cmd"; fi +if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" $PIP --version cmd="$PIP install --use-pep517 -U -r requirements.txt" -if $interactive ; then askContinueYn "$cmd"; fi +if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" cmd="$PIP install --use-pep517 -U -r requirements-dev.txt" -if $interactive ; then askContinueYn "$cmd"; fi +if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" cmd="$PIP install -U pipenv" -if $interactive ; then askContinueYn "$cmd"; fi +if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" $PIPENV --version cmd="$PIPENV update && $PIPENV --rm && $PIPENV install" -if $interactive ; then askContinueYn "$cmd"; fi +if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" $PIPENV run python --version @@ -204,11 +204,33 @@ $PIPENV run python --version # Test cmd="pytest" -if $interactive ; then askContinueYn "$cmd"; fi +if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" cmd="$PIPENV run pytest" -if $interactive ; then askContinueYn "$cmd"; fi +if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi +eval "$cmd" + +# Build docker + +cmd="docker build . -t extract_otp_secret_keys_no_qr_reader -f Dockerfile_no_qr_reader --pull" +if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi +eval "$cmd" + +cmd="docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extract_otp_secret_keys_no_qr_reader test_extract_otp_secret_keys_pytest.py -k 'not qreader' -vvv --relaxed" +if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi +eval "$cmd" + +cmd="docker build . -t extract_otp_secret_keys --pull" +if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi +eval "$cmd" + +cmd="docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extract_otp_secret_keys" +if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi +eval "$cmd" + +cmd="docker image prune" +if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" quit