diff --git a/Dockerfile b/Dockerfile index 37f7f1e..94d0c26 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,11 @@ -FROM python:3.11-alpine +FROM python:3.11-slim-bullseye WORKDIR /extract COPY . . -RUN pip install -r requirements.txt +RUN apt-get update && apt-get install -y libzbar0 python3-opencv \ + && pip install -r requirements.txt WORKDIR /files diff --git a/Dockerfile_no_qr_reader b/Dockerfile_no_qr_reader new file mode 100644 index 0000000..dac895d --- /dev/null +++ b/Dockerfile_no_qr_reader @@ -0,0 +1,11 @@ +FROM python:3.11-alpine + +WORKDIR /extract + +COPY . . + +RUN pip install protobuf qrcode Pillow + +WORKDIR /files + +ENTRYPOINT [ "python", "/extract/extract_otp_secret_keys.py" ] diff --git a/README.md b/README.md index ffea8a8..55b5a54 100644 --- a/README.md +++ b/README.md @@ -303,10 +303,22 @@ Install [Docker](https://docs.docker.com/get-docker/). Build and run the app within the container: ```bash -docker build . -t extract_otp -docker run --rm -v "$(pwd)":/files:ro extract_otp -p example_export.txt +docker build . -t extract_otp_secret_keys --pull +docker run --rm -v "$(pwd)":/files:ro extract_otp_secret_keys example_export.txt +docker run --rm -v "$(pwd)":/files:ro extract_otp_secret_keys example_export.png ``` +docker run --rm -v "$(pwd)":/files:ro -i extract_otp_secret_keys = < example_export.png +docker run --entrypoint /bin/bash -it --rm -v "$(pwd)":/files:ro extract_otp_secret_keys +docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extract_otp_secret_keys + +docker build . -t extract_otp_secret_keys_no_qr_reader -f Dockerfile_no_qr_reader --pull +docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extract_otp_secret_keys_no_qr_reader +docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extract_otp_secret_keys_no_qr_reader test_extract_otp_secret_keys_pytest.py -k "not qreader" +docker run --rm -v "$(pwd)":/files:ro extract_otp_secret_keys_no_qr_reader example_export.txt +docker run --rm -v "$(pwd)":/files:ro -i extract_otp_secret_keys_no_qr_reader - < example_export.txt +docker build . -t extract_otp_secret_keys_no_qr_reader -f Dockerfile_no_qr_reader --pull && docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extract_otp_secret_keys_no_qr_reader test_extract_otp_secret_keys_pytest.py -k "not qreader" -vvv --relaxed -s + ## Tests ### PyTest diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..9c66a08 --- /dev/null +++ b/conftest.py @@ -0,0 +1,10 @@ +import pytest + + +def pytest_addoption(parser): + parser.addoption( "--relaxed", action='store_true', help="run tests in relaxed mode") + + +@pytest.fixture +def relaxed(request): + return request.config.getoption("--relaxed") diff --git a/extract_otp_secret_keys.py b/extract_otp_secret_keys.py index 14a7c4b..ff3995e 100644 --- a/extract_otp_secret_keys.py +++ b/extract_otp_secret_keys.py @@ -51,11 +51,12 @@ import re import sys import urllib.parse as urlparse -import cv2 -import numpy - import protobuf_generated_python.google_auth_pb2 +# These dynamic import are below: +# import cv2 +# import numpy +# from qreader import QReader def sys_main(): main(sys.argv[1:]) @@ -66,6 +67,7 @@ def main(sys_args): # allow to use sys.stdout with with (avoid closing) sys.stdout.close = lambda: None + # sys.stdout.reconfigure(encoding='utf-8') args = parse_args(sys_args) verbose = args.verbose if args.verbose else 0 @@ -147,7 +149,7 @@ def get_lines_from_file(filename): if filename != '=': check_file_exists(filename) lines = read_lines_from_text_file(filename) - if lines: + if lines or filename == '-': return lines # could not process text file, try reading as image @@ -166,6 +168,8 @@ def read_lines_from_text_file(filename): abort('\nBinary input was given in stdin, please use = instead of - as infile argument for images.') # unfortunately yield line leads to random test fails lines.append(line) + if not lines: + eprint("WARN: {} is empty".format(filename.replace('-', 'stdin'))) return lines except UnicodeDecodeError: if filename == '-': @@ -178,6 +182,12 @@ def read_lines_from_text_file(filename): def convert_img_to_line(filename): + try: + import cv2 + import numpy + except Exception as e: + eprint("WARNING: No cv2 or numpy module installed. Exception: {}".format(str(e))) + return [] if verbose: print('Reading image {}'.format(filename)) try: if filename != '=': @@ -188,11 +198,15 @@ def convert_img_to_line(filename): except AttributeError: # Workaround for pytest, since pytest cannot monkeypatch sys.stdin.buffer stdin = sys.stdin.read() + if not stdin: + eprint("WARN: stdin is empty") try: - array = numpy.frombuffer(stdin, dtype='uint8') + img_array = numpy.frombuffer(stdin, dtype='uint8') except TypeError as e: abort('\nERROR: Cannot read binary stdin buffer. Exception: {}'.format(str(e))) - image = cv2.imdecode(array, cv2.IMREAD_UNCHANGED) + if not img_array.size: + return [] + image = cv2.imdecode(img_array, cv2.IMREAD_UNCHANGED) if image is None: abort('\nERROR: Unable to open file for reading.\ninput file: {}'.format(filename)) diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..f9a13fe --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +markers = + qreader: QR image reader tests diff --git a/run_pytest.sh b/run_pytest.sh new file mode 100755 index 0000000..e77ffae --- /dev/null +++ b/run_pytest.sh @@ -0,0 +1,3 @@ +#!/bin/sh +cd /extract +pip install -U pytest && pytest "$@" diff --git a/test/empty_file.txt b/test/empty_file.txt new file mode 100644 index 0000000..e69de29 diff --git a/test_extract_otp_secret_keys_pytest.py b/test_extract_otp_secret_keys_pytest.py index 9cc6a66..4a2648f 100644 --- a/test_extract_otp_secret_keys_pytest.py +++ b/test_extract_otp_secret_keys_pytest.py @@ -20,14 +20,13 @@ import io import os +import re import sys -from pytest import mark, raises +import pytest import extract_otp_secret_keys -from utils import (file_exits, read_binary_file_as_stream, read_csv, - read_csv_str, read_file_to_str, read_json, read_json_str, - remove_dir_with_files, remove_files) +from utils import * def test_extract_stdout(capsys): @@ -41,6 +40,7 @@ def test_extract_stdout(capsys): assert captured.err == '' +@pytest.mark.qreader def test_extract_multiple_files_and_mixed(capsys): # Act extract_otp_secret_keys.main([ @@ -58,7 +58,7 @@ def test_extract_multiple_files_and_mixed(capsys): def test_extract_non_existent_file(capsys): # Act - with raises(SystemExit) as e: + with pytest.raises(SystemExit) as e: extract_otp_secret_keys.main(['test/non_existent_file.txt']) # Assert @@ -86,12 +86,58 @@ def test_extract_stdin_stdout(capsys, monkeypatch): assert captured.err == '' +def test_extract_stdin_empty(capsys, monkeypatch): + # Arrange + monkeypatch.setattr('sys.stdin', io.StringIO()) + + # Act + extract_otp_secret_keys.main(['-']) + + # Assert + captured = capsys.readouterr() + + assert captured.out == '' + assert captured.err == 'WARN: stdin is empty\n' + + +def test_extract_empty_file(capsys): + # Act + with pytest.raises(SystemExit) as e: + extract_otp_secret_keys.main(['test/empty_file.txt']) + + # Assert + captured = capsys.readouterr() + + expected_stderr = 'WARN: test/empty_file.txt is empty\n\nERROR: Unable to open file for reading.\ninput file: test/empty_file.txt\n' + + assert captured.err == expected_stderr + assert captured.out == '' + assert e.value.code == 1 + assert e.type == SystemExit + + +@pytest.mark.qreader +def test_extract_stdin_img_empty(capsys, monkeypatch): + # Arrange + monkeypatch.setattr('sys.stdin', io.BytesIO()) + + # Act + extract_otp_secret_keys.main(['=']) + + # Assert + captured = capsys.readouterr() + + assert captured.out == '' + assert captured.err == 'WARN: stdin is empty\n' + + +@pytest.mark.qreader def test_extract_stdin_stdout_wrong_symbol(capsys, monkeypatch): # Arrange monkeypatch.setattr('sys.stdin', io.StringIO(read_file_to_str('example_export.txt'))) # Act - with raises(SystemExit) as e: + with pytest.raises(SystemExit) as e: extract_otp_secret_keys.main(['=']) # Assert @@ -359,8 +405,11 @@ def test_extract_saveqr(capsys): cleanup() -@mark.skipif(sys.implementation.name == 'pypy', reason="Encoding problems in verbose mode in pypy.") -def test_extract_verbose(capsys): +def test_normalize_bytes(): + assert replace_escaped_octal_utf8_bytes_with_str('Before\\\\302\\\\277\\\\303\nname: enc: \\302\\277\\303\\244\\303\\204\\303\\251\\303\\211?\nAfter') == 'Before\\\\302\\\\277\\\\303\nname: enc: ¿äÄéÉ?\nAfter' + + +def test_extract_verbose(capsys, relaxed): # Act extract_otp_secret_keys.main(['-v', 'example_export.txt']) @@ -369,7 +418,13 @@ def test_extract_verbose(capsys): expected_stdout = read_file_to_str('test/print_verbose_output.txt') - assert captured.out == expected_stdout + if relaxed or sys.implementation.name == 'pypy': + print('\nRelaxed mode\n') + + assert replace_escaped_octal_utf8_bytes_with_str(captured.out) == replace_escaped_octal_utf8_bytes_with_str(expected_stdout) + assert quick_and_dirty_workaround_encoding_problem(captured.out) == quick_and_dirty_workaround_encoding_problem(expected_stdout) + else: + assert captured.out == expected_stdout assert captured.err == '' @@ -388,7 +443,7 @@ def test_extract_debug(capsys): def test_extract_help(capsys): - with raises(SystemExit) as e: + with pytest.raises(SystemExit) as e: # Act extract_otp_secret_keys.main(['-h']) @@ -404,7 +459,7 @@ def test_extract_help(capsys): def test_extract_no_arguments(capsys): # Act - with raises(SystemExit) as e: + with pytest.raises(SystemExit) as e: extract_otp_secret_keys.main([]) # Assert @@ -419,7 +474,7 @@ def test_extract_no_arguments(capsys): def test_verbose_and_quiet(capsys): - with raises(SystemExit) as e: + with pytest.raises(SystemExit) as e: # Act extract_otp_secret_keys.main(['-v', '-q', 'example_export.txt']) @@ -434,7 +489,7 @@ def test_verbose_and_quiet(capsys): def test_wrong_data(capsys): - with raises(SystemExit) as e: + with pytest.raises(SystemExit) as e: # Act extract_otp_secret_keys.main(['test/test_export_wrong_data.txt']) @@ -453,7 +508,7 @@ data=XXXX def test_wrong_content(capsys): - with raises(SystemExit) as e: + with pytest.raises(SystemExit) as e: # Act extract_otp_secret_keys.main(['test/test_export_wrong_content.txt']) @@ -509,6 +564,7 @@ def test_add_pre_suffix(capsys): assert extract_otp_secret_keys.add_pre_suffix("name", "totp") == "name.totp" +@pytest.mark.qreader def test_img_qr_reader_from_file_happy_path(capsys): # Act extract_otp_secret_keys.main(['test/test_googleauth_export.png']) @@ -519,7 +575,7 @@ def test_img_qr_reader_from_file_happy_path(capsys): assert captured.out == EXPECTED_STDOUT_FROM_EXAMPLE_EXPORT_PNG assert captured.err == '' - +@pytest.mark.qreader def test_img_qr_reader_from_stdin(capsys, monkeypatch): # Arrange # sys.stdin.buffer should be monkey patched, but it does not work @@ -553,13 +609,14 @@ Type: totp assert captured.err == '' +@pytest.mark.qreader def test_img_qr_reader_from_stdin_wrong_symbol(capsys, monkeypatch): # Arrange # sys.stdin.buffer should be monkey patched, but it does not work monkeypatch.setattr('sys.stdin', read_binary_file_as_stream('test/test_googleauth_export.png')) # Act - with raises(SystemExit) as e: + with pytest.raises(SystemExit) as e: extract_otp_secret_keys.main(['-']) # Assert @@ -573,9 +630,10 @@ def test_img_qr_reader_from_stdin_wrong_symbol(capsys, monkeypatch): assert e.type == SystemExit +@pytest.mark.qreader def test_img_qr_reader_no_qr_code_in_image(capsys): # Act - with raises(SystemExit) as e: + with pytest.raises(SystemExit) as e: extract_otp_secret_keys.main(['test/lena_std.tif']) # Assert @@ -589,9 +647,10 @@ def test_img_qr_reader_no_qr_code_in_image(capsys): assert e.type == SystemExit +@pytest.mark.qreader def test_img_qr_reader_nonexistent_file(capsys): # Act - with raises(SystemExit) as e: + with pytest.raises(SystemExit) as e: extract_otp_secret_keys.main(['test/nonexistent.bmp']) # Assert @@ -607,7 +666,7 @@ def test_img_qr_reader_nonexistent_file(capsys): def test_non_image_file(capsys): # Act - with raises(SystemExit) as e: + with pytest.raises(SystemExit) as e: extract_otp_secret_keys.main(['test/text_masquerading_as_image.jpeg']) # Assert diff --git a/utils.py b/utils.py index ffd86e5..ddcf313 100644 --- a/utils.py +++ b/utils.py @@ -14,12 +14,13 @@ # along with this program. If not, see . import csv +import glob +import io import json import os +import re import shutil -import io import sys -import glob # Ref. https://stackoverflow.com/a/16571630 @@ -107,3 +108,17 @@ def read_binary_file_as_stream(filename): """Returns binary file content.""" with open(filename, "rb",) as infile: return io.BytesIO(infile.read()) + +def replace_escaped_octal_utf8_bytes_with_str(str): + encoded_name_strings = re.findall(r'name: .*$', str, flags=re.MULTILINE) + for encoded_name_string in encoded_name_strings: + escaped_bytes = re.findall(r'((?:\\[0-9]+)+)', encoded_name_string) + for byte_sequence in escaped_bytes: + unicode_str = b''.join([int(byte, 8).to_bytes(1) for byte in byte_sequence.split('\\') if byte]).decode('utf-8') + print("Replace '{}' by '{}'".format(byte_sequence, unicode_str)) + str = str.replace(byte_sequence, unicode_str) + return str + + +def quick_and_dirty_workaround_encoding_problem(str): + return re.sub(r'name: "encoding: .*$', '', str, flags=re.MULTILINE)